agent-gauntlet 0.9.0 → 0.10.1

package/README.md

@@ -11,30 +11,17 @@
 
 Agent Gauntlet is a configurable “feedback loop” runner for AI-assisted development workflows.
 
-You configure which paths in your repo should trigger which validations — shell commands like tests and linters, plus AI-powered code reviews. When files change, Gauntlet automatically runs the relevant validations and reports results.
+You configure which paths in your repo should trigger which validations — shell commands like tests and linters, plus AI-powered local code reviews. When files change, Gauntlet automatically runs the relevant validations and reports results.
 
 For AI reviews, it uses the CLI tool of your choice: Gemini, Codex, Claude Code, GitHub Copilot, or Cursor. 
 
 ## Features
 
 - **Agent validation loop**: Keep your coding agent on track with automated feedback loops. Detect problems — deterministically and/or non-deterministically — and let your agent fix and Gauntlet verify.
-- **Multi-agent collaboration**: Enable one AI agent to automatically request code reviews from another. For example, if Claude made changes, Gauntlet can request a review from Codex or Gemini — spreading token usage across your subscriptions instead of burning through one.
+- **Local cross-agent code reviews**: Enable one AI agent to automatically request code reviews from another. For example, if Claude made changes, Gauntlet can request a review from Codex — spreading token usage across your subscriptions instead of burning through one.
+  - Multiple AI review adapters have been evaluated for quality and efficiency. Claude and Codex deliver optimal review quality with superior token efficiency. For detailed metrics, see [Eval Results](docs/eval-results.md).
 - **Leverage existing subscriptions**: Agent Gauntlet is *free* and tool-agnostic, leveraging the AI CLI tools you already have installed.
-- **Easy CI setup**: Define your checks once, run them locally and in GitHub.
-
-### vs AI Code Review Tools
-
-Unlike traditional code review tools designed for PR workflows, Agent Gauntlet provides real-time feedback loops for autonomous coding agents.
-
-| Use Case | Recommended |
-| :--- | :--- |
-| Autonomous agent development | **Agent Gauntlet** |
-| Traditional PR review with human reviewers | Other tools |
-| IDE-integrated review while coding | Other tools |
-| Enterprise with strict compliance requirements | Other tools |
-| Budget-conscious teams with existing AI CLI tools | **Agent Gauntlet** |
-
-[Full comparison →](docs/feature_comparison.md)
+- **Easy CI setup**: Define your CI gates once, run them locally and in GitHub.
 
 ## Common Workflows
 
@@ -42,15 +29,30 @@ Agent Gauntlet supports three workflows, ranging from simple CLI execution to fu
 
 - **CLI Mode** — Run checks via command line; ideal for CI pipelines and scripts.
 - **Assistant Mode** — AI assistant runs validation loop, fixing issues iteratively.
-- **Agentic Mode** — Autonomous agent validates and fixes in real-time via stop hook.
+- **Agentic Mode** — Autonomous agent validates and fixes in real-time via stop hook (experimental).
 
 ![Agent Gauntlet Workflows](docs/images/workflows.png)
 
-## Quick Start
+### Example Workflow
+
+1. Claude implements a feature
+2. Agent Gauntlet reports quality issues detected by static code analysis and Codex reviewer agent
+3. Claude fixes issues
+4. Agent Gauntlet verifies
+
+### Comparison vs Other Tools
 
-1. **Install**: `bun add -g agent-gauntlet`
-2. **Initialize**: `agent-gauntlet init`
-3. **Run**: `agent-gauntlet run`
+### AI Code Review Tools
+
+Agent Gauntlet is not a replacement for tools that provide AI pull request code reviews. It provides real-time feedback loops for autonomous coding agents, combining deterministic static checks (build, lint, test) with multi-agent AI reviews in a single pipeline. This enables agents to iterate and self-correct until all checks and reviews pass, without human intervention.
+
+[Full comparison →](docs/feature_comparison.md)
+
+### Spec-Driven Workflow Tools
+
+It is recommended to use Agent Gauntlet in conjunction with other spec-driven development tools. We believe is the ideal implementation of the validation step in any Spec → Implement → Validate workflow.
+
+## Quick Start
 
 For basic usage and configuration guide, see the [Quick Start Guide](docs/quick-start.md).
 
@@ -59,7 +61,7 @@ For basic usage and configuration guide, see the [Quick Start Guide](docs/quick-
 - [Quick Start Guide](docs/quick-start.md) — installation, basic usage, and config layout
 - [User Guide](docs/user-guide.md) — full usage details
 - [Configuration Reference](docs/config-reference.md) — all configuration fields + defaults
-- [Stop Hook Guide](docs/stop-hook-guide.md) — integrate with Claude Code's stop hook
+- [Stop Hook Guide](docs/stop-hook-guide.md) — integrate with Claude Code's stop hook (experimental).
 - [CLI Invocation Details](docs/cli-invocation-details.md) — how we securely invoke AI CLIs
 - [Feature Comparison](docs/feature_comparison.md) — how Agent Gauntlet compares to other tools
 - [Development Guide](docs/development.md) — how to build and develop this project

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-gauntlet",
-  "version": "0.9.0",
+  "version": "0.10.1",
   "description": "A CLI tool for testing AI coding agents",
   "license": "Apache-2.0",
   "author": "Paul Caplan",
@@ -34,11 +34,21 @@
   "scripts": {
     "build": "bun build --compile --minify --sourcemap ./src/index.ts --outfile bin/agent-gauntlet",
     "test": "bun test",
+    "test:e2e": "bun test/integration/stop-hook-e2e.ts",
     "lint": "biome check src",
+    "typecheck": "tsc --noEmit && tsc --noEmit -p test/tsconfig.json",
     "changeset": "changeset",
     "version": "changeset version",
-    "release": "npm publish",
-    "gen-changeset": "bun scripts/gen-changeset.ts"
+    "release": "changeset publish",
+    "run": "bun src/index.ts run",
+    "check": "bun src/index.ts check",
+    "clean": "bun src/index.ts clean",
+    "review": "bun src/index.ts review",
+    "detect": "bun src/index.ts detect",
+    "list": "bun src/index.ts list",
+    "health": "bun src/index.ts health",
+    "validate": "bun src/index.ts validate",
+    "wait-ci": "bun src/index.ts wait-ci"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.3.11",
@@ -57,4 +67,4 @@
     "yaml": "^2.8.2",
     "zod": "^4.3.5"
   }
-}
+}

package/src/built-in-reviews/code-quality.md

@@ -0,0 +1,25 @@
+# Code Quality Review
+
+You are a senior software engineer performing a code review. Your primary goal is to identify **real problems** that could cause bugs, security vulnerabilities, or performance issues in production. Do not report style, formatting, naming conventions, or maintainability suggestions unless you see something egregious.
+
+## Focus Areas (in priority order)
+
+1. **Bugs** — Logic errors, null/undefined issues, race conditions, unhandled edge cases, resource leaks
+2. **Security** — Injection vulnerabilities, auth/authz flaws, sensitive data exposure, input validation gaps
+3. **Performance** — Algorithmic complexity issues, N+1 queries, blocking operations, memory problems
+4. **Maintainability** — Unclear code, missing error handling, duplication
+
+## Do NOT Report
+
+- Style, formatting, or naming preferences
+- Missing documentation, comments, or type annotations
+- Suggestions for "better" abstractions or patterns that aren't broken
+- Hypothetical issues that require unlikely preconditions
+- Issues in code that wasn't changed in this diff
+
+## Guidelines
+
+- **Threshold**: only report issues you would block a PR over
+- Explain **why** each issue is a problem with a concrete failure scenario
+- Provide a **concrete fix** with corrected code
+- If the status quo works correctly, it's not a violation

package/src/built-in-reviews/index.ts

@@ -0,0 +1,28 @@
+// @ts-expect-error Bun text import
+import codeQualityContent from "./code-quality.md" with { type: "text" };
+
+const BUILT_IN_PREFIX = "built-in:";
+
+const builtInSources: Record<string, string> = {
+	"code-quality": codeQualityContent,
+};
+
+/**
+ * Check if a review name uses the built-in prefix.
+ */
+export function isBuiltInReview(name: string): boolean {
+	return name.startsWith(BUILT_IN_PREFIX);
+}
+
+/**
+ * Load a built-in review prompt by name. Returns the raw markdown content.
+ */
+export function loadBuiltInReview(name: string): string {
+	const source = builtInSources[name];
+
+	if (!source) {
+		throw new Error(`Unknown built-in review: "${name}"`);
+	}
+
+	return source;
+}

package/src/cli-adapters/claude.ts

@@ -4,11 +4,196 @@ import os from "node:os";
 import path from "node:path";
 import { promisify } from "node:util";
 import { GAUNTLET_STOP_HOOK_ACTIVE_ENV } from "../commands/stop-hook.js";
+import { getDebugLogger } from "../utils/debug-log.js";
 import { type CLIAdapter, runStreamingCommand } from "./index.js";
+import { CLAUDE_THINKING_TOKENS } from "./thinking-budget.js";
 
 const execAsync = promisify(exec);
 const MAX_BUFFER_BYTES = 10 * 1024 * 1024;
 
+// Matches OTel console exporter metric blocks dumped to stdout at process exit.
+// Requires `descriptor`, `dataPointType`, and `dataPoints` fields which are
+// unique to OTel SDK output and won't appear in normal code review content.
+// Optionally matches [otel] prefix that some exporters add.
+const OTEL_METRIC_BLOCK_RE =
+	/(?:\[otel\]\s*)?\{\s*\n\s*descriptor:\s*\{[\s\S]*?dataPointType:\s*\d+[\s\S]*?dataPoints:\s*\[[\s\S]*?\]\s*,?\s*\n\}/g;
+
+interface OtelUsage {
+	cost?: number;
+	input?: number;
+	output?: number;
+	cacheRead?: number;
+	cacheCreation?: number;
+	toolCalls?: number;
+	toolContentBytes?: number;
+	apiRequests?: number;
+}
+
+const TOKEN_TYPES = ["input", "output", "cacheRead", "cacheCreation"] as const;
+
+function parseCostBlock(block: string): number | undefined {
+	const match = block.match(/value:\s*([\d.]+)/);
+	return match?.[1] ? Number.parseFloat(match[1]) : undefined;
+}
+
+function parseTokenBlock(block: string): Partial<OtelUsage> {
+	const result: Partial<OtelUsage> = {};
+	const re = /type:\s*"(\w+)"[\s\S]*?value:\s*(\d+)(?:,|\s*\})/g;
+	for (const match of block.matchAll(re)) {
+		const type = match[1] as (typeof TOKEN_TYPES)[number] | undefined;
+		const value = match[2];
+		if (!type || !value) continue;
+		if (TOKEN_TYPES.includes(type)) {
+			result[type] = Number.parseInt(value, 10);
+		}
+	}
+	return result;
+}
+
+function parseOtelMetrics(blocks: string[]): OtelUsage {
+	const usage: OtelUsage = {};
+	for (const block of blocks) {
+		const nameMatch = block.match(/name:\s*"([^"]+)"/);
+		if (!nameMatch) continue;
+
+		if (nameMatch[1] === "claude_code.cost.usage") {
+			usage.cost = parseCostBlock(block);
+		} else if (nameMatch[1] === "claude_code.token.usage") {
+			Object.assign(usage, parseTokenBlock(block));
+		}
+	}
+	return usage;
+}
+
+// Matches OTel console log exporter event records emitted by Claude Code.
+// The Node.js SDK console exporter uses util.inspect() format with unquoted keys
+// and single-quoted strings. Blocks start with `resource:` and contain a `body:`
+// field with the event name (e.g. 'claude_code.tool_result').
+const OTEL_LOG_BLOCK_RE =
+	/\{\s*\n\s*resource:\s*\{[\s\S]*?body:\s*'claude_code\.\w+'[\s\S]*?\n\}/g;
+
+/** Pre-compiled regexes for extracting single-quoted attribute values from OTel log blocks. */
+const OTEL_ATTR_RE = {
+	body: /body:\s*'([^']*)'/,
+	tool_result_size_bytes: /tool_result_size_bytes:\s*'([^']*)'/,
+	input_tokens: /input_tokens:\s*'([^']*)'/,
+	output_tokens: /output_tokens:\s*'([^']*)'/,
+	cache_read_tokens: /cache_read_tokens:\s*'([^']*)'/,
+	cache_creation_tokens: /cache_creation_tokens:\s*'([^']*)'/,
+	cost_usd: /cost_usd:\s*'([^']*)'/,
+} as const;
+
+/** Maps OTel api_request attribute regexes to OtelUsage fields. */
+const API_REQUEST_FIELDS: Array<[RegExp, keyof OtelUsage]> = [
+	[OTEL_ATTR_RE.input_tokens, "input"],
+	[OTEL_ATTR_RE.output_tokens, "output"],
+	[OTEL_ATTR_RE.cache_read_tokens, "cacheRead"],
+	[OTEL_ATTR_RE.cache_creation_tokens, "cacheCreation"],
+	[OTEL_ATTR_RE.cost_usd, "cost"],
+];
+
+/** Accumulate a tool_result log block into usage. */
+function accumulateToolResult(block: string, usage: OtelUsage): void {
+	usage.toolCalls = (usage.toolCalls || 0) + 1;
+	const bytes = block.match(OTEL_ATTR_RE.tool_result_size_bytes)?.[1];
+	if (bytes !== undefined) {
+		usage.toolContentBytes = (usage.toolContentBytes || 0) + Number(bytes);
+	}
+}
+
+/** Accumulate an api_request log block into usage. */
+function accumulateApiRequest(block: string, usage: OtelUsage): void {
+	usage.apiRequests = (usage.apiRequests || 0) + 1;
+	for (const [re, field] of API_REQUEST_FIELDS) {
+		const val = block.match(re)?.[1];
+		if (val !== undefined) {
+			usage[field] = (usage[field] || 0) + Number(val);
+		}
+	}
+}
+
+/** Accumulate tool_result and api_request event data from OTel log blocks. */
+function parseOtelLogEvents(raw: string, usage: OtelUsage): void {
+	const blocks = raw.match(OTEL_LOG_BLOCK_RE);
+	if (!blocks) return;
+	for (const block of blocks) {
+		const body = block.match(OTEL_ATTR_RE.body)?.[1];
+		if (body === "claude_code.tool_result") {
+			accumulateToolResult(block, usage);
+		} else if (body === "claude_code.api_request") {
+			accumulateApiRequest(block, usage);
+		}
+	}
+}
+
+const OTEL_SUMMARY_FIELDS: Array<[keyof OtelUsage, string]> = [
+	["input", "in"],
+	["output", "out"],
+	["cacheRead", "cacheRead"],
+	["cacheCreation", "cacheWrite"],
+	["toolCalls", "tool_calls"],
+	["toolContentBytes", "tool_content_bytes"],
+	["apiRequests", "api_requests"],
+];
+
+function formatOtelSummary(usage: OtelUsage): string | null {
+	if (usage.cost === undefined && usage.input === undefined) return null;
+
+	const parts: string[] = [];
+	if (usage.cost !== undefined) parts.push(`cost=$${usage.cost.toFixed(4)}`);
+	for (const [key, label] of OTEL_SUMMARY_FIELDS) {
+		if (usage[key] !== undefined) parts.push(`${label}=${usage[key]}`);
+	}
+
+	return `[otel] ${parts.join(" ")}`;
+}
+
+function extractOtelMetrics(
+	raw: string,
+	onLog?: (msg: string) => void,
+): string {
+	const metricBlocks = raw.match(OTEL_METRIC_BLOCK_RE);
+	const usage = metricBlocks ? parseOtelMetrics(metricBlocks) : {};
+
+	// Also parse log events for tool call and API request counts
+	parseOtelLogEvents(raw, usage);
+
+	const summary = formatOtelSummary(usage);
+	if (summary) {
+		onLog?.(`\n${summary}\n`);
+		process.stderr.write(`${summary}\n`);
+		getDebugLogger()?.logTelemetry({ adapter: "claude", summary });
+	}
+
+	return raw
+		.replace(OTEL_METRIC_BLOCK_RE, "")
+		.replace(OTEL_LOG_BLOCK_RE, "")
+		.trimEnd();
+}
+
+/** Build OTel environment overrides for console export. */
+function buildOtelEnv(): Record<string, string> {
+	const env: Record<string, string> = {};
+	if (!process.env.CLAUDE_CODE_ENABLE_TELEMETRY) {
+		env.CLAUDE_CODE_ENABLE_TELEMETRY = "1";
+	}
+	if (!process.env.OTEL_METRICS_EXPORTER) {
+		env.OTEL_METRICS_EXPORTER = "console";
+	}
+	if (!process.env.OTEL_LOGS_EXPORTER) {
+		env.OTEL_LOGS_EXPORTER = "console";
+	}
+	return env;
+}
+
+/** Strip OTel metric and log blocks from raw output. */
+function stripOtelBlocks(raw: string): string {
+	return raw
+		.replace(OTEL_METRIC_BLOCK_RE, "")
+		.replace(OTEL_LOG_BLOCK_RE, "")
+		.trimEnd();
+}
+
 export class ClaudeAdapter implements CLIAdapter {
 	name = "claude";
 
@@ -43,21 +228,26 @@ export class ClaudeAdapter implements CLIAdapter {
 	}
 
 	getUserCommandDir(): string | null {
-		// Claude supports user-level commands at ~/.claude/commands
 		return path.join(os.homedir(), ".claude", "commands");
 	}
 
+	getProjectSkillDir(): string | null {
+		return ".claude/skills";
+	}
+
+	getUserSkillDir(): string | null {
+		return path.join(os.homedir(), ".claude", "skills");
+	}
+
 	getCommandExtension(): string {
 		return ".md";
 	}
 
 	canUseSymlink(): boolean {
-		// Claude uses the same Markdown format as our canonical file
 		return true;
 	}
 
 	transformCommand(markdownContent: string): string {
-		// Claude uses the same Markdown format, no transformation needed
 		return markdownContent;
 	}
 
@@ -67,59 +257,71 @@ export class ClaudeAdapter implements CLIAdapter {
 		model?: string;
 		timeoutMs?: number;
 		onOutput?: (chunk: string) => void;
+		allowToolUse?: boolean;
+		thinkingBudget?: string;
 	}): Promise<string> {
 		const fullContent = `${opts.prompt}\n\n--- DIFF ---\n${opts.diff}`;
 
 		const tmpDir = os.tmpdir();
-		// Include process.pid for uniqueness across concurrent processes
 		const tmpFile = path.join(
 			tmpDir,
 			`gauntlet-claude-${process.pid}-${Date.now()}.txt`,
 		);
 		await fs.writeFile(tmpFile, fullContent);
 
-		// Recommended invocation per spec:
-		// -p: non-interactive print mode
-		// --allowedTools: explicitly restricts to read-only tools
-		// --max-turns: caps agentic turns
-		const args = [
-			"-p",
-			"--allowedTools",
-			"Read,Glob,Grep",
-			"--max-turns",
-			"10",
-		];
+		const args = ["-p"];
+		if (opts.allowToolUse === false) {
+			args.push("--tools", "");
+		} else {
+			args.push("--allowedTools", "Read,Glob,Grep");
+		}
+		args.push("--max-turns", "10");
+
+		const otelEnv = buildOtelEnv();
+		const thinkingEnv: Record<string, string> = {};
+		if (opts.thinkingBudget && opts.thinkingBudget in CLAUDE_THINKING_TOKENS) {
+			thinkingEnv.MAX_THINKING_TOKENS = String(
+				CLAUDE_THINKING_TOKENS[opts.thinkingBudget],
+			);
+		}
 
 		const cleanup = () => fs.unlink(tmpFile).catch(() => {});
+		const execEnv = {
+			...process.env,
+			[GAUNTLET_STOP_HOOK_ACTIVE_ENV]: "1",
+			...otelEnv,
+			...thinkingEnv,
+		};
 
-		// If onOutput callback is provided, use spawn for real-time streaming
 		if (opts.onOutput) {
-			return runStreamingCommand({
+			const outputBuffer: string[] = [];
+			const raw = await runStreamingCommand({
 				command: "claude",
 				args,
 				tmpFile,
 				timeoutMs: opts.timeoutMs,
-				onOutput: opts.onOutput,
-				cleanup,
-				env: {
-					...process.env,
-					[GAUNTLET_STOP_HOOK_ACTIVE_ENV]: "1",
+				onOutput: (chunk: string) => {
+					outputBuffer.push(chunk);
 				},
+				cleanup,
+				env: execEnv,
 			});
+			const cleanedOutput = extractOtelMetrics(
+				outputBuffer.join(""),
+				opts.onOutput,
+			);
+			opts.onOutput(cleanedOutput);
+			return stripOtelBlocks(raw);
 		}
 
-		// Otherwise use exec for buffered output
 		try {
-			const cmd = `cat "${tmpFile}" | claude -p --allowedTools "Read,Glob,Grep" --max-turns 10`;
+			const cmd = `cat "${tmpFile}" | claude ${args.map((a) => (a === "" ? '""' : a)).join(" ")}`;
 			const { stdout } = await execAsync(cmd, {
 				timeout: opts.timeoutMs,
 				maxBuffer: MAX_BUFFER_BYTES,
-				env: {
-					...process.env,
-					[GAUNTLET_STOP_HOOK_ACTIVE_ENV]: "1",
-				},
+				env: execEnv,
 			});
-			return stdout;
+			return extractOtelMetrics(stdout);
 		} finally {
 			await cleanup();
 		}