agent-gauntlet 0.1.10 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +1 -1
  2. package/package.json +4 -2
  3. package/src/cli-adapters/claude.ts +139 -108
  4. package/src/cli-adapters/codex.ts +141 -117
  5. package/src/cli-adapters/cursor.ts +152 -0
  6. package/src/cli-adapters/gemini.ts +171 -139
  7. package/src/cli-adapters/github-copilot.ts +153 -0
  8. package/src/cli-adapters/index.ts +77 -48
  9. package/src/commands/check.test.ts +24 -20
  10. package/src/commands/check.ts +65 -59
  11. package/src/commands/detect.test.ts +38 -32
  12. package/src/commands/detect.ts +74 -61
  13. package/src/commands/health.test.ts +67 -53
  14. package/src/commands/health.ts +167 -145
  15. package/src/commands/help.test.ts +37 -37
  16. package/src/commands/help.ts +30 -22
  17. package/src/commands/index.ts +9 -9
  18. package/src/commands/init.test.ts +118 -107
  19. package/src/commands/init.ts +514 -417
  20. package/src/commands/list.test.ts +87 -70
  21. package/src/commands/list.ts +28 -24
  22. package/src/commands/rerun.ts +142 -119
  23. package/src/commands/review.test.ts +26 -20
  24. package/src/commands/review.ts +65 -59
  25. package/src/commands/run.test.ts +22 -20
  26. package/src/commands/run.ts +64 -58
  27. package/src/commands/shared.ts +44 -35
  28. package/src/config/loader.test.ts +112 -90
  29. package/src/config/loader.ts +132 -123
  30. package/src/config/schema.ts +49 -47
  31. package/src/config/types.ts +15 -13
  32. package/src/config/validator.ts +521 -454
  33. package/src/core/change-detector.ts +122 -104
  34. package/src/core/entry-point.test.ts +60 -62
  35. package/src/core/entry-point.ts +76 -67
  36. package/src/core/job.ts +69 -59
  37. package/src/core/runner.ts +261 -230
  38. package/src/gates/check.ts +78 -69
  39. package/src/gates/result.ts +7 -7
  40. package/src/gates/review.test.ts +174 -138
  41. package/src/gates/review.ts +716 -561
  42. package/src/index.ts +16 -15
  43. package/src/output/console.ts +253 -214
  44. package/src/output/logger.ts +64 -52
  45. package/src/templates/run_gauntlet.template.md +18 -0
  46. package/src/utils/diff-parser.ts +64 -62
  47. package/src/utils/log-parser.ts +227 -206
  48. package/src/utils/sanitizer.ts +1 -1
package/src/index.ts CHANGED
@@ -1,23 +1,24 @@
1
1
  #!/usr/bin/env bun
2
- import { Command } from 'commander';
2
+ import { Command } from "commander";
3
+ import packageJson from "../package.json" with { type: "json" };
3
4
  import {
4
- registerRunCommand,
5
- registerRerunCommand,
6
- registerCheckCommand,
7
- registerReviewCommand,
8
- registerDetectCommand,
9
- registerListCommand,
10
- registerHealthCommand,
11
- registerInitCommand,
12
- registerHelpCommand,
13
- } from './commands/index.js';
5
+ registerCheckCommand,
6
+ registerDetectCommand,
7
+ registerHealthCommand,
8
+ registerHelpCommand,
9
+ registerInitCommand,
10
+ registerListCommand,
11
+ registerRerunCommand,
12
+ registerReviewCommand,
13
+ registerRunCommand,
14
+ } from "./commands/index.js";
14
15
 
15
16
  const program = new Command();
16
17
 
17
18
  program
18
- .name('agent-gauntlet')
19
- .description('AI-assisted quality gates')
20
- .version('0.1.0');
19
+ .name("agent-gauntlet")
20
+ .description("AI-assisted quality gates")
21
+ .version(packageJson.version);
21
22
 
22
23
  // Register all commands
23
24
  registerRunCommand(program);
@@ -32,7 +33,7 @@ registerHelpCommand(program);
32
33
 
33
34
  // Default action: help
34
35
  if (process.argv.length < 3) {
35
- process.argv.push('help');
36
+ process.argv.push("help");
36
37
  }
37
38
 
38
39
  program.parse(process.argv);
@@ -1,217 +1,256 @@
1
- import chalk from 'chalk';
2
- import fs from 'node:fs/promises';
3
- import { GateResult } from '../gates/result.js';
4
- import { Job } from '../core/job.js';
1
+ import fs from "node:fs/promises";
2
+ import chalk from "chalk";
3
+ import type { Job } from "../core/job.js";
4
+ import type { GateResult } from "../gates/result.js";
5
5
 
6
6
  export class ConsoleReporter {
7
- onJobStart(job: Job) {
8
- console.log(chalk.blue(`[START] ${job.id}`));
9
- }
10
-
11
- onJobComplete(job: Job, result: GateResult) {
12
- const duration = (result.duration / 1000).toFixed(2) + 's';
13
- const message = result.message ?? '';
14
-
15
- if (result.status === 'pass') {
16
- console.log(chalk.green(`[PASS] ${job.id} (${duration})`));
17
- } else if (result.status === 'fail') {
18
- console.log(chalk.red(`[FAIL] ${job.id} (${duration}) - ${message}`));
19
- } else {
20
- console.log(chalk.magenta(`[ERROR] ${job.id} (${duration}) - ${message}`));
21
- }
22
- }
23
-
24
- async printSummary(results: GateResult[]) {
25
- console.log('\n' + chalk.bold('--- Gauntlet Summary ---'));
26
-
27
- const passed = results.filter(r => r.status === 'pass');
28
- const failed = results.filter(r => r.status === 'fail');
29
- const errored = results.filter(r => r.status === 'error');
30
-
31
- console.log(`Total: ${results.length}`);
32
- console.log(chalk.green(`Passed: ${passed.length}`));
33
- if (failed.length > 0) console.log(chalk.red(`Failed: ${failed.length}`));
34
- if (errored.length > 0) console.log(chalk.magenta(`Errored: ${errored.length}`));
35
-
36
- if (failed.length > 0 || errored.length > 0) {
37
- console.log('\n' + chalk.bold('=== Failure Details ===\n'));
38
-
39
- for (const result of [...failed, ...errored]) {
40
- const details = await this.extractFailureDetails(result);
41
- this.printFailureDetails(result, details);
42
- }
43
- }
44
- }
45
-
46
- /** @internal Public for testing */
47
- async extractFailureDetails(result: GateResult): Promise<string[]> {
48
- const logPaths = result.logPaths || (result.logPath ? [result.logPath] : []);
49
-
50
- if (logPaths.length === 0) {
51
- return [result.message ?? 'Unknown error'];
52
- }
53
-
54
- const allDetails: string[] = [];
55
- for (const logPath of logPaths) {
56
- try {
57
- const logContent = await fs.readFile(logPath, 'utf-8');
58
- const details = this.parseLogContent(logContent, result.jobId);
59
- allDetails.push(...details);
60
- } catch (error: any) {
61
- allDetails.push(`(Could not read log file: ${logPath})`);
62
- }
63
- }
64
-
65
- return allDetails.length > 0 ? allDetails : [result.message ?? 'Unknown error'];
66
- }
67
-
68
- private parseLogContent(logContent: string, jobId: string): string[] {
69
- const lines = logContent.split('\n');
70
- const details: string[] = [];
71
-
72
- // Check if this is a review log
73
- if (jobId.startsWith('review:')) {
74
- // Look for parsed violations section (formatted output)
75
- // Use regex to be flexible about adapter name in parentheses
76
- // Matches: "--- Parsed Result ---" or "--- Parsed Result (adapter) ---"
77
- const parsedResultRegex = /---\s*Parsed Result(?:\s+\(([^)]+)\))?\s*---/;
78
- const match = logContent.match(parsedResultRegex);
79
-
80
- if (match && match.index !== undefined) {
81
- const violationsStart = match.index;
82
- const violationsSection = logContent.substring(violationsStart);
83
- const sectionLines = violationsSection.split('\n');
84
-
85
- for (let i = 0; i < sectionLines.length; i++) {
86
- const line = sectionLines[i];
87
- // Match numbered violation lines: "1. file:line - issue" (line can be a number or '?')
88
- const violationMatch = line.match(/^\d+\.\s+(.+?):(\d+|\?)\s+-\s+(.+)$/);
89
- if (violationMatch) {
90
- const file = violationMatch[1];
91
- const lineNum = violationMatch[2];
92
- const issue = violationMatch[3];
93
- details.push(` ${chalk.cyan(file)}:${chalk.yellow(lineNum)} - ${issue}`);
94
-
95
- // Check next line for "Fix:" suggestion
96
- if (i + 1 < sectionLines.length) {
97
- const nextLine = sectionLines[i + 1].trim();
98
- if (nextLine.startsWith('Fix:')) {
99
- const fix = nextLine.substring(4).trim();
100
- details.push(` ${chalk.dim('Fix:')} ${fix}`);
101
- i++; // Skip the fix line
102
- }
103
- }
104
- }
105
- }
106
- }
107
-
108
- // If no parsed violations, look for JSON violations (handles both minified and pretty-printed)
109
- if (details.length === 0) {
110
- // Find the first '{' and last '}' to extract JSON object
111
- const jsonStart = logContent.indexOf('{');
112
- const jsonEnd = logContent.lastIndexOf('}');
113
- if (jsonStart !== -1 && jsonEnd !== -1 && jsonEnd > jsonStart) {
114
- try {
115
- const jsonStr = logContent.substring(jsonStart, jsonEnd + 1);
116
- const json = JSON.parse(jsonStr);
117
- if (json.status === 'fail' && json.violations && Array.isArray(json.violations)) {
118
- json.violations.forEach((v: any) => {
119
- const file = v.file || 'unknown';
120
- const line = v.line || '?';
121
- const issue = v.issue || 'Unknown issue';
122
- details.push(` ${chalk.cyan(file)}:${chalk.yellow(line)} - ${issue}`);
123
- if (v.fix) {
124
- details.push(` ${chalk.dim('Fix:')} ${v.fix}`);
125
- }
126
- });
127
- }
128
- } catch {
129
- // JSON parse failed, fall through to other parsing
130
- }
131
- }
132
- }
133
-
134
- // If still no details, look for error messages
135
- if (details.length === 0) {
136
- // Try to find the actual error message (first non-empty line after "Error:")
137
- const errorIndex = logContent.indexOf('Error:');
138
- if (errorIndex !== -1) {
139
- const afterError = logContent.substring(errorIndex + 6).trim();
140
- const firstErrorLine = afterError.split('\n')[0].trim();
141
- if (firstErrorLine && !firstErrorLine.startsWith('Usage:') && !firstErrorLine.startsWith('Commands:')) {
142
- details.push(` ${firstErrorLine}`);
143
- }
144
- }
145
-
146
- // Also check for "Result: error" lines
147
- if (details.length === 0) {
148
- const resultMatch = logContent.match(/Result:\s*error(?:\s*-\s*(.+?))?(?:\n|$)/);
149
- if (resultMatch && resultMatch[1]) {
150
- details.push(` ${resultMatch[1]}`);
151
- }
152
- }
153
- }
154
- } else {
155
- // This is a check log
156
- // Look for STDERR section
157
- const stderrStart = logContent.indexOf('STDERR:');
158
- if (stderrStart !== -1) {
159
- const stderrSection = logContent.substring(stderrStart + 7).trim();
160
- const stderrLines = stderrSection.split('\n').filter(line => {
161
- // Skip empty lines and command output markers
162
- return line.trim() &&
163
- !line.includes('STDOUT:') &&
164
- !line.includes('Command failed:') &&
165
- !line.includes('Result:');
166
- });
167
- if (stderrLines.length > 0) {
168
- details.push(...stderrLines.slice(0, 10).map(line => ` ${line.trim()}`));
169
- }
170
- }
171
-
172
- // If no STDERR, look for error messages
173
- if (details.length === 0) {
174
- const errorMatch = logContent.match(/Command failed:\s*(.+?)(?:\n|$)/);
175
- if (errorMatch) {
176
- details.push(` ${errorMatch[1]}`);
177
- } else {
178
- // Look for any line with "Result: fail" or "Result: error"
179
- const resultMatch = logContent.match(/Result:\s*(fail|error)\s*-\s*(.+?)(?:\n|$)/);
180
- if (resultMatch) {
181
- details.push(` ${resultMatch[2]}`);
182
- }
183
- }
184
- }
185
- }
186
-
187
- // If we still have no details, use the message from the result
188
- if (details.length === 0) {
189
- details.push(' (See log file for details)');
190
- }
191
-
192
- return details;
193
- }
194
-
195
- private printFailureDetails(result: GateResult, details: string[]) {
196
- const statusColor = result.status === 'error' ? chalk.magenta : chalk.red;
197
- const statusLabel = result.status === 'error' ? 'ERROR' : 'FAIL';
198
-
199
- console.log(statusColor(`[${statusLabel}] ${result.jobId}`));
200
- if (result.message) {
201
- console.log(chalk.dim(` Summary: ${result.message}`));
202
- }
203
-
204
- if (details.length > 0) {
205
- console.log(chalk.dim(' Details:'));
206
- details.forEach(detail => console.log(detail));
207
- }
208
-
209
- if (result.logPaths && result.logPaths.length > 0) {
210
- result.logPaths.forEach(p => console.log(chalk.dim(` Log: ${p}`)));
211
- } else if (result.logPath) {
212
- console.log(chalk.dim(` Log: ${result.logPath}`));
213
- }
214
-
215
- console.log(''); // Empty line between failures
216
- }
7
+ onJobStart(job: Job) {
8
+ console.log(chalk.blue(`[START] ${job.id}`));
9
+ }
10
+
11
+ onJobComplete(job: Job, result: GateResult) {
12
+ const duration = `${(result.duration / 1000).toFixed(2)}s`;
13
+ const message = result.message ?? "";
14
+
15
+ if (result.status === "pass") {
16
+ console.log(chalk.green(`[PASS] ${job.id} (${duration})`));
17
+ } else if (result.status === "fail") {
18
+ console.log(chalk.red(`[FAIL] ${job.id} (${duration}) - ${message}`));
19
+ } else {
20
+ console.log(
21
+ chalk.magenta(`[ERROR] ${job.id} (${duration}) - ${message}`),
22
+ );
23
+ }
24
+ }
25
+
26
+ async printSummary(results: GateResult[]) {
27
+ console.log(`\n${chalk.bold("--- Gauntlet Summary ---")}`);
28
+
29
+ const passed = results.filter((r) => r.status === "pass");
30
+ const failed = results.filter((r) => r.status === "fail");
31
+ const errored = results.filter((r) => r.status === "error");
32
+
33
+ console.log(`Total: ${results.length}`);
34
+ console.log(chalk.green(`Passed: ${passed.length}`));
35
+ if (failed.length > 0) console.log(chalk.red(`Failed: ${failed.length}`));
36
+ if (errored.length > 0)
37
+ console.log(chalk.magenta(`Errored: ${errored.length}`));
38
+
39
+ if (failed.length > 0 || errored.length > 0) {
40
+ console.log(`\n${chalk.bold("=== Failure Details ===\n")}`);
41
+
42
+ for (const result of [...failed, ...errored]) {
43
+ const details = await this.extractFailureDetails(result);
44
+ this.printFailureDetails(result, details);
45
+ }
46
+ }
47
+ }
48
+
49
+ /** @internal Public for testing */
50
+ async extractFailureDetails(result: GateResult): Promise<string[]> {
51
+ const logPaths =
52
+ result.logPaths || (result.logPath ? [result.logPath] : []);
53
+
54
+ if (logPaths.length === 0) {
55
+ return [result.message ?? "Unknown error"];
56
+ }
57
+
58
+ const allDetails: string[] = [];
59
+ for (const logPath of logPaths) {
60
+ try {
61
+ const logContent = await fs.readFile(logPath, "utf-8");
62
+ const details = this.parseLogContent(logContent, result.jobId);
63
+ allDetails.push(...details);
64
+ } catch (_error: unknown) {
65
+ allDetails.push(`(Could not read log file: ${logPath})`);
66
+ }
67
+ }
68
+
69
+ return allDetails.length > 0
70
+ ? allDetails
71
+ : [result.message ?? "Unknown error"];
72
+ }
73
+
74
+ private parseLogContent(logContent: string, jobId: string): string[] {
75
+ const _lines = logContent.split("\n");
76
+ const details: string[] = [];
77
+
78
+ // Check if this is a review log
79
+ if (jobId.startsWith("review:")) {
80
+ // Look for parsed violations section (formatted output)
81
+ // Use regex to be flexible about adapter name in parentheses
82
+ // Matches: "--- Parsed Result ---" or "--- Parsed Result (adapter) ---"
83
+ const parsedResultRegex = /---\s*Parsed Result(?:\s+\(([^)]+)\))?\s*---/;
84
+ const match = logContent.match(parsedResultRegex);
85
+
86
+ if (match && match.index !== undefined) {
87
+ const violationsStart = match.index;
88
+ const violationsSection = logContent.substring(violationsStart);
89
+ const sectionLines = violationsSection.split("\n");
90
+
91
+ for (let i = 0; i < sectionLines.length; i++) {
92
+ const line = sectionLines[i];
93
+ // Match numbered violation lines: "1. file:line - issue" (line can be a number or '?')
94
+ const violationMatch = line.match(
95
+ /^\d+\.\s+(.+?):(\d+|\?)\s+-\s+(.+)$/,
96
+ );
97
+ if (violationMatch) {
98
+ const file = violationMatch[1];
99
+ const lineNum = violationMatch[2];
100
+ const issue = violationMatch[3];
101
+ details.push(
102
+ ` ${chalk.cyan(file)}:${chalk.yellow(lineNum)} - ${issue}`,
103
+ );
104
+
105
+ // Check next line for "Fix:" suggestion
106
+ if (i + 1 < sectionLines.length) {
107
+ const nextLine = sectionLines[i + 1].trim();
108
+ if (nextLine.startsWith("Fix:")) {
109
+ const fix = nextLine.substring(4).trim();
110
+ details.push(` ${chalk.dim("Fix:")} ${fix}`);
111
+ i++; // Skip the fix line
112
+ }
113
+ }
114
+ }
115
+ }
116
+ }
117
+
118
+ // If no parsed violations, look for JSON violations (handles both minified and pretty-printed)
119
+ if (details.length === 0) {
120
+ // Find the first '{' and last '}' to extract JSON object
121
+ const jsonStart = logContent.indexOf("{");
122
+ const jsonEnd = logContent.lastIndexOf("}");
123
+ if (jsonStart !== -1 && jsonEnd !== -1 && jsonEnd > jsonStart) {
124
+ try {
125
+ const jsonStr = logContent.substring(jsonStart, jsonEnd + 1);
126
+ const json = JSON.parse(jsonStr);
127
+ if (
128
+ json.status === "fail" &&
129
+ json.violations &&
130
+ Array.isArray(json.violations)
131
+ ) {
132
+ json.violations.forEach(
133
+ (v: {
134
+ file?: string;
135
+ line?: number | string;
136
+ issue?: string;
137
+ fix?: string;
138
+ }) => {
139
+ const file = v.file || "unknown";
140
+ const line = v.line || "?";
141
+ const issue = v.issue || "Unknown issue";
142
+ details.push(
143
+ ` ${chalk.cyan(file)}:${chalk.yellow(line)} - ${issue}`,
144
+ );
145
+ if (v.fix) {
146
+ details.push(` ${chalk.dim("Fix:")} ${v.fix}`);
147
+ }
148
+ },
149
+ );
150
+ }
151
+ } catch {
152
+ // JSON parse failed, fall through to other parsing
153
+ }
154
+ }
155
+ }
156
+
157
+ // If still no details, look for error messages
158
+ if (details.length === 0) {
159
+ // Try to find the actual error message (first non-empty line after "Error:")
160
+ const errorIndex = logContent.indexOf("Error:");
161
+ if (errorIndex !== -1) {
162
+ const afterError = logContent.substring(errorIndex + 6).trim();
163
+ const firstErrorLine = afterError.split("\n")[0].trim();
164
+ if (
165
+ firstErrorLine &&
166
+ !firstErrorLine.startsWith("Usage:") &&
167
+ !firstErrorLine.startsWith("Commands:")
168
+ ) {
169
+ details.push(` ${firstErrorLine}`);
170
+ }
171
+ }
172
+
173
+ // Also check for "Result: error" lines
174
+ if (details.length === 0) {
175
+ const resultMatch = logContent.match(
176
+ /Result:\s*error(?:\s*-\s*(.+?))?(?:\n|$)/,
177
+ );
178
+ if (resultMatch?.[1]) {
179
+ details.push(` ${resultMatch[1]}`);
180
+ }
181
+ }
182
+ }
183
+ } else {
184
+ // This is a check log
185
+ // Look for STDERR section
186
+ const stderrStart = logContent.indexOf("STDERR:");
187
+ if (stderrStart !== -1) {
188
+ const stderrSection = logContent.substring(stderrStart + 7).trim();
189
+ const stderrLines = stderrSection.split("\n").filter((line) => {
190
+ // Skip empty lines and command output markers
191
+ return (
192
+ line.trim() &&
193
+ !line.includes("STDOUT:") &&
194
+ !line.includes("Command failed:") &&
195
+ !line.includes("Result:")
196
+ );
197
+ });
198
+ if (stderrLines.length > 0) {
199
+ details.push(
200
+ ...stderrLines.slice(0, 10).map((line) => ` ${line.trim()}`),
201
+ );
202
+ }
203
+ }
204
+
205
+ // If no STDERR, look for error messages
206
+ if (details.length === 0) {
207
+ const errorMatch = logContent.match(/Command failed:\s*(.+?)(?:\n|$)/);
208
+ if (errorMatch) {
209
+ details.push(` ${errorMatch[1]}`);
210
+ } else {
211
+ // Look for any line with "Result: fail" or "Result: error"
212
+ const resultMatch = logContent.match(
213
+ /Result:\s*(fail|error)\s*-\s*(.+?)(?:\n|$)/,
214
+ );
215
+ if (resultMatch) {
216
+ details.push(` ${resultMatch[2]}`);
217
+ }
218
+ }
219
+ }
220
+ }
221
+
222
+ // If we still have no details, use the message from the result
223
+ if (details.length === 0) {
224
+ details.push(" (See log file for details)");
225
+ }
226
+
227
+ return details;
228
+ }
229
+
230
+ private printFailureDetails(result: GateResult, details: string[]) {
231
+ const statusColor = result.status === "error" ? chalk.magenta : chalk.red;
232
+ const statusLabel = result.status === "error" ? "ERROR" : "FAIL";
233
+
234
+ console.log(statusColor(`[${statusLabel}] ${result.jobId}`));
235
+ if (result.message) {
236
+ console.log(chalk.dim(` Summary: ${result.message}`));
237
+ }
238
+
239
+ if (details.length > 0) {
240
+ console.log(chalk.dim(" Details:"));
241
+ details.forEach((detail) => {
242
+ console.log(detail);
243
+ });
244
+ }
245
+
246
+ if (result.logPaths && result.logPaths.length > 0) {
247
+ result.logPaths.forEach((p) => {
248
+ console.log(chalk.dim(` Log: ${p}`));
249
+ });
250
+ } else if (result.logPath) {
251
+ console.log(chalk.dim(` Log: ${result.logPath}`));
252
+ }
253
+
254
+ console.log(""); // Empty line between failures
255
+ }
217
256
  }