coding-agent-benchmarks 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +474 -0
- package/dist/adapters/claudeCodeCLI.d.ts +19 -0
- package/dist/adapters/claudeCodeCLI.d.ts.map +1 -0
- package/dist/adapters/claudeCodeCLI.js +106 -0
- package/dist/adapters/claudeCodeCLI.js.map +1 -0
- package/dist/adapters/copilotCLI.d.ts +19 -0
- package/dist/adapters/copilotCLI.d.ts.map +1 -0
- package/dist/adapters/copilotCLI.js +104 -0
- package/dist/adapters/copilotCLI.js.map +1 -0
- package/dist/config/defaultScenarios.d.ts +6 -0
- package/dist/config/defaultScenarios.d.ts.map +1 -0
- package/dist/config/defaultScenarios.js +209 -0
- package/dist/config/defaultScenarios.js.map +1 -0
- package/dist/config/loader.d.ts +13 -0
- package/dist/config/loader.d.ts.map +1 -0
- package/dist/config/loader.js +153 -0
- package/dist/config/loader.js.map +1 -0
- package/dist/evaluator.d.ts +45 -0
- package/dist/evaluator.d.ts.map +1 -0
- package/dist/evaluator.js +226 -0
- package/dist/evaluator.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +38 -0
- package/dist/index.js.map +1 -0
- package/dist/runner.d.ts +6 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +233 -0
- package/dist/runner.js.map +1 -0
- package/dist/types.d.ts +354 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +6 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/baselineManager.d.ts +53 -0
- package/dist/utils/baselineManager.d.ts.map +1 -0
- package/dist/utils/baselineManager.js +220 -0
- package/dist/utils/baselineManager.js.map +1 -0
- package/dist/utils/gitUtils.d.ts +39 -0
- package/dist/utils/gitUtils.d.ts.map +1 -0
- package/dist/utils/gitUtils.js +121 -0
- package/dist/utils/gitUtils.js.map +1 -0
- package/dist/utils/githubAuth.d.ts +22 -0
- package/dist/utils/githubAuth.d.ts.map +1 -0
- package/dist/utils/githubAuth.js +79 -0
- package/dist/utils/githubAuth.js.map +1 -0
- package/dist/utils/workspaceUtils.d.ts +32 -0
- package/dist/utils/workspaceUtils.d.ts.map +1 -0
- package/dist/utils/workspaceUtils.js +121 -0
- package/dist/utils/workspaceUtils.js.map +1 -0
- package/dist/validators/eslintValidator.d.ts +22 -0
- package/dist/validators/eslintValidator.d.ts.map +1 -0
- package/dist/validators/eslintValidator.js +217 -0
- package/dist/validators/eslintValidator.js.map +1 -0
- package/dist/validators/llmJudge.d.ts +28 -0
- package/dist/validators/llmJudge.d.ts.map +1 -0
- package/dist/validators/llmJudge.js +241 -0
- package/dist/validators/llmJudge.js.map +1 -0
- package/dist/validators/patternValidator.d.ts +27 -0
- package/dist/validators/patternValidator.d.ts.map +1 -0
- package/dist/validators/patternValidator.js +233 -0
- package/dist/validators/patternValidator.js.map +1 -0
- package/package.json +50 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Git utilities for tracking file changes
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Get git status in porcelain format
|
|
6
|
+
* @param workspaceRoot The workspace root directory
|
|
7
|
+
* @returns Git status output
|
|
8
|
+
*/
|
|
9
|
+
export declare const getGitStatusPorcelain: (workspaceRoot: string) => string;
|
|
10
|
+
/**
|
|
11
|
+
* Parse git status porcelain output to get changed files
|
|
12
|
+
* @param statusOutput Output from git status --porcelain
|
|
13
|
+
* @returns Array of file paths that were changed
|
|
14
|
+
*/
|
|
15
|
+
export declare const parseGitStatus: (statusOutput: string) => string[];
|
|
16
|
+
/**
|
|
17
|
+
* Get list of files that have changed in the working directory
|
|
18
|
+
* @param workspaceRoot The workspace root directory
|
|
19
|
+
* @returns Array of file paths that were changed
|
|
20
|
+
*/
|
|
21
|
+
export declare const getChangedFiles: (workspaceRoot: string) => string[];
|
|
22
|
+
/**
|
|
23
|
+
* Get the git root directory
|
|
24
|
+
* @returns Absolute path to git root
|
|
25
|
+
*/
|
|
26
|
+
export declare const getGitRoot: () => string;
|
|
27
|
+
/**
|
|
28
|
+
* Check if a directory is inside a git repository
|
|
29
|
+
* @param directory Directory to check
|
|
30
|
+
* @returns True if inside a git repository
|
|
31
|
+
*/
|
|
32
|
+
export declare const isGitRepository: (directory: string) => boolean;
|
|
33
|
+
/**
|
|
34
|
+
* Reset git working directory to clean state (discard all changes)
|
|
35
|
+
* WARNING: This will discard all uncommitted changes
|
|
36
|
+
* @param workspaceRoot The workspace root directory
|
|
37
|
+
*/
|
|
38
|
+
export declare const resetGitWorkingDirectory: (workspaceRoot: string) => void;
|
|
39
|
+
//# sourceMappingURL=gitUtils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gitUtils.d.ts","sourceRoot":"","sources":["../../src/utils/gitUtils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH;;;;GAIG;AACH,eAAO,MAAM,qBAAqB,GAAI,eAAe,MAAM,KAAG,MAU7D,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,cAAc,MAAM,KAAG,MAAM,EAqB3D,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,eAAe,GAAI,eAAe,MAAM,KAAG,MAAM,EAG7D,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,UAAU,QAAO,MAS7B,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,eAAe,GAAI,WAAW,MAAM,KAAG,OAWnD,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,wBAAwB,GAAI,eAAe,MAAM,KAAG,IAgBhE,CAAC"}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Git utilities for tracking file changes
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.resetGitWorkingDirectory = exports.isGitRepository = exports.getGitRoot = exports.getChangedFiles = exports.parseGitStatus = exports.getGitStatusPorcelain = void 0;
|
|
7
|
+
const child_process_1 = require("child_process");
|
|
8
|
+
/**
|
|
9
|
+
* Get git status in porcelain format
|
|
10
|
+
* @param workspaceRoot The workspace root directory
|
|
11
|
+
* @returns Git status output
|
|
12
|
+
*/
|
|
13
|
+
const getGitStatusPorcelain = (workspaceRoot) => {
|
|
14
|
+
try {
|
|
15
|
+
return (0, child_process_1.execSync)('git status --porcelain', {
|
|
16
|
+
cwd: workspaceRoot,
|
|
17
|
+
encoding: 'utf-8',
|
|
18
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
throw new Error(`Failed to get git status: ${error}`);
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
exports.getGitStatusPorcelain = getGitStatusPorcelain;
|
|
26
|
+
/**
|
|
27
|
+
* Parse git status porcelain output to get changed files
|
|
28
|
+
* @param statusOutput Output from git status --porcelain
|
|
29
|
+
* @returns Array of file paths that were changed
|
|
30
|
+
*/
|
|
31
|
+
const parseGitStatus = (statusOutput) => {
|
|
32
|
+
const lines = statusOutput.trim().split('\n').filter(line => line.length > 0);
|
|
33
|
+
const files = [];
|
|
34
|
+
for (const line of lines) {
|
|
35
|
+
// Git porcelain format: XY filename
|
|
36
|
+
// X = index status, Y = working tree status
|
|
37
|
+
// We extract the filename (everything after first 3 characters)
|
|
38
|
+
if (line.length > 3) {
|
|
39
|
+
const filename = line.substring(3).trim();
|
|
40
|
+
// Handle renamed files (format: "old -> new")
|
|
41
|
+
if (filename.includes(' -> ')) {
|
|
42
|
+
const newFilename = filename.split(' -> ')[1];
|
|
43
|
+
files.push(newFilename);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
files.push(filename);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return files;
|
|
51
|
+
};
|
|
52
|
+
exports.parseGitStatus = parseGitStatus;
|
|
53
|
+
/**
|
|
54
|
+
* Get list of files that have changed in the working directory
|
|
55
|
+
* @param workspaceRoot The workspace root directory
|
|
56
|
+
* @returns Array of file paths that were changed
|
|
57
|
+
*/
|
|
58
|
+
const getChangedFiles = (workspaceRoot) => {
|
|
59
|
+
const statusOutput = (0, exports.getGitStatusPorcelain)(workspaceRoot);
|
|
60
|
+
return (0, exports.parseGitStatus)(statusOutput);
|
|
61
|
+
};
|
|
62
|
+
exports.getChangedFiles = getChangedFiles;
|
|
63
|
+
/**
|
|
64
|
+
* Get the git root directory
|
|
65
|
+
* @returns Absolute path to git root
|
|
66
|
+
*/
|
|
67
|
+
const getGitRoot = () => {
|
|
68
|
+
try {
|
|
69
|
+
return (0, child_process_1.execSync)('git rev-parse --show-toplevel', {
|
|
70
|
+
encoding: 'utf-8',
|
|
71
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
72
|
+
}).trim();
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
throw new Error('Not inside a git repository');
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
exports.getGitRoot = getGitRoot;
|
|
79
|
+
/**
|
|
80
|
+
* Check if a directory is inside a git repository
|
|
81
|
+
* @param directory Directory to check
|
|
82
|
+
* @returns True if inside a git repository
|
|
83
|
+
*/
|
|
84
|
+
const isGitRepository = (directory) => {
|
|
85
|
+
try {
|
|
86
|
+
(0, child_process_1.execSync)('git rev-parse --git-dir', {
|
|
87
|
+
cwd: directory,
|
|
88
|
+
encoding: 'utf-8',
|
|
89
|
+
stdio: ['pipe', 'pipe', 'ignore'],
|
|
90
|
+
});
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
exports.isGitRepository = isGitRepository;
|
|
98
|
+
/**
|
|
99
|
+
* Reset git working directory to clean state (discard all changes)
|
|
100
|
+
* WARNING: This will discard all uncommitted changes
|
|
101
|
+
* @param workspaceRoot The workspace root directory
|
|
102
|
+
*/
|
|
103
|
+
const resetGitWorkingDirectory = (workspaceRoot) => {
|
|
104
|
+
try {
|
|
105
|
+
// Reset all tracked files
|
|
106
|
+
(0, child_process_1.execSync)('git reset --hard HEAD', {
|
|
107
|
+
cwd: workspaceRoot,
|
|
108
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
109
|
+
});
|
|
110
|
+
// Remove all untracked files and directories
|
|
111
|
+
(0, child_process_1.execSync)('git clean -fd', {
|
|
112
|
+
cwd: workspaceRoot,
|
|
113
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
throw new Error(`Failed to reset git working directory: ${error}`);
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
exports.resetGitWorkingDirectory = resetGitWorkingDirectory;
|
|
121
|
+
//# sourceMappingURL=gitUtils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gitUtils.js","sourceRoot":"","sources":["../../src/utils/gitUtils.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,iDAAyC;AAEzC;;;;GAIG;AACI,MAAM,qBAAqB,GAAG,CAAC,aAAqB,EAAU,EAAE;IACrE,IAAI,CAAC;QACH,OAAO,IAAA,wBAAQ,EAAC,wBAAwB,EAAE;YACxC,GAAG,EAAE,aAAa;YAClB,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;IACxD,CAAC;AACH,CAAC,CAAC;AAVW,QAAA,qBAAqB,yBAUhC;AAEF;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAC,YAAoB,EAAY,EAAE;IAC/D,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,oCAAoC;QACpC,4CAA4C;QAC5C,gEAAgE;QAChE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1C,8CAA8C;YAC9C,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9C,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AArBW,QAAA,cAAc,kBAqBzB;AAEF;;;;GAIG;AACI,MAAM,eAAe,GAAG,CAAC,aAAqB,EAAY,EAAE;IACjE,MAAM,YAAY,GAAG,IAAA,6BAAqB,EAAC,aAAa,CAAC,CAAC;IAC1D,OAAO,IAAA,sBAAc,EAAC,YAAY,CAAC,CAAC;AACtC,CAAC,CAAC;AAHW,QAAA,eAAe,mBAG1B;AAEF;;;GAGG;AACI,MAAM,UAAU,GAAG,GAAW,EAAE;IACrC,IAAI,CAAC;QACH,OAAO,IAAA,wBAAQ,EAAC,+BAA+B,EAAE;YAC/C,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACjD,CAAC;AACH,CAAC,CAAC;AATW,QAAA,UAAU,cASrB;AAEF;;;;GAIG;AACI,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAW,EAAE;IAC5D,IAAI,CAAC;QACH,IAAA,wBAAQ,EAAC,yBAAyB,EAAE;YAClC,GAAG,EAAE,SAAS;YACd,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;SAClC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC,CAAC;AAXW,QAAA,eAAe,mBAW1B;AAEF;;;;GAIG;AACI,MAAM,wBAAwB,GAAG,CAAC,aAAqB,EAAQ,EAAE;IACtE,IAAI,CAAC;QACH,0BAA0B;QAC1B,IAAA,wBAAQ,EAAC,uBAAuB,EAAE;YAChC,GAAG,EAAE,aAAa;YAClB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;QAEH,6CAA6C;QAC7C,IAAA,wBAAQ,EAAC,eAAe,EAAE;YACxB,GAAG,EAAE,aAAa;YAClB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,0CAA0C,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC,CAAC;AAhBW,QAAA,wBAAwB,4BAgBnC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GitHub authentication utilities
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Get GitHub token from environment or GitHub CLI
|
|
6
|
+
* @returns GitHub token or undefined if not found
|
|
7
|
+
*/
|
|
8
|
+
export declare const getGitHubToken: () => string | undefined;
|
|
9
|
+
/**
|
|
10
|
+
* Check if GitHub authentication is available
|
|
11
|
+
* @returns Object with availability status and method used
|
|
12
|
+
*/
|
|
13
|
+
export declare const checkGitHubAuth: () => {
|
|
14
|
+
available: boolean;
|
|
15
|
+
method?: "env" | "gh-cli";
|
|
16
|
+
message: string;
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* Print helpful instructions for setting up GitHub authentication
|
|
20
|
+
*/
|
|
21
|
+
export declare const printGitHubAuthHelp: () => void;
|
|
22
|
+
//# sourceMappingURL=githubAuth.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"githubAuth.d.ts","sourceRoot":"","sources":["../../src/utils/githubAuth.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH;;;GAGG;AACH,eAAO,MAAM,cAAc,QAAO,MAAM,GAAG,SAqB1C,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAO;IACjC,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;CA2BjB,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,mBAAmB,QAAO,IAUtC,CAAC"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* GitHub authentication utilities
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.printGitHubAuthHelp = exports.checkGitHubAuth = exports.getGitHubToken = void 0;
|
|
7
|
+
const child_process_1 = require("child_process");
|
|
8
|
+
/**
|
|
9
|
+
* Get GitHub token from environment or GitHub CLI
|
|
10
|
+
* @returns GitHub token or undefined if not found
|
|
11
|
+
*/
|
|
12
|
+
const getGitHubToken = () => {
|
|
13
|
+
// 1. Try environment variable first
|
|
14
|
+
if (process.env.GITHUB_TOKEN) {
|
|
15
|
+
return process.env.GITHUB_TOKEN;
|
|
16
|
+
}
|
|
17
|
+
// 2. Try GitHub CLI
|
|
18
|
+
try {
|
|
19
|
+
const token = (0, child_process_1.execSync)('gh auth token', {
|
|
20
|
+
encoding: 'utf-8',
|
|
21
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
22
|
+
}).trim();
|
|
23
|
+
if (token && token.startsWith('gh')) {
|
|
24
|
+
return token;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
// GitHub CLI not installed or not authenticated
|
|
29
|
+
}
|
|
30
|
+
return undefined;
|
|
31
|
+
};
|
|
32
|
+
exports.getGitHubToken = getGitHubToken;
|
|
33
|
+
/**
|
|
34
|
+
* Check if GitHub authentication is available
|
|
35
|
+
* @returns Object with availability status and method used
|
|
36
|
+
*/
|
|
37
|
+
const checkGitHubAuth = () => {
|
|
38
|
+
if (process.env.GITHUB_TOKEN) {
|
|
39
|
+
return {
|
|
40
|
+
available: true,
|
|
41
|
+
method: 'env',
|
|
42
|
+
message: 'Using GITHUB_TOKEN from environment variable',
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
try {
|
|
46
|
+
(0, child_process_1.execSync)('gh auth token', {
|
|
47
|
+
encoding: 'utf-8',
|
|
48
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
49
|
+
});
|
|
50
|
+
return {
|
|
51
|
+
available: true,
|
|
52
|
+
method: 'gh-cli',
|
|
53
|
+
message: 'Using token from GitHub CLI (gh auth token)',
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return {
|
|
58
|
+
available: false,
|
|
59
|
+
message: 'GitHub token not found. See: https://github.com/settings/tokens',
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
exports.checkGitHubAuth = checkGitHubAuth;
|
|
64
|
+
/**
|
|
65
|
+
* Print helpful instructions for setting up GitHub authentication
|
|
66
|
+
*/
|
|
67
|
+
const printGitHubAuthHelp = () => {
|
|
68
|
+
console.log('\n⚠️ GitHub token required for LLM-as-judge validation\n');
|
|
69
|
+
console.log('Option 1: Set GITHUB_TOKEN environment variable');
|
|
70
|
+
console.log(' 1. Create token: https://github.com/settings/tokens');
|
|
71
|
+
console.log(' 2. Select scope: models:read');
|
|
72
|
+
console.log(' 3. Set: export GITHUB_TOKEN=ghp_xxxxxxxxxxxx\n');
|
|
73
|
+
console.log('Option 2: Use GitHub CLI (automatic)');
|
|
74
|
+
console.log(' 1. Install: brew install gh (or from https://cli.github.com)');
|
|
75
|
+
console.log(' 2. Authenticate: gh auth login');
|
|
76
|
+
console.log(' 3. Token will be used automatically\n');
|
|
77
|
+
};
|
|
78
|
+
exports.printGitHubAuthHelp = printGitHubAuthHelp;
|
|
79
|
+
//# sourceMappingURL=githubAuth.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"githubAuth.js","sourceRoot":"","sources":["../../src/utils/githubAuth.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,iDAAyC;AAEzC;;;GAGG;AACI,MAAM,cAAc,GAAG,GAAuB,EAAE;IACrD,oCAAoC;IACpC,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QAC7B,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAClC,CAAC;IAED,oBAAoB;IACpB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAA,wBAAQ,EAAC,eAAe,EAAE;YACtC,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEV,IAAI,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gDAAgD;IAClD,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AArBW,QAAA,cAAc,kBAqBzB;AAEF;;;GAGG;AACI,MAAM,eAAe,GAAG,GAI7B,EAAE;IACF,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QAC7B,OAAO;YACL,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,8CAA8C;SACxD,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,IAAA,wBAAQ,EAAC,eAAe,EAAE;YACxB,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;QAEH,OAAO;YACL,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,6CAA6C;SACvD,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,SAAS,EAAE,KAAK;YAChB,OAAO,EAAE,iEAAiE;SAC3E,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AA9BW,QAAA,eAAe,mBA8B1B;AAEF;;GAEG;AACI,MAAM,mBAAmB,GAAG,GAAS,EAAE;IAC5C,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;IACzE,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,gEAAgE,CAAC,CAAC;IAC9E,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;AACzD,CAAC,CAAC;AAVW,QAAA,mBAAmB,uBAU9B"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Utilities for workspace root resolution
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Resolve the workspace root directory
|
|
6
|
+
* Priority:
|
|
7
|
+
* 1. Explicit workspaceRoot from config
|
|
8
|
+
* 2. Git repository root
|
|
9
|
+
* 3. Current working directory
|
|
10
|
+
*
|
|
11
|
+
* @param explicitRoot Optional explicit workspace root from config
|
|
12
|
+
* @returns Absolute path to workspace root
|
|
13
|
+
*/
|
|
14
|
+
export declare const resolveWorkspaceRoot: (explicitRoot?: string) => string;
|
|
15
|
+
/**
|
|
16
|
+
* Resolve file paths relative to workspace root
|
|
17
|
+
* @param workspaceRoot Workspace root directory
|
|
18
|
+
* @param relativePaths Relative file paths
|
|
19
|
+
* @returns Absolute file paths
|
|
20
|
+
*/
|
|
21
|
+
export declare const resolveFilePaths: (workspaceRoot: string, relativePaths: readonly string[]) => string[];
|
|
22
|
+
/**
|
|
23
|
+
* Read context files and return their contents
|
|
24
|
+
* @param workspaceRoot Workspace root directory
|
|
25
|
+
* @param contextFiles Array of context file paths (relative or absolute)
|
|
26
|
+
* @returns Array of file contents with metadata
|
|
27
|
+
*/
|
|
28
|
+
export declare const readContextFiles: (workspaceRoot: string, contextFiles: readonly string[]) => Array<{
|
|
29
|
+
path: string;
|
|
30
|
+
content: string;
|
|
31
|
+
}>;
|
|
32
|
+
//# sourceMappingURL=workspaceUtils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workspaceUtils.d.ts","sourceRoot":"","sources":["../../src/utils/workspaceUtils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;;;;;;;;GASG;AACH,eAAO,MAAM,oBAAoB,GAAI,eAAe,MAAM,KAAG,MAsB5D,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAC3B,eAAe,MAAM,EACrB,eAAe,SAAS,MAAM,EAAE,KAC/B,MAAM,EASR,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAC3B,eAAe,MAAM,EACrB,cAAc,SAAS,MAAM,EAAE,KAC9B,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAsBzC,CAAC"}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Utilities for workspace root resolution
|
|
4
|
+
*/
|
|
5
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
8
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
9
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
10
|
+
}
|
|
11
|
+
Object.defineProperty(o, k2, desc);
|
|
12
|
+
}) : (function(o, m, k, k2) {
|
|
13
|
+
if (k2 === undefined) k2 = k;
|
|
14
|
+
o[k2] = m[k];
|
|
15
|
+
}));
|
|
16
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
17
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
18
|
+
}) : function(o, v) {
|
|
19
|
+
o["default"] = v;
|
|
20
|
+
});
|
|
21
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
22
|
+
var ownKeys = function(o) {
|
|
23
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
24
|
+
var ar = [];
|
|
25
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
26
|
+
return ar;
|
|
27
|
+
};
|
|
28
|
+
return ownKeys(o);
|
|
29
|
+
};
|
|
30
|
+
return function (mod) {
|
|
31
|
+
if (mod && mod.__esModule) return mod;
|
|
32
|
+
var result = {};
|
|
33
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
34
|
+
__setModuleDefault(result, mod);
|
|
35
|
+
return result;
|
|
36
|
+
};
|
|
37
|
+
})();
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.readContextFiles = exports.resolveFilePaths = exports.resolveWorkspaceRoot = void 0;
|
|
40
|
+
const path = __importStar(require("path"));
|
|
41
|
+
const fs = __importStar(require("fs"));
|
|
42
|
+
const gitUtils_1 = require("./gitUtils");
|
|
43
|
+
/**
|
|
44
|
+
* Resolve the workspace root directory
|
|
45
|
+
* Priority:
|
|
46
|
+
* 1. Explicit workspaceRoot from config
|
|
47
|
+
* 2. Git repository root
|
|
48
|
+
* 3. Current working directory
|
|
49
|
+
*
|
|
50
|
+
* @param explicitRoot Optional explicit workspace root from config
|
|
51
|
+
* @returns Absolute path to workspace root
|
|
52
|
+
*/
|
|
53
|
+
const resolveWorkspaceRoot = (explicitRoot) => {
|
|
54
|
+
// 1. Use explicit root if provided
|
|
55
|
+
if (explicitRoot) {
|
|
56
|
+
const resolved = path.resolve(explicitRoot);
|
|
57
|
+
if (fs.existsSync(resolved)) {
|
|
58
|
+
return resolved;
|
|
59
|
+
}
|
|
60
|
+
throw new Error(`Specified workspace root does not exist: ${resolved}`);
|
|
61
|
+
}
|
|
62
|
+
// 2. Try to get git root
|
|
63
|
+
try {
|
|
64
|
+
const gitRoot = (0, gitUtils_1.getGitRoot)();
|
|
65
|
+
if (gitRoot) {
|
|
66
|
+
return gitRoot;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
// Not a git repository, continue
|
|
71
|
+
}
|
|
72
|
+
// 3. Fall back to current working directory
|
|
73
|
+
return process.cwd();
|
|
74
|
+
};
|
|
75
|
+
exports.resolveWorkspaceRoot = resolveWorkspaceRoot;
|
|
76
|
+
/**
|
|
77
|
+
* Resolve file paths relative to workspace root
|
|
78
|
+
* @param workspaceRoot Workspace root directory
|
|
79
|
+
* @param relativePaths Relative file paths
|
|
80
|
+
* @returns Absolute file paths
|
|
81
|
+
*/
|
|
82
|
+
const resolveFilePaths = (workspaceRoot, relativePaths) => {
|
|
83
|
+
return relativePaths.map(relativePath => {
|
|
84
|
+
// If already absolute, return as-is
|
|
85
|
+
if (path.isAbsolute(relativePath)) {
|
|
86
|
+
return relativePath;
|
|
87
|
+
}
|
|
88
|
+
// Otherwise, resolve relative to workspace root
|
|
89
|
+
return path.resolve(workspaceRoot, relativePath);
|
|
90
|
+
});
|
|
91
|
+
};
|
|
92
|
+
exports.resolveFilePaths = resolveFilePaths;
|
|
93
|
+
/**
|
|
94
|
+
* Read context files and return their contents
|
|
95
|
+
* @param workspaceRoot Workspace root directory
|
|
96
|
+
* @param contextFiles Array of context file paths (relative or absolute)
|
|
97
|
+
* @returns Array of file contents with metadata
|
|
98
|
+
*/
|
|
99
|
+
const readContextFiles = (workspaceRoot, contextFiles) => {
|
|
100
|
+
const absolutePaths = (0, exports.resolveFilePaths)(workspaceRoot, contextFiles);
|
|
101
|
+
const results = [];
|
|
102
|
+
for (const absolutePath of absolutePaths) {
|
|
103
|
+
try {
|
|
104
|
+
if (!fs.existsSync(absolutePath)) {
|
|
105
|
+
console.warn(`Context file not found: ${absolutePath}`);
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
const content = fs.readFileSync(absolutePath, 'utf-8');
|
|
109
|
+
results.push({
|
|
110
|
+
path: path.relative(workspaceRoot, absolutePath),
|
|
111
|
+
content,
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
console.warn(`Failed to read context file ${absolutePath}: ${error}`);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return results;
|
|
119
|
+
};
|
|
120
|
+
exports.readContextFiles = readContextFiles;
|
|
121
|
+
//# sourceMappingURL=workspaceUtils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workspaceUtils.js","sourceRoot":"","sources":["../../src/utils/workspaceUtils.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,2CAA6B;AAC7B,uCAAyB;AACzB,yCAAyD;AAEzD;;;;;;;;;GASG;AACI,MAAM,oBAAoB,GAAG,CAAC,YAAqB,EAAU,EAAE;IACpE,mCAAmC;IACnC,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC5C,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,OAAO,QAAQ,CAAC;QAClB,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAA,qBAAU,GAAE,CAAC;QAC7B,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,OAAO,CAAC;QACjB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,iCAAiC;IACnC,CAAC;IAED,4CAA4C;IAC5C,OAAO,OAAO,CAAC,GAAG,EAAE,CAAC;AACvB,CAAC,CAAC;AAtBW,QAAA,oBAAoB,wBAsB/B;AAEF;;;;;GAKG;AACI,MAAM,gBAAgB,GAAG,CAC9B,aAAqB,EACrB,aAAgC,EACtB,EAAE;IACZ,OAAO,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE;QACtC,oCAAoC;QACpC,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAClC,OAAO,YAAY,CAAC;QACtB,CAAC;QACD,gDAAgD;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAZW,QAAA,gBAAgB,oBAY3B;AAEF;;;;;GAKG;AACI,MAAM,gBAAgB,GAAG,CAC9B,aAAqB,EACrB,YAA+B,EACW,EAAE;IAC5C,MAAM,aAAa,GAAG,IAAA,wBAAgB,EAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACpE,MAAM,OAAO,GAA6C,EAAE,CAAC;IAE7D,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;gBACjC,OAAO,CAAC,IAAI,CAAC,2BAA2B,YAAY,EAAE,CAAC,CAAC;gBACxD,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACvD,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,YAAY,CAAC;gBAChD,OAAO;aACR,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,+BAA+B,YAAY,KAAK,KAAK,EAAE,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAzBW,QAAA,gBAAgB,oBAyB3B"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ESLint validator for generated code
|
|
3
|
+
*/
|
|
4
|
+
import { CodeValidator, TestScenario, ValidationResult } from '../types';
|
|
5
|
+
export declare class ESLintValidator implements CodeValidator {
|
|
6
|
+
readonly type: "eslint";
|
|
7
|
+
private workspaceRoot;
|
|
8
|
+
constructor(workspaceRoot?: string);
|
|
9
|
+
/**
|
|
10
|
+
* Check if ESLint is available in the project
|
|
11
|
+
*/
|
|
12
|
+
private checkESLintAvailability;
|
|
13
|
+
/**
|
|
14
|
+
* Validate generated code using ESLint
|
|
15
|
+
*/
|
|
16
|
+
validate(files: readonly string[], scenario: TestScenario): Promise<ValidationResult>;
|
|
17
|
+
/**
|
|
18
|
+
* Calculate score based on number and severity of violations
|
|
19
|
+
*/
|
|
20
|
+
private calculateScore;
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=eslintValidator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eslintValidator.d.ts","sourceRoot":"","sources":["../../src/validators/eslintValidator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,gBAAgB,EAAa,MAAM,UAAU,CAAC;AAIpF,qBAAa,eAAgB,YAAW,aAAa;IACnD,SAAgB,IAAI,EAAG,QAAQ,CAAU;IACzC,OAAO,CAAC,aAAa,CAAS;gBAElB,aAAa,CAAC,EAAE,MAAM;IAIlC;;OAEG;YACW,uBAAuB;IAarC;;OAEG;IACG,QAAQ,CACZ,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,QAAQ,EAAE,YAAY,GACrB,OAAO,CAAC,gBAAgB,CAAC;IA+I5B;;OAEG;IACH,OAAO,CAAC,cAAc;CAsBvB"}
|