coding-agent-benchmarks 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -1
- package/dist/User.d.ts +10 -0
- package/dist/User.d.ts.map +1 -0
- package/dist/User.js +3 -0
- package/dist/User.js.map +1 -0
- package/dist/adapters/claudeCodeCLI.d.ts +10 -1
- package/dist/adapters/claudeCodeCLI.d.ts.map +1 -1
- package/dist/adapters/claudeCodeCLI.js +86 -25
- package/dist/adapters/claudeCodeCLI.js.map +1 -1
- package/dist/adapters/copilotCLI.d.ts +11 -2
- package/dist/adapters/copilotCLI.d.ts.map +1 -1
- package/dist/adapters/copilotCLI.js +99 -36
- package/dist/adapters/copilotCLI.js.map +1 -1
- package/dist/evaluator.d.ts +20 -2
- package/dist/evaluator.d.ts.map +1 -1
- package/dist/evaluator.js +58 -44
- package/dist/evaluator.js.map +1 -1
- package/dist/reporter.d.ts +35 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporter.js +232 -0
- package/dist/reporter.js.map +1 -0
- package/dist/runner.js +75 -63
- package/dist/runner.js.map +1 -1
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/gitUtils.d.ts +7 -6
- package/dist/utils/gitUtils.d.ts.map +1 -1
- package/dist/utils/gitUtils.js +27 -24
- package/dist/utils/gitUtils.js.map +1 -1
- package/package.json +21 -4
package/dist/runner.js
CHANGED
|
@@ -45,33 +45,32 @@ const copilotCLI_1 = require("./adapters/copilotCLI");
|
|
|
45
45
|
const claudeCodeCLI_1 = require("./adapters/claudeCodeCLI");
|
|
46
46
|
const llmJudge_1 = require("./validators/llmJudge");
|
|
47
47
|
const githubAuth_1 = require("./utils/githubAuth");
|
|
48
|
+
const reporter_1 = require("./reporter");
|
|
48
49
|
const program = new commander_1.Command();
|
|
49
50
|
program
|
|
50
|
-
.name(
|
|
51
|
-
.description(
|
|
52
|
-
.version(
|
|
51
|
+
.name("coding-agent-benchmarks")
|
|
52
|
+
.description("Evaluate coding agents against coding standards and best practices")
|
|
53
|
+
.version("0.1.0");
|
|
53
54
|
/**
|
|
54
55
|
* Evaluate command
|
|
55
56
|
*/
|
|
56
57
|
program
|
|
57
|
-
.command(
|
|
58
|
-
.description(
|
|
59
|
-
.option(
|
|
60
|
-
.option(
|
|
61
|
-
.option(
|
|
62
|
-
.option(
|
|
63
|
-
.option(
|
|
64
|
-
.option(
|
|
65
|
-
.option(
|
|
66
|
-
.option(
|
|
67
|
-
.option(
|
|
68
|
-
.option(
|
|
69
|
-
.option(
|
|
58
|
+
.command("evaluate")
|
|
59
|
+
.description("Run benchmark evaluations")
|
|
60
|
+
.option("--scenario <pattern>", "Filter scenarios by ID pattern (supports wildcards)")
|
|
61
|
+
.option("--category <categories>", "Filter by category (comma-separated)")
|
|
62
|
+
.option("--tag <tags>", "Filter by tags (comma-separated)")
|
|
63
|
+
.option("--adapter <type>", "Code generation adapter (copilot or claude-code)", "copilot")
|
|
64
|
+
.option("--model <model>", "LLM model for judge (default: openai/gpt-4.1)")
|
|
65
|
+
.option("--threshold <number>", "Minimum passing score", "0.8")
|
|
66
|
+
.option("--verbose", "Show detailed output")
|
|
67
|
+
.option("--output <file>", "Export JSON report to file")
|
|
68
|
+
.option("--save-baseline", "Save results as baseline")
|
|
69
|
+
.option("--compare-baseline", "Compare results with baseline")
|
|
70
|
+
.option("--workspace-root <path>", "Workspace root directory")
|
|
70
71
|
.action(async (options) => {
|
|
71
72
|
try {
|
|
72
|
-
// Load configuration
|
|
73
73
|
const { config, scenarios } = await (0, loader_1.loadConfig)(options.workspaceRoot || process.cwd());
|
|
74
|
-
// Create evaluator
|
|
75
74
|
const evaluator = new evaluator_1.Evaluator({
|
|
76
75
|
adapter: options.adapter,
|
|
77
76
|
model: options.model,
|
|
@@ -81,6 +80,25 @@ program
|
|
|
81
80
|
saveBaseline: options.saveBaseline,
|
|
82
81
|
compareBaseline: options.compareBaseline,
|
|
83
82
|
});
|
|
83
|
+
const reporter = new reporter_1.ProgressReporter({ verbose: options.verbose });
|
|
84
|
+
evaluator.on("evaluation:start", (scenarioList) => {
|
|
85
|
+
reporter.start(scenarioList);
|
|
86
|
+
});
|
|
87
|
+
evaluator.on("scenario:start", (scenarioId) => {
|
|
88
|
+
reporter.onScenarioStart(scenarioId);
|
|
89
|
+
});
|
|
90
|
+
evaluator.on("scenario:generating", (scenarioId) => {
|
|
91
|
+
// Phase already set in onScenarioStart, but could be used for more granular updates
|
|
92
|
+
});
|
|
93
|
+
evaluator.on("scenario:validating", (scenarioId) => {
|
|
94
|
+
reporter.onScenarioValidating(scenarioId);
|
|
95
|
+
});
|
|
96
|
+
evaluator.on("scenario:complete", (scenarioId, result) => {
|
|
97
|
+
reporter.onScenarioComplete(scenarioId, result);
|
|
98
|
+
});
|
|
99
|
+
evaluator.on("log", (message) => {
|
|
100
|
+
reporter.log(message);
|
|
101
|
+
});
|
|
84
102
|
// Check adapter availability
|
|
85
103
|
const isAvailable = await evaluator.checkAdapterAvailability();
|
|
86
104
|
if (!isAvailable) {
|
|
@@ -92,29 +110,21 @@ program
|
|
|
92
110
|
const filteredScenarios = evaluator.filterScenarios(scenarios, {
|
|
93
111
|
scenarioPattern: options.scenario,
|
|
94
112
|
category: options.category,
|
|
95
|
-
tags: options.tag
|
|
113
|
+
tags: options.tag
|
|
114
|
+
? options.tag.split(",").map((t) => t.trim())
|
|
115
|
+
: undefined,
|
|
96
116
|
});
|
|
97
117
|
if (filteredScenarios.length === 0) {
|
|
98
|
-
console.log(
|
|
118
|
+
console.log("No scenarios match the specified filters");
|
|
99
119
|
return;
|
|
100
120
|
}
|
|
101
121
|
// Run evaluation
|
|
102
122
|
const report = await evaluator.evaluate(filteredScenarios);
|
|
103
|
-
//
|
|
104
|
-
|
|
105
|
-
console.log('EVALUATION SUMMARY');
|
|
106
|
-
console.log('='.repeat(60));
|
|
107
|
-
console.log(`Total scenarios: ${report.summary.total}`);
|
|
108
|
-
console.log(`Passed: ${report.summary.passed}`);
|
|
109
|
-
console.log(`Failed: ${report.summary.failed}`);
|
|
110
|
-
console.log(`Skipped: ${report.summary.skipped}`);
|
|
111
|
-
console.log(`Average score: ${report.summary.averageScore.toFixed(2)}`);
|
|
112
|
-
console.log(`Total violations: ${report.summary.totalViolations}`);
|
|
113
|
-
console.log(`Total duration: ${(report.totalDuration / 1000).toFixed(1)}s`);
|
|
114
|
-
console.log('='.repeat(60));
|
|
123
|
+
// Finish reporter and display summary
|
|
124
|
+
reporter.finish(report);
|
|
115
125
|
// Export JSON report if requested
|
|
116
126
|
if (options.output) {
|
|
117
|
-
fs.writeFileSync(options.output, JSON.stringify(report, null, 2),
|
|
127
|
+
fs.writeFileSync(options.output, JSON.stringify(report, null, 2), "utf-8");
|
|
118
128
|
console.log(`\nReport exported to: ${options.output}`);
|
|
119
129
|
}
|
|
120
130
|
// Exit with error code if any scenarios failed
|
|
@@ -131,30 +141,32 @@ program
|
|
|
131
141
|
* List command
|
|
132
142
|
*/
|
|
133
143
|
program
|
|
134
|
-
.command(
|
|
135
|
-
.description(
|
|
136
|
-
.option(
|
|
137
|
-
.option(
|
|
144
|
+
.command("list")
|
|
145
|
+
.description("List available test scenarios")
|
|
146
|
+
.option("--category <categories>", "Filter by category")
|
|
147
|
+
.option("--tag <tags>", "Filter by tags (comma-separated)")
|
|
138
148
|
.action(async (options) => {
|
|
139
149
|
try {
|
|
140
150
|
const { scenarios } = await (0, loader_1.loadConfig)();
|
|
141
151
|
let filtered = scenarios;
|
|
142
152
|
// Filter by category
|
|
143
153
|
if (options.category) {
|
|
144
|
-
const categories = options.category
|
|
145
|
-
|
|
154
|
+
const categories = options.category
|
|
155
|
+
.split(",")
|
|
156
|
+
.map((c) => c.trim());
|
|
157
|
+
filtered = filtered.filter((s) => categories.includes(s.category));
|
|
146
158
|
}
|
|
147
159
|
// Filter by tags
|
|
148
160
|
if (options.tag) {
|
|
149
|
-
const tags = options.tag.split(
|
|
150
|
-
filtered = filtered.filter(s => tags.some((tag) => s.tags.includes(tag)));
|
|
161
|
+
const tags = options.tag.split(",").map((t) => t.trim());
|
|
162
|
+
filtered = filtered.filter((s) => tags.some((tag) => s.tags.includes(tag)));
|
|
151
163
|
}
|
|
152
164
|
console.log(`\nAvailable scenarios (${filtered.length}):\n`);
|
|
153
165
|
for (const scenario of filtered) {
|
|
154
166
|
console.log(` ${scenario.id}`);
|
|
155
167
|
console.log(` Category: ${scenario.category}`);
|
|
156
168
|
console.log(` Severity: ${scenario.severity}`);
|
|
157
|
-
console.log(` Tags: ${scenario.tags.join(
|
|
169
|
+
console.log(` Tags: ${scenario.tags.join(", ")}`);
|
|
158
170
|
console.log(` Description: ${scenario.description}`);
|
|
159
171
|
console.log();
|
|
160
172
|
}
|
|
@@ -168,34 +180,34 @@ program
|
|
|
168
180
|
* Check command
|
|
169
181
|
*/
|
|
170
182
|
program
|
|
171
|
-
.command(
|
|
172
|
-
.description(
|
|
183
|
+
.command("check")
|
|
184
|
+
.description("Check if coding agent CLIs and GitHub auth are available")
|
|
173
185
|
.action(async () => {
|
|
174
|
-
console.log(
|
|
186
|
+
console.log("Checking adapter availability...\n");
|
|
175
187
|
const adapters = [
|
|
176
|
-
{ name:
|
|
177
|
-
{ name:
|
|
188
|
+
{ name: "GitHub Copilot CLI", type: "copilot" },
|
|
189
|
+
{ name: "Claude Code CLI", type: "claude-code" },
|
|
178
190
|
];
|
|
179
191
|
for (const { name, type } of adapters) {
|
|
180
192
|
let adapter;
|
|
181
|
-
if (type ===
|
|
193
|
+
if (type === "copilot") {
|
|
182
194
|
adapter = new copilotCLI_1.CopilotCLIAdapter();
|
|
183
195
|
}
|
|
184
196
|
else {
|
|
185
197
|
adapter = new claudeCodeCLI_1.ClaudeCodeCLIAdapter();
|
|
186
198
|
}
|
|
187
199
|
const available = await adapter.checkAvailability();
|
|
188
|
-
const status = available ?
|
|
200
|
+
const status = available ? "✓ Available" : "✗ Not found";
|
|
189
201
|
console.log(` ${name}: ${status}`);
|
|
190
202
|
}
|
|
191
|
-
console.log(
|
|
203
|
+
console.log("\nChecking GitHub authentication...\n");
|
|
192
204
|
const authStatus = (0, githubAuth_1.checkGitHubAuth)();
|
|
193
|
-
const authIcon = authStatus.available ?
|
|
205
|
+
const authIcon = authStatus.available ? "✓" : "✗";
|
|
194
206
|
console.log(` ${authIcon} ${authStatus.message}`);
|
|
195
207
|
if (!authStatus.available) {
|
|
196
|
-
console.log(
|
|
197
|
-
console.log(
|
|
198
|
-
console.log(
|
|
208
|
+
console.log("\n 💡 GitHub token is required for LLM-as-judge validation");
|
|
209
|
+
console.log(" Setup: https://github.com/settings/tokens (scope: models:read)");
|
|
210
|
+
console.log(" Or install GitHub CLI: brew install gh && gh auth login");
|
|
199
211
|
}
|
|
200
212
|
console.log();
|
|
201
213
|
});
|
|
@@ -203,23 +215,23 @@ program
|
|
|
203
215
|
* Test LLM command
|
|
204
216
|
*/
|
|
205
217
|
program
|
|
206
|
-
.command(
|
|
207
|
-
.description(
|
|
208
|
-
.option(
|
|
218
|
+
.command("test-llm")
|
|
219
|
+
.description("Test LLM judge with a custom prompt")
|
|
220
|
+
.option("--model <model>", "LLM model to use (default: openai/gpt-4.1)")
|
|
209
221
|
.action(async (options) => {
|
|
210
222
|
try {
|
|
211
|
-
console.log(
|
|
212
|
-
console.log(
|
|
223
|
+
console.log("Testing LLM judge...\n");
|
|
224
|
+
console.log("Enter your prompt (Ctrl+D when done):\n");
|
|
213
225
|
// Read prompt from stdin
|
|
214
226
|
const chunks = [];
|
|
215
|
-
process.stdin.on(
|
|
227
|
+
process.stdin.on("data", (chunk) => {
|
|
216
228
|
chunks.push(chunk.toString());
|
|
217
229
|
});
|
|
218
|
-
process.stdin.on(
|
|
219
|
-
const prompt = chunks.join(
|
|
230
|
+
process.stdin.on("end", async () => {
|
|
231
|
+
const prompt = chunks.join("");
|
|
220
232
|
const validator = new llmJudge_1.LLMJudgeValidator(undefined, options.model);
|
|
221
233
|
const result = await validator.testJudge(prompt, options.model);
|
|
222
|
-
console.log(
|
|
234
|
+
console.log("\nLLM Response:\n");
|
|
223
235
|
console.log(result);
|
|
224
236
|
});
|
|
225
237
|
}
|
package/dist/runner.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":";;AAEA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,yCAAoC;AACpC,uCAAyB;AACzB,4CAA6C;AAC7C,2CAAwC;AAExC,sDAA0D;AAC1D,4DAAgE;AAChE,oDAA0D;AAC1D,mDAAqD;
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":";;AAEA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,yCAAoC;AACpC,uCAAyB;AACzB,4CAA6C;AAC7C,2CAAwC;AAExC,sDAA0D;AAC1D,4DAAgE;AAChE,oDAA0D;AAC1D,mDAAqD;AACrD,yCAA8C;AAE9C,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,yBAAyB,CAAC;KAC/B,WAAW,CACV,oEAAoE,CACrE;KACA,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB;;GAEG;AACH,OAAO;KACJ,OAAO,CAAC,UAAU,CAAC;KACnB,WAAW,CAAC,2BAA2B,CAAC;KACxC,MAAM,CACL,sBAAsB,EACtB,qDAAqD,CACtD;KACA,MAAM,CAAC,yBAAyB,EAAE,sCAAsC,CAAC;KACzE,MAAM,CAAC,cAAc,EAAE,kCAAkC,CAAC;KAC1D,MAAM,CACL,kBAAkB,EAClB,kDAAkD,EAClD,SAAS,CACV;KACA,MAAM,CAAC,iBAAiB,EAAE,+CAA+C,CAAC;KAC1E,MAAM,CAAC,sBAAsB,EAAE,uBAAuB,EAAE,KAAK,CAAC;KAC9D,MAAM,CAAC,WAAW,EAAE,sBAAsB,CAAC;KAC3C,MAAM,CAAC,iBAAiB,EAAE,4BAA4B,CAAC;KACvD,MAAM,CAAC,iBAAiB,EAAE,0BAA0B,CAAC;KACrD,MAAM,CAAC,oBAAoB,EAAE,+BAA+B,CAAC;KAC7D,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,CAAC;KAC7D,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAA,mBAAU,EAC5C,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,GAAG,EAAE,CACvC,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,qBAAS,CAAC;YAC9B,OAAO,EAAE,OAAO,CAAC,OAAsB;YACvC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,eAAe,EAAE,OAAO,CAAC,eAAe;SACzC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,IAAI,2BAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAEpE,SAAS,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,YAAY,EAAE,EAAE;YAChD,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,UAAU,EAAE,EAAE;YAC5C,QAAQ,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,UAAU,EAAE,EAAE;YACjD,oFAAoF;QACtF,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,UAAU,EAAE,EAAE;YACjD,QAAQ,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE;YACvD,QAAQ,CAAC,kBAAkB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE;YAC9B,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,6BAA6B;QAC7B,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,wBAAwB,EAAE,CAAC;QAC/D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CAAC,UAAU,OAAO,CAAC,OAAO,gBAAgB,CAAC,CAAC;YACzD,OAAO,CAAC,KAAK,CACX,kBAAkB,OAAO,CAAC,OAAO,0BAA0B,CAC5D,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,mBAAmB;QACnB,MAAM,iBAAiB,GAAG,SAAS,CAAC,eAAe,CAAC,SAAS,EAAE;YAC7D,eAAe,EAAE,OAAO,CAAC,QAAQ;YACjC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,OAAO,CAAC,GAAG;gBACf,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACrD,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;YACxD,OAAO;QACT,CAAC;QAED,iBAAiB;QACjB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAE3D,sCAAsC;QACtC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAExB,kCAAkC;QAClC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,EAAE,CAAC,aAAa,CACd,OAAO,CAAC,MAAM,EACd,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAC/B,OAAO,CACR,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,yBAAyB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YAC5D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL;;GAEG;AACH,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,+BAA+B,CAAC;KAC5C,MAAM,CAAC,yBAAyB,EAAE,oBAAoB,CAAC;KACvD,MAAM,CAAC,cAAc,EAAE,kCAAkC,CAAC;KAC1D,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAA,mBAAU,GAAE,CAAC;QAEzC,IAAI,QAAQ,GAAG,SAAS,CAAC;QAEzB,qBAAqB;QACrB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ;iBAChC,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QACrE,CAAC;QAED,iBAAiB;QACjB,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACjE,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/B,IAAI,CAAC,IAAI,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CACjD,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,CAAC,MAAM,MAAM,CAAC,CAAC;QAE7D,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,oBAAoB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YACxD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL;;GAEG;AACH,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,0DAA0D,CAAC;KACvE,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAElD,MAAM,QAAQ,GAA+C;QAC3D,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,SAAS,EAAE;QAC/C,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,aAAa,EAAE;KACjD,CAAC;IAEF,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,OAAO,CAAC;QACZ,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,GAAG,IAAI,8BAAiB,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,IAAI,oCAAoB,EAAE,CAAC;QACvC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,KAAK,MAAM,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,IAAA,4BAAe,GAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClD,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;IAEnD,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAC1B,OAAO,CAAC,GAAG,CACT,6DAA6D,CAC9D,CAAC;QACF,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;IAC3E,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC,CAAC,CAAC;AAEL;;GAEG;AACH,OAAO;KACJ,OAAO,CAAC,UAAU,CAAC;KACnB,WAAW,CAAC,qCAAqC,CAAC;KAClD,MAAM,CAAC,iBAAiB,EAAE,4CAA4C,CAAC;KACvE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;QAEvD,yBAAyB;QACzB,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE;YACjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE/B,MAAM,SAAS,GAAG,IAAI,4BAAiB,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YAClE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YAEhE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Core type definitions for coding-agent-benchmarks
|
|
3
3
|
*/
|
|
4
|
+
/**
|
|
5
|
+
* User entity with metadata
|
|
6
|
+
*/
|
|
7
|
+
export interface User {
|
|
8
|
+
id: number;
|
|
9
|
+
name: string;
|
|
10
|
+
email: string;
|
|
11
|
+
metadata: Record<string, unknown>;
|
|
12
|
+
}
|
|
4
13
|
/**
|
|
5
14
|
* Generic categories for test scenarios
|
|
6
15
|
*/
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,GAAG,cAAc,GAAG,aAAa,GAAG,SAAS,CAAC;AAE3G;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,OAAO,GAAG,OAAO,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,aAAa,CAAC;AAEpD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,iBAAiB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEtC;;OAEG;IACH,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAErC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAErC;;OAEG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEpC;;OAEG;IACH,yBAAyB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAE9C;;OAEG;IACH,wBAAwB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC9C;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAE7B;;OAEG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAE9B;;OAEG;IACH,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IAEvB;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;OAEG;IACH,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IAExB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEjC;;OAEG;IACH,kBAAkB,EAAE,kBAAkB,CAAC;IAEvC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;OAEG;IACH,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAEzC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,UAAU,EAAE,SAAS,EAAE,CAAC;IAExB;;OAEG;IACH,aAAa,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAElD;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IAEvB;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAEhB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,iBAAiB,EAAE,gBAAgB,EAAE,CAAC;IAEtC;;OAEG;IACH,UAAU,EAAE,SAAS,EAAE,CAAC;IAExB;;OAEG;IACH,aAAa,CAAC,EAAE;QACd,KAAK,EAAE;YACL,IAAI,EAAE,MAAM,CAAC;YACb,OAAO,EAAE,MAAM,CAAC;SACjB,EAAE,CAAC;KACL,CAAC;IAEF;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,kBAAkB,CAAC,EAAE;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,OAAO,CAAC;KACxB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,OAAO,EAAE,WAAW,CAAC;IAErB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAE5B;;OAEG;IACH,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IAEF;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,IAAI,EAAE,WAAW,CAAC;IAElB;;OAEG;IACH,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtC;;;;;;OAMG;IACH,QAAQ,CACN,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAEzC;;;;;OAKG;IACH,QAAQ,CACN,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,QAAQ,EAAE,YAAY,GACrB,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAE3B;;OAEG;IACH,cAAc,CAAC,EAAE,WAAW,CAAC;IAE7B;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE/B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,GAAG,cAAc,GAAG,aAAa,GAAG,SAAS,CAAC;AAE3G;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,OAAO,GAAG,OAAO,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,aAAa,CAAC;AAEpD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,iBAAiB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEtC;;OAEG;IACH,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAErC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAErC;;OAEG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEpC;;OAEG;IACH,yBAAyB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAE9C;;OAEG;IACH,wBAAwB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC9C;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAE7B;;OAEG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAE9B;;OAEG;IACH,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IAEvB;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;OAEG;IACH,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IAExB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEjC;;OAEG;IACH,kBAAkB,EAAE,kBAAkB,CAAC;IAEvC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;OAEG;IACH,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAEzC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,UAAU,EAAE,SAAS,EAAE,CAAC;IAExB;;OAEG;IACH,aAAa,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAElD;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IAEvB;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAEhB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,iBAAiB,EAAE,gBAAgB,EAAE,CAAC;IAEtC;;OAEG;IACH,UAAU,EAAE,SAAS,EAAE,CAAC;IAExB;;OAEG;IACH,aAAa,CAAC,EAAE;QACd,KAAK,EAAE;YACL,IAAI,EAAE,MAAM,CAAC;YACb,OAAO,EAAE,MAAM,CAAC;SACjB,EAAE,CAAC;KACL,CAAC;IAEF;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,kBAAkB,CAAC,EAAE;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,OAAO,CAAC;KACxB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,OAAO,EAAE,WAAW,CAAC;IAErB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAE5B;;OAEG;IACH,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IAEF;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,IAAI,EAAE,WAAW,CAAC;IAElB;;OAEG;IACH,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtC;;;;;;OAMG;IACH,QAAQ,CACN,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAEzC;;;;;OAKG;IACH,QAAQ,CACN,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,QAAQ,EAAE,YAAY,GACrB,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAE3B;;OAEG;IACH,cAAc,CAAC,EAAE,WAAW,CAAC;IAE7B;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE/B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}
|
package/dist/utils/gitUtils.d.ts
CHANGED
|
@@ -19,6 +19,13 @@ export declare const parseGitStatus: (statusOutput: string) => string[];
|
|
|
19
19
|
* @returns Array of file paths that were changed
|
|
20
20
|
*/
|
|
21
21
|
export declare const getChangedFiles: (workspaceRoot: string) => string[];
|
|
22
|
+
/**
|
|
23
|
+
* Get files that changed between two git status snapshots
|
|
24
|
+
* @param before Git status output before operation
|
|
25
|
+
* @param after Git status output after operation
|
|
26
|
+
* @returns Array of file paths that were added or modified
|
|
27
|
+
*/
|
|
28
|
+
export declare const getChangedFilesDiff: (before: string, after: string) => string[];
|
|
22
29
|
/**
|
|
23
30
|
* Get the git root directory
|
|
24
31
|
* @returns Absolute path to git root
|
|
@@ -30,10 +37,4 @@ export declare const getGitRoot: () => string;
|
|
|
30
37
|
* @returns True if inside a git repository
|
|
31
38
|
*/
|
|
32
39
|
export declare const isGitRepository: (directory: string) => boolean;
|
|
33
|
-
/**
|
|
34
|
-
* Reset git working directory to clean state (discard all changes)
|
|
35
|
-
* WARNING: This will discard all uncommitted changes
|
|
36
|
-
* @param workspaceRoot The workspace root directory
|
|
37
|
-
*/
|
|
38
|
-
export declare const resetGitWorkingDirectory: (workspaceRoot: string) => void;
|
|
39
40
|
//# sourceMappingURL=gitUtils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gitUtils.d.ts","sourceRoot":"","sources":["../../src/utils/gitUtils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH;;;;GAIG;AACH,eAAO,MAAM,qBAAqB,GAAI,eAAe,MAAM,KAAG,MAU7D,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,cAAc,MAAM,KAAG,MAAM,EAqB3D,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,eAAe,GAAI,eAAe,MAAM,KAAG,MAAM,EAG7D,CAAC;AAEF
|
|
1
|
+
{"version":3,"file":"gitUtils.d.ts","sourceRoot":"","sources":["../../src/utils/gitUtils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH;;;;GAIG;AACH,eAAO,MAAM,qBAAqB,GAAI,eAAe,MAAM,KAAG,MAU7D,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,cAAc,MAAM,KAAG,MAAM,EAqB3D,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,eAAe,GAAI,eAAe,MAAM,KAAG,MAAM,EAG7D,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB,GAAI,QAAQ,MAAM,EAAE,OAAO,MAAM,KAAG,MAAM,EAqBzE,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,UAAU,QAAO,MAS7B,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,eAAe,GAAI,WAAW,MAAM,KAAG,OAWnD,CAAC"}
|
package/dist/utils/gitUtils.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Git utilities for tracking file changes
|
|
4
4
|
*/
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.
|
|
6
|
+
exports.isGitRepository = exports.getGitRoot = exports.getChangedFilesDiff = exports.getChangedFiles = exports.parseGitStatus = exports.getGitStatusPorcelain = void 0;
|
|
7
7
|
const child_process_1 = require("child_process");
|
|
8
8
|
/**
|
|
9
9
|
* Get git status in porcelain format
|
|
@@ -60,6 +60,32 @@ const getChangedFiles = (workspaceRoot) => {
|
|
|
60
60
|
return (0, exports.parseGitStatus)(statusOutput);
|
|
61
61
|
};
|
|
62
62
|
exports.getChangedFiles = getChangedFiles;
|
|
63
|
+
/**
|
|
64
|
+
* Get files that changed between two git status snapshots
|
|
65
|
+
* @param before Git status output before operation
|
|
66
|
+
* @param after Git status output after operation
|
|
67
|
+
* @returns Array of file paths that were added or modified
|
|
68
|
+
*/
|
|
69
|
+
const getChangedFilesDiff = (before, after) => {
|
|
70
|
+
const beforeLines = new Set(before.split('\n').filter(Boolean));
|
|
71
|
+
const afterLines = after.split('\n').filter(Boolean);
|
|
72
|
+
const newOrModified = [];
|
|
73
|
+
for (const line of afterLines) {
|
|
74
|
+
if (!beforeLines.has(line)) {
|
|
75
|
+
// Extract file path (e.g. "?? src/file.tsx" becomes "src/file.tsx")
|
|
76
|
+
const match = /^.{3}(.+)$/.exec(line);
|
|
77
|
+
if (match) {
|
|
78
|
+
const filePath = match[1];
|
|
79
|
+
// Skip directories (end with /)
|
|
80
|
+
if (!filePath.endsWith('/')) {
|
|
81
|
+
newOrModified.push(filePath);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return newOrModified;
|
|
87
|
+
};
|
|
88
|
+
exports.getChangedFilesDiff = getChangedFilesDiff;
|
|
63
89
|
/**
|
|
64
90
|
* Get the git root directory
|
|
65
91
|
* @returns Absolute path to git root
|
|
@@ -95,27 +121,4 @@ const isGitRepository = (directory) => {
|
|
|
95
121
|
}
|
|
96
122
|
};
|
|
97
123
|
exports.isGitRepository = isGitRepository;
|
|
98
|
-
/**
|
|
99
|
-
* Reset git working directory to clean state (discard all changes)
|
|
100
|
-
* WARNING: This will discard all uncommitted changes
|
|
101
|
-
* @param workspaceRoot The workspace root directory
|
|
102
|
-
*/
|
|
103
|
-
const resetGitWorkingDirectory = (workspaceRoot) => {
|
|
104
|
-
try {
|
|
105
|
-
// Reset all tracked files
|
|
106
|
-
(0, child_process_1.execSync)('git reset --hard HEAD', {
|
|
107
|
-
cwd: workspaceRoot,
|
|
108
|
-
stdio: ['pipe', 'pipe', 'pipe'],
|
|
109
|
-
});
|
|
110
|
-
// Remove all untracked files and directories
|
|
111
|
-
(0, child_process_1.execSync)('git clean -fd', {
|
|
112
|
-
cwd: workspaceRoot,
|
|
113
|
-
stdio: ['pipe', 'pipe', 'pipe'],
|
|
114
|
-
});
|
|
115
|
-
}
|
|
116
|
-
catch (error) {
|
|
117
|
-
throw new Error(`Failed to reset git working directory: ${error}`);
|
|
118
|
-
}
|
|
119
|
-
};
|
|
120
|
-
exports.resetGitWorkingDirectory = resetGitWorkingDirectory;
|
|
121
124
|
//# sourceMappingURL=gitUtils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gitUtils.js","sourceRoot":"","sources":["../../src/utils/gitUtils.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,iDAAyC;AAEzC;;;;GAIG;AACI,MAAM,qBAAqB,GAAG,CAAC,aAAqB,EAAU,EAAE;IACrE,IAAI,CAAC;QACH,OAAO,IAAA,wBAAQ,EAAC,wBAAwB,EAAE;YACxC,GAAG,EAAE,aAAa;YAClB,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;IACxD,CAAC;AACH,CAAC,CAAC;AAVW,QAAA,qBAAqB,yBAUhC;AAEF;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAC,YAAoB,EAAY,EAAE;IAC/D,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,oCAAoC;QACpC,4CAA4C;QAC5C,gEAAgE;QAChE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1C,8CAA8C;YAC9C,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9C,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AArBW,QAAA,cAAc,kBAqBzB;AAEF;;;;GAIG;AACI,MAAM,eAAe,GAAG,CAAC,aAAqB,EAAY,EAAE;IACjE,MAAM,YAAY,GAAG,IAAA,6BAAqB,EAAC,aAAa,CAAC,CAAC;IAC1D,OAAO,IAAA,sBAAc,EAAC,YAAY,CAAC,CAAC;AACtC,CAAC,CAAC;AAHW,QAAA,eAAe,mBAG1B;AAEF
|
|
1
|
+
{"version":3,"file":"gitUtils.js","sourceRoot":"","sources":["../../src/utils/gitUtils.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,iDAAyC;AAEzC;;;;GAIG;AACI,MAAM,qBAAqB,GAAG,CAAC,aAAqB,EAAU,EAAE;IACrE,IAAI,CAAC;QACH,OAAO,IAAA,wBAAQ,EAAC,wBAAwB,EAAE;YACxC,GAAG,EAAE,aAAa;YAClB,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;IACxD,CAAC;AACH,CAAC,CAAC;AAVW,QAAA,qBAAqB,yBAUhC;AAEF;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAC,YAAoB,EAAY,EAAE;IAC/D,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,oCAAoC;QACpC,4CAA4C;QAC5C,gEAAgE;QAChE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1C,8CAA8C;YAC9C,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9C,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AArBW,QAAA,cAAc,kBAqBzB;AAEF;;;;GAIG;AACI,MAAM,eAAe,GAAG,CAAC,aAAqB,EAAY,EAAE;IACjE,MAAM,YAAY,GAAG,IAAA,6BAAqB,EAAC,aAAa,CAAC,CAAC;IAC1D,OAAO,IAAA,sBAAc,EAAC,YAAY,CAAC,CAAC;AACtC,CAAC,CAAC;AAHW,QAAA,eAAe,mBAG1B;AAEF;;;;;GAKG;AACI,MAAM,mBAAmB,GAAG,CAAC,MAAc,EAAE,KAAa,EAAY,EAAE;IAC7E,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IAChE,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAErD,MAAM,aAAa,GAAa,EAAE,CAAC;IAEnC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,oEAAoE;YACpE,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,gCAAgC;gBAChC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5B,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC/B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,aAAa,CAAC;AACvB,CAAC,CAAC;AArBW,QAAA,mBAAmB,uBAqB9B;AAEF;;;GAGG;AACI,MAAM,UAAU,GAAG,GAAW,EAAE;IACrC,IAAI,CAAC;QACH,OAAO,IAAA,wBAAQ,EAAC,+BAA+B,EAAE;YAC/C,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACjD,CAAC;AACH,CAAC,CAAC;AATW,QAAA,UAAU,cASrB;AAEF;;;;GAIG;AACI,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAW,EAAE;IAC5D,IAAI,CAAC;QACH,IAAA,wBAAQ,EAAC,yBAAyB,EAAE;YAClC,GAAG,EAAE,SAAS;YACd,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;SAClC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC,CAAC;AAXW,QAAA,eAAe,mBAW1B"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "coding-agent-benchmarks",
|
|
3
|
-
"version": "0.1.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.1.4",
|
|
4
|
+
"description": "Testing coding agents (GitHub Copilot CLI, Claude Code, etc.) with your repo's context to evaluate their code generation quality.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
7
7
|
"bin": {
|
|
@@ -18,10 +18,24 @@
|
|
|
18
18
|
"benchmark",
|
|
19
19
|
"evaluation",
|
|
20
20
|
"copilot",
|
|
21
|
+
"github-copilot",
|
|
22
|
+
"claude",
|
|
21
23
|
"claude-code",
|
|
22
24
|
"ai",
|
|
25
|
+
"llm",
|
|
26
|
+
"openai",
|
|
27
|
+
"gpt",
|
|
23
28
|
"testing",
|
|
24
|
-
"code-quality"
|
|
29
|
+
"code-quality",
|
|
30
|
+
"code-generation",
|
|
31
|
+
"code-standards",
|
|
32
|
+
"lint",
|
|
33
|
+
"eslint",
|
|
34
|
+
"ci-cd",
|
|
35
|
+
"automation",
|
|
36
|
+
"developer-tools",
|
|
37
|
+
"prompt-testing",
|
|
38
|
+
"ai-testing"
|
|
25
39
|
],
|
|
26
40
|
"author": "chiItepin",
|
|
27
41
|
"license": "MIT",
|
|
@@ -38,12 +52,15 @@
|
|
|
38
52
|
"LICENSE"
|
|
39
53
|
],
|
|
40
54
|
"dependencies": {
|
|
41
|
-
"
|
|
55
|
+
"chalk": "^4.1.2",
|
|
56
|
+
"commander": "^13.1.0",
|
|
57
|
+
"log-update": "^4.0.0"
|
|
42
58
|
},
|
|
43
59
|
"devDependencies": {
|
|
44
60
|
"@types/node": "^25.0.9",
|
|
45
61
|
"eslint": "^9.39.2",
|
|
46
62
|
"prettier": "^3.8.0",
|
|
63
|
+
"tsup": "^8.5.1",
|
|
47
64
|
"tsx": "^4.21.0",
|
|
48
65
|
"typescript": "^5.9.3"
|
|
49
66
|
}
|