coding-agent-benchmarks 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -1
- package/dist/User.d.ts +10 -0
- package/dist/User.d.ts.map +1 -0
- package/dist/User.js +3 -0
- package/dist/User.js.map +1 -0
- package/dist/adapters/claudeCodeCLI.d.ts +10 -1
- package/dist/adapters/claudeCodeCLI.d.ts.map +1 -1
- package/dist/adapters/claudeCodeCLI.js +86 -25
- package/dist/adapters/claudeCodeCLI.js.map +1 -1
- package/dist/adapters/copilotCLI.d.ts +11 -2
- package/dist/adapters/copilotCLI.d.ts.map +1 -1
- package/dist/adapters/copilotCLI.js +99 -36
- package/dist/adapters/copilotCLI.js.map +1 -1
- package/dist/evaluator.d.ts +20 -2
- package/dist/evaluator.d.ts.map +1 -1
- package/dist/evaluator.js +58 -44
- package/dist/evaluator.js.map +1 -1
- package/dist/reporter.d.ts +35 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporter.js +232 -0
- package/dist/reporter.js.map +1 -0
- package/dist/runner.js +75 -63
- package/dist/runner.js.map +1 -1
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/gitUtils.d.ts +7 -6
- package/dist/utils/gitUtils.d.ts.map +1 -1
- package/dist/utils/gitUtils.js +27 -24
- package/dist/utils/gitUtils.js.map +1 -1
- package/package.json +21 -4
package/dist/evaluator.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.Evaluator = void 0;
|
|
7
|
+
const events_1 = require("events");
|
|
7
8
|
const copilotCLI_1 = require("./adapters/copilotCLI");
|
|
8
9
|
const claudeCodeCLI_1 = require("./adapters/claudeCodeCLI");
|
|
9
10
|
const patternValidator_1 = require("./validators/patternValidator");
|
|
@@ -11,23 +12,39 @@ const llmJudge_1 = require("./validators/llmJudge");
|
|
|
11
12
|
const eslintValidator_1 = require("./validators/eslintValidator");
|
|
12
13
|
const workspaceUtils_1 = require("./utils/workspaceUtils");
|
|
13
14
|
const baselineManager_1 = require("./utils/baselineManager");
|
|
14
|
-
class Evaluator {
|
|
15
|
+
class Evaluator extends events_1.EventEmitter {
|
|
15
16
|
constructor(options) {
|
|
17
|
+
super();
|
|
16
18
|
this.options = options;
|
|
17
19
|
this.workspaceRoot = (0, workspaceUtils_1.resolveWorkspaceRoot)(options.workspaceRoot);
|
|
18
20
|
this.baselineManager = new baselineManager_1.BaselineManager(this.workspaceRoot);
|
|
19
21
|
// Create adapter based on type
|
|
20
22
|
this.adapter = this.createAdapter(options.adapter);
|
|
21
23
|
}
|
|
24
|
+
/**
|
|
25
|
+
* Type-safe event emitter methods
|
|
26
|
+
*/
|
|
27
|
+
on(event, listener) {
|
|
28
|
+
return super.on(event, listener);
|
|
29
|
+
}
|
|
30
|
+
emit(event, ...args) {
|
|
31
|
+
return super.emit(event, ...args);
|
|
32
|
+
}
|
|
22
33
|
/**
|
|
23
34
|
* Create adapter instance based on type
|
|
24
35
|
*/
|
|
25
36
|
createAdapter(type) {
|
|
26
37
|
switch (type) {
|
|
27
|
-
case
|
|
28
|
-
return new copilotCLI_1.CopilotCLIAdapter(
|
|
29
|
-
|
|
30
|
-
|
|
38
|
+
case "copilot":
|
|
39
|
+
return new copilotCLI_1.CopilotCLIAdapter({
|
|
40
|
+
workspaceRoot: this.workspaceRoot,
|
|
41
|
+
model: this.options.model,
|
|
42
|
+
});
|
|
43
|
+
case "claude-code":
|
|
44
|
+
return new claudeCodeCLI_1.ClaudeCodeCLIAdapter({
|
|
45
|
+
workspaceRoot: this.workspaceRoot,
|
|
46
|
+
model: this.options.model,
|
|
47
|
+
});
|
|
31
48
|
default:
|
|
32
49
|
throw new Error(`Unknown adapter type: ${type}`);
|
|
33
50
|
}
|
|
@@ -45,18 +62,18 @@ class Evaluator {
|
|
|
45
62
|
let filtered = scenarios;
|
|
46
63
|
// Filter by scenario ID pattern
|
|
47
64
|
if (filters.scenarioPattern) {
|
|
48
|
-
const pattern = filters.scenarioPattern.replace(/\*/g,
|
|
65
|
+
const pattern = filters.scenarioPattern.replace(/\*/g, ".*");
|
|
49
66
|
const regex = new RegExp(pattern);
|
|
50
|
-
filtered = filtered.filter(s => regex.test(s.id));
|
|
67
|
+
filtered = filtered.filter((s) => regex.test(s.id));
|
|
51
68
|
}
|
|
52
69
|
// Filter by category
|
|
53
70
|
if (filters.category) {
|
|
54
|
-
const categories = filters.category.split(
|
|
55
|
-
filtered = filtered.filter(s => categories.includes(s.category));
|
|
71
|
+
const categories = filters.category.split(",").map((c) => c.trim());
|
|
72
|
+
filtered = filtered.filter((s) => categories.includes(s.category));
|
|
56
73
|
}
|
|
57
74
|
// Filter by tags
|
|
58
75
|
if (filters.tags && filters.tags.length > 0) {
|
|
59
|
-
filtered = filtered.filter(s => filters.tags.some(tag => s.tags.includes(tag)));
|
|
76
|
+
filtered = filtered.filter((s) => filters.tags.some((tag) => s.tags.includes(tag)));
|
|
60
77
|
}
|
|
61
78
|
return filtered;
|
|
62
79
|
}
|
|
@@ -67,12 +84,13 @@ class Evaluator {
|
|
|
67
84
|
const startTime = Date.now();
|
|
68
85
|
try {
|
|
69
86
|
if (this.options.verbose) {
|
|
70
|
-
|
|
71
|
-
|
|
87
|
+
this.emit("log", `\nEvaluating scenario: ${scenario.id}`);
|
|
88
|
+
this.emit("log", ` Description: ${scenario.description}`);
|
|
72
89
|
}
|
|
73
|
-
//
|
|
90
|
+
// Emit generating phase
|
|
91
|
+
this.emit("scenario:generating", scenario.id);
|
|
74
92
|
if (this.options.verbose) {
|
|
75
|
-
|
|
93
|
+
this.emit("log", " Generating code...");
|
|
76
94
|
}
|
|
77
95
|
// Resolve timeout (null = no timeout, undefined = use defaults)
|
|
78
96
|
let timeout;
|
|
@@ -90,8 +108,10 @@ class Evaluator {
|
|
|
90
108
|
}
|
|
91
109
|
const generatedFiles = await this.adapter.generate(scenario.prompt, scenario.contextFiles, timeout);
|
|
92
110
|
if (this.options.verbose) {
|
|
93
|
-
|
|
111
|
+
this.emit("log", ` Generated ${generatedFiles.length} file(s)`);
|
|
94
112
|
}
|
|
113
|
+
// Emit validating phase
|
|
114
|
+
this.emit("scenario:validating", scenario.id);
|
|
95
115
|
// Run validators
|
|
96
116
|
const validationResults = [];
|
|
97
117
|
// Pattern validator
|
|
@@ -99,29 +119,30 @@ class Evaluator {
|
|
|
99
119
|
const patternResult = await patternValidator.validate(generatedFiles, scenario);
|
|
100
120
|
validationResults.push(patternResult);
|
|
101
121
|
if (this.options.verbose && patternResult.score >= 0) {
|
|
102
|
-
|
|
122
|
+
this.emit("log", ` Pattern validation: ${patternResult.score.toFixed(2)}`);
|
|
103
123
|
}
|
|
104
124
|
// LLM judge validator
|
|
105
125
|
const llmValidator = new llmJudge_1.LLMJudgeValidator(this.workspaceRoot, this.options.model);
|
|
106
126
|
const llmResult = await llmValidator.validate(generatedFiles, scenario);
|
|
107
127
|
validationResults.push(llmResult);
|
|
108
128
|
if (this.options.verbose && llmResult.score >= 0) {
|
|
109
|
-
|
|
129
|
+
this.emit("log", ` LLM judge: ${llmResult.score.toFixed(2)}`);
|
|
110
130
|
}
|
|
111
131
|
// ESLint validator
|
|
112
132
|
const eslintValidator = new eslintValidator_1.ESLintValidator(this.workspaceRoot);
|
|
113
133
|
const eslintResult = await eslintValidator.validate(generatedFiles, scenario);
|
|
114
134
|
validationResults.push(eslintResult);
|
|
115
135
|
if (this.options.verbose && eslintResult.score >= 0) {
|
|
116
|
-
|
|
136
|
+
this.emit("log", ` ESLint: ${eslintResult.score.toFixed(2)}`);
|
|
117
137
|
}
|
|
118
138
|
// Calculate overall score (average of non-skipped validators)
|
|
119
|
-
const activeResults = validationResults.filter(r => r.score >= 0);
|
|
139
|
+
const activeResults = validationResults.filter((r) => r.score >= 0);
|
|
120
140
|
const overallScore = activeResults.length > 0
|
|
121
|
-
? activeResults.reduce((sum, r) => sum + r.score, 0) /
|
|
141
|
+
? activeResults.reduce((sum, r) => sum + r.score, 0) /
|
|
142
|
+
activeResults.length
|
|
122
143
|
: 0;
|
|
123
144
|
// Collect all violations
|
|
124
|
-
const allViolations = validationResults.flatMap(r => r.violations);
|
|
145
|
+
const allViolations = validationResults.flatMap((r) => r.violations);
|
|
125
146
|
// Check if passed (score above threshold and no violations)
|
|
126
147
|
const passed = overallScore >= 0.8 && allViolations.length === 0;
|
|
127
148
|
const result = {
|
|
@@ -134,26 +155,26 @@ class Evaluator {
|
|
|
134
155
|
};
|
|
135
156
|
// Compare with baseline if requested
|
|
136
157
|
if (this.options.compareBaseline) {
|
|
137
|
-
const comparison = this.baselineManager.compareWithBaseline(result, this.options.adapter, this.options.model ||
|
|
158
|
+
const comparison = this.baselineManager.compareWithBaseline(result, this.options.adapter, this.options.model || "default");
|
|
138
159
|
if (comparison) {
|
|
139
160
|
result.baselineComparison = comparison;
|
|
140
161
|
}
|
|
141
162
|
}
|
|
142
163
|
// Save baseline if requested
|
|
143
164
|
if (this.options.saveBaseline) {
|
|
144
|
-
this.baselineManager.saveBaseline(result, this.options.adapter, this.options.model ||
|
|
165
|
+
this.baselineManager.saveBaseline(result, this.options.adapter, this.options.model || "default");
|
|
145
166
|
}
|
|
146
167
|
return result;
|
|
147
168
|
}
|
|
148
169
|
catch (error) {
|
|
149
170
|
const errorMessage = String(error);
|
|
150
|
-
const isTimeout = errorMessage.includes(
|
|
171
|
+
const isTimeout = errorMessage.includes("timed out");
|
|
151
172
|
// Create a violation for timeout errors
|
|
152
173
|
const violations = isTimeout
|
|
153
174
|
? [
|
|
154
175
|
{
|
|
155
|
-
type:
|
|
156
|
-
message:
|
|
176
|
+
type: "pattern",
|
|
177
|
+
message: "Code generation timed out",
|
|
157
178
|
severity: scenario.severity,
|
|
158
179
|
details: errorMessage,
|
|
159
180
|
},
|
|
@@ -176,30 +197,21 @@ class Evaluator {
|
|
|
176
197
|
async evaluate(scenarios) {
|
|
177
198
|
const startTime = Date.now();
|
|
178
199
|
const results = [];
|
|
179
|
-
|
|
200
|
+
// Emit evaluation start event
|
|
201
|
+
this.emit("evaluation:start", scenarios);
|
|
180
202
|
for (let i = 0; i < scenarios.length; i++) {
|
|
181
203
|
const scenario = scenarios[i];
|
|
182
|
-
|
|
204
|
+
// Emit scenario start event
|
|
205
|
+
this.emit("scenario:start", scenario.id, scenario);
|
|
183
206
|
const result = await this.evaluateScenario(scenario);
|
|
184
207
|
results.push(result);
|
|
185
|
-
//
|
|
186
|
-
|
|
187
|
-
console.log(` ✓ PASSED (score: ${result.score.toFixed(2)})`);
|
|
188
|
-
}
|
|
189
|
-
else {
|
|
190
|
-
console.log(` ✗ FAILED (score: ${result.score.toFixed(2)})`);
|
|
191
|
-
if (result.violations.length > 0) {
|
|
192
|
-
console.log(` ${result.violations.length} violation(s)`);
|
|
193
|
-
}
|
|
194
|
-
if (result.error) {
|
|
195
|
-
console.log(` Error: ${result.error}`);
|
|
196
|
-
}
|
|
197
|
-
}
|
|
208
|
+
// Emit scenario complete event
|
|
209
|
+
this.emit("scenario:complete", scenario.id, result);
|
|
198
210
|
}
|
|
199
211
|
// Calculate summary statistics
|
|
200
|
-
const passed = results.filter(r => r.passed).length;
|
|
201
|
-
const failed = results.filter(r => !r.passed && !r.error).length;
|
|
202
|
-
const skipped = results.filter(r => r.error).length;
|
|
212
|
+
const passed = results.filter((r) => r.passed).length;
|
|
213
|
+
const failed = results.filter((r) => !r.passed && !r.error).length;
|
|
214
|
+
const skipped = results.filter((r) => r.error).length;
|
|
203
215
|
const totalViolations = results.reduce((sum, r) => sum + r.violations.length, 0);
|
|
204
216
|
const averageScore = results.length > 0
|
|
205
217
|
? results.reduce((sum, r) => sum + r.score, 0) / results.length
|
|
@@ -219,6 +231,8 @@ class Evaluator {
|
|
|
219
231
|
},
|
|
220
232
|
totalDuration: Date.now() - startTime,
|
|
221
233
|
};
|
|
234
|
+
// Emit evaluation complete event
|
|
235
|
+
this.emit("evaluation:complete", report);
|
|
222
236
|
return report;
|
|
223
237
|
}
|
|
224
238
|
}
|
package/dist/evaluator.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":";AAAA;;GAEG;;;
|
|
1
|
+
{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,mCAAsC;AAStC,sDAA0D;AAC1D,4DAAgE;AAChE,oEAAiE;AACjE,oDAA0D;AAC1D,kEAA+D;AAC/D,2DAA8D;AAC9D,6DAA0D;AAyB1D,MAAa,SAAU,SAAQ,qBAAY;IAMzC,YAAY,OAAyB;QACnC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACjE,IAAI,CAAC,eAAe,GAAG,IAAI,iCAAe,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE/D,+BAA+B;QAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACrD,CAAC;IAED;;OAEG;IACM,EAAE,CACT,KAAQ,EACR,QAA4B;QAE5B,OAAO,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACnC,CAAC;IAEQ,IAAI,CACX,KAAQ,EACR,GAAG,IAAoC;QAEvC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,IAAiB;QACrC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,SAAS;gBACZ,OAAO,IAAI,8BAAiB,CAAC;oBAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;iBAC1B,CAAC,CAAC;YACL,KAAK,aAAa;gBAChB,OAAO,IAAI,oCAAoB,CAAC;oBAC9B,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;iBAC1B,CAAC,CAAC;YACL;gBACE,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,wBAAwB;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,eAAe,CACb,SAAyB,EACzB,OAIC;QAED,IAAI,QAAQ,GAAG,SAAS,CAAC;QAEzB,gCAAgC;QAChC,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YAC7D,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,qBAAqB;QACrB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACpE,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QACrE,CAAC;QAED,iBAAiB;QACjB,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/B,OAAO,CAAC,IAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAClD,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,QAAsB;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,0BAA0B,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC1D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,kBAAkB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YAC7D,CAAC;YAED,wBAAwB;YACxB,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YAE9C,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAC;YAC3C,CAAC;YAED,gEAAgE;YAChE,IAAI,OAAsB,CAAC;YAC3B,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBACnC,6DAA6D;gBAC7D,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC7B,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;gBACrD,wDAAwD;gBACxD,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,8BAA8B;gBAC9B,OAAO,GAAG,MAAM,CAAC;YACnB,CAAC;YAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAChD,QAAQ,CAAC,MAAM,EACf,QAAQ,CAAC,YAAY,EACrB,OAAO,CACR,CAAC;YAEF,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,eAAe,cAAc,CAAC,MAAM,UAAU,CAAC,CAAC;YACnE,CAAC;YAED,wBAAwB;YACxB,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YAE9C,iBAAiB;YACjB,MAAM,iBAAiB,GAAuB,EAAE,CAAC;YAEjD,oBAAoB;YACpB,MAAM,gBAAgB,GAAG,IAAI,mCAAgB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAClE,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,QAAQ,CACnD,cAAc,EACd,QAAQ,CACT,CAAC;YACF,iBAAiB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAEtC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,aAAa,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;gBACrD,IAAI,CAAC,IAAI,CACP,KAAK,EACL,yBAAyB,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC1D,CAAC;YACJ,CAAC;YAED,sBAAsB;YACtB,MAAM,YAAY,GAAG,IAAI,4BAAiB,CACxC,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,OAAO,CAAC,KAAK,CACnB,CAAC;YACF,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YACxE,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAElC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;gBACjD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,gBAAgB,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YAED,mBAAmB;YACnB,MAAM,eAAe,GAAG,IAAI,iCAAe,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAChE,MAAM,YAAY,GAAG,MAAM,eAAe,CAAC,QAAQ,CACjD,cAAc,EACd,QAAQ,CACT,CAAC;YACF,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAErC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,YAAY,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;gBACpD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YAED,8DAA8D;YAC9D,MAAM,aAAa,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YACpE,MAAM,YAAY,GAChB,aAAa,CAAC,MAAM,GAAG,CAAC;gBACtB,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;oBAClD,aAAa,CAAC,MAAM;gBACtB,CAAC,CAAC,CAAC,CAAC;YAER,yBAAyB;YACzB,MAAM,aAAa,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAErE,4DAA4D;YAC5D,MAAM,MAAM,GAAG,YAAY,IAAI,GAAG,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC;YAEjE,MAAM,MAAM,GAAqB;gBAC/B,QAAQ;gBACR,MAAM;gBACN,KAAK,EAAE,YAAY;gBACnB,iBAAiB;gBACjB,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aACjC,CAAC;YAEF,qCAAqC;YACrC,IAAI,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBACjC,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CACzD,MAAM,EACN,IAAI,CAAC,OAAO,CAAC,OAAO,EACpB,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,SAAS,CAChC,CAAC;gBACF,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,CAAC,kBAAkB,GAAG,UAAU,CAAC;gBACzC,CAAC;YACH,CAAC;YAED,6BAA6B;YAC7B,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;gBAC9B,IAAI,CAAC,eAAe,CAAC,YAAY,CAC/B,MAAM,EACN,IAAI,CAAC,OAAO,CAAC,OAAO,EACpB,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,SAAS,CAChC,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAErD,wCAAwC;YACxC,MAAM,UAAU,GAAG,SAAS;gBAC1B,CAAC,CAAC;oBACE;wBACE,IAAI,EAAE,SAAkB;wBACxB,OAAO,EAAE,2BAA2B;wBACpC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;wBAC3B,OAAO,EAAE,YAAY;qBACtB;iBACF;gBACH,CAAC,CAAC,EAAE,CAAC;YAEP,OAAO;gBACL,QAAQ;gBACR,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,iBAAiB,EAAE,EAAE;gBACrB,UAAU;gBACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAChC,KAAK,EAAE,sBAAsB,KAAK,EAAE;aACrC,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,SAAyB;QACtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAuB,EAAE,CAAC;QAEvC,8BAA8B;QAC9B,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAC;QAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAE9B,4BAA4B;YAC5B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;YAEnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YACrD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAErB,+BAA+B;YAC/B,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACtD,CAAC;QAED,+BAA+B;QAC/B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACnE,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACtD,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CACpC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EACrC,CAAC,CACF,CAAC;QACF,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC;YAChB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;YAC/D,CAAC,CAAC,CAAC,CAAC;QAER,MAAM,MAAM,GAAqB;YAC/B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;YAC7B,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;YACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO;YACP,OAAO,EAAE;gBACP,KAAK,EAAE,SAAS,CAAC,MAAM;gBACvB,MAAM;gBACN,MAAM;gBACN,OAAO;gBACP,YAAY;gBACZ,eAAe;aAChB;YACD,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACtC,CAAC;QAEF,iCAAiC;QACjC,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;QAEzC,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAzTD,8BAyTC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { TestScenario, EvaluationResult, EvaluationReport } from "./types";
|
|
2
|
+
export type ExecutionPhase = "pending" | "generating" | "validating" | "complete";
|
|
3
|
+
export type ScenarioStatus = "runs" | "pass" | "fail" | "skip";
|
|
4
|
+
export declare class ProgressReporter {
|
|
5
|
+
private isInteractive;
|
|
6
|
+
private verbose;
|
|
7
|
+
private scenarios;
|
|
8
|
+
private scenarioOrder;
|
|
9
|
+
private currentIndex;
|
|
10
|
+
private totalScenarios;
|
|
11
|
+
private spinnerFrame;
|
|
12
|
+
private spinnerInterval?;
|
|
13
|
+
private startTime;
|
|
14
|
+
private verboseBuffer;
|
|
15
|
+
constructor(options?: {
|
|
16
|
+
verbose?: boolean;
|
|
17
|
+
});
|
|
18
|
+
start(scenarios: TestScenario[]): void;
|
|
19
|
+
onScenarioStart(scenarioId: string): void;
|
|
20
|
+
onScenarioValidating(scenarioId: string): void;
|
|
21
|
+
onScenarioComplete(scenarioId: string, result: EvaluationResult): void;
|
|
22
|
+
log(message: string): void;
|
|
23
|
+
finish(report: EvaluationReport): void;
|
|
24
|
+
private getScenarioIndex;
|
|
25
|
+
private startSpinner;
|
|
26
|
+
private stopSpinner;
|
|
27
|
+
private render;
|
|
28
|
+
private formatScenarioLine;
|
|
29
|
+
private printScenarioResult;
|
|
30
|
+
private printViolations;
|
|
31
|
+
private flushVerboseBuffer;
|
|
32
|
+
private printSummary;
|
|
33
|
+
private formatDuration;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=reporter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAEjB,MAAM,SAAS,CAAC;AAEjB,MAAM,MAAM,cAAc,GACtB,SAAS,GACT,YAAY,GACZ,YAAY,GACZ,UAAU,CAAC;AAEf,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAY/D,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,SAAS,CAAyC;IAC1D,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,eAAe,CAAC,CAAiC;IACzD,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,aAAa,CAAgB;gBAEzB,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAO;IAK/C,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,IAAI;IAsBtC,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAoBzC,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAa9C,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAiBtE,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAQ1B,MAAM,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAUtC,OAAO,CAAC,gBAAgB;IAIxB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,MAAM;IAgBd,OAAO,CAAC,kBAAkB;IAkC1B,OAAO,CAAC,mBAAmB;IAe3B,OAAO,CAAC,eAAe;IAevB,OAAO,CAAC,kBAAkB;IAW1B,OAAO,CAAC,YAAY;IAkCpB,OAAO,CAAC,cAAc;CAOvB"}
|
package/dist/reporter.js
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.ProgressReporter = void 0;
|
|
7
|
+
const chalk_1 = __importDefault(require("chalk"));
|
|
8
|
+
const log_update_1 = __importDefault(require("log-update"));
|
|
9
|
+
const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
|
|
10
|
+
class ProgressReporter {
|
|
11
|
+
constructor(options = {}) {
|
|
12
|
+
this.scenarios = new Map();
|
|
13
|
+
this.scenarioOrder = [];
|
|
14
|
+
this.currentIndex = 0;
|
|
15
|
+
this.totalScenarios = 0;
|
|
16
|
+
this.spinnerFrame = 0;
|
|
17
|
+
this.startTime = 0;
|
|
18
|
+
this.verboseBuffer = [];
|
|
19
|
+
this.isInteractive = process.stdout.isTTY === true;
|
|
20
|
+
this.verbose = options.verbose ?? false;
|
|
21
|
+
}
|
|
22
|
+
start(scenarios) {
|
|
23
|
+
this.startTime = Date.now();
|
|
24
|
+
this.totalScenarios = scenarios.length;
|
|
25
|
+
this.scenarioOrder = scenarios.map((s) => s.id);
|
|
26
|
+
this.currentIndex = 0;
|
|
27
|
+
for (const scenario of scenarios) {
|
|
28
|
+
this.scenarios.set(scenario.id, {
|
|
29
|
+
scenario,
|
|
30
|
+
phase: "pending",
|
|
31
|
+
status: "runs",
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
if (this.isInteractive) {
|
|
35
|
+
console.log(`\nEvaluating ${scenarios.length} scenario(s)...\n`);
|
|
36
|
+
this.startSpinner();
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
console.log(`Evaluating ${scenarios.length} scenario(s)...`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
onScenarioStart(scenarioId) {
|
|
43
|
+
const state = this.scenarios.get(scenarioId);
|
|
44
|
+
if (!state)
|
|
45
|
+
return;
|
|
46
|
+
state.phase = "generating";
|
|
47
|
+
state.status = "runs";
|
|
48
|
+
state.startTime = Date.now();
|
|
49
|
+
if (this.isInteractive) {
|
|
50
|
+
this.render();
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
console.log(`\n[${this.getScenarioIndex(scenarioId)}/${this.totalScenarios}] ${scenarioId}`);
|
|
54
|
+
if (this.verbose) {
|
|
55
|
+
console.log(` Generating code...`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
onScenarioValidating(scenarioId) {
|
|
60
|
+
const state = this.scenarios.get(scenarioId);
|
|
61
|
+
if (!state)
|
|
62
|
+
return;
|
|
63
|
+
state.phase = "validating";
|
|
64
|
+
if (this.isInteractive) {
|
|
65
|
+
this.render();
|
|
66
|
+
}
|
|
67
|
+
else if (this.verbose) {
|
|
68
|
+
console.log(` Validating...`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
onScenarioComplete(scenarioId, result) {
|
|
72
|
+
const state = this.scenarios.get(scenarioId);
|
|
73
|
+
if (!state)
|
|
74
|
+
return;
|
|
75
|
+
state.phase = "complete";
|
|
76
|
+
state.result = result;
|
|
77
|
+
state.status = result.error ? "skip" : result.passed ? "pass" : "fail";
|
|
78
|
+
if (this.isInteractive) {
|
|
79
|
+
this.render();
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
this.printScenarioResult(scenarioId, result);
|
|
83
|
+
}
|
|
84
|
+
this.flushVerboseBuffer();
|
|
85
|
+
}
|
|
86
|
+
log(message) {
|
|
87
|
+
if (this.isInteractive) {
|
|
88
|
+
this.verboseBuffer.push(message);
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
console.log(message);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
finish(report) {
|
|
95
|
+
this.stopSpinner();
|
|
96
|
+
if (this.isInteractive) {
|
|
97
|
+
log_update_1.default.done();
|
|
98
|
+
}
|
|
99
|
+
this.printSummary(report);
|
|
100
|
+
}
|
|
101
|
+
getScenarioIndex(scenarioId) {
|
|
102
|
+
return this.scenarioOrder.indexOf(scenarioId) + 1;
|
|
103
|
+
}
|
|
104
|
+
startSpinner() {
|
|
105
|
+
if (!this.isInteractive)
|
|
106
|
+
return;
|
|
107
|
+
this.spinnerInterval = setInterval(() => {
|
|
108
|
+
this.spinnerFrame = (this.spinnerFrame + 1) % SPINNER_FRAMES.length;
|
|
109
|
+
this.render();
|
|
110
|
+
}, 80);
|
|
111
|
+
}
|
|
112
|
+
stopSpinner() {
|
|
113
|
+
if (this.spinnerInterval) {
|
|
114
|
+
clearInterval(this.spinnerInterval);
|
|
115
|
+
this.spinnerInterval = undefined;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
render() {
|
|
119
|
+
if (!this.isInteractive)
|
|
120
|
+
return;
|
|
121
|
+
const lines = [];
|
|
122
|
+
for (const scenarioId of this.scenarioOrder) {
|
|
123
|
+
const state = this.scenarios.get(scenarioId);
|
|
124
|
+
if (!state)
|
|
125
|
+
continue;
|
|
126
|
+
const line = this.formatScenarioLine(state);
|
|
127
|
+
lines.push(line);
|
|
128
|
+
}
|
|
129
|
+
(0, log_update_1.default)(lines.join("\n"));
|
|
130
|
+
}
|
|
131
|
+
formatScenarioLine(state) {
|
|
132
|
+
const index = this.getScenarioIndex(state.scenario.id);
|
|
133
|
+
const prefix = `[${index}/${this.totalScenarios}]`;
|
|
134
|
+
switch (state.status) {
|
|
135
|
+
case "runs": {
|
|
136
|
+
const spinner = SPINNER_FRAMES[this.spinnerFrame];
|
|
137
|
+
const phaseText = state.phase === "generating" ? "generating..." : "validating...";
|
|
138
|
+
const elapsed = state.startTime
|
|
139
|
+
? this.formatDuration(Date.now() - state.startTime)
|
|
140
|
+
: "";
|
|
141
|
+
return `${chalk_1.default.yellow(spinner)} ${prefix} ${state.scenario.id} ${chalk_1.default.dim(phaseText)} ${chalk_1.default.dim(elapsed)}`;
|
|
142
|
+
}
|
|
143
|
+
case "pass": {
|
|
144
|
+
const score = state.result?.score.toFixed(2) ?? "0.00";
|
|
145
|
+
const duration = state.result
|
|
146
|
+
? this.formatDuration(state.result.duration)
|
|
147
|
+
: "";
|
|
148
|
+
return `${chalk_1.default.green("✓")} ${prefix} ${state.scenario.id} ${chalk_1.default.green("PASS")} ${chalk_1.default.dim(`(score: ${score})`)} ${chalk_1.default.dim(duration)}`;
|
|
149
|
+
}
|
|
150
|
+
case "fail": {
|
|
151
|
+
const score = state.result?.score.toFixed(2) ?? "0.00";
|
|
152
|
+
const duration = state.result
|
|
153
|
+
? this.formatDuration(state.result.duration)
|
|
154
|
+
: "";
|
|
155
|
+
return `${chalk_1.default.red("✗")} ${prefix} ${state.scenario.id} ${chalk_1.default.red("FAIL")} ${chalk_1.default.dim(`(score: ${score})`)} ${chalk_1.default.dim(duration)}`;
|
|
156
|
+
}
|
|
157
|
+
case "skip": {
|
|
158
|
+
return `${chalk_1.default.yellow("○")} ${prefix} ${state.scenario.id} ${chalk_1.default.yellow("SKIP")} ${chalk_1.default.dim("(error)")}`;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
printScenarioResult(scenarioId, result) {
|
|
163
|
+
if (result.passed) {
|
|
164
|
+
console.log(` ✓ PASSED (score: ${result.score.toFixed(2)})`);
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
console.log(` ✗ FAILED (score: ${result.score.toFixed(2)})`);
|
|
168
|
+
this.printViolations(result.violations);
|
|
169
|
+
if (result.error) {
|
|
170
|
+
console.log(` Error: ${result.error}`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
printViolations(violations) {
|
|
175
|
+
if (violations.length === 0)
|
|
176
|
+
return;
|
|
177
|
+
console.log(` ${violations.length} violation(s):\n`);
|
|
178
|
+
violations.forEach((v, idx) => {
|
|
179
|
+
console.log(` ${idx + 1}. [${v.type}] ${v.message}`);
|
|
180
|
+
if (v.file) {
|
|
181
|
+
console.log(` File: ${v.file}${v.line ? `:${v.line}` : ""}`);
|
|
182
|
+
}
|
|
183
|
+
if (v.details) {
|
|
184
|
+
console.log(` Details: ${v.details}`);
|
|
185
|
+
}
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
flushVerboseBuffer() {
|
|
189
|
+
if (!this.isInteractive || this.verboseBuffer.length === 0)
|
|
190
|
+
return;
|
|
191
|
+
log_update_1.default.clear();
|
|
192
|
+
this.verboseBuffer.forEach((message) => console.log(message));
|
|
193
|
+
this.verboseBuffer = [];
|
|
194
|
+
this.render();
|
|
195
|
+
}
|
|
196
|
+
printSummary(report) {
|
|
197
|
+
const { summary, totalDuration } = report;
|
|
198
|
+
if (this.isInteractive) {
|
|
199
|
+
report.results.forEach((result) => {
|
|
200
|
+
if (!result.passed) {
|
|
201
|
+
console.log(`\n${chalk_1.default.red("✗")} ${result.scenario.id}`);
|
|
202
|
+
this.printViolations(result.violations);
|
|
203
|
+
if (result.error) {
|
|
204
|
+
console.log(` Error: ${result.error}`);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
console.log("\n" + "=".repeat(60));
|
|
210
|
+
console.log("EVALUATION SUMMARY");
|
|
211
|
+
console.log("=".repeat(60));
|
|
212
|
+
console.log(`Total scenarios: ${summary.total}`);
|
|
213
|
+
console.log(`Passed: ${this.isInteractive ? chalk_1.default.green(summary.passed.toString()) : summary.passed}`);
|
|
214
|
+
console.log(`Failed: ${this.isInteractive ? chalk_1.default.red(summary.failed.toString()) : summary.failed}`);
|
|
215
|
+
console.log(`Skipped: ${this.isInteractive ? chalk_1.default.yellow(summary.skipped.toString()) : summary.skipped}`);
|
|
216
|
+
console.log(`Average score: ${summary.averageScore.toFixed(2)}`);
|
|
217
|
+
console.log(`Total violations: ${summary.totalViolations}`);
|
|
218
|
+
console.log(`Total duration: ${(totalDuration / 1000).toFixed(1)}s`);
|
|
219
|
+
console.log("=".repeat(60));
|
|
220
|
+
}
|
|
221
|
+
formatDuration(ms) {
|
|
222
|
+
if (ms < 1000)
|
|
223
|
+
return `${ms}ms`;
|
|
224
|
+
if (ms < 60000)
|
|
225
|
+
return `${(ms / 1000).toFixed(1)}s`;
|
|
226
|
+
const minutes = Math.floor(ms / 60000);
|
|
227
|
+
const seconds = ((ms % 60000) / 1000).toFixed(0);
|
|
228
|
+
return `${minutes}m ${seconds}s`;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
exports.ProgressReporter = ProgressReporter;
|
|
232
|
+
//# sourceMappingURL=reporter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":";;;;;;AAAA,kDAA0B;AAC1B,4DAAmC;AAwBnC,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E,MAAa,gBAAgB;IAY3B,YAAY,UAAiC,EAAE;QATvC,cAAS,GAA+B,IAAI,GAAG,EAAE,CAAC;QAClD,kBAAa,GAAa,EAAE,CAAC;QAC7B,iBAAY,GAAW,CAAC,CAAC;QACzB,mBAAc,GAAW,CAAC,CAAC;QAC3B,iBAAY,GAAW,CAAC,CAAC;QAEzB,cAAS,GAAW,CAAC,CAAC;QACtB,kBAAa,GAAa,EAAE,CAAC;QAGnC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC;QACnD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,SAAyB;QAC7B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC;QACvC,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE;gBAC9B,QAAQ;gBACR,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,gBAAgB,SAAS,CAAC,MAAM,mBAAmB,CAAC,CAAC;YACjE,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,MAAM,iBAAiB,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,eAAe,CAAC,UAAkB;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC7C,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,KAAK,CAAC,KAAK,GAAG,YAAY,CAAC;QAC3B,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;QACtB,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CACT,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,cAAc,KAAK,UAAU,EAAE,CAChF,CAAC;YACF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;IACH,CAAC;IAED,oBAAoB,CAAC,UAAkB;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC7C,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,KAAK,CAAC,KAAK,GAAG,YAAY,CAAC;QAE3B,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC;aAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,kBAAkB,CAAC,UAAkB,EAAE,MAAwB;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC7C,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,KAAK,CAAC,KAAK,GAAG,UAAU,CAAC;QACzB,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;QACtB,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAEvE,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC5B,CAAC;IAED,GAAG,CAAC,OAAe;QACjB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,MAAwB;QAC7B,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,oBAAS,CAAC,IAAI,EAAE,CAAC;QACnB,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAEO,gBAAgB,CAAC,UAAkB;QACzC,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACpD,CAAC;IAEO,YAAY;QAClB,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,OAAO;QAEhC,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC,GAAG,EAAE;YACtC,IAAI,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;YACpE,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC,EAAE,EAAE,CAAC,CAAC;IACT,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACpC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACnC,CAAC;IACH,CAAC;IAEO,MAAM;QACZ,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,OAAO;QAEhC,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC7C,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAC5C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;QAED,IAAA,oBAAS,EAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9B,CAAC;IAEO,kBAAkB,CAAC,KAAoB;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,KAAK,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC;QAEnD,QAAQ,KAAK,CAAC,MAAM,EAAE,CAAC;YACrB,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAClD,MAAM,SAAS,GACb,KAAK,CAAC,KAAK,KAAK,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,eAAe,CAAC;gBACnE,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS;oBAC7B,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC;oBACnD,CAAC,CAAC,EAAE,CAAC;gBACP,OAAO,GAAG,eAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,EAAE,IAAI,eAAK,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,eAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACjH,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC;gBACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM;oBAC3B,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;oBAC5C,CAAC,CAAC,EAAE,CAAC;gBACP,OAAO,GAAG,eAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,EAAE,IAAI,eAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,eAAK,CAAC,GAAG,CAAC,WAAW,KAAK,GAAG,CAAC,IAAI,eAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9I,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC;gBACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM;oBAC3B,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;oBAC5C,CAAC,CAAC,EAAE,CAAC;gBACP,OAAO,GAAG,eAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,EAAE,IAAI,eAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,eAAK,CAAC,GAAG,CAAC,WAAW,KAAK,GAAG,CAAC,IAAI,eAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1I,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,OAAO,GAAG,eAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,EAAE,IAAI,eAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,eAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/G,CAAC;QACH,CAAC;IACH,CAAC;IAEO,mBAAmB,CACzB,UAAkB,EAClB,MAAwB;QAExB,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChE,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC9D,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACxC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,UAAuB;QAC7C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEpC,OAAO,CAAC,GAAG,CAAC,OAAO,UAAU,CAAC,MAAM,kBAAkB,CAAC,CAAC;QACxD,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;YAC5B,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACxD,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBACX,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrE,CAAC;YACD,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB;QACxB,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEnE,oBAAS,CAAC,KAAK,EAAE,CAAC;QAElB,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;QAC9D,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QAExB,IAAI,CAAC,MAAM,EAAE,CAAC;IAChB,CAAC;IAEO,YAAY,CAAC,MAAwB;QAC3C,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,MAAM,CAAC;QAE1C,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;gBAChC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACnB,OAAO,CAAC,GAAG,CAAC,KAAK,eAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;oBACzD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;oBACxC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;wBACjB,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;oBAC1C,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CACT,WAAW,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,eAAK,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAC1F,CAAC;QACF,OAAO,CAAC,GAAG,CACT,WAAW,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CACxF,CAAC;QACF,OAAO,CAAC,GAAG,CACT,YAAY,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,eAAK,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAC9F,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,kBAAkB,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACjE,OAAO,CAAC,GAAG,CAAC,qBAAqB,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9B,CAAC;IAEO,cAAc,CAAC,EAAU;QAC/B,IAAI,EAAE,GAAG,IAAI;YAAE,OAAO,GAAG,EAAE,IAAI,CAAC;QAChC,IAAI,EAAE,GAAG,KAAK;YAAE,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACjD,OAAO,GAAG,OAAO,KAAK,OAAO,GAAG,CAAC;IACnC,CAAC;CACF;AAnQD,4CAmQC"}
|