@skillmark/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-entry-point.d.ts +3 -0
- package/dist/cli-entry-point.d.ts.map +1 -0
- package/dist/cli-entry-point.js +207 -0
- package/dist/cli-entry-point.js.map +1 -0
- package/dist/commands/auth-setup-and-token-storage-command.d.ts +21 -0
- package/dist/commands/auth-setup-and-token-storage-command.d.ts.map +1 -0
- package/dist/commands/auth-setup-and-token-storage-command.js +166 -0
- package/dist/commands/auth-setup-and-token-storage-command.js.map +1 -0
- package/dist/commands/publish-results-command.d.ts +21 -0
- package/dist/commands/publish-results-command.d.ts.map +1 -0
- package/dist/commands/publish-results-command.js +256 -0
- package/dist/commands/publish-results-command.js.map +1 -0
- package/dist/commands/run-benchmark-command.d.ts +6 -0
- package/dist/commands/run-benchmark-command.d.ts.map +1 -0
- package/dist/commands/run-benchmark-command.js +331 -0
- package/dist/commands/run-benchmark-command.js.map +1 -0
- package/dist/commands/view-leaderboard-command.d.ts +8 -0
- package/dist/commands/view-leaderboard-command.d.ts.map +1 -0
- package/dist/commands/view-leaderboard-command.js +169 -0
- package/dist/commands/view-leaderboard-command.js.map +1 -0
- package/dist/config/api-key-config-reader.d.ts +14 -0
- package/dist/config/api-key-config-reader.d.ts.map +1 -0
- package/dist/config/api-key-config-reader.js +107 -0
- package/dist/config/api-key-config-reader.js.map +1 -0
- package/dist/config/api-key-config-reader.test.d.ts +2 -0
- package/dist/config/api-key-config-reader.test.d.ts.map +1 -0
- package/dist/config/api-key-config-reader.test.js +21 -0
- package/dist/config/api-key-config-reader.test.js.map +1 -0
- package/dist/engine/claude-cli-executor.d.ts +33 -0
- package/dist/engine/claude-cli-executor.d.ts.map +1 -0
- package/dist/engine/claude-cli-executor.js +251 -0
- package/dist/engine/claude-cli-executor.js.map +1 -0
- package/dist/engine/concept-accuracy-scorer.d.ts +24 -0
- package/dist/engine/concept-accuracy-scorer.d.ts.map +1 -0
- package/dist/engine/concept-accuracy-scorer.js +186 -0
- package/dist/engine/concept-accuracy-scorer.js.map +1 -0
- package/dist/engine/concept-accuracy-scorer.test.d.ts +2 -0
- package/dist/engine/concept-accuracy-scorer.test.d.ts.map +1 -0
- package/dist/engine/concept-accuracy-scorer.test.js +230 -0
- package/dist/engine/concept-accuracy-scorer.test.js.map +1 -0
- package/dist/engine/enhanced-test-prompt-builder.d.ts +30 -0
- package/dist/engine/enhanced-test-prompt-builder.d.ts.map +1 -0
- package/dist/engine/enhanced-test-prompt-builder.js +134 -0
- package/dist/engine/enhanced-test-prompt-builder.js.map +1 -0
- package/dist/engine/markdown-test-definition-parser.d.ts +18 -0
- package/dist/engine/markdown-test-definition-parser.d.ts.map +1 -0
- package/dist/engine/markdown-test-definition-parser.js +525 -0
- package/dist/engine/markdown-test-definition-parser.js.map +1 -0
- package/dist/engine/markdown-test-definition-parser.test.d.ts +2 -0
- package/dist/engine/markdown-test-definition-parser.test.d.ts.map +1 -0
- package/dist/engine/markdown-test-definition-parser.test.js +265 -0
- package/dist/engine/markdown-test-definition-parser.test.js.map +1 -0
- package/dist/engine/retry-with-degrade-utils.d.ts +58 -0
- package/dist/engine/retry-with-degrade-utils.d.ts.map +1 -0
- package/dist/engine/retry-with-degrade-utils.js +86 -0
- package/dist/engine/retry-with-degrade-utils.js.map +1 -0
- package/dist/engine/skill-content-collector.d.ts +53 -0
- package/dist/engine/skill-content-collector.d.ts.map +1 -0
- package/dist/engine/skill-content-collector.js +157 -0
- package/dist/engine/skill-content-collector.js.map +1 -0
- package/dist/engine/skill-creator-invoker.d.ts +36 -0
- package/dist/engine/skill-creator-invoker.d.ts.map +1 -0
- package/dist/engine/skill-creator-invoker.js +222 -0
- package/dist/engine/skill-creator-invoker.js.map +1 -0
- package/dist/engine/transcript-jsonl-parser.d.ts +28 -0
- package/dist/engine/transcript-jsonl-parser.d.ts.map +1 -0
- package/dist/engine/transcript-jsonl-parser.js +175 -0
- package/dist/engine/transcript-jsonl-parser.js.map +1 -0
- package/dist/sources/git-repository-skill-source-handler.d.ts +18 -0
- package/dist/sources/git-repository-skill-source-handler.d.ts.map +1 -0
- package/dist/sources/git-repository-skill-source-handler.js +119 -0
- package/dist/sources/git-repository-skill-source-handler.js.map +1 -0
- package/dist/sources/local-skill-source-handler.d.ts +21 -0
- package/dist/sources/local-skill-source-handler.d.ts.map +1 -0
- package/dist/sources/local-skill-source-handler.js +138 -0
- package/dist/sources/local-skill-source-handler.js.map +1 -0
- package/dist/sources/local-skill-source-handler.test.d.ts +2 -0
- package/dist/sources/local-skill-source-handler.test.d.ts.map +1 -0
- package/dist/sources/local-skill-source-handler.test.js +55 -0
- package/dist/sources/local-skill-source-handler.test.js.map +1 -0
- package/dist/sources/skillsh-registry-source-handler.d.ts +18 -0
- package/dist/sources/skillsh-registry-source-handler.d.ts.map +1 -0
- package/dist/sources/skillsh-registry-source-handler.js +130 -0
- package/dist/sources/skillsh-registry-source-handler.js.map +1 -0
- package/dist/sources/unified-skill-source-resolver.d.ts +20 -0
- package/dist/sources/unified-skill-source-resolver.d.ts.map +1 -0
- package/dist/sources/unified-skill-source-resolver.js +64 -0
- package/dist/sources/unified-skill-source-resolver.js.map +1 -0
- package/dist/sources/unified-skill-source-resolver.test.d.ts +2 -0
- package/dist/sources/unified-skill-source-resolver.test.d.ts.map +1 -0
- package/dist/sources/unified-skill-source-resolver.test.js +84 -0
- package/dist/sources/unified-skill-source-resolver.test.js.map +1 -0
- package/dist/types/benchmark-types.d.ts +142 -0
- package/dist/types/benchmark-types.d.ts.map +1 -0
- package/dist/types/benchmark-types.js +5 -0
- package/dist/types/benchmark-types.js.map +1 -0
- package/dist/types/index.d.ts +5 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +5 -0
- package/dist/types/index.js.map +1 -0
- package/package.json +38 -0
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Publish results command - uploads benchmark results to leaderboard API
|
|
3
|
+
*/
|
|
4
|
+
import { readFile, readdir } from 'node:fs/promises';
|
|
5
|
+
import { resolve, join } from 'node:path';
|
|
6
|
+
import chalk from 'chalk';
|
|
7
|
+
import ora from 'ora';
|
|
8
|
+
/** Default API endpoint */
|
|
9
|
+
const DEFAULT_ENDPOINT = 'https://skillmark.sh/api';
|
|
10
|
+
/**
|
|
11
|
+
* Execute the publish command
|
|
12
|
+
*/
|
|
13
|
+
export async function publishResults(resultPath, options) {
|
|
14
|
+
const spinner = ora();
|
|
15
|
+
try {
|
|
16
|
+
// 1. Load result file
|
|
17
|
+
spinner.start('Loading result file...');
|
|
18
|
+
const fullPath = resolve(resultPath);
|
|
19
|
+
const content = await readFile(fullPath, 'utf-8');
|
|
20
|
+
const result = JSON.parse(content);
|
|
21
|
+
spinner.succeed(`Loaded: ${result.skillName} (${result.model})`);
|
|
22
|
+
// 2. Validate result
|
|
23
|
+
spinner.start('Validating result...');
|
|
24
|
+
validateResult(result);
|
|
25
|
+
spinner.succeed('Result validated');
|
|
26
|
+
// 3. Upload to API
|
|
27
|
+
spinner.start('Uploading to leaderboard...');
|
|
28
|
+
const endpoint = options.endpoint || DEFAULT_ENDPOINT;
|
|
29
|
+
const response = await uploadResult(result, options.apiKey, endpoint);
|
|
30
|
+
spinner.succeed('Uploaded successfully');
|
|
31
|
+
// 4. Display result
|
|
32
|
+
console.log(chalk.green('\n✓ Result published to leaderboard\n'));
|
|
33
|
+
console.log(`${chalk.gray('Skill:')} ${result.skillName}`);
|
|
34
|
+
console.log(`${chalk.gray('Accuracy:')} ${result.aggregatedMetrics.accuracy.toFixed(1)}%`);
|
|
35
|
+
console.log(`${chalk.gray('Model:')} ${result.model}`);
|
|
36
|
+
if (response.leaderboardUrl) {
|
|
37
|
+
console.log(`\n${chalk.blue('View leaderboard:')} ${response.leaderboardUrl}`);
|
|
38
|
+
}
|
|
39
|
+
if (response.rank) {
|
|
40
|
+
console.log(`${chalk.gray('Rank:')} #${response.rank}`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
spinner.fail('Publish failed');
|
|
45
|
+
throw error;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Validate result before publishing
|
|
50
|
+
*/
|
|
51
|
+
function validateResult(result) {
|
|
52
|
+
// Check required fields
|
|
53
|
+
if (!result.skillId) {
|
|
54
|
+
throw new Error('Invalid result: missing skillId');
|
|
55
|
+
}
|
|
56
|
+
if (!result.skillName) {
|
|
57
|
+
throw new Error('Invalid result: missing skillName');
|
|
58
|
+
}
|
|
59
|
+
if (!result.model || !['haiku', 'sonnet', 'opus'].includes(result.model)) {
|
|
60
|
+
throw new Error('Invalid result: invalid model');
|
|
61
|
+
}
|
|
62
|
+
if (!result.aggregatedMetrics) {
|
|
63
|
+
throw new Error('Invalid result: missing aggregatedMetrics');
|
|
64
|
+
}
|
|
65
|
+
if (!result.hash) {
|
|
66
|
+
throw new Error('Invalid result: missing verification hash');
|
|
67
|
+
}
|
|
68
|
+
if (!result.version) {
|
|
69
|
+
throw new Error('Invalid result: missing version');
|
|
70
|
+
}
|
|
71
|
+
// Validate metrics ranges
|
|
72
|
+
const m = result.aggregatedMetrics;
|
|
73
|
+
if (m.accuracy < 0 || m.accuracy > 100) {
|
|
74
|
+
throw new Error('Invalid result: accuracy out of range');
|
|
75
|
+
}
|
|
76
|
+
if (m.tokensTotal < 0) {
|
|
77
|
+
throw new Error('Invalid result: negative token count');
|
|
78
|
+
}
|
|
79
|
+
if (m.costUsd < 0) {
|
|
80
|
+
throw new Error('Invalid result: negative cost');
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Upload result to API
|
|
85
|
+
*/
|
|
86
|
+
async function uploadResult(result, apiKey, endpoint) {
|
|
87
|
+
const url = `${endpoint}/results`;
|
|
88
|
+
const response = await fetch(url, {
|
|
89
|
+
method: 'POST',
|
|
90
|
+
headers: {
|
|
91
|
+
'Content-Type': 'application/json',
|
|
92
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
93
|
+
'X-Skillmark-Version': result.version,
|
|
94
|
+
},
|
|
95
|
+
body: JSON.stringify({
|
|
96
|
+
skillId: result.skillId,
|
|
97
|
+
skillName: result.skillName,
|
|
98
|
+
source: result.skillSource,
|
|
99
|
+
model: result.model,
|
|
100
|
+
accuracy: result.aggregatedMetrics.accuracy,
|
|
101
|
+
tokensTotal: result.aggregatedMetrics.tokensTotal,
|
|
102
|
+
durationMs: result.aggregatedMetrics.durationMs,
|
|
103
|
+
costUsd: result.aggregatedMetrics.costUsd,
|
|
104
|
+
runs: result.runs,
|
|
105
|
+
hash: result.hash,
|
|
106
|
+
timestamp: result.timestamp,
|
|
107
|
+
rawJson: JSON.stringify(result),
|
|
108
|
+
}),
|
|
109
|
+
});
|
|
110
|
+
if (!response.ok) {
|
|
111
|
+
const errorText = await response.text();
|
|
112
|
+
throw new Error(`API error (${response.status}): ${errorText}`);
|
|
113
|
+
}
|
|
114
|
+
return response.json();
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Publish results with auto-key (from run command with --publish flag)
|
|
118
|
+
* Includes test files and skill.sh link detection
|
|
119
|
+
*/
|
|
120
|
+
export async function publishResultsWithAutoKey(result, options) {
|
|
121
|
+
const spinner = ora();
|
|
122
|
+
try {
|
|
123
|
+
// 1. Validate result
|
|
124
|
+
spinner.start('Validating result...');
|
|
125
|
+
validateResult(result);
|
|
126
|
+
spinner.succeed('Result validated');
|
|
127
|
+
// 2. Load test files if path provided
|
|
128
|
+
let testFiles = [];
|
|
129
|
+
if (options.testsPath) {
|
|
130
|
+
spinner.start('Loading test files...');
|
|
131
|
+
testFiles = await loadTestFiles(options.testsPath);
|
|
132
|
+
spinner.succeed(`Loaded ${testFiles.length} test file(s)`);
|
|
133
|
+
}
|
|
134
|
+
// 3. Detect skill.sh link
|
|
135
|
+
const skillshLink = detectSkillshLink(result.skillSource);
|
|
136
|
+
// 4. Upload to API
|
|
137
|
+
spinner.start('Uploading to leaderboard...');
|
|
138
|
+
const endpoint = options.endpoint || DEFAULT_ENDPOINT;
|
|
139
|
+
const response = await uploadResultWithExtras(result, options.apiKey, endpoint, testFiles, skillshLink);
|
|
140
|
+
spinner.succeed('Uploaded successfully');
|
|
141
|
+
// 5. Display result
|
|
142
|
+
console.log(chalk.green('\n✓ Result published to leaderboard\n'));
|
|
143
|
+
console.log(`${chalk.gray('Skill:')} ${result.skillName}`);
|
|
144
|
+
console.log(`${chalk.gray('Accuracy:')} ${result.aggregatedMetrics.accuracy.toFixed(1)}%`);
|
|
145
|
+
console.log(`${chalk.gray('Model:')} ${result.model}`);
|
|
146
|
+
if (response.submitter?.github) {
|
|
147
|
+
console.log(`${chalk.gray('Submitter:')} @${response.submitter.github}`);
|
|
148
|
+
}
|
|
149
|
+
if (response.leaderboardUrl) {
|
|
150
|
+
console.log(`\n${chalk.blue('View leaderboard:')} ${response.leaderboardUrl}`);
|
|
151
|
+
}
|
|
152
|
+
if (response.rank) {
|
|
153
|
+
console.log(`${chalk.gray('Rank:')} #${response.rank}`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
spinner.fail('Publish failed');
|
|
158
|
+
throw error;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Load test files from directory
|
|
163
|
+
*/
|
|
164
|
+
async function loadTestFiles(testsPath) {
|
|
165
|
+
const fullPath = resolve(testsPath);
|
|
166
|
+
const files = [];
|
|
167
|
+
try {
|
|
168
|
+
const entries = await readdir(fullPath, { withFileTypes: true });
|
|
169
|
+
for (const entry of entries) {
|
|
170
|
+
if (entry.isFile() && entry.name.endsWith('.md')) {
|
|
171
|
+
const filePath = join(fullPath, entry.name);
|
|
172
|
+
const content = await readFile(filePath, 'utf-8');
|
|
173
|
+
files.push({
|
|
174
|
+
name: entry.name,
|
|
175
|
+
content,
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
catch {
|
|
181
|
+
// Directory doesn't exist or can't be read, return empty
|
|
182
|
+
}
|
|
183
|
+
return files;
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Detect if source is from skill.sh registry and return link
|
|
187
|
+
*/
|
|
188
|
+
function detectSkillshLink(source) {
|
|
189
|
+
// Check for skill.sh pattern: skill.sh/user/skill-name
|
|
190
|
+
const skillshMatch = source.match(/^skill\.sh\/([^/]+)\/([^/]+)$/i);
|
|
191
|
+
if (skillshMatch) {
|
|
192
|
+
return `https://skill.sh/${skillshMatch[1]}/${skillshMatch[2]}`;
|
|
193
|
+
}
|
|
194
|
+
// Check for full URL pattern
|
|
195
|
+
const urlMatch = source.match(/^https?:\/\/skill\.sh\/([^/]+)\/([^/]+)/i);
|
|
196
|
+
if (urlMatch) {
|
|
197
|
+
return `https://skill.sh/${urlMatch[1]}/${urlMatch[2]}`;
|
|
198
|
+
}
|
|
199
|
+
return null;
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Upload result with test files and skill.sh link
|
|
203
|
+
*/
|
|
204
|
+
async function uploadResultWithExtras(result, apiKey, endpoint, testFiles, skillshLink) {
|
|
205
|
+
const url = `${endpoint}/results`;
|
|
206
|
+
const response = await fetch(url, {
|
|
207
|
+
method: 'POST',
|
|
208
|
+
headers: {
|
|
209
|
+
'Content-Type': 'application/json',
|
|
210
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
211
|
+
'X-Skillmark-Version': result.version,
|
|
212
|
+
},
|
|
213
|
+
body: JSON.stringify({
|
|
214
|
+
skillId: result.skillId,
|
|
215
|
+
skillName: result.skillName,
|
|
216
|
+
source: result.skillSource,
|
|
217
|
+
model: result.model,
|
|
218
|
+
accuracy: result.aggregatedMetrics.accuracy,
|
|
219
|
+
tokensTotal: result.aggregatedMetrics.tokensTotal,
|
|
220
|
+
tokensInput: result.aggregatedMetrics.tokensInput,
|
|
221
|
+
tokensOutput: result.aggregatedMetrics.tokensOutput,
|
|
222
|
+
durationMs: result.aggregatedMetrics.durationMs,
|
|
223
|
+
costUsd: result.aggregatedMetrics.costUsd,
|
|
224
|
+
toolCount: result.aggregatedMetrics.toolCount,
|
|
225
|
+
runs: result.runs,
|
|
226
|
+
hash: result.hash,
|
|
227
|
+
timestamp: result.timestamp,
|
|
228
|
+
rawJson: JSON.stringify(result),
|
|
229
|
+
testFiles: testFiles.length > 0 ? testFiles : undefined,
|
|
230
|
+
skillshLink: skillshLink || undefined,
|
|
231
|
+
}),
|
|
232
|
+
});
|
|
233
|
+
if (!response.ok) {
|
|
234
|
+
const errorText = await response.text();
|
|
235
|
+
throw new Error(`API error (${response.status}): ${errorText}`);
|
|
236
|
+
}
|
|
237
|
+
return response.json();
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* Verify API key is valid
|
|
241
|
+
*/
|
|
242
|
+
export async function verifyApiKey(apiKey, endpoint = DEFAULT_ENDPOINT) {
|
|
243
|
+
try {
|
|
244
|
+
const response = await fetch(`${endpoint}/verify`, {
|
|
245
|
+
method: 'POST',
|
|
246
|
+
headers: {
|
|
247
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
248
|
+
},
|
|
249
|
+
});
|
|
250
|
+
return response.ok;
|
|
251
|
+
}
|
|
252
|
+
catch {
|
|
253
|
+
return false;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
//# sourceMappingURL=publish-results-command.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"publish-results-command.js","sourceRoot":"","sources":["../../src/commands/publish-results-command.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAY,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AAGtB,2BAA2B;AAC3B,MAAM,gBAAgB,GAAG,0BAA0B,CAAC;AAepD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,UAAkB,EAClB,OAAuB;IAEvB,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;IAEtB,IAAI,CAAC;QACH,sBAAsB;QACtB,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,MAAM,GAAoB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpD,OAAO,CAAC,OAAO,CAAC,WAAW,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC;QAEjE,qBAAqB;QACrB,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtC,cAAc,CAAC,MAAM,CAAC,CAAC;QACvB,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAEpC,mBAAmB;QACnB,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC;QACtD,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACtE,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;QAEzC,oBAAoB;QACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3F,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAE1D,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/B,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,MAAuB;IAC7C,wBAAwB;IACxB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,0BAA0B;IAC1B,MAAM,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACnC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,GAAG,GAAG,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,CAAC,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CACzB,MAAuB,EACvB,MAAc,EACd,QAAgB;IAEhB,MAAM,GAAG,GAAG,GAAG,QAAQ,UAAU,CAAC;IAElC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,eAAe,EAAE,UAAU,MAAM,EAAE;YACnC,qBAAqB,EAAE,MAAM,CAAC,OAAO;SACtC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,MAAM,CAAC,WAAW;YAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,iBAAiB,CAAC,QAAQ;YAC3C,WAAW,EAAE,MAAM,CAAC,iBAAiB,CAAC,WAAW;YACjD,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,UAAU;YAC/C,OAAO,EAAE,MAAM,CAAC,iBAAiB,CAAC,OAAO;YACzC,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;SAChC,CAAC;KACH,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,cAAc,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,EAAyD,CAAC;AAChF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,MAAuB,EACvB,OAA2B;IAE3B,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;IAEtB,IAAI,CAAC;QACH,qBAAqB;QACrB,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtC,cAAc,CAAC,MAAM,CAAC,CAAC;QACvB,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAEpC,sCAAsC;QACtC,IAAI,SAAS,GAAqB,EAAE,CAAC;QACrC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;YACvC,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACnD,OAAO,CAAC,OAAO,CAAC,UAAU,SAAS,CAAC,MAAM,eAAe,CAAC,CAAC;QAC7D,CAAC;QAED,0BAA0B;QAC1B,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAE1D,mBAAmB;QACnB,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC;QACtD,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,MAAM,EACN,OAAO,CAAC,MAAM,EACd,QAAQ,EACR,SAAS,EACT,WAAW,CACZ,CAAC;QACF,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;QAEzC,oBAAoB;QACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3F,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAE1D,IAAI,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/B,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa,CAAC,SAAiB;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,KAAK,GAAqB,EAAE,CAAC;IAEnC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEjE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAClD,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,OAAO;iBACR,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,yDAAyD;IAC3D,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,MAAc;IACvC,uDAAuD;IACvD,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpE,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,oBAAoB,YAAY,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,CAAC;IAED,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC1E,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,oBAAoB,QAAQ,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,sBAAsB,CACnC,MAAuB,EACvB,MAAc,EACd,QAAgB,EAChB,SAA2B,EAC3B,WAA0B;IAM1B,MAAM,GAAG,GAAG,GAAG,QAAQ,UAAU,CAAC;IAElC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,eAAe,EAAE,UAAU,MAAM,EAAE;YACnC,qBAAqB,EAAE,MAAM,CAAC,OAAO;SACtC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,MAAM,CAAC,WAAW;YAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,iBAAiB,CAAC,QAAQ;YAC3C,WAAW,EAAE,MAAM,CAAC,iBAAiB,CAAC,WAAW;YACjD,WAAW,EAAE,MAAM,CAAC,iBAAiB,CAAC,WAAW;YACjD,YAAY,EAAE,MAAM,CAAC,iBAAiB,CAAC,YAAY;YACnD,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,UAAU;YAC/C,OAAO,EAAE,MAAM,CAAC,iBAAiB,CAAC,OAAO;YACzC,SAAS,EAAE,MAAM,CAAC,iBAAiB,CAAC,SAAS;YAC7C,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAC/B,SAAS,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YACvD,WAAW,EAAE,WAAW,IAAI,SAAS;SACtC,CAAC;KACH,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,cAAc,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,EAIlB,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAc,EACd,WAAmB,gBAAgB;IAEnC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,QAAQ,SAAS,EAAE;YACjD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,eAAe,EAAE,UAAU,MAAM,EAAE;aACpC;SACF,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,EAAE,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { BenchmarkResult, RunOptions } from '../types/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Execute the run command
|
|
4
|
+
*/
|
|
5
|
+
export declare function runBenchmark(skillSource: string, options: RunOptions): Promise<BenchmarkResult>;
|
|
6
|
+
//# sourceMappingURL=run-benchmark-command.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-benchmark-command.d.ts","sourceRoot":"","sources":["../../src/commands/run-benchmark-command.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EACV,eAAe,EAEf,UAAU,EAEX,MAAM,mBAAmB,CAAC;AAmE3B;;GAEG;AACH,wBAAsB,YAAY,CAChC,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,eAAe,CAAC,CAuM1B"}
|
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run benchmark command - executes benchmarks against a skill
|
|
3
|
+
*/
|
|
4
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
5
|
+
import { join, resolve } from 'node:path';
|
|
6
|
+
import { createHash } from 'node:crypto';
|
|
7
|
+
import chalk from 'chalk';
|
|
8
|
+
import ora from 'ora';
|
|
9
|
+
import { resolveSkillSource, formatSourceDisplay } from '../sources/unified-skill-source-resolver.js';
|
|
10
|
+
import { loadTestsFromDirectory, discoverTests } from '../engine/markdown-test-definition-parser.js';
|
|
11
|
+
import { executeTest, cleanupTranscripts } from '../engine/claude-cli-executor.js';
|
|
12
|
+
import { scoreResponse, aggregateMetrics } from '../engine/concept-accuracy-scorer.js';
|
|
13
|
+
import { getStoredToken, runAuth } from './auth-setup-and-token-storage-command.js';
|
|
14
|
+
/** CLI version */
|
|
15
|
+
const VERSION = '0.1.0';
|
|
16
|
+
/**
|
|
17
|
+
* Verify Claude CLI authentication before running benchmarks
|
|
18
|
+
*/
|
|
19
|
+
async function verifyClaudeAuth() {
|
|
20
|
+
const { spawn } = await import('node:child_process');
|
|
21
|
+
// Get stored token (from env or config file)
|
|
22
|
+
const storedToken = await getStoredToken();
|
|
23
|
+
return new Promise((resolve) => {
|
|
24
|
+
// Prepare environment with token if available
|
|
25
|
+
const env = { ...process.env };
|
|
26
|
+
if (storedToken) {
|
|
27
|
+
env.CLAUDE_CODE_OAUTH_TOKEN = storedToken;
|
|
28
|
+
}
|
|
29
|
+
// Quick test with minimal prompt
|
|
30
|
+
const proc = spawn('claude', ['-p', 'Say OK', '--output-format', 'json', '--model', 'haiku'], {
|
|
31
|
+
env,
|
|
32
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
33
|
+
timeout: 30000,
|
|
34
|
+
});
|
|
35
|
+
let stdout = '';
|
|
36
|
+
proc.stdout.on('data', (data) => { stdout += data.toString(); });
|
|
37
|
+
const timeout = setTimeout(() => {
|
|
38
|
+
proc.kill();
|
|
39
|
+
resolve({ ok: false, error: 'Authentication check timed out' });
|
|
40
|
+
}, 30000);
|
|
41
|
+
proc.on('close', () => {
|
|
42
|
+
clearTimeout(timeout);
|
|
43
|
+
try {
|
|
44
|
+
const result = JSON.parse(stdout);
|
|
45
|
+
if (result.is_error && result.result?.includes('Invalid API key')) {
|
|
46
|
+
resolve({
|
|
47
|
+
ok: false,
|
|
48
|
+
error: `Claude CLI not authenticated.\n` +
|
|
49
|
+
` Run: skillmark auth\n` +
|
|
50
|
+
` Or set CLAUDE_CODE_OAUTH_TOKEN environment variable.`,
|
|
51
|
+
});
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
resolve({ ok: true, token: storedToken });
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
resolve({ ok: true, token: storedToken }); // Assume ok if we can't parse
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
proc.on('error', (err) => {
|
|
61
|
+
clearTimeout(timeout);
|
|
62
|
+
resolve({ ok: false, error: `Claude CLI not found: ${err.message}` });
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Execute the run command
|
|
68
|
+
*/
|
|
69
|
+
export async function runBenchmark(skillSource, options) {
|
|
70
|
+
const spinner = ora();
|
|
71
|
+
try {
|
|
72
|
+
// 0. Verify Claude CLI authentication
|
|
73
|
+
spinner.start('Verifying Claude CLI authentication...');
|
|
74
|
+
let authCheck = await verifyClaudeAuth();
|
|
75
|
+
if (!authCheck.ok) {
|
|
76
|
+
spinner.warn('Not authenticated - starting auth setup...');
|
|
77
|
+
console.log('');
|
|
78
|
+
await runAuth();
|
|
79
|
+
console.log('');
|
|
80
|
+
// Verify again after auth
|
|
81
|
+
spinner.start('Verifying authentication...');
|
|
82
|
+
authCheck = await verifyClaudeAuth();
|
|
83
|
+
if (!authCheck.ok) {
|
|
84
|
+
spinner.fail('Authentication still failed');
|
|
85
|
+
throw new Error(authCheck.error);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
spinner.succeed('Claude CLI authenticated');
|
|
89
|
+
// 1. Resolve skill source
|
|
90
|
+
spinner.start('Resolving skill source...');
|
|
91
|
+
const skill = await resolveSkillSource(skillSource);
|
|
92
|
+
spinner.succeed(`Resolved: ${formatSourceDisplay(skill)}`);
|
|
93
|
+
// 2. Load test definitions
|
|
94
|
+
spinner.start('Loading test definitions...');
|
|
95
|
+
let tests;
|
|
96
|
+
if (options.tests) {
|
|
97
|
+
// Use explicit tests path
|
|
98
|
+
tests = await loadTestsFromDirectory(resolve(options.tests));
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
// Auto-discover tests in skill directory
|
|
102
|
+
tests = await discoverTests(skill.localPath);
|
|
103
|
+
}
|
|
104
|
+
if (tests.length === 0) {
|
|
105
|
+
spinner.fail('No test files found');
|
|
106
|
+
throw new Error('No test files found. Specify --tests <path> or add tests/ directory to skill.');
|
|
107
|
+
}
|
|
108
|
+
spinner.succeed(`Loaded ${tests.length} test(s)`);
|
|
109
|
+
// 3. Create output directory
|
|
110
|
+
const outputDir = resolve(options.output);
|
|
111
|
+
await mkdir(outputDir, { recursive: true });
|
|
112
|
+
// 4. Run benchmarks
|
|
113
|
+
const verbose = options.verbose ?? false;
|
|
114
|
+
console.log(chalk.blue(`\nRunning ${tests.length} test(s) × ${options.runs} run(s) with ${options.model}${verbose ? ' (verbose)' : ''}\n`));
|
|
115
|
+
const allResults = [];
|
|
116
|
+
const workDir = process.cwd();
|
|
117
|
+
for (let run = 1; run <= options.runs; run++) {
|
|
118
|
+
console.log(chalk.gray(`── Run ${run}/${options.runs} ──`));
|
|
119
|
+
for (const test of tests) {
|
|
120
|
+
const testStart = Date.now();
|
|
121
|
+
if (verbose) {
|
|
122
|
+
console.log(chalk.cyan(`\n▶ Starting: ${test.name}`));
|
|
123
|
+
console.log(chalk.gray(` Type: ${test.type} | Timeout: ${test.timeout}s | Concepts: ${test.concepts.length}`));
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
spinner.start(`Testing: ${test.name}`);
|
|
127
|
+
}
|
|
128
|
+
// Elapsed timer for verbose mode (declared outside try for catch access)
|
|
129
|
+
let elapsedTimer = null;
|
|
130
|
+
try {
|
|
131
|
+
if (verbose) {
|
|
132
|
+
const cliStart = Date.now();
|
|
133
|
+
process.stdout.write(chalk.gray(` Invoking Claude CLI... 0s`));
|
|
134
|
+
elapsedTimer = setInterval(() => {
|
|
135
|
+
const elapsed = Math.floor((Date.now() - cliStart) / 1000);
|
|
136
|
+
process.stdout.write(`\r${chalk.gray(` Invoking Claude CLI... ${elapsed}s`)}`);
|
|
137
|
+
}, 1000);
|
|
138
|
+
}
|
|
139
|
+
const execution = await executeTest(test, skill.localPath, options.model, workDir);
|
|
140
|
+
// Clear elapsed timer
|
|
141
|
+
if (elapsedTimer) {
|
|
142
|
+
clearInterval(elapsedTimer);
|
|
143
|
+
process.stdout.write('\n');
|
|
144
|
+
}
|
|
145
|
+
if (!execution.success) {
|
|
146
|
+
if (verbose) {
|
|
147
|
+
console.log(chalk.red(` ✗ Failed: ${execution.error}`));
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
spinner.fail(`${test.name}: ${execution.error}`);
|
|
151
|
+
}
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
if (verbose) {
|
|
155
|
+
const elapsed = ((Date.now() - testStart) / 1000).toFixed(1);
|
|
156
|
+
console.log(chalk.gray(` Execution complete (${elapsed}s)`));
|
|
157
|
+
console.log(chalk.gray(` Tokens: ${execution.inputTokens + execution.outputTokens} | Tools: ${execution.toolCount} | Cost: $${execution.costUsd.toFixed(4)}`));
|
|
158
|
+
}
|
|
159
|
+
// Calculate metrics
|
|
160
|
+
const metrics = {
|
|
161
|
+
accuracy: 0, // Calculated by scorer
|
|
162
|
+
tokensTotal: execution.inputTokens + execution.outputTokens,
|
|
163
|
+
tokensInput: execution.inputTokens,
|
|
164
|
+
tokensOutput: execution.outputTokens,
|
|
165
|
+
durationMs: execution.durationMs,
|
|
166
|
+
toolCount: execution.toolCount,
|
|
167
|
+
costUsd: execution.costUsd,
|
|
168
|
+
};
|
|
169
|
+
// Score response
|
|
170
|
+
const result = scoreResponse(test, execution.response, metrics);
|
|
171
|
+
allResults.push(result);
|
|
172
|
+
// Display result
|
|
173
|
+
const status = result.passed ? chalk.green('✓') : chalk.yellow('○');
|
|
174
|
+
const accuracy = result.metrics.accuracy.toFixed(1);
|
|
175
|
+
if (verbose) {
|
|
176
|
+
console.log(chalk.gray(` Scoring response...`));
|
|
177
|
+
console.log(` ${status} Result: ${accuracy}% accuracy (${result.matchedConcepts.length}/${test.concepts.length} concepts)`);
|
|
178
|
+
if (result.matchedConcepts.length > 0) {
|
|
179
|
+
console.log(chalk.green(` Matched: ${result.matchedConcepts.join(', ')}`));
|
|
180
|
+
}
|
|
181
|
+
if (result.missedConcepts.length > 0) {
|
|
182
|
+
console.log(chalk.yellow(` Missed: ${result.missedConcepts.join(', ')}`));
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
else {
|
|
186
|
+
spinner.succeed(`${status} ${test.name}: ${accuracy}% (${result.matchedConcepts.length}/${test.concepts.length} concepts)`);
|
|
187
|
+
// Show missed concepts if any
|
|
188
|
+
if (result.missedConcepts.length > 0 && result.missedConcepts.length <= 3) {
|
|
189
|
+
console.log(chalk.gray(` Missed: ${result.missedConcepts.join(', ')}`));
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
catch (error) {
|
|
194
|
+
// Clear elapsed timer on error
|
|
195
|
+
if (elapsedTimer) {
|
|
196
|
+
clearInterval(elapsedTimer);
|
|
197
|
+
process.stdout.write('\n');
|
|
198
|
+
}
|
|
199
|
+
if (verbose) {
|
|
200
|
+
console.log(chalk.red(` ✗ Error: ${error instanceof Error ? error.message : 'Unknown error'}`));
|
|
201
|
+
}
|
|
202
|
+
else {
|
|
203
|
+
spinner.fail(`${test.name}: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
// 5. Aggregate results
|
|
209
|
+
const aggregated = aggregateMetrics(allResults);
|
|
210
|
+
const benchmarkResult = {
|
|
211
|
+
skillId: createSkillId(skill.name, skill.original),
|
|
212
|
+
skillName: skill.name,
|
|
213
|
+
skillSource: skill.original,
|
|
214
|
+
model: options.model,
|
|
215
|
+
runs: options.runs,
|
|
216
|
+
testResults: allResults,
|
|
217
|
+
aggregatedMetrics: aggregated,
|
|
218
|
+
timestamp: new Date().toISOString(),
|
|
219
|
+
version: VERSION,
|
|
220
|
+
};
|
|
221
|
+
// Add verification hash
|
|
222
|
+
benchmarkResult.hash = generateResultHash(benchmarkResult);
|
|
223
|
+
// 6. Write outputs
|
|
224
|
+
spinner.start('Writing results...');
|
|
225
|
+
const jsonPath = join(outputDir, 'result.json');
|
|
226
|
+
const mdPath = join(outputDir, 'report.md');
|
|
227
|
+
await writeFile(jsonPath, JSON.stringify(benchmarkResult, null, 2));
|
|
228
|
+
await writeFile(mdPath, generateMarkdownReport(benchmarkResult));
|
|
229
|
+
spinner.succeed(`Results saved to ${outputDir}`);
|
|
230
|
+
// 7. Print summary
|
|
231
|
+
printSummary(benchmarkResult);
|
|
232
|
+
// Cleanup
|
|
233
|
+
await cleanupTranscripts(workDir);
|
|
234
|
+
return benchmarkResult;
|
|
235
|
+
}
|
|
236
|
+
catch (error) {
|
|
237
|
+
spinner.fail('Benchmark failed');
|
|
238
|
+
throw error;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* Create a unique skill ID
|
|
243
|
+
*/
|
|
244
|
+
function createSkillId(name, source) {
|
|
245
|
+
const hash = createHash('sha256')
|
|
246
|
+
.update(`${name}:${source}`)
|
|
247
|
+
.digest('hex')
|
|
248
|
+
.slice(0, 12);
|
|
249
|
+
return `${name.toLowerCase().replace(/[^a-z0-9]/g, '-')}-${hash}`;
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Generate verification hash for result
|
|
253
|
+
*/
|
|
254
|
+
function generateResultHash(result) {
|
|
255
|
+
const data = {
|
|
256
|
+
skillId: result.skillId,
|
|
257
|
+
model: result.model,
|
|
258
|
+
runs: result.runs,
|
|
259
|
+
accuracy: result.aggregatedMetrics.accuracy,
|
|
260
|
+
tokensTotal: result.aggregatedMetrics.tokensTotal,
|
|
261
|
+
timestamp: result.timestamp,
|
|
262
|
+
};
|
|
263
|
+
return createHash('sha256').update(JSON.stringify(data)).digest('hex').slice(0, 16);
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Generate markdown report
|
|
267
|
+
*/
|
|
268
|
+
function generateMarkdownReport(result) {
|
|
269
|
+
const m = result.aggregatedMetrics;
|
|
270
|
+
let md = `# Skillmark Benchmark Report
|
|
271
|
+
|
|
272
|
+
## Summary
|
|
273
|
+
|
|
274
|
+
| Metric | Value |
|
|
275
|
+
|--------|-------|
|
|
276
|
+
| Skill | ${result.skillName} |
|
|
277
|
+
| Source | ${result.skillSource} |
|
|
278
|
+
| Model | ${result.model} |
|
|
279
|
+
| Runs | ${result.runs} |
|
|
280
|
+
| Accuracy | ${m.accuracy.toFixed(1)}% |
|
|
281
|
+
| Total Tokens | ${m.tokensTotal.toLocaleString()} |
|
|
282
|
+
| Duration | ${(m.durationMs / 1000).toFixed(1)}s |
|
|
283
|
+
| Cost | $${m.costUsd.toFixed(4)} |
|
|
284
|
+
| Tool Calls | ${m.toolCount} |
|
|
285
|
+
|
|
286
|
+
## Test Results
|
|
287
|
+
|
|
288
|
+
`;
|
|
289
|
+
// Group results by test name
|
|
290
|
+
const byTest = new Map();
|
|
291
|
+
for (const r of result.testResults) {
|
|
292
|
+
const existing = byTest.get(r.test.name) || [];
|
|
293
|
+
existing.push(r);
|
|
294
|
+
byTest.set(r.test.name, existing);
|
|
295
|
+
}
|
|
296
|
+
for (const [testName, results] of byTest) {
|
|
297
|
+
const avgAccuracy = results.reduce((sum, r) => sum + r.metrics.accuracy, 0) / results.length;
|
|
298
|
+
const status = avgAccuracy >= 70 ? '✓' : '○';
|
|
299
|
+
md += `### ${status} ${testName}
|
|
300
|
+
|
|
301
|
+
- **Accuracy**: ${avgAccuracy.toFixed(1)}%
|
|
302
|
+
- **Concepts Matched**: ${results[0].matchedConcepts.join(', ') || 'None'}
|
|
303
|
+
- **Concepts Missed**: ${results[0].missedConcepts.join(', ') || 'None'}
|
|
304
|
+
|
|
305
|
+
`;
|
|
306
|
+
}
|
|
307
|
+
md += `---
|
|
308
|
+
|
|
309
|
+
*Generated by Skillmark v${result.version} at ${result.timestamp}*
|
|
310
|
+
`;
|
|
311
|
+
return md;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Print summary to console
|
|
315
|
+
*/
|
|
316
|
+
function printSummary(result) {
|
|
317
|
+
const m = result.aggregatedMetrics;
|
|
318
|
+
console.log(chalk.bold('\n═══ Benchmark Summary ═══\n'));
|
|
319
|
+
console.log(`${chalk.gray('Skill:')} ${result.skillName}`);
|
|
320
|
+
console.log(`${chalk.gray('Model:')} ${result.model}`);
|
|
321
|
+
console.log(`${chalk.gray('Runs:')} ${result.runs}`);
|
|
322
|
+
console.log('');
|
|
323
|
+
const accuracyColor = m.accuracy >= 80 ? chalk.green : m.accuracy >= 60 ? chalk.yellow : chalk.red;
|
|
324
|
+
console.log(`${chalk.gray('Accuracy:')} ${accuracyColor(m.accuracy.toFixed(1) + '%')}`);
|
|
325
|
+
console.log(`${chalk.gray('Tokens:')} ${m.tokensTotal.toLocaleString()} (in: ${m.tokensInput.toLocaleString()}, out: ${m.tokensOutput.toLocaleString()})`);
|
|
326
|
+
console.log(`${chalk.gray('Duration:')} ${(m.durationMs / 1000).toFixed(1)}s`);
|
|
327
|
+
console.log(`${chalk.gray('Cost:')} $${m.costUsd.toFixed(4)}`);
|
|
328
|
+
console.log(`${chalk.gray('Tools:')} ${m.toolCount} calls`);
|
|
329
|
+
console.log(chalk.gray('\n═════════════════════════\n'));
|
|
330
|
+
}
|
|
331
|
+
//# sourceMappingURL=run-benchmark-command.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-benchmark-command.js","sourceRoot":"","sources":["../../src/commands/run-benchmark-command.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AAOtB,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,6CAA6C,CAAC;AACtG,OAAO,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,8CAA8C,CAAC;AACrG,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACnF,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,2CAA2C,CAAC;AAEpF,kBAAkB;AAClB,MAAM,OAAO,GAAG,OAAO,CAAC;AAExB;;GAEG;AACH,KAAK,UAAU,gBAAgB;IAC7B,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAErD,6CAA6C;IAC7C,MAAM,WAAW,GAAG,MAAM,cAAc,EAAE,CAAC;IAE3C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,8CAA8C;QAC9C,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC/B,IAAI,WAAW,EAAE,CAAC;YAChB,GAAG,CAAC,uBAAuB,GAAG,WAAW,CAAC;QAC5C,CAAC;QAED,iCAAiC;QACjC,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE;YAC5F,GAAG;YACH,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;YACjC,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEjE,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;YAC9B,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC,CAAC;QAClE,CAAC,EAAE,KAAK,CAAC,CAAC;QAEV,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACpB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClC,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;oBAClE,OAAO,CAAC;wBACN,EAAE,EAAE,KAAK;wBACT,KAAK,EAAE,iCAAiC;4BACtC,yBAAyB;4BACzB,wDAAwD;qBAC3D,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;YAC5C,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,8BAA8B;YAC3E,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACvB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,yBAAyB,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,WAAmB,EACnB,OAAmB;IAEnB,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;IAEtB,IAAI,CAAC;QACH,sCAAsC;QACtC,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACxD,IAAI,SAAS,GAAG,MAAM,gBAAgB,EAAE,CAAC;QACzC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChB,MAAM,OAAO,EAAE,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChB,0BAA0B;YAC1B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC7C,SAAS,GAAG,MAAM,gBAAgB,EAAE,CAAC;YACrC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;gBAClB,OAAO,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;gBAC5C,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QACD,OAAO,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;QAE5C,0BAA0B;QAC1B,OAAO,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,WAAW,CAAC,CAAC;QACpD,OAAO,CAAC,OAAO,CAAC,aAAa,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAE3D,2BAA2B;QAC3B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC7C,IAAI,KAAK,CAAC;QACV,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,0BAA0B;YAC1B,KAAK,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,yCAAyC;YACzC,KAAK,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,OAAO,CAAC,UAAU,KAAK,CAAC,MAAM,UAAU,CAAC,CAAC;QAElD,6BAA6B;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5C,oBAAoB;QACpB,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;QACzC,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,MAAM,cAAc,OAAO,CAAC,IAAI,gBAAgB,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAC/H,CAAC;QAEF,MAAM,UAAU,GAAiB,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAE9B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;YAE5D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;oBACtD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,eAAe,IAAI,CAAC,OAAO,iBAAiB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAClH,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBACzC,CAAC;gBAED,yEAAyE;gBACzE,IAAI,YAAY,GAA0C,IAAI,CAAC;gBAC/D,IAAI,CAAC;oBACH,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC,CAAC;wBAChE,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE;4BAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC;4BAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,4BAA4B,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC;wBAClF,CAAC,EAAE,IAAI,CAAC,CAAC;oBACX,CAAC;oBAED,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBAEnF,sBAAsB;oBACtB,IAAI,YAAY,EAAE,CAAC;wBACjB,aAAa,CAAC,YAAY,CAAC,CAAC;wBAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC7B,CAAC;oBAED,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;wBACvB,IAAI,OAAO,EAAE,CAAC;4BACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;wBAC3D,CAAC;6BAAM,CAAC;4BACN,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;wBACnD,CAAC;wBACD,SAAS;oBACX,CAAC;oBAED,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;wBAC7D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,yBAAyB,OAAO,IAAI,CAAC,CAAC,CAAC;wBAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,YAAY,aAAa,SAAS,CAAC,SAAS,aAAa,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBAClK,CAAC;oBAED,oBAAoB;oBACpB,MAAM,OAAO,GAAqB;wBAChC,QAAQ,EAAE,CAAC,EAAE,uBAAuB;wBACpC,WAAW,EAAE,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,YAAY;wBAC3D,WAAW,EAAE,SAAS,CAAC,WAAW;wBAClC,YAAY,EAAE,SAAS,CAAC,YAAY;wBACpC,UAAU,EAAE,SAAS,CAAC,UAAU;wBAChC,SAAS,EAAE,SAAS,CAAC,SAAS;wBAC9B,OAAO,EAAE,SAAS,CAAC,OAAO;qBAC3B,CAAC;oBAEF,iBAAiB;oBACjB,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;oBAChE,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAExB,iBAAiB;oBACjB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBACpE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;oBAEpD,IAAI,OAAO,EAAE,CAAC;wBACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,CAAC;wBACjD,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,YAAY,QAAQ,eAAe,MAAM,CAAC,eAAe,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,YAAY,CAAC,CAAC;wBAC7H,IAAI,MAAM,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACtC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;wBAC9E,CAAC;wBACD,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACrC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;wBAC7E,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,OAAO,CACb,GAAG,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,MAAM,MAAM,CAAC,eAAe,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,YAAY,CAC3G,CAAC;wBAEF,8BAA8B;wBAC9B,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;4BAC1E,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;wBAC5E,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,+BAA+B;oBAC/B,IAAI,YAAY,EAAE,CAAC;wBACjB,aAAa,CAAC,YAAY,CAAC,CAAC;wBAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC7B,CAAC;oBACD,IAAI,OAAO,EAAE,CAAC;wBACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;oBACnG,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;oBAC5F,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,uBAAuB;QACvB,MAAM,UAAU,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;QAEhD,MAAM,eAAe,GAAoB;YACvC,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC;YAClD,SAAS,EAAE,KAAK,CAAC,IAAI;YACrB,WAAW,EAAE,KAAK,CAAC,QAAQ;YAC3B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,WAAW,EAAE,UAAU;YACvB,iBAAiB,EAAE,UAAU;YAC7B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,OAAO;SACjB,CAAC;QAEF,wBAAwB;QACxB,eAAe,CAAC,IAAI,GAAG,kBAAkB,CAAC,eAAe,CAAC,CAAC;QAE3D,mBAAmB;QACnB,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAEpC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAE5C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACpE,MAAM,SAAS,CAAC,MAAM,EAAE,sBAAsB,CAAC,eAAe,CAAC,CAAC,CAAC;QAEjE,OAAO,CAAC,OAAO,CAAC,oBAAoB,SAAS,EAAE,CAAC,CAAC;QAEjD,mBAAmB;QACnB,YAAY,CAAC,eAAe,CAAC,CAAC;QAE9B,UAAU;QACV,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAElC,OAAO,eAAe,CAAC;IACzB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACjC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,IAAY,EAAE,MAAc;IACjD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;SAC9B,MAAM,CAAC,GAAG,IAAI,IAAI,MAAM,EAAE,CAAC;SAC3B,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChB,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;AACpE,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,MAAuB;IACjD,MAAM,IAAI,GAAG;QACX,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,QAAQ,EAAE,MAAM,CAAC,iBAAiB,CAAC,QAAQ;QAC3C,WAAW,EAAE,MAAM,CAAC,iBAAiB,CAAC,WAAW;QACjD,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;IACF,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACtF,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAEnC,IAAI,EAAE,GAAG;;;;;;YAMC,MAAM,CAAC,SAAS;aACf,MAAM,CAAC,WAAW;YACnB,MAAM,CAAC,KAAK;WACb,MAAM,CAAC,IAAI;eACP,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;mBACjB,CAAC,CAAC,WAAW,CAAC,cAAc,EAAE;eAClC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACnC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;iBACf,CAAC,CAAC,SAAS;;;;CAI3B,CAAC;IAEA,6BAA6B;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,EAAE,CAAC;QACzC,MAAM,WAAW,GACf,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QAC3E,MAAM,MAAM,GAAG,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAE7C,EAAE,IAAI,OAAO,MAAM,IAAI,QAAQ;;kBAEjB,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;0BACd,OAAO,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM;yBAChD,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM;;CAEtE,CAAC;IACA,CAAC;IAED,EAAE,IAAI;;2BAEmB,MAAM,CAAC,OAAO,OAAO,MAAM,CAAC,SAAS;CAC/D,CAAC;IAEA,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,MAAuB;IAC3C,MAAM,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAEnC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,CAAC;IAEzD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,MAAM,aAAa,GAAG,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACnG,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1F,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC,WAAW,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC,YAAY,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IAC/J,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACjF,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,SAAS,QAAQ,CAAC,CAAC;IAEjE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,CAAC;AAC3D,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"view-leaderboard-command.d.ts","sourceRoot":"","sources":["../../src/commands/view-leaderboard-command.ts"],"names":[],"mappings":"AAUA;;GAEG;AACH,wBAAsB,eAAe,CACnC,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAClD,OAAO,CAAC,IAAI,CAAC,CAwBf"}
|