@skillmark/cli 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/dist/cli-entry-point.d.ts +3 -0
  2. package/dist/cli-entry-point.d.ts.map +1 -0
  3. package/dist/cli-entry-point.js +207 -0
  4. package/dist/cli-entry-point.js.map +1 -0
  5. package/dist/commands/auth-setup-and-token-storage-command.d.ts +21 -0
  6. package/dist/commands/auth-setup-and-token-storage-command.d.ts.map +1 -0
  7. package/dist/commands/auth-setup-and-token-storage-command.js +166 -0
  8. package/dist/commands/auth-setup-and-token-storage-command.js.map +1 -0
  9. package/dist/commands/publish-results-command.d.ts +21 -0
  10. package/dist/commands/publish-results-command.d.ts.map +1 -0
  11. package/dist/commands/publish-results-command.js +256 -0
  12. package/dist/commands/publish-results-command.js.map +1 -0
  13. package/dist/commands/run-benchmark-command.d.ts +6 -0
  14. package/dist/commands/run-benchmark-command.d.ts.map +1 -0
  15. package/dist/commands/run-benchmark-command.js +331 -0
  16. package/dist/commands/run-benchmark-command.js.map +1 -0
  17. package/dist/commands/view-leaderboard-command.d.ts +8 -0
  18. package/dist/commands/view-leaderboard-command.d.ts.map +1 -0
  19. package/dist/commands/view-leaderboard-command.js +169 -0
  20. package/dist/commands/view-leaderboard-command.js.map +1 -0
  21. package/dist/config/api-key-config-reader.d.ts +14 -0
  22. package/dist/config/api-key-config-reader.d.ts.map +1 -0
  23. package/dist/config/api-key-config-reader.js +107 -0
  24. package/dist/config/api-key-config-reader.js.map +1 -0
  25. package/dist/config/api-key-config-reader.test.d.ts +2 -0
  26. package/dist/config/api-key-config-reader.test.d.ts.map +1 -0
  27. package/dist/config/api-key-config-reader.test.js +21 -0
  28. package/dist/config/api-key-config-reader.test.js.map +1 -0
  29. package/dist/engine/claude-cli-executor.d.ts +33 -0
  30. package/dist/engine/claude-cli-executor.d.ts.map +1 -0
  31. package/dist/engine/claude-cli-executor.js +251 -0
  32. package/dist/engine/claude-cli-executor.js.map +1 -0
  33. package/dist/engine/concept-accuracy-scorer.d.ts +24 -0
  34. package/dist/engine/concept-accuracy-scorer.d.ts.map +1 -0
  35. package/dist/engine/concept-accuracy-scorer.js +186 -0
  36. package/dist/engine/concept-accuracy-scorer.js.map +1 -0
  37. package/dist/engine/concept-accuracy-scorer.test.d.ts +2 -0
  38. package/dist/engine/concept-accuracy-scorer.test.d.ts.map +1 -0
  39. package/dist/engine/concept-accuracy-scorer.test.js +230 -0
  40. package/dist/engine/concept-accuracy-scorer.test.js.map +1 -0
  41. package/dist/engine/enhanced-test-prompt-builder.d.ts +30 -0
  42. package/dist/engine/enhanced-test-prompt-builder.d.ts.map +1 -0
  43. package/dist/engine/enhanced-test-prompt-builder.js +134 -0
  44. package/dist/engine/enhanced-test-prompt-builder.js.map +1 -0
  45. package/dist/engine/markdown-test-definition-parser.d.ts +18 -0
  46. package/dist/engine/markdown-test-definition-parser.d.ts.map +1 -0
  47. package/dist/engine/markdown-test-definition-parser.js +525 -0
  48. package/dist/engine/markdown-test-definition-parser.js.map +1 -0
  49. package/dist/engine/markdown-test-definition-parser.test.d.ts +2 -0
  50. package/dist/engine/markdown-test-definition-parser.test.d.ts.map +1 -0
  51. package/dist/engine/markdown-test-definition-parser.test.js +265 -0
  52. package/dist/engine/markdown-test-definition-parser.test.js.map +1 -0
  53. package/dist/engine/retry-with-degrade-utils.d.ts +58 -0
  54. package/dist/engine/retry-with-degrade-utils.d.ts.map +1 -0
  55. package/dist/engine/retry-with-degrade-utils.js +86 -0
  56. package/dist/engine/retry-with-degrade-utils.js.map +1 -0
  57. package/dist/engine/skill-content-collector.d.ts +53 -0
  58. package/dist/engine/skill-content-collector.d.ts.map +1 -0
  59. package/dist/engine/skill-content-collector.js +157 -0
  60. package/dist/engine/skill-content-collector.js.map +1 -0
  61. package/dist/engine/skill-creator-invoker.d.ts +36 -0
  62. package/dist/engine/skill-creator-invoker.d.ts.map +1 -0
  63. package/dist/engine/skill-creator-invoker.js +222 -0
  64. package/dist/engine/skill-creator-invoker.js.map +1 -0
  65. package/dist/engine/transcript-jsonl-parser.d.ts +28 -0
  66. package/dist/engine/transcript-jsonl-parser.d.ts.map +1 -0
  67. package/dist/engine/transcript-jsonl-parser.js +175 -0
  68. package/dist/engine/transcript-jsonl-parser.js.map +1 -0
  69. package/dist/sources/git-repository-skill-source-handler.d.ts +18 -0
  70. package/dist/sources/git-repository-skill-source-handler.d.ts.map +1 -0
  71. package/dist/sources/git-repository-skill-source-handler.js +119 -0
  72. package/dist/sources/git-repository-skill-source-handler.js.map +1 -0
  73. package/dist/sources/local-skill-source-handler.d.ts +21 -0
  74. package/dist/sources/local-skill-source-handler.d.ts.map +1 -0
  75. package/dist/sources/local-skill-source-handler.js +138 -0
  76. package/dist/sources/local-skill-source-handler.js.map +1 -0
  77. package/dist/sources/local-skill-source-handler.test.d.ts +2 -0
  78. package/dist/sources/local-skill-source-handler.test.d.ts.map +1 -0
  79. package/dist/sources/local-skill-source-handler.test.js +55 -0
  80. package/dist/sources/local-skill-source-handler.test.js.map +1 -0
  81. package/dist/sources/skillsh-registry-source-handler.d.ts +18 -0
  82. package/dist/sources/skillsh-registry-source-handler.d.ts.map +1 -0
  83. package/dist/sources/skillsh-registry-source-handler.js +130 -0
  84. package/dist/sources/skillsh-registry-source-handler.js.map +1 -0
  85. package/dist/sources/unified-skill-source-resolver.d.ts +20 -0
  86. package/dist/sources/unified-skill-source-resolver.d.ts.map +1 -0
  87. package/dist/sources/unified-skill-source-resolver.js +64 -0
  88. package/dist/sources/unified-skill-source-resolver.js.map +1 -0
  89. package/dist/sources/unified-skill-source-resolver.test.d.ts +2 -0
  90. package/dist/sources/unified-skill-source-resolver.test.d.ts.map +1 -0
  91. package/dist/sources/unified-skill-source-resolver.test.js +84 -0
  92. package/dist/sources/unified-skill-source-resolver.test.js.map +1 -0
  93. package/dist/types/benchmark-types.d.ts +142 -0
  94. package/dist/types/benchmark-types.d.ts.map +1 -0
  95. package/dist/types/benchmark-types.js +5 -0
  96. package/dist/types/benchmark-types.js.map +1 -0
  97. package/dist/types/index.d.ts +5 -0
  98. package/dist/types/index.d.ts.map +1 -0
  99. package/dist/types/index.js +5 -0
  100. package/dist/types/index.js.map +1 -0
  101. package/package.json +38 -0
@@ -0,0 +1,256 @@
1
+ /**
2
+ * Publish results command - uploads benchmark results to leaderboard API
3
+ */
4
+ import { readFile, readdir } from 'node:fs/promises';
5
+ import { resolve, join } from 'node:path';
6
+ import chalk from 'chalk';
7
+ import ora from 'ora';
8
+ /** Default API endpoint */
9
+ const DEFAULT_ENDPOINT = 'https://skillmark.sh/api';
10
+ /**
11
+ * Execute the publish command
12
+ */
13
+ export async function publishResults(resultPath, options) {
14
+ const spinner = ora();
15
+ try {
16
+ // 1. Load result file
17
+ spinner.start('Loading result file...');
18
+ const fullPath = resolve(resultPath);
19
+ const content = await readFile(fullPath, 'utf-8');
20
+ const result = JSON.parse(content);
21
+ spinner.succeed(`Loaded: ${result.skillName} (${result.model})`);
22
+ // 2. Validate result
23
+ spinner.start('Validating result...');
24
+ validateResult(result);
25
+ spinner.succeed('Result validated');
26
+ // 3. Upload to API
27
+ spinner.start('Uploading to leaderboard...');
28
+ const endpoint = options.endpoint || DEFAULT_ENDPOINT;
29
+ const response = await uploadResult(result, options.apiKey, endpoint);
30
+ spinner.succeed('Uploaded successfully');
31
+ // 4. Display result
32
+ console.log(chalk.green('\n✓ Result published to leaderboard\n'));
33
+ console.log(`${chalk.gray('Skill:')} ${result.skillName}`);
34
+ console.log(`${chalk.gray('Accuracy:')} ${result.aggregatedMetrics.accuracy.toFixed(1)}%`);
35
+ console.log(`${chalk.gray('Model:')} ${result.model}`);
36
+ if (response.leaderboardUrl) {
37
+ console.log(`\n${chalk.blue('View leaderboard:')} ${response.leaderboardUrl}`);
38
+ }
39
+ if (response.rank) {
40
+ console.log(`${chalk.gray('Rank:')} #${response.rank}`);
41
+ }
42
+ }
43
+ catch (error) {
44
+ spinner.fail('Publish failed');
45
+ throw error;
46
+ }
47
+ }
48
+ /**
49
+ * Validate result before publishing
50
+ */
51
+ function validateResult(result) {
52
+ // Check required fields
53
+ if (!result.skillId) {
54
+ throw new Error('Invalid result: missing skillId');
55
+ }
56
+ if (!result.skillName) {
57
+ throw new Error('Invalid result: missing skillName');
58
+ }
59
+ if (!result.model || !['haiku', 'sonnet', 'opus'].includes(result.model)) {
60
+ throw new Error('Invalid result: invalid model');
61
+ }
62
+ if (!result.aggregatedMetrics) {
63
+ throw new Error('Invalid result: missing aggregatedMetrics');
64
+ }
65
+ if (!result.hash) {
66
+ throw new Error('Invalid result: missing verification hash');
67
+ }
68
+ if (!result.version) {
69
+ throw new Error('Invalid result: missing version');
70
+ }
71
+ // Validate metrics ranges
72
+ const m = result.aggregatedMetrics;
73
+ if (m.accuracy < 0 || m.accuracy > 100) {
74
+ throw new Error('Invalid result: accuracy out of range');
75
+ }
76
+ if (m.tokensTotal < 0) {
77
+ throw new Error('Invalid result: negative token count');
78
+ }
79
+ if (m.costUsd < 0) {
80
+ throw new Error('Invalid result: negative cost');
81
+ }
82
+ }
83
+ /**
84
+ * Upload result to API
85
+ */
86
+ async function uploadResult(result, apiKey, endpoint) {
87
+ const url = `${endpoint}/results`;
88
+ const response = await fetch(url, {
89
+ method: 'POST',
90
+ headers: {
91
+ 'Content-Type': 'application/json',
92
+ 'Authorization': `Bearer ${apiKey}`,
93
+ 'X-Skillmark-Version': result.version,
94
+ },
95
+ body: JSON.stringify({
96
+ skillId: result.skillId,
97
+ skillName: result.skillName,
98
+ source: result.skillSource,
99
+ model: result.model,
100
+ accuracy: result.aggregatedMetrics.accuracy,
101
+ tokensTotal: result.aggregatedMetrics.tokensTotal,
102
+ durationMs: result.aggregatedMetrics.durationMs,
103
+ costUsd: result.aggregatedMetrics.costUsd,
104
+ runs: result.runs,
105
+ hash: result.hash,
106
+ timestamp: result.timestamp,
107
+ rawJson: JSON.stringify(result),
108
+ }),
109
+ });
110
+ if (!response.ok) {
111
+ const errorText = await response.text();
112
+ throw new Error(`API error (${response.status}): ${errorText}`);
113
+ }
114
+ return response.json();
115
+ }
116
+ /**
117
+ * Publish results with auto-key (from run command with --publish flag)
118
+ * Includes test files and skill.sh link detection
119
+ */
120
+ export async function publishResultsWithAutoKey(result, options) {
121
+ const spinner = ora();
122
+ try {
123
+ // 1. Validate result
124
+ spinner.start('Validating result...');
125
+ validateResult(result);
126
+ spinner.succeed('Result validated');
127
+ // 2. Load test files if path provided
128
+ let testFiles = [];
129
+ if (options.testsPath) {
130
+ spinner.start('Loading test files...');
131
+ testFiles = await loadTestFiles(options.testsPath);
132
+ spinner.succeed(`Loaded ${testFiles.length} test file(s)`);
133
+ }
134
+ // 3. Detect skill.sh link
135
+ const skillshLink = detectSkillshLink(result.skillSource);
136
+ // 4. Upload to API
137
+ spinner.start('Uploading to leaderboard...');
138
+ const endpoint = options.endpoint || DEFAULT_ENDPOINT;
139
+ const response = await uploadResultWithExtras(result, options.apiKey, endpoint, testFiles, skillshLink);
140
+ spinner.succeed('Uploaded successfully');
141
+ // 5. Display result
142
+ console.log(chalk.green('\n✓ Result published to leaderboard\n'));
143
+ console.log(`${chalk.gray('Skill:')} ${result.skillName}`);
144
+ console.log(`${chalk.gray('Accuracy:')} ${result.aggregatedMetrics.accuracy.toFixed(1)}%`);
145
+ console.log(`${chalk.gray('Model:')} ${result.model}`);
146
+ if (response.submitter?.github) {
147
+ console.log(`${chalk.gray('Submitter:')} @${response.submitter.github}`);
148
+ }
149
+ if (response.leaderboardUrl) {
150
+ console.log(`\n${chalk.blue('View leaderboard:')} ${response.leaderboardUrl}`);
151
+ }
152
+ if (response.rank) {
153
+ console.log(`${chalk.gray('Rank:')} #${response.rank}`);
154
+ }
155
+ }
156
+ catch (error) {
157
+ spinner.fail('Publish failed');
158
+ throw error;
159
+ }
160
+ }
161
+ /**
162
+ * Load test files from directory
163
+ */
164
+ async function loadTestFiles(testsPath) {
165
+ const fullPath = resolve(testsPath);
166
+ const files = [];
167
+ try {
168
+ const entries = await readdir(fullPath, { withFileTypes: true });
169
+ for (const entry of entries) {
170
+ if (entry.isFile() && entry.name.endsWith('.md')) {
171
+ const filePath = join(fullPath, entry.name);
172
+ const content = await readFile(filePath, 'utf-8');
173
+ files.push({
174
+ name: entry.name,
175
+ content,
176
+ });
177
+ }
178
+ }
179
+ }
180
+ catch {
181
+ // Directory doesn't exist or can't be read, return empty
182
+ }
183
+ return files;
184
+ }
185
+ /**
186
+ * Detect if source is from skill.sh registry and return link
187
+ */
188
+ function detectSkillshLink(source) {
189
+ // Check for skill.sh pattern: skill.sh/user/skill-name
190
+ const skillshMatch = source.match(/^skill\.sh\/([^/]+)\/([^/]+)$/i);
191
+ if (skillshMatch) {
192
+ return `https://skill.sh/${skillshMatch[1]}/${skillshMatch[2]}`;
193
+ }
194
+ // Check for full URL pattern
195
+ const urlMatch = source.match(/^https?:\/\/skill\.sh\/([^/]+)\/([^/]+)/i);
196
+ if (urlMatch) {
197
+ return `https://skill.sh/${urlMatch[1]}/${urlMatch[2]}`;
198
+ }
199
+ return null;
200
+ }
201
+ /**
202
+ * Upload result with test files and skill.sh link
203
+ */
204
+ async function uploadResultWithExtras(result, apiKey, endpoint, testFiles, skillshLink) {
205
+ const url = `${endpoint}/results`;
206
+ const response = await fetch(url, {
207
+ method: 'POST',
208
+ headers: {
209
+ 'Content-Type': 'application/json',
210
+ 'Authorization': `Bearer ${apiKey}`,
211
+ 'X-Skillmark-Version': result.version,
212
+ },
213
+ body: JSON.stringify({
214
+ skillId: result.skillId,
215
+ skillName: result.skillName,
216
+ source: result.skillSource,
217
+ model: result.model,
218
+ accuracy: result.aggregatedMetrics.accuracy,
219
+ tokensTotal: result.aggregatedMetrics.tokensTotal,
220
+ tokensInput: result.aggregatedMetrics.tokensInput,
221
+ tokensOutput: result.aggregatedMetrics.tokensOutput,
222
+ durationMs: result.aggregatedMetrics.durationMs,
223
+ costUsd: result.aggregatedMetrics.costUsd,
224
+ toolCount: result.aggregatedMetrics.toolCount,
225
+ runs: result.runs,
226
+ hash: result.hash,
227
+ timestamp: result.timestamp,
228
+ rawJson: JSON.stringify(result),
229
+ testFiles: testFiles.length > 0 ? testFiles : undefined,
230
+ skillshLink: skillshLink || undefined,
231
+ }),
232
+ });
233
+ if (!response.ok) {
234
+ const errorText = await response.text();
235
+ throw new Error(`API error (${response.status}): ${errorText}`);
236
+ }
237
+ return response.json();
238
+ }
239
+ /**
240
+ * Verify API key is valid
241
+ */
242
+ export async function verifyApiKey(apiKey, endpoint = DEFAULT_ENDPOINT) {
243
+ try {
244
+ const response = await fetch(`${endpoint}/verify`, {
245
+ method: 'POST',
246
+ headers: {
247
+ 'Authorization': `Bearer ${apiKey}`,
248
+ },
249
+ });
250
+ return response.ok;
251
+ }
252
+ catch {
253
+ return false;
254
+ }
255
+ }
256
+ //# sourceMappingURL=publish-results-command.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"publish-results-command.js","sourceRoot":"","sources":["../../src/commands/publish-results-command.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAY,MAAM,WAAW,CAAC;AACpD,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AAGtB,2BAA2B;AAC3B,MAAM,gBAAgB,GAAG,0BAA0B,CAAC;AAepD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,UAAkB,EAClB,OAAuB;IAEvB,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;IAEtB,IAAI,CAAC;QACH,sBAAsB;QACtB,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,MAAM,GAAoB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpD,OAAO,CAAC,OAAO,CAAC,WAAW,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC;QAEjE,qBAAqB;QACrB,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtC,cAAc,CAAC,MAAM,CAAC,CAAC;QACvB,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAEpC,mBAAmB;QACnB,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC;QACtD,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACtE,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;QAEzC,oBAAoB;QACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3F,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAE1D,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/B,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,MAAuB;IAC7C,wBAAwB;IACxB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,0BAA0B;IAC1B,MAAM,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACnC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,GAAG,GAAG,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,CAAC,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CACzB,MAAuB,EACvB,MAAc,EACd,QAAgB;IAEhB,MAAM,GAAG,GAAG,GAAG,QAAQ,UAAU,CAAC;IAElC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,eAAe,EAAE,UAAU,MAAM,EAAE;YACnC,qBAAqB,EAAE,MAAM,CAAC,OAAO;SACtC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,MAAM,CAAC,WAAW;YAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,iBAAiB,CAAC,QAAQ;YAC3C,WAAW,EAAE,MAAM,CAAC,iBAAiB,CAAC,WAAW;YACjD,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,UAAU;YAC/C,OAAO,EAAE,MAAM,CAAC,iBAAiB,CAAC,OAAO;YACzC,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;SAChC,CAAC;KACH,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,cAAc,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,EAAyD,CAAC;AAChF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,MAAuB,EACvB,OAA2B;IAE3B,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;IAEtB,IAAI,CAAC;QACH,qBAAqB;QACrB,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtC,cAAc,CAAC,MAAM,CAAC,CAAC;QACvB,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAEpC,sCAAsC;QACtC,IAAI,SAAS,GAAqB,EAAE,CAAC;QACrC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;YACvC,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACnD,OAAO,CAAC,OAAO,CAAC,UAAU,SAAS,CAAC,MAAM,eAAe,CAAC,CAAC;QAC7D,CAAC;QAED,0BAA0B;QAC1B,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAE1D,mBAAmB;QACnB,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC;QACtD,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC3C,MAAM,EACN,OAAO,CAAC,MAAM,EACd,QAAQ,EACR,SAAS,EACT,WAAW,CACZ,CAAC;QACF,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;QAEzC,oBAAoB;QACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3F,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAE1D,IAAI,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/B,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa,CAAC,SAAiB;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,KAAK,GAAqB,EAAE,CAAC;IAEnC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEjE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAClD,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,OAAO;iBACR,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,yDAAyD;IAC3D,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,MAAc;IACvC,uDAAuD;IACvD,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpE,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,oBAAoB,YAAY,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,CAAC;IAED,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC1E,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,oBAAoB,QAAQ,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,sBAAsB,CACnC,MAAuB,EACvB,MAAc,EACd,QAAgB,EAChB,SAA2B,EAC3B,WAA0B;IAM1B,MAAM,GAAG,GAAG,GAAG,QAAQ,UAAU,CAAC;IAElC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,eAAe,EAAE,UAAU,MAAM,EAAE;YACnC,qBAAqB,EAAE,MAAM,CAAC,OAAO;SACtC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,MAAM,CAAC,WAAW;YAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,iBAAiB,CAAC,QAAQ;YAC3C,WAAW,EAAE,MAAM,CAAC,iBAAiB,CAAC,WAAW;YACjD,WAAW,EAAE,MAAM,CAAC,iBAAiB,CAAC,WAAW;YACjD,YAAY,EAAE,MAAM,CAAC,iBAAiB,CAAC,YAAY;YACnD,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,UAAU;YAC/C,OAAO,EAAE,MAAM,CAAC,iBAAiB,CAAC,OAAO;YACzC,SAAS,EAAE,MAAM,CAAC,iBAAiB,CAAC,SAAS;YAC7C,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAC/B,SAAS,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YACvD,WAAW,EAAE,WAAW,IAAI,SAAS;SACtC,CAAC;KACH,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,cAAc,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,EAIlB,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAc,EACd,WAAmB,gBAAgB;IAEnC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,QAAQ,SAAS,EAAE;YACjD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,eAAe,EAAE,UAAU,MAAM,EAAE;aACpC;SACF,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,EAAE,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
@@ -0,0 +1,6 @@
1
+ import type { BenchmarkResult, RunOptions } from '../types/index.js';
2
+ /**
3
+ * Execute the run command
4
+ */
5
+ export declare function runBenchmark(skillSource: string, options: RunOptions): Promise<BenchmarkResult>;
6
+ //# sourceMappingURL=run-benchmark-command.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run-benchmark-command.d.ts","sourceRoot":"","sources":["../../src/commands/run-benchmark-command.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EACV,eAAe,EAEf,UAAU,EAEX,MAAM,mBAAmB,CAAC;AAmE3B;;GAEG;AACH,wBAAsB,YAAY,CAChC,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,eAAe,CAAC,CAuM1B"}
@@ -0,0 +1,331 @@
1
+ /**
2
+ * Run benchmark command - executes benchmarks against a skill
3
+ */
4
+ import { mkdir, writeFile } from 'node:fs/promises';
5
+ import { join, resolve } from 'node:path';
6
+ import { createHash } from 'node:crypto';
7
+ import chalk from 'chalk';
8
+ import ora from 'ora';
9
+ import { resolveSkillSource, formatSourceDisplay } from '../sources/unified-skill-source-resolver.js';
10
+ import { loadTestsFromDirectory, discoverTests } from '../engine/markdown-test-definition-parser.js';
11
+ import { executeTest, cleanupTranscripts } from '../engine/claude-cli-executor.js';
12
+ import { scoreResponse, aggregateMetrics } from '../engine/concept-accuracy-scorer.js';
13
+ import { getStoredToken, runAuth } from './auth-setup-and-token-storage-command.js';
14
+ /** CLI version */
15
+ const VERSION = '0.1.0';
16
+ /**
17
+ * Verify Claude CLI authentication before running benchmarks
18
+ */
19
+ async function verifyClaudeAuth() {
20
+ const { spawn } = await import('node:child_process');
21
+ // Get stored token (from env or config file)
22
+ const storedToken = await getStoredToken();
23
+ return new Promise((resolve) => {
24
+ // Prepare environment with token if available
25
+ const env = { ...process.env };
26
+ if (storedToken) {
27
+ env.CLAUDE_CODE_OAUTH_TOKEN = storedToken;
28
+ }
29
+ // Quick test with minimal prompt
30
+ const proc = spawn('claude', ['-p', 'Say OK', '--output-format', 'json', '--model', 'haiku'], {
31
+ env,
32
+ stdio: ['ignore', 'pipe', 'pipe'],
33
+ timeout: 30000,
34
+ });
35
+ let stdout = '';
36
+ proc.stdout.on('data', (data) => { stdout += data.toString(); });
37
+ const timeout = setTimeout(() => {
38
+ proc.kill();
39
+ resolve({ ok: false, error: 'Authentication check timed out' });
40
+ }, 30000);
41
+ proc.on('close', () => {
42
+ clearTimeout(timeout);
43
+ try {
44
+ const result = JSON.parse(stdout);
45
+ if (result.is_error && result.result?.includes('Invalid API key')) {
46
+ resolve({
47
+ ok: false,
48
+ error: `Claude CLI not authenticated.\n` +
49
+ ` Run: skillmark auth\n` +
50
+ ` Or set CLAUDE_CODE_OAUTH_TOKEN environment variable.`,
51
+ });
52
+ return;
53
+ }
54
+ resolve({ ok: true, token: storedToken });
55
+ }
56
+ catch {
57
+ resolve({ ok: true, token: storedToken }); // Assume ok if we can't parse
58
+ }
59
+ });
60
+ proc.on('error', (err) => {
61
+ clearTimeout(timeout);
62
+ resolve({ ok: false, error: `Claude CLI not found: ${err.message}` });
63
+ });
64
+ });
65
+ }
66
+ /**
67
+ * Execute the run command
68
+ */
69
+ export async function runBenchmark(skillSource, options) {
70
+ const spinner = ora();
71
+ try {
72
+ // 0. Verify Claude CLI authentication
73
+ spinner.start('Verifying Claude CLI authentication...');
74
+ let authCheck = await verifyClaudeAuth();
75
+ if (!authCheck.ok) {
76
+ spinner.warn('Not authenticated - starting auth setup...');
77
+ console.log('');
78
+ await runAuth();
79
+ console.log('');
80
+ // Verify again after auth
81
+ spinner.start('Verifying authentication...');
82
+ authCheck = await verifyClaudeAuth();
83
+ if (!authCheck.ok) {
84
+ spinner.fail('Authentication still failed');
85
+ throw new Error(authCheck.error);
86
+ }
87
+ }
88
+ spinner.succeed('Claude CLI authenticated');
89
+ // 1. Resolve skill source
90
+ spinner.start('Resolving skill source...');
91
+ const skill = await resolveSkillSource(skillSource);
92
+ spinner.succeed(`Resolved: ${formatSourceDisplay(skill)}`);
93
+ // 2. Load test definitions
94
+ spinner.start('Loading test definitions...');
95
+ let tests;
96
+ if (options.tests) {
97
+ // Use explicit tests path
98
+ tests = await loadTestsFromDirectory(resolve(options.tests));
99
+ }
100
+ else {
101
+ // Auto-discover tests in skill directory
102
+ tests = await discoverTests(skill.localPath);
103
+ }
104
+ if (tests.length === 0) {
105
+ spinner.fail('No test files found');
106
+ throw new Error('No test files found. Specify --tests <path> or add tests/ directory to skill.');
107
+ }
108
+ spinner.succeed(`Loaded ${tests.length} test(s)`);
109
+ // 3. Create output directory
110
+ const outputDir = resolve(options.output);
111
+ await mkdir(outputDir, { recursive: true });
112
+ // 4. Run benchmarks
113
+ const verbose = options.verbose ?? false;
114
+ console.log(chalk.blue(`\nRunning ${tests.length} test(s) × ${options.runs} run(s) with ${options.model}${verbose ? ' (verbose)' : ''}\n`));
115
+ const allResults = [];
116
+ const workDir = process.cwd();
117
+ for (let run = 1; run <= options.runs; run++) {
118
+ console.log(chalk.gray(`── Run ${run}/${options.runs} ──`));
119
+ for (const test of tests) {
120
+ const testStart = Date.now();
121
+ if (verbose) {
122
+ console.log(chalk.cyan(`\n▶ Starting: ${test.name}`));
123
+ console.log(chalk.gray(` Type: ${test.type} | Timeout: ${test.timeout}s | Concepts: ${test.concepts.length}`));
124
+ }
125
+ else {
126
+ spinner.start(`Testing: ${test.name}`);
127
+ }
128
+ // Elapsed timer for verbose mode (declared outside try for catch access)
129
+ let elapsedTimer = null;
130
+ try {
131
+ if (verbose) {
132
+ const cliStart = Date.now();
133
+ process.stdout.write(chalk.gray(` Invoking Claude CLI... 0s`));
134
+ elapsedTimer = setInterval(() => {
135
+ const elapsed = Math.floor((Date.now() - cliStart) / 1000);
136
+ process.stdout.write(`\r${chalk.gray(` Invoking Claude CLI... ${elapsed}s`)}`);
137
+ }, 1000);
138
+ }
139
+ const execution = await executeTest(test, skill.localPath, options.model, workDir);
140
+ // Clear elapsed timer
141
+ if (elapsedTimer) {
142
+ clearInterval(elapsedTimer);
143
+ process.stdout.write('\n');
144
+ }
145
+ if (!execution.success) {
146
+ if (verbose) {
147
+ console.log(chalk.red(` ✗ Failed: ${execution.error}`));
148
+ }
149
+ else {
150
+ spinner.fail(`${test.name}: ${execution.error}`);
151
+ }
152
+ continue;
153
+ }
154
+ if (verbose) {
155
+ const elapsed = ((Date.now() - testStart) / 1000).toFixed(1);
156
+ console.log(chalk.gray(` Execution complete (${elapsed}s)`));
157
+ console.log(chalk.gray(` Tokens: ${execution.inputTokens + execution.outputTokens} | Tools: ${execution.toolCount} | Cost: $${execution.costUsd.toFixed(4)}`));
158
+ }
159
+ // Calculate metrics
160
+ const metrics = {
161
+ accuracy: 0, // Calculated by scorer
162
+ tokensTotal: execution.inputTokens + execution.outputTokens,
163
+ tokensInput: execution.inputTokens,
164
+ tokensOutput: execution.outputTokens,
165
+ durationMs: execution.durationMs,
166
+ toolCount: execution.toolCount,
167
+ costUsd: execution.costUsd,
168
+ };
169
+ // Score response
170
+ const result = scoreResponse(test, execution.response, metrics);
171
+ allResults.push(result);
172
+ // Display result
173
+ const status = result.passed ? chalk.green('✓') : chalk.yellow('○');
174
+ const accuracy = result.metrics.accuracy.toFixed(1);
175
+ if (verbose) {
176
+ console.log(chalk.gray(` Scoring response...`));
177
+ console.log(` ${status} Result: ${accuracy}% accuracy (${result.matchedConcepts.length}/${test.concepts.length} concepts)`);
178
+ if (result.matchedConcepts.length > 0) {
179
+ console.log(chalk.green(` Matched: ${result.matchedConcepts.join(', ')}`));
180
+ }
181
+ if (result.missedConcepts.length > 0) {
182
+ console.log(chalk.yellow(` Missed: ${result.missedConcepts.join(', ')}`));
183
+ }
184
+ }
185
+ else {
186
+ spinner.succeed(`${status} ${test.name}: ${accuracy}% (${result.matchedConcepts.length}/${test.concepts.length} concepts)`);
187
+ // Show missed concepts if any
188
+ if (result.missedConcepts.length > 0 && result.missedConcepts.length <= 3) {
189
+ console.log(chalk.gray(` Missed: ${result.missedConcepts.join(', ')}`));
190
+ }
191
+ }
192
+ }
193
+ catch (error) {
194
+ // Clear elapsed timer on error
195
+ if (elapsedTimer) {
196
+ clearInterval(elapsedTimer);
197
+ process.stdout.write('\n');
198
+ }
199
+ if (verbose) {
200
+ console.log(chalk.red(` ✗ Error: ${error instanceof Error ? error.message : 'Unknown error'}`));
201
+ }
202
+ else {
203
+ spinner.fail(`${test.name}: ${error instanceof Error ? error.message : 'Unknown error'}`);
204
+ }
205
+ }
206
+ }
207
+ }
208
+ // 5. Aggregate results
209
+ const aggregated = aggregateMetrics(allResults);
210
+ const benchmarkResult = {
211
+ skillId: createSkillId(skill.name, skill.original),
212
+ skillName: skill.name,
213
+ skillSource: skill.original,
214
+ model: options.model,
215
+ runs: options.runs,
216
+ testResults: allResults,
217
+ aggregatedMetrics: aggregated,
218
+ timestamp: new Date().toISOString(),
219
+ version: VERSION,
220
+ };
221
+ // Add verification hash
222
+ benchmarkResult.hash = generateResultHash(benchmarkResult);
223
+ // 6. Write outputs
224
+ spinner.start('Writing results...');
225
+ const jsonPath = join(outputDir, 'result.json');
226
+ const mdPath = join(outputDir, 'report.md');
227
+ await writeFile(jsonPath, JSON.stringify(benchmarkResult, null, 2));
228
+ await writeFile(mdPath, generateMarkdownReport(benchmarkResult));
229
+ spinner.succeed(`Results saved to ${outputDir}`);
230
+ // 7. Print summary
231
+ printSummary(benchmarkResult);
232
+ // Cleanup
233
+ await cleanupTranscripts(workDir);
234
+ return benchmarkResult;
235
+ }
236
+ catch (error) {
237
+ spinner.fail('Benchmark failed');
238
+ throw error;
239
+ }
240
+ }
241
+ /**
242
+ * Create a unique skill ID
243
+ */
244
+ function createSkillId(name, source) {
245
+ const hash = createHash('sha256')
246
+ .update(`${name}:${source}`)
247
+ .digest('hex')
248
+ .slice(0, 12);
249
+ return `${name.toLowerCase().replace(/[^a-z0-9]/g, '-')}-${hash}`;
250
+ }
251
+ /**
252
+ * Generate verification hash for result
253
+ */
254
+ function generateResultHash(result) {
255
+ const data = {
256
+ skillId: result.skillId,
257
+ model: result.model,
258
+ runs: result.runs,
259
+ accuracy: result.aggregatedMetrics.accuracy,
260
+ tokensTotal: result.aggregatedMetrics.tokensTotal,
261
+ timestamp: result.timestamp,
262
+ };
263
+ return createHash('sha256').update(JSON.stringify(data)).digest('hex').slice(0, 16);
264
+ }
265
+ /**
266
+ * Generate markdown report
267
+ */
268
+ function generateMarkdownReport(result) {
269
+ const m = result.aggregatedMetrics;
270
+ let md = `# Skillmark Benchmark Report
271
+
272
+ ## Summary
273
+
274
+ | Metric | Value |
275
+ |--------|-------|
276
+ | Skill | ${result.skillName} |
277
+ | Source | ${result.skillSource} |
278
+ | Model | ${result.model} |
279
+ | Runs | ${result.runs} |
280
+ | Accuracy | ${m.accuracy.toFixed(1)}% |
281
+ | Total Tokens | ${m.tokensTotal.toLocaleString()} |
282
+ | Duration | ${(m.durationMs / 1000).toFixed(1)}s |
283
+ | Cost | $${m.costUsd.toFixed(4)} |
284
+ | Tool Calls | ${m.toolCount} |
285
+
286
+ ## Test Results
287
+
288
+ `;
289
+ // Group results by test name
290
+ const byTest = new Map();
291
+ for (const r of result.testResults) {
292
+ const existing = byTest.get(r.test.name) || [];
293
+ existing.push(r);
294
+ byTest.set(r.test.name, existing);
295
+ }
296
+ for (const [testName, results] of byTest) {
297
+ const avgAccuracy = results.reduce((sum, r) => sum + r.metrics.accuracy, 0) / results.length;
298
+ const status = avgAccuracy >= 70 ? '✓' : '○';
299
+ md += `### ${status} ${testName}
300
+
301
+ - **Accuracy**: ${avgAccuracy.toFixed(1)}%
302
+ - **Concepts Matched**: ${results[0].matchedConcepts.join(', ') || 'None'}
303
+ - **Concepts Missed**: ${results[0].missedConcepts.join(', ') || 'None'}
304
+
305
+ `;
306
+ }
307
+ md += `---
308
+
309
+ *Generated by Skillmark v${result.version} at ${result.timestamp}*
310
+ `;
311
+ return md;
312
+ }
313
+ /**
314
+ * Print summary to console
315
+ */
316
+ function printSummary(result) {
317
+ const m = result.aggregatedMetrics;
318
+ console.log(chalk.bold('\n═══ Benchmark Summary ═══\n'));
319
+ console.log(`${chalk.gray('Skill:')} ${result.skillName}`);
320
+ console.log(`${chalk.gray('Model:')} ${result.model}`);
321
+ console.log(`${chalk.gray('Runs:')} ${result.runs}`);
322
+ console.log('');
323
+ const accuracyColor = m.accuracy >= 80 ? chalk.green : m.accuracy >= 60 ? chalk.yellow : chalk.red;
324
+ console.log(`${chalk.gray('Accuracy:')} ${accuracyColor(m.accuracy.toFixed(1) + '%')}`);
325
+ console.log(`${chalk.gray('Tokens:')} ${m.tokensTotal.toLocaleString()} (in: ${m.tokensInput.toLocaleString()}, out: ${m.tokensOutput.toLocaleString()})`);
326
+ console.log(`${chalk.gray('Duration:')} ${(m.durationMs / 1000).toFixed(1)}s`);
327
+ console.log(`${chalk.gray('Cost:')} $${m.costUsd.toFixed(4)}`);
328
+ console.log(`${chalk.gray('Tools:')} ${m.toolCount} calls`);
329
+ console.log(chalk.gray('\n═════════════════════════\n'));
330
+ }
331
+ //# sourceMappingURL=run-benchmark-command.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run-benchmark-command.js","sourceRoot":"","sources":["../../src/commands/run-benchmark-command.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AAOtB,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,6CAA6C,CAAC;AACtG,OAAO,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,8CAA8C,CAAC;AACrG,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACnF,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,2CAA2C,CAAC;AAEpF,kBAAkB;AAClB,MAAM,OAAO,GAAG,OAAO,CAAC;AAExB;;GAEG;AACH,KAAK,UAAU,gBAAgB;IAC7B,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAErD,6CAA6C;IAC7C,MAAM,WAAW,GAAG,MAAM,cAAc,EAAE,CAAC;IAE3C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,8CAA8C;QAC9C,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC/B,IAAI,WAAW,EAAE,CAAC;YAChB,GAAG,CAAC,uBAAuB,GAAG,WAAW,CAAC;QAC5C,CAAC;QAED,iCAAiC;QACjC,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE;YAC5F,GAAG;YACH,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;YACjC,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEjE,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;YAC9B,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC,CAAC;QAClE,CAAC,EAAE,KAAK,CAAC,CAAC;QAEV,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACpB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClC,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;oBAClE,OAAO,CAAC;wBACN,EAAE,EAAE,KAAK;wBACT,KAAK,EAAE,iCAAiC;4BACtC,yBAAyB;4BACzB,wDAAwD;qBAC3D,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;YAC5C,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,8BAA8B;YAC3E,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACvB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,yBAAyB,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,WAAmB,EACnB,OAAmB;IAEnB,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;IAEtB,IAAI,CAAC;QACH,sCAAsC;QACtC,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACxD,IAAI,SAAS,GAAG,MAAM,gBAAgB,EAAE,CAAC;QACzC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChB,MAAM,OAAO,EAAE,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChB,0BAA0B;YAC1B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC7C,SAAS,GAAG,MAAM,gBAAgB,EAAE,CAAC;YACrC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;gBAClB,OAAO,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;gBAC5C,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QACD,OAAO,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;QAE5C,0BAA0B;QAC1B,OAAO,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,WAAW,CAAC,CAAC;QACpD,OAAO,CAAC,OAAO,CAAC,aAAa,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAE3D,2BAA2B;QAC3B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC7C,IAAI,KAAK,CAAC;QACV,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,0BAA0B;YAC1B,KAAK,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,yCAAyC;YACzC,KAAK,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,OAAO,CAAC,UAAU,KAAK,CAAC,MAAM,UAAU,CAAC,CAAC;QAElD,6BAA6B;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5C,oBAAoB;QACpB,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;QACzC,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,MAAM,cAAc,OAAO,CAAC,IAAI,gBAAgB,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAC/H,CAAC;QAEF,MAAM,UAAU,GAAiB,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAE9B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;YAE5D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;oBACtD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,eAAe,IAAI,CAAC,OAAO,iBAAiB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAClH,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBACzC,CAAC;gBAED,yEAAyE;gBACzE,IAAI,YAAY,GAA0C,IAAI,CAAC;gBAC/D,IAAI,CAAC;oBACH,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC,CAAC;wBAChE,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE;4BAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC;4BAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,4BAA4B,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC;wBAClF,CAAC,EAAE,IAAI,CAAC,CAAC;oBACX,CAAC;oBAED,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBAEnF,sBAAsB;oBACtB,IAAI,YAAY,EAAE,CAAC;wBACjB,aAAa,CAAC,YAAY,CAAC,CAAC;wBAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC7B,CAAC;oBAED,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;wBACvB,IAAI,OAAO,EAAE,CAAC;4BACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;wBAC3D,CAAC;6BAAM,CAAC;4BACN,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;wBACnD,CAAC;wBACD,SAAS;oBACX,CAAC;oBAED,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;wBAC7D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,yBAAyB,OAAO,IAAI,CAAC,CAAC,CAAC;wBAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,YAAY,aAAa,SAAS,CAAC,SAAS,aAAa,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBAClK,CAAC;oBAED,oBAAoB;oBACpB,MAAM,OAAO,GAAqB;wBAChC,QAAQ,EAAE,CAAC,EAAE,uBAAuB;wBACpC,WAAW,EAAE,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,YAAY;wBAC3D,WAAW,EAAE,SAAS,CAAC,WAAW;wBAClC,YAAY,EAAE,SAAS,CAAC,YAAY;wBACpC,UAAU,EAAE,SAAS,CAAC,UAAU;wBAChC,SAAS,EAAE,SAAS,CAAC,SAAS;wBAC9B,OAAO,EAAE,SAAS,CAAC,OAAO;qBAC3B,CAAC;oBAEF,iBAAiB;oBACjB,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;oBAChE,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAExB,iBAAiB;oBACjB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBACpE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;oBAEpD,IAAI,OAAO,EAAE,CAAC;wBACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,CAAC;wBACjD,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,YAAY,QAAQ,eAAe,MAAM,CAAC,eAAe,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,YAAY,CAAC,CAAC;wBAC7H,IAAI,MAAM,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACtC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;wBAC9E,CAAC;wBACD,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACrC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;wBAC7E,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,OAAO,CACb,GAAG,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,MAAM,MAAM,CAAC,eAAe,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,YAAY,CAC3G,CAAC;wBAEF,8BAA8B;wBAC9B,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;4BAC1E,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;wBAC5E,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,+BAA+B;oBAC/B,IAAI,YAAY,EAAE,CAAC;wBACjB,aAAa,CAAC,YAAY,CAAC,CAAC;wBAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC7B,CAAC;oBACD,IAAI,OAAO,EAAE,CAAC;wBACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;oBACnG,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;oBAC5F,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,uBAAuB;QACvB,MAAM,UAAU,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;QAEhD,MAAM,eAAe,GAAoB;YACvC,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC;YAClD,SAAS,EAAE,KAAK,CAAC,IAAI;YACrB,WAAW,EAAE,KAAK,CAAC,QAAQ;YAC3B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,WAAW,EAAE,UAAU;YACvB,iBAAiB,EAAE,UAAU;YAC7B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,OAAO;SACjB,CAAC;QAEF,wBAAwB;QACxB,eAAe,CAAC,IAAI,GAAG,kBAAkB,CAAC,eAAe,CAAC,CAAC;QAE3D,mBAAmB;QACnB,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAEpC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAE5C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACpE,MAAM,SAAS,CAAC,MAAM,EAAE,sBAAsB,CAAC,eAAe,CAAC,CAAC,CAAC;QAEjE,OAAO,CAAC,OAAO,CAAC,oBAAoB,SAAS,EAAE,CAAC,CAAC;QAEjD,mBAAmB;QACnB,YAAY,CAAC,eAAe,CAAC,CAAC;QAE9B,UAAU;QACV,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAElC,OAAO,eAAe,CAAC;IACzB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACjC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,IAAY,EAAE,MAAc;IACjD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC;SAC9B,MAAM,CAAC,GAAG,IAAI,IAAI,MAAM,EAAE,CAAC;SAC3B,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChB,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;AACpE,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,MAAuB;IACjD,MAAM,IAAI,GAAG;QACX,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,QAAQ,EAAE,MAAM,CAAC,iBAAiB,CAAC,QAAQ;QAC3C,WAAW,EAAE,MAAM,CAAC,iBAAiB,CAAC,WAAW;QACjD,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;IACF,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACtF,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAEnC,IAAI,EAAE,GAAG;;;;;;YAMC,MAAM,CAAC,SAAS;aACf,MAAM,CAAC,WAAW;YACnB,MAAM,CAAC,KAAK;WACb,MAAM,CAAC,IAAI;eACP,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;mBACjB,CAAC,CAAC,WAAW,CAAC,cAAc,EAAE;eAClC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACnC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;iBACf,CAAC,CAAC,SAAS;;;;CAI3B,CAAC;IAEA,6BAA6B;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,EAAE,CAAC;QACzC,MAAM,WAAW,GACf,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QAC3E,MAAM,MAAM,GAAG,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAE7C,EAAE,IAAI,OAAO,MAAM,IAAI,QAAQ;;kBAEjB,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;0BACd,OAAO,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM;yBAChD,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM;;CAEtE,CAAC;IACA,CAAC;IAED,EAAE,IAAI;;2BAEmB,MAAM,CAAC,OAAO,OAAO,MAAM,CAAC,SAAS;CAC/D,CAAC;IAEA,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,MAAuB;IAC3C,MAAM,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAEnC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,CAAC;IAEzD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,MAAM,aAAa,GAAG,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACnG,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1F,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC,WAAW,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC,YAAY,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IAC/J,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACjF,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,SAAS,QAAQ,CAAC,CAAC;IAEjE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,CAAC;AAC3D,CAAC"}
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Execute the leaderboard command
3
+ */
4
+ export declare function viewLeaderboard(skillName?: string, options?: {
5
+ endpoint?: string;
6
+ limit?: number;
7
+ }): Promise<void>;
8
+ //# sourceMappingURL=view-leaderboard-command.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"view-leaderboard-command.d.ts","sourceRoot":"","sources":["../../src/commands/view-leaderboard-command.ts"],"names":[],"mappings":"AAUA;;GAEG;AACH,wBAAsB,eAAe,CACnC,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAClD,OAAO,CAAC,IAAI,CAAC,CAwBf"}