aitelier 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/add.d.ts +3 -0
- package/dist/commands/add.d.ts.map +1 -0
- package/dist/commands/add.js +141 -0
- package/dist/commands/add.js.map +1 -0
- package/dist/commands/add.test.d.ts +2 -0
- package/dist/commands/add.test.d.ts.map +1 -0
- package/dist/commands/add.test.js +203 -0
- package/dist/commands/add.test.js.map +1 -0
- package/dist/commands/eval.d.ts +3 -0
- package/dist/commands/eval.d.ts.map +1 -0
- package/dist/commands/eval.js +505 -0
- package/dist/commands/eval.js.map +1 -0
- package/dist/commands/eval.test.d.ts +2 -0
- package/dist/commands/eval.test.d.ts.map +1 -0
- package/dist/commands/eval.test.js +804 -0
- package/dist/commands/eval.test.js.map +1 -0
- package/dist/commands/format.d.ts +3 -0
- package/dist/commands/format.d.ts.map +1 -0
- package/dist/commands/format.js +198 -0
- package/dist/commands/format.js.map +1 -0
- package/dist/commands/format.test.d.ts +2 -0
- package/dist/commands/format.test.d.ts.map +1 -0
- package/dist/commands/format.test.js +568 -0
- package/dist/commands/format.test.js.map +1 -0
- package/dist/commands/init.d.ts +3 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +94 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/init.test.d.ts +2 -0
- package/dist/commands/init.test.d.ts.map +1 -0
- package/dist/commands/init.test.js +123 -0
- package/dist/commands/init.test.js.map +1 -0
- package/dist/commands/rate.d.ts +3 -0
- package/dist/commands/rate.d.ts.map +1 -0
- package/dist/commands/rate.js +209 -0
- package/dist/commands/rate.js.map +1 -0
- package/dist/commands/rate.test.d.ts +2 -0
- package/dist/commands/rate.test.d.ts.map +1 -0
- package/dist/commands/rate.test.js +393 -0
- package/dist/commands/rate.test.js.map +1 -0
- package/dist/commands/split.d.ts +3 -0
- package/dist/commands/split.d.ts.map +1 -0
- package/dist/commands/split.js +210 -0
- package/dist/commands/split.js.map +1 -0
- package/dist/commands/split.test.d.ts +2 -0
- package/dist/commands/split.test.d.ts.map +1 -0
- package/dist/commands/split.test.js +503 -0
- package/dist/commands/split.test.js.map +1 -0
- package/dist/commands/stats.d.ts +3 -0
- package/dist/commands/stats.d.ts.map +1 -0
- package/dist/commands/stats.js +209 -0
- package/dist/commands/stats.js.map +1 -0
- package/dist/commands/stats.test.d.ts +2 -0
- package/dist/commands/stats.test.d.ts.map +1 -0
- package/dist/commands/stats.test.js +476 -0
- package/dist/commands/stats.test.js.map +1 -0
- package/dist/commands/status.d.ts +3 -0
- package/dist/commands/status.d.ts.map +1 -0
- package/dist/commands/status.js +132 -0
- package/dist/commands/status.js.map +1 -0
- package/dist/commands/status.test.d.ts +2 -0
- package/dist/commands/status.test.d.ts.map +1 -0
- package/dist/commands/status.test.js +443 -0
- package/dist/commands/status.test.js.map +1 -0
- package/dist/commands/train.d.ts +3 -0
- package/dist/commands/train.d.ts.map +1 -0
- package/dist/commands/train.js +139 -0
- package/dist/commands/train.js.map +1 -0
- package/dist/commands/train.test.d.ts +2 -0
- package/dist/commands/train.test.d.ts.map +1 -0
- package/dist/commands/train.test.js +288 -0
- package/dist/commands/train.test.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +27 -0
- package/dist/index.js.map +1 -0
- package/dist/providers/openai.d.ts +9 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +20 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/together.d.ts +11 -0
- package/dist/providers/together.d.ts.map +1 -0
- package/dist/providers/together.js +118 -0
- package/dist/providers/together.js.map +1 -0
- package/dist/providers/types.d.ts +28 -0
- package/dist/providers/types.d.ts.map +1 -0
- package/dist/providers/types.js +2 -0
- package/dist/providers/types.js.map +1 -0
- package/dist/storage/config.d.ts +24 -0
- package/dist/storage/config.d.ts.map +1 -0
- package/dist/storage/config.js +11 -0
- package/dist/storage/config.js.map +1 -0
- package/dist/storage/dataset.d.ts +14 -0
- package/dist/storage/dataset.d.ts.map +1 -0
- package/dist/storage/dataset.js +18 -0
- package/dist/storage/dataset.js.map +1 -0
- package/package.json +58 -0
|
@@ -0,0 +1,505 @@
|
|
|
1
|
+
import inquirer from 'inquirer';
|
|
2
|
+
import { access, mkdir } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { loadConfig } from '../storage/config.js';
|
|
5
|
+
import { readExamples } from '../storage/dataset.js';
|
|
6
|
+
import { TogetherProvider } from '../providers/together.js';
|
|
7
|
+
import { writeFile } from 'node:fs/promises';
|
|
8
|
+
const CONFIG_FILE = '.ftpipeline.json';
|
|
9
|
+
const VAL_FILE = 'data/val.jsonl';
|
|
10
|
+
const EVALS_DIR = 'data/evals';
|
|
11
|
+
export function registerEval(program) {
|
|
12
|
+
program
|
|
13
|
+
.command('eval')
|
|
14
|
+
.description('Evaluate fine-tuned model on validation examples')
|
|
15
|
+
.option('--compare', 'Compare base model vs fine-tuned model (blind A/B test)')
|
|
16
|
+
.action(async (options) => {
|
|
17
|
+
try {
|
|
18
|
+
if (options.compare) {
|
|
19
|
+
await evalCompareCommand();
|
|
20
|
+
}
|
|
21
|
+
else {
|
|
22
|
+
await evalCommand();
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
|
|
27
|
+
process.exit(1);
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
async function evalCommand() {
|
|
32
|
+
const cwd = process.cwd();
|
|
33
|
+
// Check if project is initialized
|
|
34
|
+
try {
|
|
35
|
+
await access(join(cwd, CONFIG_FILE));
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
throw new Error('Project not initialized. Run `ft init` first to create .ftpipeline.json');
|
|
39
|
+
}
|
|
40
|
+
// Load config
|
|
41
|
+
const config = await loadConfig(cwd);
|
|
42
|
+
// Find most recent completed model
|
|
43
|
+
const completedRuns = config.runs.filter((r) => r.status === 'completed' && r.modelId);
|
|
44
|
+
if (completedRuns.length === 0) {
|
|
45
|
+
console.log('\n' + '═'.repeat(70));
|
|
46
|
+
console.log('No Completed Models Found');
|
|
47
|
+
console.log('═'.repeat(70));
|
|
48
|
+
console.log('\nNo completed fine-tuning jobs with model IDs found.');
|
|
49
|
+
console.log('\nTo start a training job, run: ft train');
|
|
50
|
+
console.log('To check job status, run: ft status');
|
|
51
|
+
console.log('═'.repeat(70) + '\n');
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const latestRun = completedRuns[completedRuns.length - 1];
|
|
55
|
+
const modelId = latestRun.modelId;
|
|
56
|
+
// Load validation examples
|
|
57
|
+
const valPath = join(cwd, VAL_FILE);
|
|
58
|
+
let valExamples;
|
|
59
|
+
try {
|
|
60
|
+
valExamples = await readExamples(valPath);
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
if (error.code === 'ENOENT') {
|
|
64
|
+
console.log('\n' + '═'.repeat(70));
|
|
65
|
+
console.log('No Validation Data Found');
|
|
66
|
+
console.log('═'.repeat(70));
|
|
67
|
+
console.log('\nNo validation examples found.');
|
|
68
|
+
console.log('\nTo create validation data, run: ft split');
|
|
69
|
+
console.log('═'.repeat(70) + '\n');
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
if (valExamples.length === 0) {
|
|
75
|
+
console.log('\n' + '═'.repeat(70));
|
|
76
|
+
console.log('No Validation Examples');
|
|
77
|
+
console.log('═'.repeat(70));
|
|
78
|
+
console.log('\nValidation file is empty.');
|
|
79
|
+
console.log('\nTo create validation data, run: ft split');
|
|
80
|
+
console.log('═'.repeat(70) + '\n');
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
// Initialize provider
|
|
84
|
+
let provider;
|
|
85
|
+
if (config.provider === 'together') {
|
|
86
|
+
provider = new TogetherProvider();
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
throw new Error(`Provider "${config.provider}" not yet supported. Use "together" for now.`);
|
|
90
|
+
}
|
|
91
|
+
// Display header
|
|
92
|
+
console.log('\n' + '═'.repeat(70));
|
|
93
|
+
console.log('Model Evaluation');
|
|
94
|
+
console.log('═'.repeat(70));
|
|
95
|
+
console.log(`Model: ${modelId}`);
|
|
96
|
+
console.log(`Validation examples: ${valExamples.length}`);
|
|
97
|
+
console.log('═'.repeat(70) + '\n');
|
|
98
|
+
// Initialize results and stats
|
|
99
|
+
const results = [];
|
|
100
|
+
const stats = {
|
|
101
|
+
evaluated: 0,
|
|
102
|
+
skipped: 0,
|
|
103
|
+
scores: [],
|
|
104
|
+
};
|
|
105
|
+
// Main evaluation loop
|
|
106
|
+
for (let i = 0; i < valExamples.length; i++) {
|
|
107
|
+
const example = valExamples[i];
|
|
108
|
+
console.log(`\nExample ${i + 1} of ${valExamples.length}`);
|
|
109
|
+
console.log('━'.repeat(70));
|
|
110
|
+
// Extract input messages (system + user, no assistant)
|
|
111
|
+
const inputMessages = example.messages.filter((m) => m.role !== 'assistant');
|
|
112
|
+
const expectedOutput = example.messages.find((m) => m.role === 'assistant')?.content || '';
|
|
113
|
+
if (!expectedOutput) {
|
|
114
|
+
console.log('Warning: No expected output found for this example. Skipping...\n');
|
|
115
|
+
stats.skipped++;
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
// Run inference
|
|
119
|
+
console.log('Running inference...');
|
|
120
|
+
let actualOutput;
|
|
121
|
+
try {
|
|
122
|
+
actualOutput = await provider.runInference(modelId, inputMessages);
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
console.log(`Error running inference: ${error instanceof Error ? error.message : String(error)}`);
|
|
126
|
+
console.log('Skipping this example...\n');
|
|
127
|
+
stats.skipped++;
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
// Display side-by-side comparison
|
|
131
|
+
displayComparison(inputMessages, expectedOutput, actualOutput);
|
|
132
|
+
// Prompt for action
|
|
133
|
+
const action = await promptAction();
|
|
134
|
+
if (action === 'quit') {
|
|
135
|
+
console.log('\nQuitting evaluation...');
|
|
136
|
+
break;
|
|
137
|
+
}
|
|
138
|
+
if (action === 'skip') {
|
|
139
|
+
stats.skipped++;
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
// Prompt for score
|
|
143
|
+
const score = await promptScore();
|
|
144
|
+
stats.evaluated++;
|
|
145
|
+
stats.scores.push(score);
|
|
146
|
+
// Save result
|
|
147
|
+
results.push({
|
|
148
|
+
exampleId: example.id,
|
|
149
|
+
input: inputMessages,
|
|
150
|
+
expectedOutput,
|
|
151
|
+
actualOutput,
|
|
152
|
+
score,
|
|
153
|
+
});
|
|
154
|
+
console.log(`✓ Scored as ${score}/5\n`);
|
|
155
|
+
}
|
|
156
|
+
// Calculate summary stats
|
|
157
|
+
if (stats.evaluated === 0) {
|
|
158
|
+
console.log('\n' + '═'.repeat(70));
|
|
159
|
+
console.log('No examples evaluated. Exiting without saving results.');
|
|
160
|
+
console.log('═'.repeat(70) + '\n');
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
const averageScore = stats.scores.reduce((sum, s) => sum + s, 0) / stats.scores.length;
|
|
164
|
+
const sendableRate = (stats.scores.filter((s) => s >= 4).length / stats.scores.length) * 100;
|
|
165
|
+
// Display summary
|
|
166
|
+
displaySummary(stats, averageScore, sendableRate);
|
|
167
|
+
// Save results to file
|
|
168
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').split('T')[0];
|
|
169
|
+
const evalsPath = join(cwd, EVALS_DIR);
|
|
170
|
+
try {
|
|
171
|
+
await mkdir(evalsPath, { recursive: true });
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
// Directory already exists
|
|
175
|
+
}
|
|
176
|
+
const evalSummary = {
|
|
177
|
+
modelId,
|
|
178
|
+
timestamp: new Date().toISOString(),
|
|
179
|
+
totalExamples: stats.evaluated,
|
|
180
|
+
averageScore,
|
|
181
|
+
sendableRate,
|
|
182
|
+
results,
|
|
183
|
+
};
|
|
184
|
+
const outputPath = join(evalsPath, `eval-${modelId.replace(/[/\\:]/g, '_')}-${timestamp}.json`);
|
|
185
|
+
await writeFile(outputPath, JSON.stringify(evalSummary, null, 2) + '\n');
|
|
186
|
+
console.log(`\nResults saved to: ${outputPath}`);
|
|
187
|
+
console.log('═'.repeat(70) + '\n');
|
|
188
|
+
}
|
|
189
|
+
function displayComparison(inputMessages, expectedOutput, actualOutput) {
|
|
190
|
+
console.log('\nInput:');
|
|
191
|
+
for (const msg of inputMessages) {
|
|
192
|
+
const label = msg.role.charAt(0).toUpperCase() + msg.role.slice(1);
|
|
193
|
+
console.log(` ${label}: ${msg.content}`);
|
|
194
|
+
}
|
|
195
|
+
console.log('\n' + '━'.repeat(70));
|
|
196
|
+
console.log('Expected Output:');
|
|
197
|
+
console.log('━'.repeat(70));
|
|
198
|
+
console.log(expectedOutput);
|
|
199
|
+
console.log('\n' + '━'.repeat(70));
|
|
200
|
+
console.log('Model Output:');
|
|
201
|
+
console.log('━'.repeat(70));
|
|
202
|
+
console.log(actualOutput);
|
|
203
|
+
console.log('━'.repeat(70));
|
|
204
|
+
}
|
|
205
|
+
async function promptAction() {
|
|
206
|
+
const answer = await inquirer.prompt([
|
|
207
|
+
{
|
|
208
|
+
type: 'list',
|
|
209
|
+
name: 'action',
|
|
210
|
+
message: 'What would you like to do?',
|
|
211
|
+
choices: [
|
|
212
|
+
{ name: 'Score this output', value: 'score' },
|
|
213
|
+
{ name: 'Skip', value: 'skip' },
|
|
214
|
+
{ name: 'Quit', value: 'quit' },
|
|
215
|
+
],
|
|
216
|
+
},
|
|
217
|
+
]);
|
|
218
|
+
return answer.action;
|
|
219
|
+
}
|
|
220
|
+
async function promptScore() {
|
|
221
|
+
const answer = await inquirer.prompt([
|
|
222
|
+
{
|
|
223
|
+
type: 'input',
|
|
224
|
+
name: 'score',
|
|
225
|
+
message: 'Score this output (1-5):',
|
|
226
|
+
validate: (input) => {
|
|
227
|
+
const num = parseFloat(input);
|
|
228
|
+
if (isNaN(num)) {
|
|
229
|
+
return 'Please enter a valid number';
|
|
230
|
+
}
|
|
231
|
+
if (num < 1 || num > 5) {
|
|
232
|
+
return 'Score must be between 1 and 5';
|
|
233
|
+
}
|
|
234
|
+
if (!Number.isInteger(num)) {
|
|
235
|
+
return 'Score must be a whole number';
|
|
236
|
+
}
|
|
237
|
+
return true;
|
|
238
|
+
},
|
|
239
|
+
},
|
|
240
|
+
]);
|
|
241
|
+
return parseInt(answer.score, 10);
|
|
242
|
+
}
|
|
243
|
+
function displaySummary(stats, averageScore, sendableRate) {
|
|
244
|
+
console.log('\n' + '═'.repeat(70));
|
|
245
|
+
console.log('Evaluation Complete');
|
|
246
|
+
console.log('═'.repeat(70));
|
|
247
|
+
console.log(`Examples evaluated: ${stats.evaluated}`);
|
|
248
|
+
console.log(`Skipped: ${stats.skipped}`);
|
|
249
|
+
console.log(`\nAverage score: ${averageScore.toFixed(2)}/5`);
|
|
250
|
+
console.log(`Sendable rate (4+): ${sendableRate.toFixed(1)}%`);
|
|
251
|
+
console.log('═'.repeat(70));
|
|
252
|
+
}
|
|
253
|
+
async function evalCompareCommand() {
|
|
254
|
+
const cwd = process.cwd();
|
|
255
|
+
// Check if project is initialized
|
|
256
|
+
try {
|
|
257
|
+
await access(join(cwd, CONFIG_FILE));
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
throw new Error('Project not initialized. Run `ft init` first to create .ftpipeline.json');
|
|
261
|
+
}
|
|
262
|
+
// Load config
|
|
263
|
+
const config = await loadConfig(cwd);
|
|
264
|
+
// Find most recent completed model
|
|
265
|
+
const completedRuns = config.runs.filter((r) => r.status === 'completed' && r.modelId);
|
|
266
|
+
if (completedRuns.length === 0) {
|
|
267
|
+
console.log('\n' + '═'.repeat(70));
|
|
268
|
+
console.log('No Completed Models Found');
|
|
269
|
+
console.log('═'.repeat(70));
|
|
270
|
+
console.log('\nNo completed fine-tuning jobs with model IDs found.');
|
|
271
|
+
console.log('\nTo start a training job, run: ft train');
|
|
272
|
+
console.log('To check job status, run: ft status');
|
|
273
|
+
console.log('═'.repeat(70) + '\n');
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
const latestRun = completedRuns[completedRuns.length - 1];
|
|
277
|
+
const fineTunedModelId = latestRun.modelId;
|
|
278
|
+
const baseModelId = config.model;
|
|
279
|
+
// Load validation examples
|
|
280
|
+
const valPath = join(cwd, VAL_FILE);
|
|
281
|
+
let valExamples;
|
|
282
|
+
try {
|
|
283
|
+
valExamples = await readExamples(valPath);
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
if (error.code === 'ENOENT') {
|
|
287
|
+
console.log('\n' + '═'.repeat(70));
|
|
288
|
+
console.log('No Validation Data Found');
|
|
289
|
+
console.log('═'.repeat(70));
|
|
290
|
+
console.log('\nNo validation examples found.');
|
|
291
|
+
console.log('\nTo create validation data, run: ft split');
|
|
292
|
+
console.log('═'.repeat(70) + '\n');
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
throw error;
|
|
296
|
+
}
|
|
297
|
+
if (valExamples.length === 0) {
|
|
298
|
+
console.log('\n' + '═'.repeat(70));
|
|
299
|
+
console.log('No Validation Examples');
|
|
300
|
+
console.log('═'.repeat(70));
|
|
301
|
+
console.log('\nValidation file is empty.');
|
|
302
|
+
console.log('\nTo create validation data, run: ft split');
|
|
303
|
+
console.log('═'.repeat(70) + '\n');
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
// Initialize provider
|
|
307
|
+
let provider;
|
|
308
|
+
if (config.provider === 'together') {
|
|
309
|
+
provider = new TogetherProvider();
|
|
310
|
+
}
|
|
311
|
+
else {
|
|
312
|
+
throw new Error(`Provider "${config.provider}" not yet supported. Use "together" for now.`);
|
|
313
|
+
}
|
|
314
|
+
// Display header
|
|
315
|
+
console.log('\n' + '═'.repeat(70));
|
|
316
|
+
console.log('Model Comparison (Blind A/B Test)');
|
|
317
|
+
console.log('═'.repeat(70));
|
|
318
|
+
console.log(`Base model: ${baseModelId}`);
|
|
319
|
+
console.log(`Fine-tuned model: ${fineTunedModelId}`);
|
|
320
|
+
console.log(`Validation examples: ${valExamples.length}`);
|
|
321
|
+
console.log('\nYou will see outputs from both models (labeled A and B).');
|
|
322
|
+
console.log("The order is randomized - you won't know which is which until the end.");
|
|
323
|
+
console.log('═'.repeat(70) + '\n');
|
|
324
|
+
// Initialize results and stats
|
|
325
|
+
const results = [];
|
|
326
|
+
const stats = {
|
|
327
|
+
evaluated: 0,
|
|
328
|
+
skipped: 0,
|
|
329
|
+
scores: [],
|
|
330
|
+
};
|
|
331
|
+
// Main comparison loop
|
|
332
|
+
for (let i = 0; i < valExamples.length; i++) {
|
|
333
|
+
const example = valExamples[i];
|
|
334
|
+
console.log(`\nExample ${i + 1} of ${valExamples.length}`);
|
|
335
|
+
console.log('━'.repeat(70));
|
|
336
|
+
// Extract input messages (system + user, no assistant)
|
|
337
|
+
const inputMessages = example.messages.filter((m) => m.role !== 'assistant');
|
|
338
|
+
const expectedOutput = example.messages.find((m) => m.role === 'assistant')?.content || '';
|
|
339
|
+
if (!expectedOutput) {
|
|
340
|
+
console.log('Warning: No expected output found for this example. Skipping...\n');
|
|
341
|
+
stats.skipped++;
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
// Run inference on both models
|
|
345
|
+
console.log('Running inference on both models...');
|
|
346
|
+
let baseOutput;
|
|
347
|
+
let fineTunedOutput;
|
|
348
|
+
try {
|
|
349
|
+
[baseOutput, fineTunedOutput] = await Promise.all([
|
|
350
|
+
provider.runInference(baseModelId, inputMessages),
|
|
351
|
+
provider.runInference(fineTunedModelId, inputMessages),
|
|
352
|
+
]);
|
|
353
|
+
}
|
|
354
|
+
catch (error) {
|
|
355
|
+
console.log(`Error running inference: ${error instanceof Error ? error.message : String(error)}`);
|
|
356
|
+
console.log('Skipping this example...\n');
|
|
357
|
+
stats.skipped++;
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
// Randomly assign to A and B
|
|
361
|
+
const modelAIsBase = Math.random() < 0.5;
|
|
362
|
+
const modelAOutput = modelAIsBase ? baseOutput : fineTunedOutput;
|
|
363
|
+
const modelBOutput = modelAIsBase ? fineTunedOutput : baseOutput;
|
|
364
|
+
// Display comparison
|
|
365
|
+
displayBlindComparison(inputMessages, expectedOutput, modelAOutput, modelBOutput);
|
|
366
|
+
// Prompt for action
|
|
367
|
+
const action = await promptAction();
|
|
368
|
+
if (action === 'quit') {
|
|
369
|
+
console.log('\nQuitting comparison...');
|
|
370
|
+
break;
|
|
371
|
+
}
|
|
372
|
+
if (action === 'skip') {
|
|
373
|
+
stats.skipped++;
|
|
374
|
+
continue;
|
|
375
|
+
}
|
|
376
|
+
// Prompt for scores
|
|
377
|
+
console.log('\n' + '━'.repeat(70));
|
|
378
|
+
console.log('Score Model A:');
|
|
379
|
+
const scoreA = await promptScore();
|
|
380
|
+
console.log(`✓ Scored as ${scoreA}/5\n`);
|
|
381
|
+
console.log('Score Model B:');
|
|
382
|
+
const scoreB = await promptScore();
|
|
383
|
+
console.log(`✓ Scored as ${scoreB}/5\n`);
|
|
384
|
+
stats.evaluated++;
|
|
385
|
+
// Save result
|
|
386
|
+
results.push({
|
|
387
|
+
exampleId: example.id,
|
|
388
|
+
input: inputMessages,
|
|
389
|
+
expectedOutput,
|
|
390
|
+
baseOutput,
|
|
391
|
+
fineTunedOutput,
|
|
392
|
+
modelAOutput,
|
|
393
|
+
modelBOutput,
|
|
394
|
+
modelAIsBase,
|
|
395
|
+
scoreA,
|
|
396
|
+
scoreB,
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
// Calculate summary stats
|
|
400
|
+
if (stats.evaluated === 0) {
|
|
401
|
+
console.log('\n' + '═'.repeat(70));
|
|
402
|
+
console.log('No examples evaluated. Exiting without saving results.');
|
|
403
|
+
console.log('═'.repeat(70) + '\n');
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
// Calculate aggregate stats
|
|
407
|
+
let baseModelTotalScore = 0;
|
|
408
|
+
let fineTunedModelTotalScore = 0;
|
|
409
|
+
let baseModelWins = 0;
|
|
410
|
+
let fineTunedModelWins = 0;
|
|
411
|
+
let ties = 0;
|
|
412
|
+
for (const result of results) {
|
|
413
|
+
const baseScore = result.modelAIsBase ? result.scoreA : result.scoreB;
|
|
414
|
+
const fineTunedScore = result.modelAIsBase ? result.scoreB : result.scoreA;
|
|
415
|
+
baseModelTotalScore += baseScore;
|
|
416
|
+
fineTunedModelTotalScore += fineTunedScore;
|
|
417
|
+
if (baseScore > fineTunedScore) {
|
|
418
|
+
baseModelWins++;
|
|
419
|
+
}
|
|
420
|
+
else if (fineTunedScore > baseScore) {
|
|
421
|
+
fineTunedModelWins++;
|
|
422
|
+
}
|
|
423
|
+
else {
|
|
424
|
+
ties++;
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
const baseModelAvgScore = baseModelTotalScore / stats.evaluated;
|
|
428
|
+
const fineTunedModelAvgScore = fineTunedModelTotalScore / stats.evaluated;
|
|
429
|
+
// Display reveal and summary
|
|
430
|
+
displayComparisonSummary(baseModelId, fineTunedModelId, stats, baseModelAvgScore, fineTunedModelAvgScore, baseModelWins, fineTunedModelWins, ties);
|
|
431
|
+
// Save results to file
|
|
432
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').split('T')[0];
|
|
433
|
+
const evalsPath = join(cwd, EVALS_DIR);
|
|
434
|
+
try {
|
|
435
|
+
await mkdir(evalsPath, { recursive: true });
|
|
436
|
+
}
|
|
437
|
+
catch {
|
|
438
|
+
// Directory already exists
|
|
439
|
+
}
|
|
440
|
+
const compareSummary = {
|
|
441
|
+
baseModelId,
|
|
442
|
+
fineTunedModelId,
|
|
443
|
+
timestamp: new Date().toISOString(),
|
|
444
|
+
totalExamples: stats.evaluated,
|
|
445
|
+
baseModelAvgScore,
|
|
446
|
+
fineTunedModelAvgScore,
|
|
447
|
+
baseModelWins,
|
|
448
|
+
fineTunedModelWins,
|
|
449
|
+
ties,
|
|
450
|
+
results,
|
|
451
|
+
};
|
|
452
|
+
const outputPath = join(evalsPath, `compare-${fineTunedModelId.replace(/[/\\:]/g, '_')}-${timestamp}.json`);
|
|
453
|
+
await writeFile(outputPath, JSON.stringify(compareSummary, null, 2) + '\n');
|
|
454
|
+
console.log(`\nResults saved to: ${outputPath}`);
|
|
455
|
+
console.log('═'.repeat(70) + '\n');
|
|
456
|
+
}
|
|
457
|
+
function displayBlindComparison(inputMessages, expectedOutput, modelAOutput, modelBOutput) {
|
|
458
|
+
console.log('\nInput:');
|
|
459
|
+
for (const msg of inputMessages) {
|
|
460
|
+
const label = msg.role.charAt(0).toUpperCase() + msg.role.slice(1);
|
|
461
|
+
console.log(` ${label}: ${msg.content}`);
|
|
462
|
+
}
|
|
463
|
+
console.log('\n' + '━'.repeat(70));
|
|
464
|
+
console.log('Expected Output:');
|
|
465
|
+
console.log('━'.repeat(70));
|
|
466
|
+
console.log(expectedOutput);
|
|
467
|
+
console.log('\n' + '━'.repeat(70));
|
|
468
|
+
console.log('Model A Output:');
|
|
469
|
+
console.log('━'.repeat(70));
|
|
470
|
+
console.log(modelAOutput);
|
|
471
|
+
console.log('\n' + '━'.repeat(70));
|
|
472
|
+
console.log('Model B Output:');
|
|
473
|
+
console.log('━'.repeat(70));
|
|
474
|
+
console.log(modelBOutput);
|
|
475
|
+
console.log('━'.repeat(70));
|
|
476
|
+
}
|
|
477
|
+
function displayComparisonSummary(baseModelId, fineTunedModelId, stats, baseModelAvgScore, fineTunedModelAvgScore, baseModelWins, fineTunedModelWins, ties) {
|
|
478
|
+
console.log('\n' + '═'.repeat(70));
|
|
479
|
+
console.log('Comparison Results - The Reveal!');
|
|
480
|
+
console.log('═'.repeat(70));
|
|
481
|
+
console.log(`Base model: ${baseModelId}`);
|
|
482
|
+
console.log(`Fine-tuned model: ${fineTunedModelId}`);
|
|
483
|
+
console.log('═'.repeat(70));
|
|
484
|
+
console.log(`\nExamples compared: ${stats.evaluated}`);
|
|
485
|
+
console.log(`Skipped: ${stats.skipped}`);
|
|
486
|
+
console.log('\n' + '━'.repeat(70));
|
|
487
|
+
console.log('Average Scores:');
|
|
488
|
+
console.log('━'.repeat(70));
|
|
489
|
+
console.log(`Base model: ${baseModelAvgScore.toFixed(2)}/5`);
|
|
490
|
+
console.log(`Fine-tuned model: ${fineTunedModelAvgScore.toFixed(2)}/5`);
|
|
491
|
+
const improvement = baseModelAvgScore > 0
|
|
492
|
+
? (((fineTunedModelAvgScore - baseModelAvgScore) / baseModelAvgScore) * 100).toFixed(1)
|
|
493
|
+
: '0.0';
|
|
494
|
+
console.log(`\nImprovement: ${improvement}%`);
|
|
495
|
+
console.log('\n' + '━'.repeat(70));
|
|
496
|
+
console.log('Head-to-Head:');
|
|
497
|
+
console.log('━'.repeat(70));
|
|
498
|
+
console.log(`Base model wins: ${baseModelWins}`);
|
|
499
|
+
console.log(`Fine-tuned model wins: ${fineTunedModelWins}`);
|
|
500
|
+
console.log(`Ties: ${ties}`);
|
|
501
|
+
const winRate = stats.evaluated > 0 ? ((fineTunedModelWins / stats.evaluated) * 100).toFixed(1) : '0.0';
|
|
502
|
+
console.log(`\nFine-tuned model win rate: ${winRate}%`);
|
|
503
|
+
console.log('═'.repeat(70));
|
|
504
|
+
}
|
|
505
|
+
//# sourceMappingURL=eval.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../src/commands/eval.ts"],"names":[],"mappings":"AACA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAG5D,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7C,MAAM,WAAW,GAAG,kBAAkB,CAAC;AACvC,MAAM,QAAQ,GAAG,gBAAgB,CAAC;AAClC,MAAM,SAAS,GAAG,YAAY,CAAC;AAmD/B,MAAM,UAAU,YAAY,CAAC,OAAgB;IAC3C,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,kDAAkD,CAAC;SAC/D,MAAM,CAAC,WAAW,EAAE,yDAAyD,CAAC;SAC9E,MAAM,CAAC,KAAK,EAAE,OAA8B,EAAE,EAAE;QAC/C,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,kBAAkB,EAAE,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,MAAM,WAAW,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,WAAW;IACxB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAE1B,kCAAkC;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;IAC7F,CAAC;IAED,cAAc;IACd,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;IAErC,mCAAmC;IACnC,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;IAEvF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACnC,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,SAAS,CAAC,OAAQ,CAAC;IAEnC,2BAA2B;IAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACpC,IAAI,WAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;YACnC,OAAO;QACT,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACnC,OAAO;IACT,CAAC;IAED,sBAAsB;IACtB,IAAI,QAAQ,CAAC;IACb,IAAI,MAAM,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QACnC,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACpC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,CAAC,QAAQ,8CAA8C,CAAC,CAAC;IAC9F,CAAC;IAED,iBAAiB;IACjB,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,EAAE,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,wBAAwB,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IAEnC,+BAA+B;IAC/B,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,KAAK,GAAiB;QAC1B,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,uBAAuB;IACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAE/B,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAE5B,uDAAuD;QACvD,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;QAC7E,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;QAE3F,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC;YACjF,KAAK,CAAC,OAAO,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QAED,gBAAgB;QAChB,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpC,IAAI,YAAoB,CAAC;QACzB,IAAI,CAAC;YACH,YAAY,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CACT,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACrF,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;YAC1C,KAAK,CAAC,OAAO,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QAED,kCAAkC;QAClC,iBAAiB,CAAC,aAAa,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC;QAE/D,oBAAoB;QACpB,MAAM,MAAM,GAAG,MAAM,YAAY,EAAE,CAAC;QAEpC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;YACxC,MAAM;QACR,CAAC;QAED,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,KAAK,CAAC,OAAO,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QAED,mBAAmB;QACnB,MAAM,KAAK,GAAG,MAAM,WAAW,EAAE,CAAC;QAClC,KAAK,CAAC,SAAS,EAAE,CAAC;QAClB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEzB,cAAc;QACd,OAAO,CAAC,IAAI,CAAC;YACX,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,KAAK,EAAE,aAAa;YACpB,cAAc;YACd,YAAY;YACZ,KAAK;SACN,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,0BAA0B;IAC1B,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACnC,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IACvF,MAAM,YAAY,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;IAE7F,kBAAkB;IAClB,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;IAElD,uBAAuB;IACvB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,2BAA2B;IAC7B,CAAC;IAED,MAAM,WAAW,GAAgB;QAC/B,OAAO;QACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,aAAa,EAAE,KAAK,CAAC,SAAS;QAC9B,YAAY;QACZ,YAAY;QACZ,OAAO;KACR,CAAC;IAEF,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,SAAS,OAAO,CAAC,CAAC;IAChG,MAAM,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAEzE,OAAO,CAAC,GAAG,CAAC,uBAAuB,UAAU,EAAE,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,iBAAiB,CACxB,aAAwB,EACxB,cAAsB,EACtB,YAAoB;IAEpB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACxB,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAE5B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAC1B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,YAAY;IACzB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAwC;QAC1E;YACE,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,4BAA4B;YACrC,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,OAAO,EAAE;gBAC7C,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE;gBAC/B,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE;aAChC;SACF;KACF,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,MAAM,CAAC;AACvB,CAAC;AAED,KAAK,UAAU,WAAW;IACxB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAoB;QACtD;YACE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,0BAA0B;YACnC,QAAQ,EAAE,CAAC,KAAa,EAAE,EAAE;gBAC1B,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC9B,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;oBACf,OAAO,6BAA6B,CAAC;gBACvC,CAAC;gBACD,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;oBACvB,OAAO,+BAA+B,CAAC;gBACzC,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC3B,OAAO,8BAA8B,CAAC;gBACxC,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC;SACF;KACF,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,cAAc,CAAC,KAAmB,EAAE,YAAoB,EAAE,YAAoB;IACrF,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,oBAAoB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,CAAC,uBAAuB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,kBAAkB;IAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAE1B,kCAAkC;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;IAC7F,CAAC;IAED,cAAc;IACd,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;IAErC,mCAAmC;IACnC,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;IAEvF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACnC,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1D,MAAM,gBAAgB,GAAG,SAAS,CAAC,OAAQ,CAAC;IAC5C,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;IAEjC,2BAA2B;IAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACpC,IAAI,WAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;YACnC,OAAO;QACT,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACnC,OAAO;IACT,CAAC;IAED,sBAAsB;IACtB,IAAI,QAAQ,CAAC;IACb,IAAI,MAAM,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QACnC,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACpC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,CAAC,QAAQ,8CAA8C,CAAC,CAAC;IAC9F,CAAC;IAED,iBAAiB;IACjB,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,eAAe,WAAW,EAAE,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,qBAAqB,gBAAgB,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,wBAAwB,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAC;IAC1E,OAAO,CAAC,GAAG,CAAC,wEAAwE,CAAC,CAAC;IACtF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IAEnC,+BAA+B;IAC/B,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,KAAK,GAAiB;QAC1B,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,uBAAuB;IACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAE/B,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAE5B,uDAAuD;QACvD,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;QAC7E,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;QAE3F,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC;YACjF,KAAK,CAAC,OAAO,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QAED,+BAA+B;QAC/B,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,IAAI,UAAkB,CAAC;QACvB,IAAI,eAAuB,CAAC;QAE5B,IAAI,CAAC;YACH,CAAC,UAAU,EAAE,eAAe,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChD,QAAQ,CAAC,YAAY,CAAC,WAAW,EAAE,aAAa,CAAC;gBACjD,QAAQ,CAAC,YAAY,CAAC,gBAAgB,EAAE,aAAa,CAAC;aACvD,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CACT,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACrF,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;YAC1C,KAAK,CAAC,OAAO,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QAED,6BAA6B;QAC7B,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;QACzC,MAAM,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC;QACjE,MAAM,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC;QAEjE,qBAAqB;QACrB,sBAAsB,CAAC,aAAa,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;QAElF,oBAAoB;QACpB,MAAM,MAAM,GAAG,MAAM,YAAY,EAAE,CAAC;QAEpC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;YACxC,MAAM;QACR,CAAC;QAED,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,KAAK,CAAC,OAAO,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QAED,oBAAoB;QACpB,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,WAAW,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,MAAM,CAAC,CAAC;QAEzC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,WAAW,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,MAAM,CAAC,CAAC;QAEzC,KAAK,CAAC,SAAS,EAAE,CAAC;QAElB,cAAc;QACd,OAAO,CAAC,IAAI,CAAC;YACX,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,KAAK,EAAE,aAAa;YACpB,cAAc;YACd,UAAU;YACV,eAAe;YACf,YAAY;YACZ,YAAY;YACZ,YAAY;YACZ,MAAM;YACN,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACnC,OAAO;IACT,CAAC;IAED,4BAA4B;IAC5B,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,wBAAwB,GAAG,CAAC,CAAC;IACjC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;QACtE,MAAM,cAAc,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;QAE3E,mBAAmB,IAAI,SAAS,CAAC;QACjC,wBAAwB,IAAI,cAAc,CAAC;QAE3C,IAAI,SAAS,GAAG,cAAc,EAAE,CAAC;YAC/B,aAAa,EAAE,CAAC;QAClB,CAAC;aAAM,IAAI,cAAc,GAAG,SAAS,EAAE,CAAC;YACtC,kBAAkB,EAAE,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,IAAI,EAAE,CAAC;QACT,CAAC;IACH,CAAC;IAED,MAAM,iBAAiB,GAAG,mBAAmB,GAAG,KAAK,CAAC,SAAS,CAAC;IAChE,MAAM,sBAAsB,GAAG,wBAAwB,GAAG,KAAK,CAAC,SAAS,CAAC;IAE1E,6BAA6B;IAC7B,wBAAwB,CACtB,WAAW,EACX,gBAAgB,EAChB,KAAK,EACL,iBAAiB,EACjB,sBAAsB,EACtB,aAAa,EACb,kBAAkB,EAClB,IAAI,CACL,CAAC;IAEF,uBAAuB;IACvB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,2BAA2B;IAC7B,CAAC;IAED,MAAM,cAAc,GAAmB;QACrC,WAAW;QACX,gBAAgB;QAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,aAAa,EAAE,KAAK,CAAC,SAAS;QAC9B,iBAAiB;QACjB,sBAAsB;QACtB,aAAa;QACb,kBAAkB;QAClB,IAAI;QACJ,OAAO;KACR,CAAC;IAEF,MAAM,UAAU,GAAG,IAAI,CACrB,SAAS,EACT,WAAW,gBAAgB,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,SAAS,OAAO,CACxE,CAAC;IACF,MAAM,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAE5E,OAAO,CAAC,GAAG,CAAC,uBAAuB,UAAU,EAAE,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,sBAAsB,CAC7B,aAAwB,EACxB,cAAsB,EACtB,YAAoB,EACpB,YAAoB;IAEpB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACxB,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAE5B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAE1B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAC1B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,wBAAwB,CAC/B,WAAmB,EACnB,gBAAwB,EACxB,KAAmB,EACnB,iBAAyB,EACzB,sBAA8B,EAC9B,aAAqB,EACrB,kBAA0B,EAC1B,IAAY;IAEZ,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,eAAe,WAAW,EAAE,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,qBAAqB,gBAAgB,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAEzC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,qBAAqB,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,qBAAqB,sBAAsB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAExE,MAAM,WAAW,GACf,iBAAiB,GAAG,CAAC;QACnB,CAAC,CAAC,CAAC,CAAC,CAAC,sBAAsB,GAAG,iBAAiB,CAAC,GAAG,iBAAiB,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QACvF,CAAC,CAAC,KAAK,CAAC;IACZ,OAAO,CAAC,GAAG,CAAC,kBAAkB,WAAW,GAAG,CAAC,CAAC;IAE9C,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,0BAA0B,aAAa,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,0BAA0B,kBAAkB,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC;IAE9C,MAAM,OAAO,GACX,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,GAAG,KAAK,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1F,OAAO,CAAC,GAAG,CAAC,gCAAgC,OAAO,GAAG,CAAC,CAAC;IAExD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval.test.d.ts","sourceRoot":"","sources":["../../src/commands/eval.test.ts"],"names":[],"mappings":""}
|