@cogitator-ai/evals 0.13.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +94 -14
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/plugin-evals/execute.d.ts +45 -0
- package/dist/plugin-evals/execute.d.ts.map +1 -0
- package/dist/plugin-evals/execute.js +508 -0
- package/dist/plugin-evals/execute.js.map +1 -0
- package/dist/plugin-evals/files.d.ts +17 -0
- package/dist/plugin-evals/files.d.ts.map +1 -0
- package/dist/plugin-evals/files.js +46 -0
- package/dist/plugin-evals/files.js.map +1 -0
- package/dist/plugin-evals/glob.d.ts +8 -0
- package/dist/plugin-evals/glob.d.ts.map +1 -0
- package/dist/plugin-evals/glob.js +63 -0
- package/dist/plugin-evals/glob.js.map +1 -0
- package/dist/plugin-evals/graders.d.ts +21 -0
- package/dist/plugin-evals/graders.d.ts.map +1 -0
- package/dist/plugin-evals/graders.js +198 -0
- package/dist/plugin-evals/graders.js.map +1 -0
- package/dist/plugin-evals/index.d.ts +4 -0
- package/dist/plugin-evals/index.d.ts.map +1 -0
- package/dist/plugin-evals/index.js +3 -0
- package/dist/plugin-evals/index.js.map +1 -0
- package/dist/plugin-evals/judge.d.ts +33 -0
- package/dist/plugin-evals/judge.d.ts.map +1 -0
- package/dist/plugin-evals/judge.js +73 -0
- package/dist/plugin-evals/judge.js.map +1 -0
- package/dist/plugin-evals/mocks.d.ts +34 -0
- package/dist/plugin-evals/mocks.d.ts.map +1 -0
- package/dist/plugin-evals/mocks.js +373 -0
- package/dist/plugin-evals/mocks.js.map +1 -0
- package/dist/plugin-evals/report.d.ts +4 -0
- package/dist/plugin-evals/report.d.ts.map +1 -0
- package/dist/plugin-evals/report.js +177 -0
- package/dist/plugin-evals/report.js.map +1 -0
- package/dist/plugin-evals/runner.d.ts +23 -0
- package/dist/plugin-evals/runner.d.ts.map +1 -0
- package/dist/plugin-evals/runner.js +582 -0
- package/dist/plugin-evals/runner.js.map +1 -0
- package/dist/plugin-evals/schema.d.ts +179 -0
- package/dist/plugin-evals/schema.d.ts.map +1 -0
- package/dist/plugin-evals/schema.js +194 -0
- package/dist/plugin-evals/schema.js.map +1 -0
- package/dist/plugin-evals/suite.d.ts +40 -0
- package/dist/plugin-evals/suite.d.ts.map +1 -0
- package/dist/plugin-evals/suite.js +476 -0
- package/dist/plugin-evals/suite.js.map +1 -0
- package/dist/plugin-evals/tools.d.ts +42 -0
- package/dist/plugin-evals/tools.d.ts.map +1 -0
- package/dist/plugin-evals/tools.js +136 -0
- package/dist/plugin-evals/tools.js.map +1 -0
- package/dist/plugin-evals/transcript.d.ts +35 -0
- package/dist/plugin-evals/transcript.d.ts.map +1 -0
- package/dist/plugin-evals/transcript.js +149 -0
- package/dist/plugin-evals/transcript.js.map +1 -0
- package/dist/plugin-evals/types.d.ts +378 -0
- package/dist/plugin-evals/types.d.ts.map +1 -0
- package/dist/plugin-evals/types.js +2 -0
- package/dist/plugin-evals/types.js.map +1 -0
- package/dist/reporters/html-format.d.ts +5 -2
- package/dist/reporters/html-format.d.ts.map +1 -1
- package/dist/reporters/html-format.js +6 -3
- package/dist/reporters/html-format.js.map +1 -1
- package/package.json +5 -3
|
@@ -0,0 +1,582 @@
|
|
|
1
|
+
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { tmpdir } from 'node:os';
|
|
3
|
+
import { createRequire } from 'node:module';
|
|
4
|
+
import { dirname, join, resolve } from 'node:path';
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
import { mean } from '../stats/percentiles.js';
|
|
7
|
+
import { executeRun } from './execute.js';
|
|
8
|
+
import { errorText } from './files.js';
|
|
9
|
+
import { grade, isPaidGrader } from './graders.js';
|
|
10
|
+
import { createPluginEvalJudge } from './judge.js';
|
|
11
|
+
import { pluginEvalHtml } from './report.js';
|
|
12
|
+
import { issuesText } from './schema.js';
|
|
13
|
+
import { loadSuite, PluginEvalError } from './suite.js';
|
|
14
|
+
import { caseGrants, parseGrant } from './tools.js';
|
|
15
|
+
const OptionsSchema = z.object({
|
|
16
|
+
target: z.string().min(1).optional(),
|
|
17
|
+
cwd: z.string().min(1).optional(),
|
|
18
|
+
model: z.string().trim().min(1).optional(),
|
|
19
|
+
judgeModel: z.string().trim().min(1).optional(),
|
|
20
|
+
runs: z.number().int().min(1).max(50).optional(),
|
|
21
|
+
concurrency: z.number().int().min(1).max(8).default(1),
|
|
22
|
+
ablation: z.enum(['none', 'with-without']).optional(),
|
|
23
|
+
threshold: z.number().min(0).max(1).default(1),
|
|
24
|
+
maxCostUsd: z.number().min(0).optional(),
|
|
25
|
+
allowTools: z.array(z.string().min(1)).default([]),
|
|
26
|
+
scaffold: z.boolean().default(false),
|
|
27
|
+
caseGlob: z.string().min(1).optional(),
|
|
28
|
+
tags: z.array(z.string().min(1)).default([]),
|
|
29
|
+
evalDir: z.string().min(1).optional(),
|
|
30
|
+
outputDir: z.string().min(1).optional(),
|
|
31
|
+
keepTemp: z.boolean().default(false),
|
|
32
|
+
pluginsRoot: z.string().min(1).optional(),
|
|
33
|
+
});
|
|
34
|
+
async function importCore() {
|
|
35
|
+
try {
|
|
36
|
+
return await import('@cogitator-ai/core');
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
throw new PluginEvalError(`Plugin evals need @cogitator-ai/core: ${errorText(error)}`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function coreVersion() {
|
|
43
|
+
try {
|
|
44
|
+
const require = createRequire(import.meta.url);
|
|
45
|
+
const entry = require.resolve('@cogitator-ai/core');
|
|
46
|
+
const json = JSON.parse(readFileSync(join(dirname(entry), '..', 'package.json'), 'utf8'));
|
|
47
|
+
return typeof json === 'object' &&
|
|
48
|
+
json !== null &&
|
|
49
|
+
'version' in json &&
|
|
50
|
+
typeof json.version === 'string'
|
|
51
|
+
? json.version
|
|
52
|
+
: 'unknown';
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return 'unknown';
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
async function installedPlugin(core, root, id) {
|
|
59
|
+
const { records, warnings } = await core.readInstalledPlugins(root ?? core.defaultPluginsRoot());
|
|
60
|
+
const matches = records.filter((record) => record.id === id || (!id.includes('@') && record.name === id));
|
|
61
|
+
if (matches.length > 1) {
|
|
62
|
+
throw new PluginEvalError(`${id} names ${matches.length} installed plugins (${matches.map((m) => m.id).join(', ')}): give name@marketplace`);
|
|
63
|
+
}
|
|
64
|
+
if (matches.length === 0 && warnings.length > 0) {
|
|
65
|
+
throw new PluginEvalError(`No installed plugin ${id} could be read: ${warnings.join(', ')}`);
|
|
66
|
+
}
|
|
67
|
+
return matches[0]?.installPath;
|
|
68
|
+
}
|
|
69
|
+
/** The manifest name and MCP servers of each plugin directory the cases load */
|
|
70
|
+
async function pluginFacts(core, suite) {
|
|
71
|
+
const dirs = new Set([suite.pluginRoot]);
|
|
72
|
+
for (const entry of suite.cases)
|
|
73
|
+
for (const dir of entry.case?.plugins ?? [])
|
|
74
|
+
dirs.add(dir);
|
|
75
|
+
const facts = new Map();
|
|
76
|
+
for (const dir of dirs) {
|
|
77
|
+
const layout = await core.readPluginLayout(dir);
|
|
78
|
+
if (layout.fatal.length > 0) {
|
|
79
|
+
throw new PluginEvalError(`The plugin at ${dir} cannot load: ${layout.fatal.join(', ')}`);
|
|
80
|
+
}
|
|
81
|
+
facts.set(dir, {
|
|
82
|
+
name: layout.manifest.name,
|
|
83
|
+
...(layout.manifest.version !== undefined && { version: layout.manifest.version }),
|
|
84
|
+
servers: new Set(Object.keys(layout.mcpServers)),
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
return facts;
|
|
88
|
+
}
|
|
89
|
+
/** Loads each plugin once the way a run does, so a plugin that fails fails the command up front */
|
|
90
|
+
async function checkPluginsLoad(core, llm, dirs) {
|
|
91
|
+
const store = mkdtempSync(join(tmpdir(), 'cogitator-plugin-eval-probe-'));
|
|
92
|
+
const probe = new core.Cogitator({
|
|
93
|
+
llm,
|
|
94
|
+
experimental_plugins: {
|
|
95
|
+
dirs,
|
|
96
|
+
root: store,
|
|
97
|
+
installed: { root: join(store, 'plugins') },
|
|
98
|
+
settings: false,
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
try {
|
|
102
|
+
await probe.experimental_plugins.load();
|
|
103
|
+
for (const info of probe.experimental_plugins.list()) {
|
|
104
|
+
if (info.origin === 'core')
|
|
105
|
+
continue;
|
|
106
|
+
if (info.status !== 'active') {
|
|
107
|
+
throw new PluginEvalError(`The plugin ${info.name} did not load: ${info.error ?? info.status}`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
finally {
|
|
112
|
+
await probe.close().catch(() => undefined);
|
|
113
|
+
rmSync(store, { recursive: true, force: true });
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
/** The graders a two-arm run keeps out of the score: plugin-fired indicators */
|
|
117
|
+
function excludedGraders(evalCase, twoArm, declared) {
|
|
118
|
+
if (!twoArm)
|
|
119
|
+
return new Set();
|
|
120
|
+
const servers = Object.keys(evalCase.mocks);
|
|
121
|
+
const pluginMocksOnly = servers.length > 0 && servers.every(declared);
|
|
122
|
+
const indicator = (grader) => {
|
|
123
|
+
if (grader.arm === 'both')
|
|
124
|
+
return false;
|
|
125
|
+
if (grader.arm === 'with-only')
|
|
126
|
+
return true;
|
|
127
|
+
if (grader.type === 'tool_used') {
|
|
128
|
+
return grader.tool === 'Skill' || grader.tool === 'use_skill';
|
|
129
|
+
}
|
|
130
|
+
if (!pluginMocksOnly)
|
|
131
|
+
return false;
|
|
132
|
+
if (grader.type === 'regex')
|
|
133
|
+
return grader.target === 'mock_calls';
|
|
134
|
+
if (grader.type === 'llm')
|
|
135
|
+
return grader.focus === 'mock_calls';
|
|
136
|
+
return false;
|
|
137
|
+
};
|
|
138
|
+
const excluded = evalCase.graders.filter(indicator);
|
|
139
|
+
if (excluded.length === evalCase.graders.length)
|
|
140
|
+
return new Set();
|
|
141
|
+
return new Set(excluded.map((grader) => grader.name));
|
|
142
|
+
}
|
|
143
|
+
function weightedScore(results) {
|
|
144
|
+
const scored = results.filter((result) => result.scored);
|
|
145
|
+
const total = scored.reduce((sum, result) => sum + result.weight, 0);
|
|
146
|
+
if (total === 0)
|
|
147
|
+
return 0;
|
|
148
|
+
return scored.reduce((sum, result) => sum + (result.passed ? result.weight : 0), 0) / total;
|
|
149
|
+
}
|
|
150
|
+
function timestamp(date) {
|
|
151
|
+
return date.toISOString().replace(/[:.]/g, '-');
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Experimental. Runs a plugin's eval suite in the format of Claude Code's `claude plugin eval`:
|
|
155
|
+
* each case's prompt several times in a fresh workspace with the plugin loaded and, for the
|
|
156
|
+
* no-plugin baseline, without it, scores every run with the case's graders (`regex`,
|
|
157
|
+
* `tool_used`, `tool_order`, `file_exists`, and `llm` and `baseline` judged by a majority of three
|
|
158
|
+
* votes) and writes `aggregate-result.json` and a self-contained `report.html`.
|
|
159
|
+
*
|
|
160
|
+
* Needs `@cogitator-ai/core`. Throws `PluginEvalError` when the suite cannot start: no plugin, no
|
|
161
|
+
* case, an invalid option. A case that fails to load is reported in the result instead.
|
|
162
|
+
*
|
|
163
|
+
* @example
|
|
164
|
+
* ```ts
|
|
165
|
+
* const { result, reportFile } = await experimental_runPluginEvals({
|
|
166
|
+
* target: './my-plugin',
|
|
167
|
+
* llm: { defaultModel: 'anthropic/claude-sonnet-5-5' },
|
|
168
|
+
* threshold: 0.8,
|
|
169
|
+
* });
|
|
170
|
+
* console.log(result.aggregates.overallScore, reportFile);
|
|
171
|
+
* ```
|
|
172
|
+
*/
|
|
173
|
+
export async function experimental_runPluginEvals(input) {
|
|
174
|
+
const parsed = OptionsSchema.safeParse(input);
|
|
175
|
+
if (!parsed.success)
|
|
176
|
+
throw new PluginEvalError(`Invalid options: ${issuesText(parsed.error)}`);
|
|
177
|
+
const options = parsed.data;
|
|
178
|
+
const started = new Date();
|
|
179
|
+
const cwd = resolve(options.cwd ?? process.cwd());
|
|
180
|
+
const signal = input.signal ?? new AbortController().signal;
|
|
181
|
+
const emit = input.onProgress ?? (() => undefined);
|
|
182
|
+
const runGrants = [];
|
|
183
|
+
for (const rule of options.allowTools) {
|
|
184
|
+
const grant = parseGrant(rule);
|
|
185
|
+
if ('error' in grant)
|
|
186
|
+
throw new PluginEvalError(`--allow-tools: ${grant.error}`);
|
|
187
|
+
runGrants.push(grant);
|
|
188
|
+
}
|
|
189
|
+
const core = await importCore();
|
|
190
|
+
const suite = await loadSuite({
|
|
191
|
+
...(options.target !== undefined && { target: options.target }),
|
|
192
|
+
cwd,
|
|
193
|
+
...(options.evalDir !== undefined && { evalDir: options.evalDir }),
|
|
194
|
+
...(options.caseGlob !== undefined && { caseGlob: options.caseGlob }),
|
|
195
|
+
tags: options.tags,
|
|
196
|
+
installed: (id) => installedPlugin(core, options.pluginsRoot, id),
|
|
197
|
+
});
|
|
198
|
+
for (const warning of suite.warnings)
|
|
199
|
+
emit({ type: 'warning', message: warning });
|
|
200
|
+
const facts = await pluginFacts(core, suite);
|
|
201
|
+
await checkPluginsLoad(core, input.llm, [...facts.keys()]);
|
|
202
|
+
const main = facts.get(suite.pluginRoot);
|
|
203
|
+
if (!main)
|
|
204
|
+
throw new PluginEvalError(`No plugin found at ${suite.pluginRoot}`);
|
|
205
|
+
const results = [];
|
|
206
|
+
const plans = [];
|
|
207
|
+
for (const entry of suite.cases) {
|
|
208
|
+
const evalCase = entry.case;
|
|
209
|
+
if (!evalCase || entry.error !== undefined) {
|
|
210
|
+
results.push({
|
|
211
|
+
name: entry.name,
|
|
212
|
+
path: entry.path,
|
|
213
|
+
tags: evalCase?.tags ?? [],
|
|
214
|
+
error: entry.error ?? 'the case did not load',
|
|
215
|
+
singleArm: true,
|
|
216
|
+
notices: [],
|
|
217
|
+
prompt: evalCase?.prompt ?? '',
|
|
218
|
+
graders: evalCase?.graders ?? [],
|
|
219
|
+
arms: { with: [], without: [] },
|
|
220
|
+
aggregates: { score: 0, passed: false, runs: 0, costUsd: 0 },
|
|
221
|
+
});
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
const notices = [];
|
|
225
|
+
const granted = caseGrants(evalCase.allowedTools, runGrants);
|
|
226
|
+
const pluginServers = new Map();
|
|
227
|
+
for (const dir of evalCase.plugins) {
|
|
228
|
+
const plugin = facts.get(dir);
|
|
229
|
+
if (plugin)
|
|
230
|
+
pluginServers.set(plugin.name, plugin.servers);
|
|
231
|
+
}
|
|
232
|
+
const declared = (server) => [...pluginServers.values()].some((set) => set.has(server));
|
|
233
|
+
const model = options.model ?? evalCase.model ?? input.llm?.defaultModel;
|
|
234
|
+
const caseResult = {
|
|
235
|
+
name: evalCase.name,
|
|
236
|
+
path: evalCase.path,
|
|
237
|
+
tags: evalCase.tags,
|
|
238
|
+
singleArm: true,
|
|
239
|
+
notices,
|
|
240
|
+
prompt: evalCase.prompt,
|
|
241
|
+
graders: evalCase.graders,
|
|
242
|
+
arms: { with: [], without: [] },
|
|
243
|
+
aggregates: { score: null, passed: false, runs: 0, costUsd: 0 },
|
|
244
|
+
};
|
|
245
|
+
results.push(caseResult);
|
|
246
|
+
if ('error' in granted) {
|
|
247
|
+
caseResult.error = granted.error;
|
|
248
|
+
caseResult.aggregates.score = 0;
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
const usedProblems = Object.entries(evalCase.mockProblems).filter(([server]) => declared(server));
|
|
252
|
+
if (usedProblems.length > 0) {
|
|
253
|
+
caseResult.error = `invalid mock: ${usedProblems.map(([, problem]) => problem).join(', ')}`;
|
|
254
|
+
caseResult.aggregates.score = 0;
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
for (const [server, problem] of Object.entries(evalCase.mockProblems)) {
|
|
258
|
+
if (!declared(server))
|
|
259
|
+
notices.push(`mocks/${server} left out: ${problem}`);
|
|
260
|
+
}
|
|
261
|
+
if (model === undefined) {
|
|
262
|
+
throw new PluginEvalError('No model for the agent under test: pass --model, set model in the case, or llm.defaultModel in the configuration');
|
|
263
|
+
}
|
|
264
|
+
const singleArm = options.ablation === 'none' ||
|
|
265
|
+
(options.ablation === undefined &&
|
|
266
|
+
evalCase.context.historyFile !== undefined &&
|
|
267
|
+
!suite.named);
|
|
268
|
+
caseResult.singleArm = singleArm;
|
|
269
|
+
if (singleArm && options.ablation === undefined) {
|
|
270
|
+
notices.push('single-arm (no delta): the case resumes a transcript');
|
|
271
|
+
}
|
|
272
|
+
if (granted.notGranted.length > 0)
|
|
273
|
+
notices.push(`not granted: ${granted.notGranted.join(', ')}`);
|
|
274
|
+
if (granted.noCounterpart.length > 0) {
|
|
275
|
+
notices.push(`no Cogitator counterpart, not given: ${granted.noCounterpart.join(', ')}`);
|
|
276
|
+
}
|
|
277
|
+
if (evalCase.context.scaffoldScript && !options.scaffold) {
|
|
278
|
+
notices.push('scaffold_script not run: pass --scaffold to run it');
|
|
279
|
+
}
|
|
280
|
+
const mocked = [];
|
|
281
|
+
for (const [plugin, servers] of pluginServers) {
|
|
282
|
+
for (const server of [...servers].sort()) {
|
|
283
|
+
const tools = evalCase.mocks[server];
|
|
284
|
+
mocked.push(`plugin_${plugin}_${server}[${tools ? tools.map((tool) => tool.tool).join(', ') : 'not started: no mock'}]`);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
for (const [server, tools] of Object.entries(evalCase.mocks)) {
|
|
288
|
+
if (!declared(server))
|
|
289
|
+
mocked.push(`${server}[${tools.map((tool) => tool.tool).join(', ')}]`);
|
|
290
|
+
}
|
|
291
|
+
if (mocked.length > 0)
|
|
292
|
+
notices.push(`mocked: ${mocked.join(', ')}`);
|
|
293
|
+
plans.push({
|
|
294
|
+
index: results.length - 1,
|
|
295
|
+
case: evalCase,
|
|
296
|
+
arms: singleArm ? ['with'] : ['with', 'without'],
|
|
297
|
+
grants: granted.grants,
|
|
298
|
+
excluded: excludedGraders(evalCase, !singleArm, declared),
|
|
299
|
+
runs: options.runs ?? evalCase.runs,
|
|
300
|
+
model,
|
|
301
|
+
pluginServers,
|
|
302
|
+
notices,
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
const needsJudge = plans.some((plan) => plan.case.graders.some(isPaidGrader));
|
|
306
|
+
const judgeRuntime = needsJudge ? new core.Cogitator({ llm: input.llm }) : undefined;
|
|
307
|
+
const judges = new Map();
|
|
308
|
+
const judgeOf = (plan) => {
|
|
309
|
+
if (!judgeRuntime)
|
|
310
|
+
return undefined;
|
|
311
|
+
const model = options.judgeModel ?? plan.model;
|
|
312
|
+
let judge = judges.get(model);
|
|
313
|
+
if (!judge) {
|
|
314
|
+
judge = createPluginEvalJudge(judgeRuntime, model);
|
|
315
|
+
judges.set(model, judge);
|
|
316
|
+
}
|
|
317
|
+
return judge;
|
|
318
|
+
};
|
|
319
|
+
const jobs = [];
|
|
320
|
+
for (const plan of plans) {
|
|
321
|
+
emit({ type: 'case', case: plan.case.name, notices: plan.notices, arms: plan.arms });
|
|
322
|
+
for (const arm of plan.arms) {
|
|
323
|
+
for (let index = 1; index <= plan.runs; index++)
|
|
324
|
+
jobs.push({ plan, arm, index });
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
let spent = 0;
|
|
328
|
+
let partialReason = null;
|
|
329
|
+
let completedCleanly = false;
|
|
330
|
+
const ceilingReached = () => options.maxCostUsd !== undefined && spent >= options.maxCostUsd;
|
|
331
|
+
const stopReason = () => {
|
|
332
|
+
if (signal.aborted)
|
|
333
|
+
return 'interrupted';
|
|
334
|
+
if (partialReason === 'auth_failed')
|
|
335
|
+
return 'auth_failed';
|
|
336
|
+
if (ceilingReached())
|
|
337
|
+
return 'cost_ceiling';
|
|
338
|
+
return undefined;
|
|
339
|
+
};
|
|
340
|
+
const gradeRun = async (job, outcome) => {
|
|
341
|
+
const { plan, arm } = job;
|
|
342
|
+
const graders = [];
|
|
343
|
+
let skippedPaid = false;
|
|
344
|
+
let cost = outcome.costUsd;
|
|
345
|
+
if (outcome.capture) {
|
|
346
|
+
for (const grader of plan.case.graders) {
|
|
347
|
+
const excluded = plan.excluded.has(grader.name);
|
|
348
|
+
if (excluded && arm === 'without')
|
|
349
|
+
continue;
|
|
350
|
+
const base = { name: grader.name, type: grader.type, weight: grader.weight };
|
|
351
|
+
if (isPaidGrader(grader) && signal.aborted) {
|
|
352
|
+
skippedPaid = true;
|
|
353
|
+
partialReason ??= 'interrupted';
|
|
354
|
+
graders.push({
|
|
355
|
+
...base,
|
|
356
|
+
passed: false,
|
|
357
|
+
scored: false,
|
|
358
|
+
skipped: true,
|
|
359
|
+
explanation: 'skipped: the run was interrupted before this judge grader',
|
|
360
|
+
});
|
|
361
|
+
continue;
|
|
362
|
+
}
|
|
363
|
+
const judge = isPaidGrader(grader) ? judgeOf(plan) : undefined;
|
|
364
|
+
if (isPaidGrader(grader) && (ceilingReached() || !judge)) {
|
|
365
|
+
skippedPaid = true;
|
|
366
|
+
if (ceilingReached())
|
|
367
|
+
partialReason ??= 'cost_ceiling';
|
|
368
|
+
graders.push({
|
|
369
|
+
...base,
|
|
370
|
+
passed: false,
|
|
371
|
+
scored: false,
|
|
372
|
+
skipped: true,
|
|
373
|
+
explanation: 'skipped: the cost ceiling was reached before this judge grader',
|
|
374
|
+
});
|
|
375
|
+
continue;
|
|
376
|
+
}
|
|
377
|
+
try {
|
|
378
|
+
const verdict = await grade(grader, outcome.capture, judge ?? noJudge, signal);
|
|
379
|
+
cost += verdict.costUsd ?? 0;
|
|
380
|
+
spent += verdict.costUsd ?? 0;
|
|
381
|
+
graders.push({ ...base, ...verdict, scored: !excluded });
|
|
382
|
+
}
|
|
383
|
+
catch (error) {
|
|
384
|
+
if (isPaidGrader(grader) && signal.aborted) {
|
|
385
|
+
skippedPaid = true;
|
|
386
|
+
partialReason ??= 'interrupted';
|
|
387
|
+
graders.push({
|
|
388
|
+
...base,
|
|
389
|
+
passed: false,
|
|
390
|
+
scored: false,
|
|
391
|
+
skipped: true,
|
|
392
|
+
explanation: 'skipped: the run was interrupted while the judge graded',
|
|
393
|
+
});
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
graders.push({
|
|
397
|
+
...base,
|
|
398
|
+
passed: false,
|
|
399
|
+
scored: !excluded,
|
|
400
|
+
explanation: `grader error: ${errorText(error)}`,
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
const capture = outcome.capture;
|
|
406
|
+
return {
|
|
407
|
+
arm,
|
|
408
|
+
index: job.index,
|
|
409
|
+
score: capture && !outcome.aborted ? weightedScore(graders) : 0,
|
|
410
|
+
error: outcome.error,
|
|
411
|
+
...(outcome.apiError && { apiError: true }),
|
|
412
|
+
...(outcome.aborted && { aborted: outcome.aborted }),
|
|
413
|
+
...(skippedPaid && { skippedPaidGraders: true }),
|
|
414
|
+
graders,
|
|
415
|
+
lastMessage: capture?.lastMessage ?? '',
|
|
416
|
+
toolCalls: capture?.toolCalls ?? [],
|
|
417
|
+
mockCalls: capture?.mockCalls ?? [],
|
|
418
|
+
filesCreated: capture?.filesCreated ?? [],
|
|
419
|
+
costUsd: cost,
|
|
420
|
+
durationSeconds: outcome.durationSeconds,
|
|
421
|
+
...(outcome.tempDir !== undefined && { tempDir: outcome.tempDir }),
|
|
422
|
+
};
|
|
423
|
+
};
|
|
424
|
+
const runs = new Map();
|
|
425
|
+
let next = 0;
|
|
426
|
+
const worker = async () => {
|
|
427
|
+
while (next < jobs.length) {
|
|
428
|
+
const job = jobs[next++];
|
|
429
|
+
if (!job)
|
|
430
|
+
break;
|
|
431
|
+
const stop = stopReason();
|
|
432
|
+
if (stop) {
|
|
433
|
+
partialReason ??= stop;
|
|
434
|
+
emit({ type: 'skipped', case: job.plan.case.name, reason: stop });
|
|
435
|
+
continue;
|
|
436
|
+
}
|
|
437
|
+
const outcome = await executeRun({
|
|
438
|
+
core,
|
|
439
|
+
case: job.plan.case,
|
|
440
|
+
arm: job.arm,
|
|
441
|
+
llm: input.llm,
|
|
442
|
+
model: job.plan.model,
|
|
443
|
+
grants: job.plan.grants,
|
|
444
|
+
pluginServers: job.plan.pluginServers,
|
|
445
|
+
scaffold: options.scaffold,
|
|
446
|
+
exec: input.exec,
|
|
447
|
+
keepTemp: options.keepTemp,
|
|
448
|
+
signal,
|
|
449
|
+
});
|
|
450
|
+
spent += outcome.costUsd;
|
|
451
|
+
if (outcome.authFailed && !completedCleanly)
|
|
452
|
+
partialReason ??= 'auth_failed';
|
|
453
|
+
if (outcome.interrupted) {
|
|
454
|
+
partialReason ??= 'interrupted';
|
|
455
|
+
outcome.cleanup();
|
|
456
|
+
continue;
|
|
457
|
+
}
|
|
458
|
+
if (!outcome.apiError)
|
|
459
|
+
completedCleanly = true;
|
|
460
|
+
try {
|
|
461
|
+
const run = await gradeRun(job, outcome);
|
|
462
|
+
runs.set(job, run);
|
|
463
|
+
emit({ type: 'run', case: job.plan.case.name, run, of: job.plan.runs });
|
|
464
|
+
}
|
|
465
|
+
finally {
|
|
466
|
+
outcome.cleanup();
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
};
|
|
470
|
+
try {
|
|
471
|
+
await Promise.all(Array.from({ length: Math.min(options.concurrency, jobs.length) }, worker));
|
|
472
|
+
}
|
|
473
|
+
finally {
|
|
474
|
+
await judgeRuntime?.close().catch(() => undefined);
|
|
475
|
+
}
|
|
476
|
+
if (signal.aborted)
|
|
477
|
+
partialReason ??= 'interrupted';
|
|
478
|
+
for (const job of jobs) {
|
|
479
|
+
const run = runs.get(job);
|
|
480
|
+
if (!run)
|
|
481
|
+
continue;
|
|
482
|
+
const target = results[job.plan.index];
|
|
483
|
+
if (target)
|
|
484
|
+
target.arms[job.arm].push(run);
|
|
485
|
+
}
|
|
486
|
+
for (const plan of plans) {
|
|
487
|
+
const result = results[plan.index];
|
|
488
|
+
if (!result || result.error !== undefined)
|
|
489
|
+
continue;
|
|
490
|
+
const withRuns = result.arms.with;
|
|
491
|
+
const withoutRuns = result.arms.without;
|
|
492
|
+
const score = withRuns.length > 0 ? mean(withRuns.map((run) => run.score)) : null;
|
|
493
|
+
const comparable = !withRuns.some((run) => run.apiError) && !withoutRuns.some((run) => run.apiError);
|
|
494
|
+
const withoutScore = result.singleArm
|
|
495
|
+
? undefined
|
|
496
|
+
: withoutRuns.length > 0 && !withoutRuns.some((run) => run.apiError)
|
|
497
|
+
? mean(withoutRuns.map((run) => run.score))
|
|
498
|
+
: null;
|
|
499
|
+
result.aggregates = {
|
|
500
|
+
score,
|
|
501
|
+
...(withoutScore !== undefined && { withoutScore }),
|
|
502
|
+
...(score !== null &&
|
|
503
|
+
withoutScore !== undefined &&
|
|
504
|
+
withoutScore !== null &&
|
|
505
|
+
comparable && { delta: score - withoutScore }),
|
|
506
|
+
passed: score !== null && score >= options.threshold - 1e-9,
|
|
507
|
+
runs: withRuns.length + withoutRuns.length,
|
|
508
|
+
costUsd: [...withRuns, ...withoutRuns].reduce((sum, run) => sum + run.costUsd, 0),
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
const allRuns = results.flatMap((result) => [...result.arms.with, ...result.arms.without]);
|
|
512
|
+
const withRuns = results.flatMap((result) => result.arms.with);
|
|
513
|
+
const deltas = results.flatMap((result) => result.aggregates.delta === undefined ? [] : [result.aggregates.delta]);
|
|
514
|
+
const baselines = results.flatMap((result) => typeof result.aggregates.withoutScore === 'number' ? [result.aggregates.withoutScore] : []);
|
|
515
|
+
const apiErrors = allRuns.filter((run) => run.apiError).length;
|
|
516
|
+
const loadErrors = results.filter((result) => result.error !== undefined).length;
|
|
517
|
+
const casesPassed = results.filter((result) => result.aggregates.passed).length;
|
|
518
|
+
const partial = partialReason !== null;
|
|
519
|
+
const outcome = partialReason === 'interrupted'
|
|
520
|
+
? 'interrupted'
|
|
521
|
+
: partial
|
|
522
|
+
? 'partial'
|
|
523
|
+
: casesPassed === results.length && apiErrors === 0 && loadErrors === 0
|
|
524
|
+
? 'passed'
|
|
525
|
+
: 'failed';
|
|
526
|
+
const result = {
|
|
527
|
+
schemaVersion: 1,
|
|
528
|
+
cogitatorVersion: coreVersion(),
|
|
529
|
+
plugin: {
|
|
530
|
+
name: main.name,
|
|
531
|
+
root: suite.pluginRoot,
|
|
532
|
+
...(main.version !== undefined && { version: main.version }),
|
|
533
|
+
},
|
|
534
|
+
evalDir: suite.evalDir,
|
|
535
|
+
startedAt: started.toISOString(),
|
|
536
|
+
outcome,
|
|
537
|
+
partial,
|
|
538
|
+
partialReason,
|
|
539
|
+
config: {
|
|
540
|
+
...(options.runs !== undefined && { runs: options.runs }),
|
|
541
|
+
concurrency: options.concurrency,
|
|
542
|
+
...(options.model !== undefined && { model: options.model }),
|
|
543
|
+
...(options.judgeModel !== undefined && { judgeModel: options.judgeModel }),
|
|
544
|
+
...(options.ablation !== undefined && { ablation: options.ablation }),
|
|
545
|
+
threshold: options.threshold,
|
|
546
|
+
...(options.maxCostUsd !== undefined && { maxCostUsd: options.maxCostUsd }),
|
|
547
|
+
allowTools: options.allowTools,
|
|
548
|
+
scaffold: options.scaffold,
|
|
549
|
+
...(options.caseGlob !== undefined && { caseFilter: options.caseGlob }),
|
|
550
|
+
tagFilter: options.tags,
|
|
551
|
+
},
|
|
552
|
+
aggregates: {
|
|
553
|
+
overallScore: mean(results.map((result) => result.aggregates.score ?? 0)),
|
|
554
|
+
casesPassed,
|
|
555
|
+
casesTotal: results.length,
|
|
556
|
+
...(deltas.length > 0 && { meanDelta: mean(deltas) }),
|
|
557
|
+
...(baselines.length > 0 && { baselineScore: mean(baselines) }),
|
|
558
|
+
perfectRuns: withRuns.length === 0
|
|
559
|
+
? 0
|
|
560
|
+
: withRuns.filter((run) => run.graders.length > 0 && run.graders.every((grader) => grader.passed)).length / withRuns.length,
|
|
561
|
+
},
|
|
562
|
+
apiErrors,
|
|
563
|
+
loadErrors,
|
|
564
|
+
warnings: suite.warnings,
|
|
565
|
+
cases: results,
|
|
566
|
+
costUsd: allRuns.reduce((sum, run) => sum + run.costUsd, 0),
|
|
567
|
+
durationSeconds: (Date.now() - started.getTime()) / 1000,
|
|
568
|
+
};
|
|
569
|
+
const outputDir = options.outputDir
|
|
570
|
+
? resolve(cwd, options.outputDir)
|
|
571
|
+
: join(suite.resultsDir, timestamp(started));
|
|
572
|
+
mkdirSync(outputDir, { recursive: true });
|
|
573
|
+
const resultFile = join(outputDir, 'aggregate-result.json');
|
|
574
|
+
const reportFile = join(outputDir, 'report.html');
|
|
575
|
+
writeFileSync(resultFile, `${JSON.stringify(result, null, 2)}\n`);
|
|
576
|
+
writeFileSync(reportFile, pluginEvalHtml(result));
|
|
577
|
+
return { result, outputDir, resultFile, reportFile };
|
|
578
|
+
}
|
|
579
|
+
const noJudge = {
|
|
580
|
+
ask: () => Promise.reject(new Error('no judge model is configured')),
|
|
581
|
+
};
|
|
582
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/plugin-evals/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtF,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAmB,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAwB,MAAM,SAAS,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAoB,MAAM,SAAS,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAkB,MAAM,SAAS,CAAC;AAiBjE,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACpC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACjC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC1C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACtD,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAClD,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACpC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACtC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACvC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACpC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC;AAEH,KAAK,UAAU,UAAU;IACvB,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,eAAe,CAAC,yCAAyC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAED,SAAS,WAAW;IAClB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;QACpD,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CACjE,CAAC;QACF,OAAO,OAAO,IAAI,KAAK,QAAQ;YAC7B,IAAI,KAAK,IAAI;YACb,SAAS,IAAI,IAAI;YACjB,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;YAChC,CAAC,CAAC,IAAI,CAAC,OAAO;YACd,CAAC,CAAC,SAAS,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,IAAU,EACV,IAAwB,EACxB,EAAU;IAEV,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;IACjG,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAC5B,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC,CAC1E,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,eAAe,CACvB,GAAG,EAAE,UAAU,OAAO,CAAC,MAAM,uBAAuB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAClH,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,eAAe,CAAC,uBAAuB,EAAE,mBAAmB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC;AACjC,CAAC;AAQD,gFAAgF;AAChF,KAAK,UAAU,WAAW,CAAC,IAAU,EAAE,KAAkB;IACvD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK;QAAE,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5F,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,eAAe,CAAC,iBAAiB,GAAG,iBAAiB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5F,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YACb,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI;YAC1B,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YAClF,OAAO,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;SACjD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,mGAAmG;AACnG,KAAK,UAAU,gBAAgB,CAC7B,IAAU,EACV,GAA6B,EAC7B,IAAuB;IAEvB,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,8BAA8B,CAAC,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC;QAC/B,GAAG;QACH,oBAAoB,EAAE;YACpB,IAAI;YACJ,IAAI,EAAE,KAAK;YACX,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE;YAC3C,QAAQ,EAAE,KAAK;SAChB;KACF,CAAC,CAAC;IACH,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;QACxC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,oBAAoB,CAAC,IAAI,EAAE,EAAE,CAAC;YACrD,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;gBAAE,SAAS;YACrC,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC7B,MAAM,IAAI,eAAe,CACvB,cAAc,IAAI,CAAC,IAAI,kBAAkB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC3C,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,SAAS,eAAe,CACtB,QAAwB,EACxB,MAAe,EACf,QAAqC;IAErC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,CAAC,MAAwB,EAAE,EAAE;QAC7C,IAAI,MAAM,CAAC,GAAG,KAAK,MAAM;YAAE,OAAO,KAAK,CAAC;QACxC,IAAI,MAAM,CAAC,GAAG,KAAK,WAAW;YAAE,OAAO,IAAI,CAAC;QAC5C,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAChC,OAAO,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,CAAC;QAChE,CAAC;QACD,IAAI,CAAC,eAAe;YAAE,OAAO,KAAK,CAAC;QACnC,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO;YAAE,OAAO,MAAM,CAAC,MAAM,KAAK,YAAY,CAAC;QACnE,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK;YAAE,OAAO,MAAM,CAAC,KAAK,KAAK,YAAY,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IACF,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IAClE,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;AACxD,CAAC;AAqBD,SAAS,aAAa,CAAC,OAA0C;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACrE,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1B,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC;AAC9F,CAAC;AAED,SAAS,SAAS,CAAC,IAAU;IAC3B,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,KAAwB;IAExB,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,eAAe,CAAC,oBAAoB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/F,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC;IAC5D,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAEnD,MAAM,SAAS,GAAgB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,OAAO,IAAI,KAAK;YAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QACjF,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,UAAU,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC;QAC5B,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QAC/D,GAAG;QACH,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;QAClE,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrE,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;KAClE,CAAC,CAAC;IACH,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ;QAAE,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAClF,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC7C,MAAM,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACzC,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,eAAe,CAAC,sBAAsB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;IAE/E,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC;QAC5B,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,EAAE;gBAC1B,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,uBAAuB;gBAC7C,SAAS,EAAE,IAAI;gBACf,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI,EAAE;gBAC9B,OAAO,EAAE,QAAQ,EAAE,OAAO,IAAI,EAAE;gBAChC,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;gBAC/B,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;aAC7D,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAC7D,MAAM,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;QACrD,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,MAAM;gBAAE,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAChG,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,EAAE,YAAY,CAAC;QACzE,MAAM,UAAU,GAAyB;YACvC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS,EAAE,IAAI;YACf,OAAO;YACP,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;YAC/B,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;SAChE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzB,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;YACvB,UAAU,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YACjC,UAAU,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;YAChC,SAAS;QACX,CAAC;QACD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAC7E,QAAQ,CAAC,MAAM,CAAC,CACjB,CAAC;QACF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,KAAK,GAAG,iBAAiB,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5F,UAAU,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;YAChC,SAAS;QACX,CAAC;QACD,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YACtE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,SAAS,MAAM,cAAc,OAAO,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,eAAe,CACvB,kHAAkH,CACnH,CAAC;QACJ,CAAC;QACD,MAAM,SAAS,GACb,OAAO,CAAC,QAAQ,KAAK,MAAM;YAC3B,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS;gBAC7B,QAAQ,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS;gBAC1C,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAClB,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC;QACjC,IAAI,SAAS,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;QACvE,CAAC;QACD,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,OAAO,CAAC,IAAI,CAAC,wCAAwC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3F,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACzD,OAAO,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;QACrE,CAAC;QACD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,aAAa,EAAE,CAAC;YAC9C,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;gBACzC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrC,MAAM,CAAC,IAAI,CACT,UAAU,MAAM,IAAI,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,sBAAsB,GAAG,CAC5G,CAAC;YACJ,CAAC;QACH,CAAC;QACD,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7D,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChG,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpE,KAAK,CAAC,IAAI,CAAC;YACT,KAAK,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;YACzB,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC;YAChD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,eAAe,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC;YACzD,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI;YACnC,KAAK;YACL,aAAa;YACb,OAAO;SACR,CAAC,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IAC9E,MAAM,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACrF,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2B,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,IAAiB,EAA+B,EAAE;QACjE,IAAI,CAAC,YAAY;YAAE,OAAO,SAAS,CAAC;QACpC,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,KAAK,CAAC;QAC/C,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,qBAAqB,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;YACnD,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACrF,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,aAAa,GAAmC,IAAI,CAAC;IACzD,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,MAAM,cAAc,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;IAC7F,MAAM,UAAU,GAAG,GAAwC,EAAE;QAC3D,IAAI,MAAM,CAAC,OAAO;YAAE,OAAO,aAAa,CAAC;QACzC,IAAI,aAAa,KAAK,aAAa;YAAE,OAAO,aAAa,CAAC;QAC1D,IAAI,cAAc,EAAE;YAAE,OAAO,cAAc,CAAC;QAC5C,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAQ,EAAE,OAAmB,EAAgC,EAAE;QACrF,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC;QAC1B,MAAM,OAAO,GAA6B,EAAE,CAAC;QAC7C,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;QAC3B,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,QAAQ,IAAI,GAAG,KAAK,SAAS;oBAAE,SAAS;gBAC5C,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC7E,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC3C,WAAW,GAAG,IAAI,CAAC;oBACnB,aAAa,KAAK,aAAa,CAAC;oBAChC,OAAO,CAAC,IAAI,CAAC;wBACX,GAAG,IAAI;wBACP,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,IAAI;wBACb,WAAW,EAAE,2DAA2D;qBACzE,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;gBACD,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC/D,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzD,WAAW,GAAG,IAAI,CAAC;oBACnB,IAAI,cAAc,EAAE;wBAAE,aAAa,KAAK,cAAc,CAAC;oBACvD,OAAO,CAAC,IAAI,CAAC;wBACX,GAAG,IAAI;wBACP,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,IAAI;wBACb,WAAW,EAAE,gEAAgE;qBAC9E,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,IAAI,OAAO,EAAE,MAAM,CAAC,CAAC;oBAC/E,IAAI,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;oBAC7B,KAAK,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;oBAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC3D,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBAC3C,WAAW,GAAG,IAAI,CAAC;wBACnB,aAAa,KAAK,aAAa,CAAC;wBAChC,OAAO,CAAC,IAAI,CAAC;4BACX,GAAG,IAAI;4BACP,MAAM,EAAE,KAAK;4BACb,MAAM,EAAE,KAAK;4BACb,OAAO,EAAE,IAAI;4BACb,WAAW,EAAE,yDAAyD;yBACvE,CAAC,CAAC;wBACH,SAAS;oBACX,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC;wBACX,GAAG,IAAI;wBACP,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,CAAC,QAAQ;wBACjB,WAAW,EAAE,iBAAiB,SAAS,CAAC,KAAK,CAAC,EAAE;qBACjD,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAChC,OAAO;YACL,GAAG;YACH,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,KAAK,EAAE,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YAC3C,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;YACpD,GAAG,CAAC,WAAW,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;YAChD,OAAO;YACP,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,EAAE;YACvC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,EAAE;YACnC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,EAAE;YACnC,YAAY,EAAE,OAAO,EAAE,YAAY,IAAI,EAAE;YACzC,OAAO,EAAE,IAAI;YACb,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;SACnE,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4B,CAAC;IACjD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,OAAO,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACzB,IAAI,CAAC,GAAG;gBAAE,MAAM;YAChB,MAAM,IAAI,GAAG,UAAU,EAAE,CAAC;YAC1B,IAAI,IAAI,EAAE,CAAC;gBACT,aAAa,KAAK,IAAI,CAAC;gBACvB,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;gBAClE,SAAS;YACX,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC;gBAC/B,IAAI;gBACJ,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI;gBACnB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK;gBACrB,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM;gBACvB,aAAa,EAAE,GAAG,CAAC,IAAI,CAAC,aAAa;gBACrC,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,MAAM;aACP,CAAC,CAAC;YACH,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC;YACzB,IAAI,OAAO,CAAC,UAAU,IAAI,CAAC,gBAAgB;gBAAE,aAAa,KAAK,aAAa,CAAC;YAC7E,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;gBACxB,aAAa,KAAK,aAAa,CAAC;gBAChC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAClB,SAAS;YACX,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,gBAAgB,GAAG,IAAI,CAAC;YAC/C,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBACzC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBACnB,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1E,CAAC;oBAAS,CAAC;gBACT,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IAChG,CAAC;YAAS,CAAC;QACT,MAAM,YAAY,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,MAAM,CAAC,OAAO;QAAE,aAAa,KAAK,aAAa,CAAC;IAEpD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7C,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS;YAAE,SAAS;QACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;QACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAClF,MAAM,UAAU,GACd,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS;YACnC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAClE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC3C,CAAC,CAAC,IAAI,CAAC;QACX,MAAM,CAAC,UAAU,GAAG;YAClB,KAAK;YACL,GAAG,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,CAAC;YACnD,GAAG,CAAC,KAAK,KAAK,IAAI;gBAChB,YAAY,KAAK,SAAS;gBAC1B,YAAY,KAAK,IAAI;gBACrB,UAAU,IAAI,EAAE,KAAK,EAAE,KAAK,GAAG,YAAY,EAAE,CAAC;YAChD,MAAM,EAAE,KAAK,KAAK,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI;YAC3D,IAAI,EAAE,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM;YAC1C,OAAO,EAAE,CAAC,GAAG,QAAQ,EAAE,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;SAClF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3F,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CACxC,MAAM,CAAC,UAAU,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CACvE,CAAC;IACF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAC3C,OAAO,MAAM,CAAC,UAAU,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAC3F,CAAC;IACF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAC/D,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IACjF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChF,MAAM,OAAO,GAAG,aAAa,KAAK,IAAI,CAAC;IACvC,MAAM,OAAO,GACX,aAAa,KAAK,aAAa;QAC7B,CAAC,CAAC,aAAa;QACf,CAAC,CAAC,OAAO;YACP,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,MAAM,IAAI,SAAS,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC;gBACrE,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ,CAAC;IAEnB,MAAM,MAAM,GAAqB;QAC/B,aAAa,EAAE,CAAC;QAChB,gBAAgB,EAAE,WAAW,EAAE;QAC/B,MAAM,EAAE;YACN,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,KAAK,CAAC,UAAU;YACtB,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;SAC7D;QACD,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE;QAChC,OAAO;QACP,OAAO;QACP,aAAa;QACb,MAAM,EAAE;YACN,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;YACzD,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;YAC5D,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC;YAC3E,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrE,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC;YAC3E,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvE,SAAS,EAAE,OAAO,CAAC,IAAI;SACxB;QACD,UAAU,EAAE;YACV,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YACzE,WAAW;YACX,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/D,WAAW,EACT,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,QAAQ,CAAC,MAAM,CACb,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAChF,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM;SACjC;QACD,SAAS;QACT,UAAU;QACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,OAAO;QACd,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3D,eAAe,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI;KACzD,CAAC;IAEF,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS;QACjC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC;QACjC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAClD,aAAa,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAClE,aAAa,CAAC,UAAU,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;IAClD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,OAAO,GAAoB;IAC/B,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;CACrE,CAAC"}
|