@cogitator-ai/evals 0.13.2 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +94 -14
  2. package/dist/index.d.ts +2 -0
  3. package/dist/index.d.ts.map +1 -1
  4. package/dist/index.js +1 -0
  5. package/dist/index.js.map +1 -1
  6. package/dist/plugin-evals/execute.d.ts +45 -0
  7. package/dist/plugin-evals/execute.d.ts.map +1 -0
  8. package/dist/plugin-evals/execute.js +508 -0
  9. package/dist/plugin-evals/execute.js.map +1 -0
  10. package/dist/plugin-evals/files.d.ts +17 -0
  11. package/dist/plugin-evals/files.d.ts.map +1 -0
  12. package/dist/plugin-evals/files.js +46 -0
  13. package/dist/plugin-evals/files.js.map +1 -0
  14. package/dist/plugin-evals/glob.d.ts +8 -0
  15. package/dist/plugin-evals/glob.d.ts.map +1 -0
  16. package/dist/plugin-evals/glob.js +63 -0
  17. package/dist/plugin-evals/glob.js.map +1 -0
  18. package/dist/plugin-evals/graders.d.ts +21 -0
  19. package/dist/plugin-evals/graders.d.ts.map +1 -0
  20. package/dist/plugin-evals/graders.js +198 -0
  21. package/dist/plugin-evals/graders.js.map +1 -0
  22. package/dist/plugin-evals/index.d.ts +4 -0
  23. package/dist/plugin-evals/index.d.ts.map +1 -0
  24. package/dist/plugin-evals/index.js +3 -0
  25. package/dist/plugin-evals/index.js.map +1 -0
  26. package/dist/plugin-evals/judge.d.ts +33 -0
  27. package/dist/plugin-evals/judge.d.ts.map +1 -0
  28. package/dist/plugin-evals/judge.js +73 -0
  29. package/dist/plugin-evals/judge.js.map +1 -0
  30. package/dist/plugin-evals/mocks.d.ts +34 -0
  31. package/dist/plugin-evals/mocks.d.ts.map +1 -0
  32. package/dist/plugin-evals/mocks.js +373 -0
  33. package/dist/plugin-evals/mocks.js.map +1 -0
  34. package/dist/plugin-evals/report.d.ts +4 -0
  35. package/dist/plugin-evals/report.d.ts.map +1 -0
  36. package/dist/plugin-evals/report.js +177 -0
  37. package/dist/plugin-evals/report.js.map +1 -0
  38. package/dist/plugin-evals/runner.d.ts +23 -0
  39. package/dist/plugin-evals/runner.d.ts.map +1 -0
  40. package/dist/plugin-evals/runner.js +582 -0
  41. package/dist/plugin-evals/runner.js.map +1 -0
  42. package/dist/plugin-evals/schema.d.ts +179 -0
  43. package/dist/plugin-evals/schema.d.ts.map +1 -0
  44. package/dist/plugin-evals/schema.js +194 -0
  45. package/dist/plugin-evals/schema.js.map +1 -0
  46. package/dist/plugin-evals/suite.d.ts +40 -0
  47. package/dist/plugin-evals/suite.d.ts.map +1 -0
  48. package/dist/plugin-evals/suite.js +476 -0
  49. package/dist/plugin-evals/suite.js.map +1 -0
  50. package/dist/plugin-evals/tools.d.ts +42 -0
  51. package/dist/plugin-evals/tools.d.ts.map +1 -0
  52. package/dist/plugin-evals/tools.js +136 -0
  53. package/dist/plugin-evals/tools.js.map +1 -0
  54. package/dist/plugin-evals/transcript.d.ts +35 -0
  55. package/dist/plugin-evals/transcript.d.ts.map +1 -0
  56. package/dist/plugin-evals/transcript.js +149 -0
  57. package/dist/plugin-evals/transcript.js.map +1 -0
  58. package/dist/plugin-evals/types.d.ts +378 -0
  59. package/dist/plugin-evals/types.d.ts.map +1 -0
  60. package/dist/plugin-evals/types.js +2 -0
  61. package/dist/plugin-evals/types.js.map +1 -0
  62. package/dist/reporters/html-format.d.ts +5 -2
  63. package/dist/reporters/html-format.d.ts.map +1 -1
  64. package/dist/reporters/html-format.js +6 -3
  65. package/dist/reporters/html-format.js.map +1 -1
  66. package/package.json +5 -3
@@ -0,0 +1,582 @@
1
+ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
2
+ import { tmpdir } from 'node:os';
3
+ import { createRequire } from 'node:module';
4
+ import { dirname, join, resolve } from 'node:path';
5
+ import { z } from 'zod';
6
+ import { mean } from '../stats/percentiles.js';
7
+ import { executeRun } from './execute.js';
8
+ import { errorText } from './files.js';
9
+ import { grade, isPaidGrader } from './graders.js';
10
+ import { createPluginEvalJudge } from './judge.js';
11
+ import { pluginEvalHtml } from './report.js';
12
+ import { issuesText } from './schema.js';
13
+ import { loadSuite, PluginEvalError } from './suite.js';
14
+ import { caseGrants, parseGrant } from './tools.js';
15
+ const OptionsSchema = z.object({
16
+ target: z.string().min(1).optional(),
17
+ cwd: z.string().min(1).optional(),
18
+ model: z.string().trim().min(1).optional(),
19
+ judgeModel: z.string().trim().min(1).optional(),
20
+ runs: z.number().int().min(1).max(50).optional(),
21
+ concurrency: z.number().int().min(1).max(8).default(1),
22
+ ablation: z.enum(['none', 'with-without']).optional(),
23
+ threshold: z.number().min(0).max(1).default(1),
24
+ maxCostUsd: z.number().min(0).optional(),
25
+ allowTools: z.array(z.string().min(1)).default([]),
26
+ scaffold: z.boolean().default(false),
27
+ caseGlob: z.string().min(1).optional(),
28
+ tags: z.array(z.string().min(1)).default([]),
29
+ evalDir: z.string().min(1).optional(),
30
+ outputDir: z.string().min(1).optional(),
31
+ keepTemp: z.boolean().default(false),
32
+ pluginsRoot: z.string().min(1).optional(),
33
+ });
34
+ async function importCore() {
35
+ try {
36
+ return await import('@cogitator-ai/core');
37
+ }
38
+ catch (error) {
39
+ throw new PluginEvalError(`Plugin evals need @cogitator-ai/core: ${errorText(error)}`);
40
+ }
41
+ }
42
+ function coreVersion() {
43
+ try {
44
+ const require = createRequire(import.meta.url);
45
+ const entry = require.resolve('@cogitator-ai/core');
46
+ const json = JSON.parse(readFileSync(join(dirname(entry), '..', 'package.json'), 'utf8'));
47
+ return typeof json === 'object' &&
48
+ json !== null &&
49
+ 'version' in json &&
50
+ typeof json.version === 'string'
51
+ ? json.version
52
+ : 'unknown';
53
+ }
54
+ catch {
55
+ return 'unknown';
56
+ }
57
+ }
58
+ async function installedPlugin(core, root, id) {
59
+ const { records, warnings } = await core.readInstalledPlugins(root ?? core.defaultPluginsRoot());
60
+ const matches = records.filter((record) => record.id === id || (!id.includes('@') && record.name === id));
61
+ if (matches.length > 1) {
62
+ throw new PluginEvalError(`${id} names ${matches.length} installed plugins (${matches.map((m) => m.id).join(', ')}): give name@marketplace`);
63
+ }
64
+ if (matches.length === 0 && warnings.length > 0) {
65
+ throw new PluginEvalError(`No installed plugin ${id} could be read: ${warnings.join(', ')}`);
66
+ }
67
+ return matches[0]?.installPath;
68
+ }
69
+ /** The manifest name and MCP servers of each plugin directory the cases load */
70
+ async function pluginFacts(core, suite) {
71
+ const dirs = new Set([suite.pluginRoot]);
72
+ for (const entry of suite.cases)
73
+ for (const dir of entry.case?.plugins ?? [])
74
+ dirs.add(dir);
75
+ const facts = new Map();
76
+ for (const dir of dirs) {
77
+ const layout = await core.readPluginLayout(dir);
78
+ if (layout.fatal.length > 0) {
79
+ throw new PluginEvalError(`The plugin at ${dir} cannot load: ${layout.fatal.join(', ')}`);
80
+ }
81
+ facts.set(dir, {
82
+ name: layout.manifest.name,
83
+ ...(layout.manifest.version !== undefined && { version: layout.manifest.version }),
84
+ servers: new Set(Object.keys(layout.mcpServers)),
85
+ });
86
+ }
87
+ return facts;
88
+ }
89
+ /** Loads each plugin once the way a run does, so a plugin that fails fails the command up front */
90
+ async function checkPluginsLoad(core, llm, dirs) {
91
+ const store = mkdtempSync(join(tmpdir(), 'cogitator-plugin-eval-probe-'));
92
+ const probe = new core.Cogitator({
93
+ llm,
94
+ experimental_plugins: {
95
+ dirs,
96
+ root: store,
97
+ installed: { root: join(store, 'plugins') },
98
+ settings: false,
99
+ },
100
+ });
101
+ try {
102
+ await probe.experimental_plugins.load();
103
+ for (const info of probe.experimental_plugins.list()) {
104
+ if (info.origin === 'core')
105
+ continue;
106
+ if (info.status !== 'active') {
107
+ throw new PluginEvalError(`The plugin ${info.name} did not load: ${info.error ?? info.status}`);
108
+ }
109
+ }
110
+ }
111
+ finally {
112
+ await probe.close().catch(() => undefined);
113
+ rmSync(store, { recursive: true, force: true });
114
+ }
115
+ }
116
+ /** The graders a two-arm run keeps out of the score: plugin-fired indicators */
117
+ function excludedGraders(evalCase, twoArm, declared) {
118
+ if (!twoArm)
119
+ return new Set();
120
+ const servers = Object.keys(evalCase.mocks);
121
+ const pluginMocksOnly = servers.length > 0 && servers.every(declared);
122
+ const indicator = (grader) => {
123
+ if (grader.arm === 'both')
124
+ return false;
125
+ if (grader.arm === 'with-only')
126
+ return true;
127
+ if (grader.type === 'tool_used') {
128
+ return grader.tool === 'Skill' || grader.tool === 'use_skill';
129
+ }
130
+ if (!pluginMocksOnly)
131
+ return false;
132
+ if (grader.type === 'regex')
133
+ return grader.target === 'mock_calls';
134
+ if (grader.type === 'llm')
135
+ return grader.focus === 'mock_calls';
136
+ return false;
137
+ };
138
+ const excluded = evalCase.graders.filter(indicator);
139
+ if (excluded.length === evalCase.graders.length)
140
+ return new Set();
141
+ return new Set(excluded.map((grader) => grader.name));
142
+ }
143
+ function weightedScore(results) {
144
+ const scored = results.filter((result) => result.scored);
145
+ const total = scored.reduce((sum, result) => sum + result.weight, 0);
146
+ if (total === 0)
147
+ return 0;
148
+ return scored.reduce((sum, result) => sum + (result.passed ? result.weight : 0), 0) / total;
149
+ }
150
+ function timestamp(date) {
151
+ return date.toISOString().replace(/[:.]/g, '-');
152
+ }
153
+ /**
154
+ * Experimental. Runs a plugin's eval suite in the format of Claude Code's `claude plugin eval`:
155
+ * each case's prompt several times in a fresh workspace with the plugin loaded and, for the
156
+ * no-plugin baseline, without it, scores every run with the case's graders (`regex`,
157
+ * `tool_used`, `tool_order`, `file_exists`, and `llm` and `baseline` judged by a majority of three
158
+ * votes) and writes `aggregate-result.json` and a self-contained `report.html`.
159
+ *
160
+ * Needs `@cogitator-ai/core`. Throws `PluginEvalError` when the suite cannot start: no plugin, no
161
+ * case, an invalid option. A case that fails to load is reported in the result instead.
162
+ *
163
+ * @example
164
+ * ```ts
165
+ * const { result, reportFile } = await experimental_runPluginEvals({
166
+ * target: './my-plugin',
167
+ * llm: { defaultModel: 'anthropic/claude-sonnet-5-5' },
168
+ * threshold: 0.8,
169
+ * });
170
+ * console.log(result.aggregates.overallScore, reportFile);
171
+ * ```
172
+ */
173
+ export async function experimental_runPluginEvals(input) {
174
+ const parsed = OptionsSchema.safeParse(input);
175
+ if (!parsed.success)
176
+ throw new PluginEvalError(`Invalid options: ${issuesText(parsed.error)}`);
177
+ const options = parsed.data;
178
+ const started = new Date();
179
+ const cwd = resolve(options.cwd ?? process.cwd());
180
+ const signal = input.signal ?? new AbortController().signal;
181
+ const emit = input.onProgress ?? (() => undefined);
182
+ const runGrants = [];
183
+ for (const rule of options.allowTools) {
184
+ const grant = parseGrant(rule);
185
+ if ('error' in grant)
186
+ throw new PluginEvalError(`--allow-tools: ${grant.error}`);
187
+ runGrants.push(grant);
188
+ }
189
+ const core = await importCore();
190
+ const suite = await loadSuite({
191
+ ...(options.target !== undefined && { target: options.target }),
192
+ cwd,
193
+ ...(options.evalDir !== undefined && { evalDir: options.evalDir }),
194
+ ...(options.caseGlob !== undefined && { caseGlob: options.caseGlob }),
195
+ tags: options.tags,
196
+ installed: (id) => installedPlugin(core, options.pluginsRoot, id),
197
+ });
198
+ for (const warning of suite.warnings)
199
+ emit({ type: 'warning', message: warning });
200
+ const facts = await pluginFacts(core, suite);
201
+ await checkPluginsLoad(core, input.llm, [...facts.keys()]);
202
+ const main = facts.get(suite.pluginRoot);
203
+ if (!main)
204
+ throw new PluginEvalError(`No plugin found at ${suite.pluginRoot}`);
205
+ const results = [];
206
+ const plans = [];
207
+ for (const entry of suite.cases) {
208
+ const evalCase = entry.case;
209
+ if (!evalCase || entry.error !== undefined) {
210
+ results.push({
211
+ name: entry.name,
212
+ path: entry.path,
213
+ tags: evalCase?.tags ?? [],
214
+ error: entry.error ?? 'the case did not load',
215
+ singleArm: true,
216
+ notices: [],
217
+ prompt: evalCase?.prompt ?? '',
218
+ graders: evalCase?.graders ?? [],
219
+ arms: { with: [], without: [] },
220
+ aggregates: { score: 0, passed: false, runs: 0, costUsd: 0 },
221
+ });
222
+ continue;
223
+ }
224
+ const notices = [];
225
+ const granted = caseGrants(evalCase.allowedTools, runGrants);
226
+ const pluginServers = new Map();
227
+ for (const dir of evalCase.plugins) {
228
+ const plugin = facts.get(dir);
229
+ if (plugin)
230
+ pluginServers.set(plugin.name, plugin.servers);
231
+ }
232
+ const declared = (server) => [...pluginServers.values()].some((set) => set.has(server));
233
+ const model = options.model ?? evalCase.model ?? input.llm?.defaultModel;
234
+ const caseResult = {
235
+ name: evalCase.name,
236
+ path: evalCase.path,
237
+ tags: evalCase.tags,
238
+ singleArm: true,
239
+ notices,
240
+ prompt: evalCase.prompt,
241
+ graders: evalCase.graders,
242
+ arms: { with: [], without: [] },
243
+ aggregates: { score: null, passed: false, runs: 0, costUsd: 0 },
244
+ };
245
+ results.push(caseResult);
246
+ if ('error' in granted) {
247
+ caseResult.error = granted.error;
248
+ caseResult.aggregates.score = 0;
249
+ continue;
250
+ }
251
+ const usedProblems = Object.entries(evalCase.mockProblems).filter(([server]) => declared(server));
252
+ if (usedProblems.length > 0) {
253
+ caseResult.error = `invalid mock: ${usedProblems.map(([, problem]) => problem).join(', ')}`;
254
+ caseResult.aggregates.score = 0;
255
+ continue;
256
+ }
257
+ for (const [server, problem] of Object.entries(evalCase.mockProblems)) {
258
+ if (!declared(server))
259
+ notices.push(`mocks/${server} left out: ${problem}`);
260
+ }
261
+ if (model === undefined) {
262
+ throw new PluginEvalError('No model for the agent under test: pass --model, set model in the case, or llm.defaultModel in the configuration');
263
+ }
264
+ const singleArm = options.ablation === 'none' ||
265
+ (options.ablation === undefined &&
266
+ evalCase.context.historyFile !== undefined &&
267
+ !suite.named);
268
+ caseResult.singleArm = singleArm;
269
+ if (singleArm && options.ablation === undefined) {
270
+ notices.push('single-arm (no delta): the case resumes a transcript');
271
+ }
272
+ if (granted.notGranted.length > 0)
273
+ notices.push(`not granted: ${granted.notGranted.join(', ')}`);
274
+ if (granted.noCounterpart.length > 0) {
275
+ notices.push(`no Cogitator counterpart, not given: ${granted.noCounterpart.join(', ')}`);
276
+ }
277
+ if (evalCase.context.scaffoldScript && !options.scaffold) {
278
+ notices.push('scaffold_script not run: pass --scaffold to run it');
279
+ }
280
+ const mocked = [];
281
+ for (const [plugin, servers] of pluginServers) {
282
+ for (const server of [...servers].sort()) {
283
+ const tools = evalCase.mocks[server];
284
+ mocked.push(`plugin_${plugin}_${server}[${tools ? tools.map((tool) => tool.tool).join(', ') : 'not started: no mock'}]`);
285
+ }
286
+ }
287
+ for (const [server, tools] of Object.entries(evalCase.mocks)) {
288
+ if (!declared(server))
289
+ mocked.push(`${server}[${tools.map((tool) => tool.tool).join(', ')}]`);
290
+ }
291
+ if (mocked.length > 0)
292
+ notices.push(`mocked: ${mocked.join(', ')}`);
293
+ plans.push({
294
+ index: results.length - 1,
295
+ case: evalCase,
296
+ arms: singleArm ? ['with'] : ['with', 'without'],
297
+ grants: granted.grants,
298
+ excluded: excludedGraders(evalCase, !singleArm, declared),
299
+ runs: options.runs ?? evalCase.runs,
300
+ model,
301
+ pluginServers,
302
+ notices,
303
+ });
304
+ }
305
+ const needsJudge = plans.some((plan) => plan.case.graders.some(isPaidGrader));
306
+ const judgeRuntime = needsJudge ? new core.Cogitator({ llm: input.llm }) : undefined;
307
+ const judges = new Map();
308
+ const judgeOf = (plan) => {
309
+ if (!judgeRuntime)
310
+ return undefined;
311
+ const model = options.judgeModel ?? plan.model;
312
+ let judge = judges.get(model);
313
+ if (!judge) {
314
+ judge = createPluginEvalJudge(judgeRuntime, model);
315
+ judges.set(model, judge);
316
+ }
317
+ return judge;
318
+ };
319
+ const jobs = [];
320
+ for (const plan of plans) {
321
+ emit({ type: 'case', case: plan.case.name, notices: plan.notices, arms: plan.arms });
322
+ for (const arm of plan.arms) {
323
+ for (let index = 1; index <= plan.runs; index++)
324
+ jobs.push({ plan, arm, index });
325
+ }
326
+ }
327
+ let spent = 0;
328
+ let partialReason = null;
329
+ let completedCleanly = false;
330
+ const ceilingReached = () => options.maxCostUsd !== undefined && spent >= options.maxCostUsd;
331
+ const stopReason = () => {
332
+ if (signal.aborted)
333
+ return 'interrupted';
334
+ if (partialReason === 'auth_failed')
335
+ return 'auth_failed';
336
+ if (ceilingReached())
337
+ return 'cost_ceiling';
338
+ return undefined;
339
+ };
340
+ const gradeRun = async (job, outcome) => {
341
+ const { plan, arm } = job;
342
+ const graders = [];
343
+ let skippedPaid = false;
344
+ let cost = outcome.costUsd;
345
+ if (outcome.capture) {
346
+ for (const grader of plan.case.graders) {
347
+ const excluded = plan.excluded.has(grader.name);
348
+ if (excluded && arm === 'without')
349
+ continue;
350
+ const base = { name: grader.name, type: grader.type, weight: grader.weight };
351
+ if (isPaidGrader(grader) && signal.aborted) {
352
+ skippedPaid = true;
353
+ partialReason ??= 'interrupted';
354
+ graders.push({
355
+ ...base,
356
+ passed: false,
357
+ scored: false,
358
+ skipped: true,
359
+ explanation: 'skipped: the run was interrupted before this judge grader',
360
+ });
361
+ continue;
362
+ }
363
+ const judge = isPaidGrader(grader) ? judgeOf(plan) : undefined;
364
+ if (isPaidGrader(grader) && (ceilingReached() || !judge)) {
365
+ skippedPaid = true;
366
+ if (ceilingReached())
367
+ partialReason ??= 'cost_ceiling';
368
+ graders.push({
369
+ ...base,
370
+ passed: false,
371
+ scored: false,
372
+ skipped: true,
373
+ explanation: 'skipped: the cost ceiling was reached before this judge grader',
374
+ });
375
+ continue;
376
+ }
377
+ try {
378
+ const verdict = await grade(grader, outcome.capture, judge ?? noJudge, signal);
379
+ cost += verdict.costUsd ?? 0;
380
+ spent += verdict.costUsd ?? 0;
381
+ graders.push({ ...base, ...verdict, scored: !excluded });
382
+ }
383
+ catch (error) {
384
+ if (isPaidGrader(grader) && signal.aborted) {
385
+ skippedPaid = true;
386
+ partialReason ??= 'interrupted';
387
+ graders.push({
388
+ ...base,
389
+ passed: false,
390
+ scored: false,
391
+ skipped: true,
392
+ explanation: 'skipped: the run was interrupted while the judge graded',
393
+ });
394
+ continue;
395
+ }
396
+ graders.push({
397
+ ...base,
398
+ passed: false,
399
+ scored: !excluded,
400
+ explanation: `grader error: ${errorText(error)}`,
401
+ });
402
+ }
403
+ }
404
+ }
405
+ const capture = outcome.capture;
406
+ return {
407
+ arm,
408
+ index: job.index,
409
+ score: capture && !outcome.aborted ? weightedScore(graders) : 0,
410
+ error: outcome.error,
411
+ ...(outcome.apiError && { apiError: true }),
412
+ ...(outcome.aborted && { aborted: outcome.aborted }),
413
+ ...(skippedPaid && { skippedPaidGraders: true }),
414
+ graders,
415
+ lastMessage: capture?.lastMessage ?? '',
416
+ toolCalls: capture?.toolCalls ?? [],
417
+ mockCalls: capture?.mockCalls ?? [],
418
+ filesCreated: capture?.filesCreated ?? [],
419
+ costUsd: cost,
420
+ durationSeconds: outcome.durationSeconds,
421
+ ...(outcome.tempDir !== undefined && { tempDir: outcome.tempDir }),
422
+ };
423
+ };
424
+ const runs = new Map();
425
+ let next = 0;
426
+ const worker = async () => {
427
+ while (next < jobs.length) {
428
+ const job = jobs[next++];
429
+ if (!job)
430
+ break;
431
+ const stop = stopReason();
432
+ if (stop) {
433
+ partialReason ??= stop;
434
+ emit({ type: 'skipped', case: job.plan.case.name, reason: stop });
435
+ continue;
436
+ }
437
+ const outcome = await executeRun({
438
+ core,
439
+ case: job.plan.case,
440
+ arm: job.arm,
441
+ llm: input.llm,
442
+ model: job.plan.model,
443
+ grants: job.plan.grants,
444
+ pluginServers: job.plan.pluginServers,
445
+ scaffold: options.scaffold,
446
+ exec: input.exec,
447
+ keepTemp: options.keepTemp,
448
+ signal,
449
+ });
450
+ spent += outcome.costUsd;
451
+ if (outcome.authFailed && !completedCleanly)
452
+ partialReason ??= 'auth_failed';
453
+ if (outcome.interrupted) {
454
+ partialReason ??= 'interrupted';
455
+ outcome.cleanup();
456
+ continue;
457
+ }
458
+ if (!outcome.apiError)
459
+ completedCleanly = true;
460
+ try {
461
+ const run = await gradeRun(job, outcome);
462
+ runs.set(job, run);
463
+ emit({ type: 'run', case: job.plan.case.name, run, of: job.plan.runs });
464
+ }
465
+ finally {
466
+ outcome.cleanup();
467
+ }
468
+ }
469
+ };
470
+ try {
471
+ await Promise.all(Array.from({ length: Math.min(options.concurrency, jobs.length) }, worker));
472
+ }
473
+ finally {
474
+ await judgeRuntime?.close().catch(() => undefined);
475
+ }
476
+ if (signal.aborted)
477
+ partialReason ??= 'interrupted';
478
+ for (const job of jobs) {
479
+ const run = runs.get(job);
480
+ if (!run)
481
+ continue;
482
+ const target = results[job.plan.index];
483
+ if (target)
484
+ target.arms[job.arm].push(run);
485
+ }
486
+ for (const plan of plans) {
487
+ const result = results[plan.index];
488
+ if (!result || result.error !== undefined)
489
+ continue;
490
+ const withRuns = result.arms.with;
491
+ const withoutRuns = result.arms.without;
492
+ const score = withRuns.length > 0 ? mean(withRuns.map((run) => run.score)) : null;
493
+ const comparable = !withRuns.some((run) => run.apiError) && !withoutRuns.some((run) => run.apiError);
494
+ const withoutScore = result.singleArm
495
+ ? undefined
496
+ : withoutRuns.length > 0 && !withoutRuns.some((run) => run.apiError)
497
+ ? mean(withoutRuns.map((run) => run.score))
498
+ : null;
499
+ result.aggregates = {
500
+ score,
501
+ ...(withoutScore !== undefined && { withoutScore }),
502
+ ...(score !== null &&
503
+ withoutScore !== undefined &&
504
+ withoutScore !== null &&
505
+ comparable && { delta: score - withoutScore }),
506
+ passed: score !== null && score >= options.threshold - 1e-9,
507
+ runs: withRuns.length + withoutRuns.length,
508
+ costUsd: [...withRuns, ...withoutRuns].reduce((sum, run) => sum + run.costUsd, 0),
509
+ };
510
+ }
511
+ const allRuns = results.flatMap((result) => [...result.arms.with, ...result.arms.without]);
512
+ const withRuns = results.flatMap((result) => result.arms.with);
513
+ const deltas = results.flatMap((result) => result.aggregates.delta === undefined ? [] : [result.aggregates.delta]);
514
+ const baselines = results.flatMap((result) => typeof result.aggregates.withoutScore === 'number' ? [result.aggregates.withoutScore] : []);
515
+ const apiErrors = allRuns.filter((run) => run.apiError).length;
516
+ const loadErrors = results.filter((result) => result.error !== undefined).length;
517
+ const casesPassed = results.filter((result) => result.aggregates.passed).length;
518
+ const partial = partialReason !== null;
519
+ const outcome = partialReason === 'interrupted'
520
+ ? 'interrupted'
521
+ : partial
522
+ ? 'partial'
523
+ : casesPassed === results.length && apiErrors === 0 && loadErrors === 0
524
+ ? 'passed'
525
+ : 'failed';
526
+ const result = {
527
+ schemaVersion: 1,
528
+ cogitatorVersion: coreVersion(),
529
+ plugin: {
530
+ name: main.name,
531
+ root: suite.pluginRoot,
532
+ ...(main.version !== undefined && { version: main.version }),
533
+ },
534
+ evalDir: suite.evalDir,
535
+ startedAt: started.toISOString(),
536
+ outcome,
537
+ partial,
538
+ partialReason,
539
+ config: {
540
+ ...(options.runs !== undefined && { runs: options.runs }),
541
+ concurrency: options.concurrency,
542
+ ...(options.model !== undefined && { model: options.model }),
543
+ ...(options.judgeModel !== undefined && { judgeModel: options.judgeModel }),
544
+ ...(options.ablation !== undefined && { ablation: options.ablation }),
545
+ threshold: options.threshold,
546
+ ...(options.maxCostUsd !== undefined && { maxCostUsd: options.maxCostUsd }),
547
+ allowTools: options.allowTools,
548
+ scaffold: options.scaffold,
549
+ ...(options.caseGlob !== undefined && { caseFilter: options.caseGlob }),
550
+ tagFilter: options.tags,
551
+ },
552
+ aggregates: {
553
+ overallScore: mean(results.map((result) => result.aggregates.score ?? 0)),
554
+ casesPassed,
555
+ casesTotal: results.length,
556
+ ...(deltas.length > 0 && { meanDelta: mean(deltas) }),
557
+ ...(baselines.length > 0 && { baselineScore: mean(baselines) }),
558
+ perfectRuns: withRuns.length === 0
559
+ ? 0
560
+ : withRuns.filter((run) => run.graders.length > 0 && run.graders.every((grader) => grader.passed)).length / withRuns.length,
561
+ },
562
+ apiErrors,
563
+ loadErrors,
564
+ warnings: suite.warnings,
565
+ cases: results,
566
+ costUsd: allRuns.reduce((sum, run) => sum + run.costUsd, 0),
567
+ durationSeconds: (Date.now() - started.getTime()) / 1000,
568
+ };
569
+ const outputDir = options.outputDir
570
+ ? resolve(cwd, options.outputDir)
571
+ : join(suite.resultsDir, timestamp(started));
572
+ mkdirSync(outputDir, { recursive: true });
573
+ const resultFile = join(outputDir, 'aggregate-result.json');
574
+ const reportFile = join(outputDir, 'report.html');
575
+ writeFileSync(resultFile, `${JSON.stringify(result, null, 2)}\n`);
576
+ writeFileSync(reportFile, pluginEvalHtml(result));
577
+ return { result, outputDir, resultFile, reportFile };
578
+ }
579
+ const noJudge = {
580
+ ask: () => Promise.reject(new Error('no judge model is configured')),
581
+ };
582
+ //# sourceMappingURL=runner.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/plugin-evals/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtF,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAmB,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAwB,MAAM,SAAS,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAoB,MAAM,SAAS,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAkB,MAAM,SAAS,CAAC;AAiBjE,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACpC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACjC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC1C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACtD,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAClD,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACpC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACtC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACvC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACpC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC;AAEH,KAAK,UAAU,UAAU;IACvB,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,eAAe,CAAC,yCAAyC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAED,SAAS,WAAW;IAClB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;QACpD,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CACjE,CAAC;QACF,OAAO,OAAO,IAAI,KAAK,QAAQ;YAC7B,IAAI,KAAK,IAAI;YACb,SAAS,IAAI,IAAI;YACjB,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;YAChC,CAAC,CAAC,IAAI,CAAC,OAAO;YACd,CAAC,CAAC,SAAS,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,IAAU,EACV,IAAwB,EACxB,EAAU;IAEV,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;IACjG,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAC5B,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC,CAC1E,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,eAAe,CACvB,GAAG,EAAE,UAAU,OAAO,CAAC,MAAM,uBAAuB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAClH,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,eAAe,CAAC,uBAAuB,EAAE,mBAAmB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC;AACjC,CAAC;AAQD,gFAAgF;AAChF,KAAK,UAAU,WAAW,CAAC,IAAU,EAAE,KAAkB;IACvD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK;QAAE,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5F,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,eAAe,CAAC,iBAAiB,GAAG,iBAAiB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5F,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YACb,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI;YAC1B,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YAClF,OAAO,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;SACjD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,mGAAmG;AACnG,KAAK,UAAU,gBAAgB,CAC7B,IAAU,EACV,GAA6B,EAC7B,IAAuB;IAEvB,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,8BAA8B,CAAC,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC;QAC/B,GAAG;QACH,oBAAoB,EAAE;YACpB,IAAI;YACJ,IAAI,EAAE,KAAK;YACX,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE;YAC3C,QAAQ,EAAE,KAAK;SAChB;KACF,CAAC,CAAC;IACH,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;QACxC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,oBAAoB,CAAC,IAAI,EAAE,EAAE,CAAC;YACrD,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;gBAAE,SAAS;YACrC,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC7B,MAAM,IAAI,eAAe,CACvB,cAAc,IAAI,CAAC,IAAI,kBAAkB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC3C,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,SAAS,eAAe,CACtB,QAAwB,EACxB,MAAe,EACf,QAAqC;IAErC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,CAAC,MAAwB,EAAE,EAAE;QAC7C,IAAI,MAAM,CAAC,GAAG,KAAK,MAAM;YAAE,OAAO,KAAK,CAAC;QACxC,IAAI,MAAM,CAAC,GAAG,KAAK,WAAW;YAAE,OAAO,IAAI,CAAC;QAC5C,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAChC,OAAO,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,CAAC;QAChE,CAAC;QACD,IAAI,CAAC,eAAe;YAAE,OAAO,KAAK,CAAC;QACnC,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO;YAAE,OAAO,MAAM,CAAC,MAAM,KAAK,YAAY,CAAC;QACnE,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK;YAAE,OAAO,MAAM,CAAC,KAAK,KAAK,YAAY,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IACF,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IAClE,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;AACxD,CAAC;AAqBD,SAAS,aAAa,CAAC,OAA0C;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACrE,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1B,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC;AAC9F,CAAC;AAED,SAAS,SAAS,CAAC,IAAU;IAC3B,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,KAAwB;IAExB,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,eAAe,CAAC,oBAAoB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/F,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC;IAC5D,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAEnD,MAAM,SAAS,GAAgB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,OAAO,IAAI,KAAK;YAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QACjF,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,UAAU,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC;QAC5B,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QAC/D,GAAG;QACH,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;QAClE,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrE,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;KAClE,CAAC,CAAC;IACH,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ;QAAE,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAClF,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC7C,MAAM,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACzC,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,eAAe,CAAC,sBAAsB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;IAE/E,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC;QAC5B,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,EAAE;gBAC1B,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,uBAAuB;gBAC7C,SAAS,EAAE,IAAI;gBACf,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI,EAAE;gBAC9B,OAAO,EAAE,QAAQ,EAAE,OAAO,IAAI,EAAE;gBAChC,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;gBAC/B,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;aAC7D,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAC7D,MAAM,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;QACrD,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,MAAM;gBAAE,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAChG,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,EAAE,YAAY,CAAC;QACzE,MAAM,UAAU,GAAyB;YACvC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS,EAAE,IAAI;YACf,OAAO;YACP,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;YAC/B,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;SAChE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzB,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;YACvB,UAAU,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YACjC,UAAU,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;YAChC,SAAS;QACX,CAAC;QACD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAC7E,QAAQ,CAAC,MAAM,CAAC,CACjB,CAAC;QACF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,KAAK,GAAG,iBAAiB,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5F,UAAU,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;YAChC,SAAS;QACX,CAAC;QACD,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YACtE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,SAAS,MAAM,cAAc,OAAO,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,eAAe,CACvB,kHAAkH,CACnH,CAAC;QACJ,CAAC;QACD,MAAM,SAAS,GACb,OAAO,CAAC,QAAQ,KAAK,MAAM;YAC3B,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS;gBAC7B,QAAQ,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS;gBAC1C,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAClB,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC;QACjC,IAAI,SAAS,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;QACvE,CAAC;QACD,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,OAAO,CAAC,IAAI,CAAC,wCAAwC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3F,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACzD,OAAO,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;QACrE,CAAC;QACD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,aAAa,EAAE,CAAC;YAC9C,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;gBACzC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrC,MAAM,CAAC,IAAI,CACT,UAAU,MAAM,IAAI,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,sBAAsB,GAAG,CAC5G,CAAC;YACJ,CAAC;QACH,CAAC;QACD,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7D,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChG,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpE,KAAK,CAAC,IAAI,CAAC;YACT,KAAK,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;YACzB,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC;YAChD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,eAAe,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC;YACzD,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI;YACnC,KAAK;YACL,aAAa;YACb,OAAO;SACR,CAAC,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IAC9E,MAAM,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACrF,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2B,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,IAAiB,EAA+B,EAAE;QACjE,IAAI,CAAC,YAAY;YAAE,OAAO,SAAS,CAAC;QACpC,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,KAAK,CAAC;QAC/C,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,qBAAqB,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;YACnD,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACrF,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,aAAa,GAAmC,IAAI,CAAC;IACzD,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,MAAM,cAAc,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;IAC7F,MAAM,UAAU,GAAG,GAAwC,EAAE;QAC3D,IAAI,MAAM,CAAC,OAAO;YAAE,OAAO,aAAa,CAAC;QACzC,IAAI,aAAa,KAAK,aAAa;YAAE,OAAO,aAAa,CAAC;QAC1D,IAAI,cAAc,EAAE;YAAE,OAAO,cAAc,CAAC;QAC5C,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAQ,EAAE,OAAmB,EAAgC,EAAE;QACrF,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC;QAC1B,MAAM,OAAO,GAA6B,EAAE,CAAC;QAC7C,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;QAC3B,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,QAAQ,IAAI,GAAG,KAAK,SAAS;oBAAE,SAAS;gBAC5C,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC7E,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC3C,WAAW,GAAG,IAAI,CAAC;oBACnB,aAAa,KAAK,aAAa,CAAC;oBAChC,OAAO,CAAC,IAAI,CAAC;wBACX,GAAG,IAAI;wBACP,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,IAAI;wBACb,WAAW,EAAE,2DAA2D;qBACzE,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;gBACD,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC/D,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzD,WAAW,GAAG,IAAI,CAAC;oBACnB,IAAI,cAAc,EAAE;wBAAE,aAAa,KAAK,cAAc,CAAC;oBACvD,OAAO,CAAC,IAAI,CAAC;wBACX,GAAG,IAAI;wBACP,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,IAAI;wBACb,WAAW,EAAE,gEAAgE;qBAC9E,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,IAAI,OAAO,EAAE,MAAM,CAAC,CAAC;oBAC/E,IAAI,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;oBAC7B,KAAK,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;oBAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC3D,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBAC3C,WAAW,GAAG,IAAI,CAAC;wBACnB,aAAa,KAAK,aAAa,CAAC;wBAChC,OAAO,CAAC,IAAI,CAAC;4BACX,GAAG,IAAI;4BACP,MAAM,EAAE,KAAK;4BACb,MAAM,EAAE,KAAK;4BACb,OAAO,EAAE,IAAI;4BACb,WAAW,EAAE,yDAAyD;yBACvE,CAAC,CAAC;wBACH,SAAS;oBACX,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC;wBACX,GAAG,IAAI;wBACP,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,CAAC,QAAQ;wBACjB,WAAW,EAAE,iBAAiB,SAAS,CAAC,KAAK,CAAC,EAAE;qBACjD,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAChC,OAAO;YACL,GAAG;YACH,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,KAAK,EAAE,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YAC3C,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;YACpD,GAAG,CAAC,WAAW,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;YAChD,OAAO;YACP,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,EAAE;YACvC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,EAAE;YACnC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,EAAE;YACnC,YAAY,EAAE,OAAO,EAAE,YAAY,IAAI,EAAE;YACzC,OAAO,EAAE,IAAI;YACb,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;SACnE,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4B,CAAC;IACjD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,OAAO,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACzB,IAAI,CAAC,GAAG;gBAAE,MAAM;YAChB,MAAM,IAAI,GAAG,UAAU,EAAE,CAAC;YAC1B,IAAI,IAAI,EAAE,CAAC;gBACT,aAAa,KAAK,IAAI,CAAC;gBACvB,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;gBAClE,SAAS;YACX,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC;gBAC/B,IAAI;gBACJ,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI;gBACnB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK;gBACrB,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM;gBACvB,aAAa,EAAE,GAAG,CAAC,IAAI,CAAC,aAAa;gBACrC,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,MAAM;aACP,CAAC,CAAC;YACH,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC;YACzB,IAAI,OAAO,CAAC,UAAU,IAAI,CAAC,gBAAgB;gBAAE,aAAa,KAAK,aAAa,CAAC;YAC7E,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;gBACxB,aAAa,KAAK,aAAa,CAAC;gBAChC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAClB,SAAS;YACX,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,gBAAgB,GAAG,IAAI,CAAC;YAC/C,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBACzC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBACnB,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1E,CAAC;oBAAS,CAAC;gBACT,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IAChG,CAAC;YAAS,CAAC;QACT,MAAM,YAAY,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,MAAM,CAAC,OAAO;QAAE,aAAa,KAAK,aAAa,CAAC;IAEpD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7C,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS;YAAE,SAAS;QACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;QACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAClF,MAAM,UAAU,GACd,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS;YACnC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAClE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC3C,CAAC,CAAC,IAAI,CAAC;QACX,MAAM,CAAC,UAAU,GAAG;YAClB,KAAK;YACL,GAAG,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,CAAC;YACnD,GAAG,CAAC,KAAK,KAAK,IAAI;gBAChB,YAAY,KAAK,SAAS;gBAC1B,YAAY,KAAK,IAAI;gBACrB,UAAU,IAAI,EAAE,KAAK,EAAE,KAAK,GAAG,YAAY,EAAE,CAAC;YAChD,MAAM,EAAE,KAAK,KAAK,IAAI,IAAI,KAAK,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI;YAC3D,IAAI,EAAE,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM;YAC1C,OAAO,EAAE,CAAC,GAAG,QAAQ,EAAE,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;SAClF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3F,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CACxC,MAAM,CAAC,UAAU,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CACvE,CAAC;IACF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAC3C,OAAO,MAAM,CAAC,UAAU,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAC3F,CAAC;IACF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAC/D,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IACjF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChF,MAAM,OAAO,GAAG,aAAa,KAAK,IAAI,CAAC;IACvC,MAAM,OAAO,GACX,aAAa,KAAK,aAAa;QAC7B,CAAC,CAAC,aAAa;QACf,CAAC,CAAC,OAAO;YACP,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,MAAM,IAAI,SAAS,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC;gBACrE,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ,CAAC;IAEnB,MAAM,MAAM,GAAqB;QAC/B,aAAa,EAAE,CAAC;QAChB,gBAAgB,EAAE,WAAW,EAAE;QAC/B,MAAM,EAAE;YACN,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,KAAK,CAAC,UAAU;YACtB,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;SAC7D;QACD,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE;QAChC,OAAO;QACP,OAAO;QACP,aAAa;QACb,MAAM,EAAE;YACN,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;YACzD,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;YAC5D,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC;YAC3E,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrE,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC;YAC3E,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvE,SAAS,EAAE,OAAO,CAAC,IAAI;SACxB;QACD,UAAU,EAAE;YACV,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YACzE,WAAW;YACX,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/D,WAAW,EACT,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,QAAQ,CAAC,MAAM,CACb,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAChF,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM;SACjC;QACD,SAAS;QACT,UAAU;QACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,OAAO;QACd,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3D,eAAe,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI;KACzD,CAAC;IAEF,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS;QACjC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC;QACjC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAClD,aAAa,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAClE,aAAa,CAAC,UAAU,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;IAClD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,OAAO,GAAoB;IAC/B,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;CACrE,CAAC"}