@zhivex-ai/core 1.7.0 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,545 @@
1
+ import { estimateTokenCost } from "./agent-trace.js";
2
+ import { embed } from "./embed.js";
3
+ import { ValidationError } from "./errors.js";
4
+ import { generateText } from "./generate-text.js";
5
+ import { cosineSimilarity } from "./retrieval.js";
6
+ export const MODEL_EVALUATION_REPORT_SCHEMA_VERSION = 1;
7
+ function assertNonEmptyIdentifier(value, path) {
8
+ if (typeof value !== "string" ||
9
+ value.trim().length === 0 ||
10
+ value !== value.trim() ||
11
+ /[\u0000-\u001f\u007f]/u.test(value)) {
12
+ throw new ValidationError(`${path} must be a non-empty string without surrounding whitespace or control characters.`);
13
+ }
14
+ }
15
+ function assertFiniteRange(value, path, minimum, maximum) {
16
+ if (typeof value !== "number" || !Number.isFinite(value) || value < minimum || value > maximum) {
17
+ throw new ValidationError(`${path} must be a finite number between ${minimum} and ${maximum}.`);
18
+ }
19
+ }
20
+ const normalizeError = (error) => error instanceof Error ? error.message : String(error);
21
+ const percentile = (values, quantile) => {
22
+ if (values.length === 0) {
23
+ return 0;
24
+ }
25
+ const sorted = [...values].sort((left, right) => left - right);
26
+ const index = Math.max(0, Math.ceil(quantile * sorted.length) - 1);
27
+ return sorted[index] ?? 0;
28
+ };
29
+ const mean = (values) => values.length === 0 ? undefined : values.reduce((sum, value) => sum + value, 0) / values.length;
30
+ const sumUsage = (usages) => {
31
+ const present = usages.filter((usage) => usage !== undefined);
32
+ if (present.length === 0) {
33
+ return undefined;
34
+ }
35
+ const fields = [
36
+ "inputTokens",
37
+ "cachedInputTokens",
38
+ "cacheWriteTokens",
39
+ "outputTokens",
40
+ "reasoningTokens",
41
+ "totalTokens"
42
+ ];
43
+ const total = {};
44
+ for (const field of fields) {
45
+ const values = present.map((usage) => usage[field]).filter((value) => value !== undefined);
46
+ if (values.length > 0) {
47
+ total[field] = values.reduce((sum, value) => sum + value, 0);
48
+ }
49
+ }
50
+ const speeds = new Set(present.map((usage) => usage.speed).filter((value) => value !== undefined));
51
+ if (speeds.size === 1) {
52
+ total.speed = [...speeds][0];
53
+ }
54
+ return total;
55
+ };
56
+ const catalogPricing = (catalog, candidate) => {
57
+ if (candidate.pricing) {
58
+ return candidate.pricing;
59
+ }
60
+ const entry = catalog?.find(candidate.model.provider, candidate.model.modelId);
61
+ if (!entry) {
62
+ return undefined;
63
+ }
64
+ return pricingFromCatalogEntry(entry, catalog?.metadata.pricing?.currency);
65
+ };
66
+ const pricingFromCatalogEntry = (entry, currency) => ({
67
+ inputCostPer1kTokens: entry.inputCostPer1kTokens,
68
+ cachedInputCostPer1kTokens: entry.cachedInputCostPer1kTokens,
69
+ cacheWriteCostPer1kTokens: entry.cacheWriteCostPer1kTokens,
70
+ outputCostPer1kTokens: entry.outputCostPer1kTokens,
71
+ costPer1kTokens: entry.costPer1kTokens,
72
+ longContextPricing: entry.longContextPricing,
73
+ currency
74
+ });
75
+ const validateSuite = (suite) => {
76
+ if (!Array.isArray(suite.cases) || suite.cases.length === 0) {
77
+ throw new ValidationError("Model evaluation requires at least one case.");
78
+ }
79
+ if (!Array.isArray(suite.candidates) || suite.candidates.length === 0) {
80
+ throw new ValidationError("Model evaluation requires at least one candidate.");
81
+ }
82
+ if (!Array.isArray(suite.scorers) || suite.scorers.length === 0) {
83
+ throw new ValidationError("Model evaluation requires at least one scorer.");
84
+ }
85
+ const caseNames = new Set();
86
+ for (const [index, testCase] of suite.cases.entries()) {
87
+ assertNonEmptyIdentifier(testCase.name, `cases[${index}].name`);
88
+ if (caseNames.has(testCase.name)) {
89
+ throw new ValidationError(`Duplicate model evaluation case name ${JSON.stringify(testCase.name)}.`);
90
+ }
91
+ caseNames.add(testCase.name);
92
+ if ((testCase.prompt === undefined) === (testCase.messages === undefined)) {
93
+ throw new ValidationError(`Case ${JSON.stringify(testCase.name)} must define exactly one of prompt or messages.`);
94
+ }
95
+ if (testCase.messages !== undefined && !Array.isArray(testCase.messages)) {
96
+ throw new ValidationError(`Case ${JSON.stringify(testCase.name)} messages must be an array.`);
97
+ }
98
+ }
99
+ const candidateIds = new Set();
100
+ for (const [index, candidate] of suite.candidates.entries()) {
101
+ assertNonEmptyIdentifier(candidate.id, `candidates[${index}].id`);
102
+ if (candidateIds.has(candidate.id)) {
103
+ throw new ValidationError(`Duplicate model evaluation candidate id ${JSON.stringify(candidate.id)}.`);
104
+ }
105
+ candidateIds.add(candidate.id);
106
+ if (!candidate.model || typeof candidate.model.generate !== "function") {
107
+ throw new ValidationError(`Candidate ${JSON.stringify(candidate.id)} must define a language model.`);
108
+ }
109
+ }
110
+ const scorerIds = new Set();
111
+ for (const [index, scorer] of suite.scorers.entries()) {
112
+ assertNonEmptyIdentifier(scorer.id, `scorers[${index}].id`);
113
+ if (scorerIds.has(scorer.id)) {
114
+ throw new ValidationError(`Duplicate model evaluation scorer id ${JSON.stringify(scorer.id)}.`);
115
+ }
116
+ scorerIds.add(scorer.id);
117
+ if (typeof scorer.score !== "function") {
118
+ throw new ValidationError(`Scorer ${JSON.stringify(scorer.id)} must define score().`);
119
+ }
120
+ }
121
+ const repetitions = suite.repetitions ?? 1;
122
+ if (!Number.isSafeInteger(repetitions) || repetitions < 1 || repetitions > 100) {
123
+ throw new ValidationError("Model evaluation repetitions must be a safe integer between 1 and 100.");
124
+ }
125
+ const maxConcurrency = suite.maxConcurrency ?? 4;
126
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 64) {
127
+ throw new ValidationError("Model evaluation maxConcurrency must be a safe integer between 1 and 64.");
128
+ }
129
+ validateThresholds(suite.thresholds);
130
+ return { repetitions, maxConcurrency };
131
+ };
132
+ const validateThresholds = (thresholds) => {
133
+ if (!thresholds) {
134
+ return;
135
+ }
136
+ if (thresholds.minMeanScore !== undefined) {
137
+ assertFiniteRange(thresholds.minMeanScore, "thresholds.minMeanScore", 0, 1);
138
+ }
139
+ if (thresholds.maxErrorRate !== undefined) {
140
+ assertFiniteRange(thresholds.maxErrorRate, "thresholds.maxErrorRate", 0, 1);
141
+ }
142
+ for (const field of ["maxTotalCost", "maxP95LatencyMs"]) {
143
+ const value = thresholds[field];
144
+ if (value !== undefined && (typeof value !== "number" || !Number.isFinite(value) || value < 0)) {
145
+ throw new ValidationError(`thresholds.${field} must be a finite, non-negative number.`);
146
+ }
147
+ }
148
+ };
149
+ const normalizeScorerResult = (scorerId, result) => {
150
+ const normalized = typeof result === "number" ? { score: result } : result;
151
+ assertFiniteRange(normalized.score, `Scorer ${JSON.stringify(scorerId)} score`, 0, 1);
152
+ return {
153
+ scorerId,
154
+ score: normalized.score,
155
+ feedback: normalized.feedback,
156
+ metadata: normalized.metadata
157
+ };
158
+ };
159
+ const evaluateSingleRun = async (testCase, candidate, repetition, suite) => {
160
+ const startedAt = Date.now();
161
+ let outputText;
162
+ let finishReason;
163
+ let usage;
164
+ let toolResults = [];
165
+ let error;
166
+ try {
167
+ const input = testCase.prompt === undefined
168
+ ? { messages: testCase.messages }
169
+ : { prompt: testCase.prompt };
170
+ const output = await generateText({
171
+ ...candidate.settings,
172
+ ...testCase.settings,
173
+ ...input,
174
+ model: candidate.model
175
+ });
176
+ outputText = output.text;
177
+ finishReason = output.finishReason;
178
+ usage = output.usage;
179
+ toolResults = output.toolResults;
180
+ }
181
+ catch (cause) {
182
+ error = normalizeError(cause);
183
+ }
184
+ const latencyMs = Math.max(0, Date.now() - startedAt);
185
+ const cost = estimateTokenCost(usage, catalogPricing(suite.catalog, candidate));
186
+ const context = {
187
+ testCase,
188
+ candidate: {
189
+ id: candidate.id,
190
+ provider: candidate.model.provider,
191
+ modelId: candidate.model.modelId,
192
+ metadata: candidate.metadata
193
+ },
194
+ repetition,
195
+ outputText,
196
+ finishReason,
197
+ usage,
198
+ toolResults,
199
+ latencyMs,
200
+ cost,
201
+ error
202
+ };
203
+ const scores = [];
204
+ for (const scorer of suite.scorers) {
205
+ try {
206
+ scores.push(normalizeScorerResult(scorer.id, await scorer.score(context)));
207
+ }
208
+ catch (cause) {
209
+ scores.push({ scorerId: scorer.id, error: normalizeError(cause) });
210
+ }
211
+ }
212
+ const numericScores = scores.flatMap((score) => score.score === undefined ? [] : [score.score]);
213
+ const scorerFailed = scores.some((score) => score.error !== undefined);
214
+ return {
215
+ caseName: testCase.name,
216
+ candidateId: candidate.id,
217
+ provider: candidate.model.provider,
218
+ modelId: candidate.model.modelId,
219
+ repetition,
220
+ ok: error === undefined && !scorerFailed,
221
+ outputText,
222
+ finishReason,
223
+ usage,
224
+ toolCalls: toolResults.map((result) => result.toolName),
225
+ latencyMs,
226
+ cost: cost.totalCost === undefined && cost.usage === undefined ? undefined : cost,
227
+ scores,
228
+ meanScore: mean(numericScores),
229
+ error,
230
+ caseMetadata: testCase.metadata,
231
+ candidateMetadata: candidate.metadata
232
+ };
233
+ };
234
+ const summarizeCandidate = (candidate, runs) => {
235
+ const successfulRuns = runs.filter((run) => run.ok).length;
236
+ const scores = runs.flatMap((run) => run.meanScore === undefined ? [] : [run.meanScore]);
237
+ const costs = runs.flatMap((run) => run.cost?.totalCost === undefined ? [] : [run.cost.totalCost]);
238
+ const hasCompleteCostEstimate = runs.length > 0 && costs.length === runs.length;
239
+ const currencies = new Set(runs.map((run) => run.cost?.currency).filter((value) => value !== undefined));
240
+ return {
241
+ candidateId: candidate.id,
242
+ provider: candidate.model.provider,
243
+ modelId: candidate.model.modelId,
244
+ runs: runs.length,
245
+ successfulRuns,
246
+ failedRuns: runs.length - successfulRuns,
247
+ errorRate: runs.length === 0 ? 0 : (runs.length - successfulRuns) / runs.length,
248
+ meanScore: mean(scores),
249
+ minScore: scores.length === 0 ? undefined : Math.min(...scores),
250
+ maxScore: scores.length === 0 ? undefined : Math.max(...scores),
251
+ p50LatencyMs: percentile(runs.map((run) => run.latencyMs), 0.5),
252
+ p95LatencyMs: percentile(runs.map((run) => run.latencyMs), 0.95),
253
+ totalCost: hasCompleteCostEstimate ? costs.reduce((sum, value) => sum + value, 0) : undefined,
254
+ meanCost: hasCompleteCostEstimate ? mean(costs) : undefined,
255
+ currency: hasCompleteCostEstimate && currencies.size === 1 ? [...currencies][0] : undefined,
256
+ usage: sumUsage(runs.map((run) => run.usage)),
257
+ metadata: candidate.metadata
258
+ };
259
+ };
260
+ export const evaluateModelEvaluationGate = (candidates, thresholds) => {
261
+ validateThresholds(thresholds);
262
+ const checks = [];
263
+ for (const candidate of candidates) {
264
+ const addCheck = (code, actual, threshold, ok, label) => checks.push({
265
+ candidateId: candidate.candidateId,
266
+ code,
267
+ ok,
268
+ actual,
269
+ threshold,
270
+ message: `${candidate.candidateId} ${label}: ${actual === undefined ? "unavailable" : actual}; threshold ${threshold}.`
271
+ });
272
+ if (thresholds.minMeanScore !== undefined) {
273
+ addCheck("min_mean_score", candidate.meanScore, thresholds.minMeanScore, candidate.meanScore !== undefined && candidate.meanScore >= thresholds.minMeanScore, "mean score");
274
+ }
275
+ if (thresholds.maxTotalCost !== undefined) {
276
+ addCheck("max_total_cost", candidate.totalCost, thresholds.maxTotalCost, candidate.totalCost !== undefined && candidate.totalCost <= thresholds.maxTotalCost, "total cost");
277
+ }
278
+ if (thresholds.maxP95LatencyMs !== undefined) {
279
+ addCheck("max_p95_latency_ms", candidate.p95LatencyMs, thresholds.maxP95LatencyMs, candidate.p95LatencyMs <= thresholds.maxP95LatencyMs, "p95 latency");
280
+ }
281
+ if (thresholds.maxErrorRate !== undefined) {
282
+ addCheck("max_error_rate", candidate.errorRate, thresholds.maxErrorRate, candidate.errorRate <= thresholds.maxErrorRate, "error rate");
283
+ }
284
+ }
285
+ return {
286
+ ok: checks.every((check) => check.ok),
287
+ thresholds: { ...thresholds },
288
+ checks
289
+ };
290
+ };
291
+ export const runModelEvaluation = async (suite) => {
292
+ const { repetitions, maxConcurrency } = validateSuite(suite);
293
+ const tasks = suite.candidates.flatMap((candidate) => suite.cases.flatMap((testCase) => Array.from({ length: repetitions }, (_, repetition) => ({
294
+ candidate,
295
+ testCase,
296
+ repetition: repetition + 1
297
+ }))));
298
+ const runs = new Array(tasks.length);
299
+ let cursor = 0;
300
+ const workers = Array.from({ length: Math.min(maxConcurrency, tasks.length) }, async () => {
301
+ while (cursor < tasks.length) {
302
+ const index = cursor;
303
+ cursor += 1;
304
+ const task = tasks[index];
305
+ runs[index] = await evaluateSingleRun(task.testCase, task.candidate, task.repetition, suite);
306
+ }
307
+ });
308
+ await Promise.all(workers);
309
+ const candidates = suite.candidates.map((candidate) => summarizeCandidate(candidate, runs.filter((run) => run.candidateId === candidate.id)));
310
+ const gate = suite.thresholds ? evaluateModelEvaluationGate(candidates, suite.thresholds) : undefined;
311
+ return {
312
+ schemaVersion: MODEL_EVALUATION_REPORT_SCHEMA_VERSION,
313
+ createdAt: new Date().toISOString(),
314
+ ok: runs.every((run) => run.ok) && (gate?.ok ?? true),
315
+ repetitions,
316
+ totalRuns: runs.length,
317
+ cases: suite.cases.map((testCase) => testCase.name),
318
+ candidates,
319
+ runs,
320
+ gate,
321
+ metadata: suite.metadata
322
+ };
323
+ };
324
+ export const compareModelEvaluationReports = (baseline, target) => {
325
+ const ids = new Set([
326
+ ...baseline.candidates.map((candidate) => candidate.candidateId),
327
+ ...target.candidates.map((candidate) => candidate.candidateId)
328
+ ]);
329
+ return {
330
+ baselineCreatedAt: baseline.createdAt,
331
+ targetCreatedAt: target.createdAt,
332
+ candidates: [...ids].sort().map((candidateId) => {
333
+ const base = baseline.candidates.find((candidate) => candidate.candidateId === candidateId);
334
+ const next = target.candidates.find((candidate) => candidate.candidateId === candidateId);
335
+ return {
336
+ candidateId,
337
+ baseline: base,
338
+ target: next,
339
+ scoreDelta: base?.meanScore === undefined || next?.meanScore === undefined
340
+ ? undefined
341
+ : next.meanScore - base.meanScore,
342
+ totalCostDelta: base?.totalCost === undefined || next?.totalCost === undefined
343
+ ? undefined
344
+ : next.totalCost - base.totalCost,
345
+ p95LatencyDeltaMs: base === undefined || next === undefined
346
+ ? undefined
347
+ : next.p95LatencyMs - base.p95LatencyMs,
348
+ errorRateDelta: base === undefined || next === undefined
349
+ ? undefined
350
+ : next.errorRate - base.errorRate
351
+ };
352
+ })
353
+ };
354
+ };
355
+ const normalizedText = (value, options) => {
356
+ const trimmed = options.trim === false ? value : value.trim();
357
+ return options.caseSensitive === false ? trimmed.toLocaleLowerCase() : trimmed;
358
+ };
359
+ const canonicalJson = (value) => {
360
+ if (value === null || typeof value !== "object") {
361
+ return JSON.stringify(value);
362
+ }
363
+ if (Array.isArray(value)) {
364
+ return `[${value.map(canonicalJson).join(",")}]`;
365
+ }
366
+ return `{${Object.keys(value)
367
+ .sort()
368
+ .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
369
+ .join(",")}}`;
370
+ };
371
+ export const createExactMatchScorer = (options = {}) => ({
372
+ id: options.id ?? "exact_match",
373
+ score(context) {
374
+ if (typeof context.testCase.reference !== "string") {
375
+ throw new ValidationError(`Case ${JSON.stringify(context.testCase.name)} requires a string reference for exact match.`);
376
+ }
377
+ const actual = normalizedText(context.outputText ?? "", options);
378
+ const expected = normalizedText(context.testCase.reference, options);
379
+ return {
380
+ score: actual === expected ? 1 : 0,
381
+ feedback: actual === expected ? undefined : "Output did not exactly match the reference."
382
+ };
383
+ }
384
+ });
385
+ export const createContainsScorer = (options = {}) => ({
386
+ id: options.id ?? "contains",
387
+ score(context) {
388
+ if (typeof context.testCase.reference !== "string") {
389
+ throw new ValidationError(`Case ${JSON.stringify(context.testCase.name)} requires a string reference for contains scoring.`);
390
+ }
391
+ const actual = normalizedText(context.outputText ?? "", { caseSensitive: options.caseSensitive, trim: false });
392
+ const expected = normalizedText(context.testCase.reference, { caseSensitive: options.caseSensitive, trim: false });
393
+ return actual.includes(expected) ? 1 : 0;
394
+ }
395
+ });
396
+ export const createRegexScorer = (options) => ({
397
+ id: options.id ?? "regex",
398
+ score(context) {
399
+ const pattern = typeof options.pattern === "function" ? options.pattern(context.testCase) : options.pattern;
400
+ pattern.lastIndex = 0;
401
+ return pattern.test(context.outputText ?? "") ? 1 : 0;
402
+ }
403
+ });
404
+ export const createJsonValueScorer = (options = {}) => ({
405
+ id: options.id ?? "json_value",
406
+ score(context) {
407
+ if (context.testCase.reference === undefined) {
408
+ throw new ValidationError(`Case ${JSON.stringify(context.testCase.name)} requires a JSON reference.`);
409
+ }
410
+ let actual;
411
+ try {
412
+ actual = JSON.parse(context.outputText ?? "");
413
+ }
414
+ catch {
415
+ return { score: 0, feedback: "Output is not valid JSON." };
416
+ }
417
+ return canonicalJson(actual) === canonicalJson(context.testCase.reference)
418
+ ? 1
419
+ : { score: 0, feedback: "JSON output did not match the reference." };
420
+ }
421
+ });
422
+ export const createJsonSchemaScorer = (schema, options = {}) => ({
423
+ id: options.id ?? "json_schema",
424
+ score(context) {
425
+ let actual;
426
+ try {
427
+ actual = JSON.parse(context.outputText ?? "");
428
+ }
429
+ catch {
430
+ return { score: 0, feedback: "Output is not valid JSON." };
431
+ }
432
+ const result = schema.safeParse(actual);
433
+ return result.success
434
+ ? 1
435
+ : { score: 0, feedback: result.error.issues.map((issue) => issue.message).join("; ") };
436
+ }
437
+ });
438
+ export const createToolCallScorer = (options = {}) => ({
439
+ id: options.id ?? "tool_calls",
440
+ score(context) {
441
+ const expected = context.testCase.expectedToolCalls;
442
+ if (!expected) {
443
+ throw new ValidationError(`Case ${JSON.stringify(context.testCase.name)} requires expectedToolCalls.`);
444
+ }
445
+ const actual = context.toolResults.map((result) => result.toolName);
446
+ const matches = options.requireExactOrder
447
+ ? JSON.stringify(actual) === JSON.stringify(expected)
448
+ : (() => {
449
+ if (actual.length !== expected.length) {
450
+ return false;
451
+ }
452
+ const expectedCounts = new Map();
453
+ for (const name of expected) {
454
+ expectedCounts.set(name, (expectedCounts.get(name) ?? 0) + 1);
455
+ }
456
+ for (const name of actual) {
457
+ const remaining = expectedCounts.get(name);
458
+ if (remaining === undefined || remaining === 0) {
459
+ return false;
460
+ }
461
+ expectedCounts.set(name, remaining - 1);
462
+ }
463
+ return [...expectedCounts.values()].every((count) => count === 0);
464
+ })();
465
+ return matches ? 1 : { score: 0, feedback: `Expected tool calls ${JSON.stringify(expected)}, received ${JSON.stringify(actual)}.` };
466
+ }
467
+ });
468
+ export const createEmbeddingSimilarityScorer = (options) => {
469
+ const referenceCache = new Map();
470
+ return {
471
+ id: options.id ?? "embedding_similarity",
472
+ async score(context) {
473
+ if (typeof context.testCase.reference !== "string") {
474
+ throw new ValidationError(`Case ${JSON.stringify(context.testCase.name)} requires a string reference for embedding similarity.`);
475
+ }
476
+ const referenceKey = `${options.model.provider}\u0000${options.model.modelId}\u0000${context.testCase.name}\u0000${context.testCase.reference}`;
477
+ let referenceEmbeddingPromise = referenceCache.get(referenceKey);
478
+ if (!referenceEmbeddingPromise) {
479
+ referenceEmbeddingPromise = embed({ model: options.model, value: context.testCase.reference }).then((result) => {
480
+ const embedding = result.embeddings[0];
481
+ if (!embedding) {
482
+ throw new ValidationError("Embedding model returned no reference embedding.");
483
+ }
484
+ return embedding;
485
+ });
486
+ referenceCache.set(referenceKey, referenceEmbeddingPromise);
487
+ }
488
+ let referenceEmbedding;
489
+ try {
490
+ referenceEmbedding = await referenceEmbeddingPromise;
491
+ }
492
+ catch (cause) {
493
+ if (referenceCache.get(referenceKey) === referenceEmbeddingPromise) {
494
+ referenceCache.delete(referenceKey);
495
+ }
496
+ throw cause;
497
+ }
498
+ const outputEmbedding = (await embed({ model: options.model, value: context.outputText ?? "" })).embeddings[0];
499
+ if (!outputEmbedding) {
500
+ throw new ValidationError("Embedding model returned no output embedding.");
501
+ }
502
+ const similarity = Math.max(0, Math.min(1, cosineSimilarity(referenceEmbedding, outputEmbedding)));
503
+ const minimum = options.minimumSimilarity;
504
+ return {
505
+ score: minimum === undefined || similarity >= minimum ? similarity : 0,
506
+ metadata: { similarity }
507
+ };
508
+ }
509
+ };
510
+ };
511
+ export const createModelJudgeScorer = (options) => ({
512
+ id: options.id ?? "model_judge",
513
+ async score(context) {
514
+ const response = await generateText({
515
+ model: options.model,
516
+ maxTokens: options.maxTokens ?? 256,
517
+ prompt: [
518
+ "Evaluate the candidate output using the rubric.",
519
+ "Return only JSON with this shape: {\"score\": number between 0 and 1, \"feedback\": string}.",
520
+ `Rubric: ${options.rubric}`,
521
+ `Input: ${context.testCase.prompt ?? JSON.stringify(context.testCase.messages)}`,
522
+ `Reference: ${JSON.stringify(context.testCase.reference)}`,
523
+ `Candidate output: ${context.outputText ?? ""}`
524
+ ].join("\n")
525
+ });
526
+ let parsed;
527
+ try {
528
+ parsed = JSON.parse(response.text);
529
+ }
530
+ catch {
531
+ throw new ValidationError("Model judge returned invalid JSON.");
532
+ }
533
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
534
+ throw new ValidationError("Model judge returned an invalid result object.");
535
+ }
536
+ const score = parsed.score;
537
+ assertFiniteRange(score, "Model judge score", 0, 1);
538
+ const feedback = parsed.feedback;
539
+ if (feedback !== undefined && typeof feedback !== "string") {
540
+ throw new ValidationError("Model judge feedback must be a string.");
541
+ }
542
+ return { score, feedback };
543
+ }
544
+ });
545
+ //# sourceMappingURL=model-evaluation.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"model-evaluation.js","sourceRoot":"","sources":["../src/model-evaluation.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAwC,MAAM,kBAAkB,CAAC;AAE3F,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAYlD,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAU,CAAC;AA0KjE,SAAS,wBAAwB,CAAC,KAAc,EAAE,IAAY;IAC5D,IACE,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QACzB,KAAK,KAAK,KAAK,CAAC,IAAI,EAAE;QACtB,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,EACpC,CAAC;QACD,MAAM,IAAI,eAAe,CAAC,GAAG,IAAI,mFAAmF,CAAC,CAAC;IACxH,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAc,EACd,IAAY,EACZ,OAAe,EACf,OAAe;IAEf,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,OAAO,IAAI,KAAK,GAAG,OAAO,EAAE,CAAC;QAC/F,MAAM,IAAI,eAAe,CAAC,GAAG,IAAI,oCAAoC,OAAO,QAAQ,OAAO,GAAG,CAAC,CAAC;IAClG,CAAC;AACH,CAAC;AAED,MAAM,cAAc,GAAG,CAAC,KAAc,EAAU,EAAE,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAE1G,MAAM,UAAU,GAAG,CAAC,MAAgB,EAAE,QAAgB,EAAU,EAAE;IAChE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACnE,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,CAAC,MAAgB,EAAsB,EAAE,CACpD,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAElG,MAAM,QAAQ,GAAG,CAAC,MAAqC,EAA0B,EAAE;IACjF,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAuB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACnF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG;QACb,aAAa;QACb,mBAAmB;QACnB,kBAAkB;QAClB,cAAc;QACd,iBAAiB;QACjB,aAAa;KACL,CAAC;IACX,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;QAC5G,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,KAAK,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAA6C,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC;IAC9I,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,OAAiC,EAAE,SAAmC,EAA4B,EAAE;IAC1H,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC,OAAO,CAAC;IAC3B,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/E,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,uBAAuB,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC7E,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAAC,KAAwB,EAAE,QAAiB,EAAgB,EAAE,CAAC,CAAC;IAC9F,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;IAChD,0BAA0B,EAAE,KAAK,CAAC,0BAA0B;IAC5D,yBAAyB,EAAE,KAAK,CAAC,yBAAyB;IAC1D,qBAAqB,EAAE,KAAK,CAAC,qBAAqB;IAClD,eAAe,EAAE,KAAK,CAAC,eAAe;IACtC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;IAC5C,QAAQ;CACT,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,CAAC,KAA2B,EAAmD,EAAE;IACrG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,eAAe,CAAC,8CAA8C,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,eAAe,CAAC,mDAAmD,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,eAAe,CAAC,gDAAgD,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,wBAAwB,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAC;QAChE,IAAI,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,eAAe,CAAC,wCAAwC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtG,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,CAAC;YAC1E,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QACpH,CAAC;QACD,IAAI,QAAQ,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzE,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5D,wBAAwB,CAAC,SAAS,CAAC,EAAE,EAAE,cAAc,KAAK,MAAM,CAAC,CAAC;QAClE,IAAI,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,eAAe,CAAC,2CAA2C,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QACxG,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,OAAO,SAAS,CAAC,KAAK,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;YACvE,MAAM,IAAI,eAAe,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,gCAAgC,CAAC,CAAC;QACvG,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,wBAAwB,CAAC,MAAM,CAAC,EAAE,EAAE,WAAW,KAAK,MAAM,CAAC,CAAC;QAC5D,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,eAAe,CAAC,wCAAwC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QAClG,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACzB,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YACvC,MAAM,IAAI,eAAe,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,uBAAuB,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;QAC/E,MAAM,IAAI,eAAe,CAAC,wEAAwE,CAAC,CAAC;IACtG,CAAC;IACD,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,IAAI,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,IAAI,cAAc,GAAG,EAAE,EAAE,CAAC;QACvF,MAAM,IAAI,eAAe,CAAC,0EAA0E,CAAC,CAAC;IACxG,CAAC;IACD,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACrC,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC;AACzC,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,UAAiD,EAAQ,EAAE;IACrF,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO;IACT,CAAC;IACD,IAAI,UAAU,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QAC1C,iBAAiB,CAAC,UAAU,CAAC,YAAY,EAAE,yBAAyB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,UAAU,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QAC1C,iBAAiB,CAAC,UAAU,CAAC,YAAY,EAAE,yBAAyB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9E,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,CAAC,cAAc,EAAE,iBAAiB,CAAU,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;YAC/F,MAAM,IAAI,eAAe,CAAC,cAAc,KAAK,yCAAyC,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,QAAgB,EAAE,MAA4C,EAAwB,EAAE;IACrH,MAAM,UAAU,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC3E,iBAAiB,CAAC,UAAU,CAAC,KAAK,EAAE,UAAU,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACtF,OAAO;QACL,QAAQ;QACR,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;KAC9B,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,KAAK,EAC7B,QAA6B,EAC7B,SAAmC,EACnC,UAAkB,EAClB,KAA2B,EACQ,EAAE;IACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,UAA8B,CAAC;IACnC,IAAI,YAAsC,CAAC;IAC3C,IAAI,KAA6B,CAAC;IAClC,IAAI,WAAW,GAA0B,EAAE,CAAC;IAC5C,IAAI,KAAyB,CAAC;IAE9B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,KAAK,SAAS;YACzC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAS,EAAE;YAClC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,GAAG,SAAS,CAAC,QAAQ;YACrB,GAAG,QAAQ,CAAC,QAAQ;YACpB,GAAG,KAAK;YACR,KAAK,EAAE,SAAS,CAAC,KAAK;SACvB,CAAC,CAAC;QACH,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACnC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACrB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;IAChF,MAAM,OAAO,GAAiC;QAC5C,QAAQ;QACR,SAAS,EAAE;YACT,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ;YAClC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO;YAChC,QAAQ,EAAE,SAAS,CAAC,QAAQ;SAC7B;QACD,UAAU;QACV,UAAU;QACV,YAAY;QACZ,KAAK;QACL,WAAW;QACX,SAAS;QACT,IAAI;QACJ,KAAK;KACN,CAAC;IAEF,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAChG,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAEvE,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,IAAI;QACvB,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ;QAClC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO;QAChC,UAAU;QACV,EAAE,EAAE,KAAK,KAAK,SAAS,IAAI,CAAC,YAAY;QACxC,UAAU;QACV,YAAY;QACZ,KAAK;QACL,SAAS,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC;QACvD,SAAS;QACT,IAAI,EAAE,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACjF,MAAM;QACN,SAAS,EAAE,IAAI,CAAC,aAAa,CAAC;QAC9B,KAAK;QACL,YAAY,EAAE,QAAQ,CAAC,QAAQ;QAC/B,iBAAiB,EAAE,SAAS,CAAC,QAAQ;KACtC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,SAAmC,EACnC,IAAgC,EACC,EAAE;IACnC,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;IACzF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IACnG,MAAM,uBAAuB,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC;IAChF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC;IAC1H,OAAO;QACL,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ;QAClC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO;QAChC,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,cAAc;QACd,UAAU,EAAE,IAAI,CAAC,MAAM,GAAG,cAAc;QACxC,SAAS,EAAE,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,GAAG,IAAI,CAAC,MAAM;QAC/E,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;QAC/D,QAAQ,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;QAC/D,YAAY,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC;QAC/D,YAAY,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC;QAChE,SAAS,EAAE,uBAAuB,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QAC7F,QAAQ,EAAE,uBAAuB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QAC3D,QAAQ,EAAE,uBAAuB,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QAC3F,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC7C,QAAQ,EAAE,SAAS,CAAC,QAAQ;KAC7B,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,2BAA2B,GAAG,CACzC,UAA6C,EAC7C,UAAqC,EACV,EAAE;IAC7B,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAC/B,MAAM,MAAM,GAA+B,EAAE,CAAC;IAC9C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,CACf,IAAkC,EAClC,MAA0B,EAC1B,SAAiB,EACjB,EAAW,EACX,KAAa,EACb,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;YACf,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,IAAI;YACJ,EAAE;YACF,MAAM;YACN,SAAS;YACT,OAAO,EAAE,GAAG,SAAS,CAAC,WAAW,IAAI,KAAK,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,eAAe,SAAS,GAAG;SACxH,CAAC,CAAC;QAEH,IAAI,UAAU,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1C,QAAQ,CACN,gBAAgB,EAChB,SAAS,CAAC,SAAS,EACnB,UAAU,CAAC,YAAY,EACvB,SAAS,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,SAAS,IAAI,UAAU,CAAC,YAAY,EACnF,YAAY,CACb,CAAC;QACJ,CAAC;QACD,IAAI,UAAU,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1C,QAAQ,CACN,gBAAgB,EAChB,SAAS,CAAC,SAAS,EACnB,UAAU,CAAC,YAAY,EACvB,SAAS,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,SAAS,IAAI,UAAU,CAAC,YAAY,EACnF,YAAY,CACb,CAAC;QACJ,CAAC;QACD,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAC7C,QAAQ,CACN,oBAAoB,EACpB,SAAS,CAAC,YAAY,EACtB,UAAU,CAAC,eAAe,EAC1B,SAAS,CAAC,YAAY,IAAI,UAAU,CAAC,eAAe,EACpD,aAAa,CACd,CAAC;QACJ,CAAC;QACD,IAAI,UAAU,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1C,QAAQ,CACN,gBAAgB,EAChB,SAAS,CAAC,SAAS,EACnB,UAAU,CAAC,YAAY,EACvB,SAAS,CAAC,SAAS,IAAI,UAAU,CAAC,YAAY,EAC9C,YAAY,CACb,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,UAAU,EAAE,EAAE,GAAG,UAAU,EAAE;QAC7B,MAAM;KACP,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EAAE,KAA2B,EAAkC,EAAE;IACtG,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CACnD,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;QACtD,SAAS;QACT,QAAQ;QACR,UAAU,EAAE,UAAU,GAAG,CAAC;KAC3B,CAAC,CAAC,CACJ,CACF,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,KAAK,CAA2B,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/D,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACxF,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,MAAM,CAAC;YACrB,MAAM,IAAI,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC/F,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3B,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CACpD,kBAAkB,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE,CAAC,CAAC,CACtF,CAAC;IACF,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,2BAA2B,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtG,OAAO;QACL,aAAa,EAAE,sCAAsC;QACrD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,IAAI,CAAC;QACrD,WAAW;QACX,SAAS,EAAE,IAAI,CAAC,MAAM;QACtB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;QACnD,UAAU;QACV,IAAI;QACJ,IAAI;QACJ,QAAQ,EAAE,KAAK,CAAC,QAAQ;KACzB,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAC3C,QAA+B,EAC/B,MAA6B,EACF,EAAE;IAC7B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC;QAClB,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,CAAC;QAChE,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,CAAC;KAC/D,CAAC,CAAC;IACH,OAAO;QACL,iBAAiB,EAAE,QAAQ,CAAC,SAAS;QACrC,eAAe,EAAE,MAAM,CAAC,SAAS;QACjC,UAAU,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;YAC9C,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,KAAK,WAAW,CAAC,CAAC;YAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,KAAK,WAAW,CAAC,CAAC;YAC1F,OAAO;gBACL,WAAW;gBACX,QAAQ,EAAE,IAAI;gBACd,MAAM,EAAE,IAAI;gBACZ,UAAU,EAAE,IAAI,EAAE,SAAS,KAAK,SAAS,IAAI,IAAI,EAAE,SAAS,KAAK,SAAS;oBACxE,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS;gBACnC,cAAc,EAAE,IAAI,EAAE,SAAS,KAAK,SAAS,IAAI,IAAI,EAAE,SAAS,KAAK,SAAS;oBAC5E,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS;gBACnC,iBAAiB,EAAE,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;oBACzD,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY;gBACzC,cAAc,EAAE,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;oBACtD,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS;aACpC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,KAAa,EAAE,OAAoD,EAAU,EAAE;IACrG,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC9D,OAAO,OAAO,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;AACjF,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,KAAgB,EAAU,EAAE;IACjD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACnD,CAAC;IACD,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;SAC1B,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,CAAE,CAAC,EAAE,CAAC;SACpE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,UAInC,EAAE,EAAyB,EAAE,CAAC,CAAC;IACjC,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,aAAa;IAC/B,KAAK,CAAC,OAAO;QACX,IAAI,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAC1H,CAAC;QACD,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACrE,OAAO;YACL,KAAK,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,QAAQ,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,6CAA6C;SAC1F,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,UAGjC,EAAE,EAAyB,EAAE,CAAC,CAAC;IACjC,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,UAAU;IAC5B,KAAK,CAAC,OAAO;QACX,IAAI,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;QAC/H,CAAC;QACD,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,EAAE,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/G,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACnH,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,OAGjC,EAAyB,EAAE,CAAC,CAAC;IAC5B,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,OAAO;IACzB,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,GAAG,OAAO,OAAO,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QAC5G,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QACtB,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,UAA2B,EAAE,EAAyB,EAAE,CAAC,CAAC;IAC9F,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,YAAY;IAC9B,KAAK,CAAC,OAAO;QACX,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAC7C,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QACxG,CAAC;QACD,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,2BAA2B,EAAE,CAAC;QAC7D,CAAC;QACD,OAAO,aAAa,CAAC,MAAmB,CAAC,KAAK,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YACrF,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,0CAA0C,EAAE,CAAC;IACzE,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,MAAkB,EAAE,UAA2B,EAAE,EAAyB,EAAE,CAAC,CAAC;IACnH,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,aAAa;IAC/B,KAAK,CAAC,OAAO;QACX,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,2BAA2B,EAAE,CAAC;QAC7D,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACxC,OAAO,MAAM,CAAC,OAAO;YACnB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC3F,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,UAAwD,EAAE,EAAyB,EAAE,CAAC,CAAC;IAC1H,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,YAAY;IAC9B,KAAK,CAAC,OAAO;QACX,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QACpD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;QACzG,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,OAAO,CAAC,iBAAiB;YACvC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;YACrD,CAAC,CAAC,CAAC,GAAG,EAAE;gBACJ,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACtC,OAAO,KAAK,CAAC;gBACf,CAAC;gBACD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;gBACjD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;oBAC5B,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAChE,CAAC;gBACD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;oBAC1B,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC3C,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;wBAC/C,OAAO,KAAK,CAAC;oBACf,CAAC;oBACD,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;gBAC1C,CAAC;gBACD,OAAO,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;YACpE,CAAC,CAAC,EAAE,CAAC;QACT,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,uBAAuB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IACtI,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,+BAA+B,GAAG,CAAC,OAI/C,EAAyB,EAAE;IAC1B,MAAM,cAAc,GAAG,IAAI,GAAG,EAA6B,CAAC;IAC5D,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,sBAAsB;QACxC,KAAK,CAAC,KAAK,CAAC,OAAO;YACjB,IAAI,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBACnD,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACnI,CAAC;YACD,MAAM,YAAY,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,SAAS,OAAO,CAAC,KAAK,CAAC,OAAO,SAAS,OAAO,CAAC,QAAQ,CAAC,IAAI,SAAS,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAChJ,IAAI,yBAAyB,GAAG,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACjE,IAAI,CAAC,yBAAyB,EAAE,CAAC;gBAC/B,yBAAyB,GAAG,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;oBAC7G,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;oBACvC,IAAI,CAAC,SAAS,EAAE,CAAC;wBACf,MAAM,IAAI,eAAe,CAAC,kDAAkD,CAAC,CAAC;oBAChF,CAAC;oBACD,OAAO,SAAS,CAAC;gBACnB,CAAC,CAAC,CAAC;gBACH,cAAc,CAAC,GAAG,CAAC,YAAY,EAAE,yBAAyB,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,kBAA4B,CAAC;YACjC,IAAI,CAAC;gBACH,kBAAkB,GAAG,MAAM,yBAAyB,CAAC;YACvD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,yBAAyB,EAAE,CAAC;oBACnE,cAAc,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;gBACtC,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,eAAe,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC/G,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,MAAM,IAAI,eAAe,CAAC,+CAA+C,CAAC,CAAC;YAC7E,CAAC;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,gBAAgB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;YACnG,MAAM,OAAO,GAAG,OAAO,CAAC,iBAAiB,CAAC;YAC1C,OAAO;gBACL,KAAK,EAAE,OAAO,KAAK,SAAS,IAAI,UAAU,IAAI,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBACtE,QAAQ,EAAE,EAAE,UAAU,EAAE;aACzB,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,OAKtC,EAAyB,EAAE,CAAC,CAAC;IAC5B,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,aAAa;IAC/B,KAAK,CAAC,KAAK,CAAC,OAAO;QACjB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;YACnC,MAAM,EAAE;gBACN,iDAAiD;gBACjD,8FAA8F;gBAC9F,WAAW,OAAO,CAAC,MAAM,EAAE;gBAC3B,UAAU,OAAO,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;gBAChF,cAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBAC1D,qBAAqB,OAAO,CAAC,UAAU,IAAI,EAAE,EAAE;aAChD,CAAC,IAAI,CAAC,IAAI,CAAC;SACb,CAAC,CAAC;QACH,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,eAAe,CAAC,oCAAoC,CAAC,CAAC;QAClE,CAAC;QACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3E,MAAM,IAAI,eAAe,CAAC,gDAAgD,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,KAAK,GAAI,MAA8B,CAAC,KAAK,CAAC;QACpD,iBAAiB,CAAC,KAAK,EAAE,mBAAmB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAI,MAAiC,CAAC,QAAQ,CAAC;QAC7D,IAAI,QAAQ,KAAK,SAAS,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,eAAe,CAAC,wCAAwC,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IAC7B,CAAC;CACF,CAAC,CAAC"}