@zhivex-ai/core 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api-stability.d.ts.map +1 -1
- package/dist/api-stability.js +18 -3
- package/dist/api-stability.js.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/model-evaluation.d.ts +185 -0
- package/dist/model-evaluation.d.ts.map +1 -0
- package/dist/model-evaluation.js +545 -0
- package/dist/model-evaluation.js.map +1 -0
- package/dist/model-resolver.d.ts +105 -0
- package/dist/model-resolver.d.ts.map +1 -0
- package/dist/model-resolver.js +396 -0
- package/dist/model-resolver.js.map +1 -0
- package/dist/raw-provider-options.d.ts +20 -0
- package/dist/raw-provider-options.d.ts.map +1 -0
- package/dist/raw-provider-options.js +63 -0
- package/dist/raw-provider-options.js.map +1 -0
- package/dist/types.d.ts +5 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,545 @@
|
|
|
1
|
+
import { estimateTokenCost } from "./agent-trace.js";
|
|
2
|
+
import { embed } from "./embed.js";
|
|
3
|
+
import { ValidationError } from "./errors.js";
|
|
4
|
+
import { generateText } from "./generate-text.js";
|
|
5
|
+
import { cosineSimilarity } from "./retrieval.js";
|
|
6
|
+
export const MODEL_EVALUATION_REPORT_SCHEMA_VERSION = 1;
|
|
7
|
+
function assertNonEmptyIdentifier(value, path) {
|
|
8
|
+
if (typeof value !== "string" ||
|
|
9
|
+
value.trim().length === 0 ||
|
|
10
|
+
value !== value.trim() ||
|
|
11
|
+
/[\u0000-\u001f\u007f]/u.test(value)) {
|
|
12
|
+
throw new ValidationError(`${path} must be a non-empty string without surrounding whitespace or control characters.`);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
function assertFiniteRange(value, path, minimum, maximum) {
|
|
16
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < minimum || value > maximum) {
|
|
17
|
+
throw new ValidationError(`${path} must be a finite number between ${minimum} and ${maximum}.`);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
const normalizeError = (error) => error instanceof Error ? error.message : String(error);
|
|
21
|
+
const percentile = (values, quantile) => {
|
|
22
|
+
if (values.length === 0) {
|
|
23
|
+
return 0;
|
|
24
|
+
}
|
|
25
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
26
|
+
const index = Math.max(0, Math.ceil(quantile * sorted.length) - 1);
|
|
27
|
+
return sorted[index] ?? 0;
|
|
28
|
+
};
|
|
29
|
+
const mean = (values) => values.length === 0 ? undefined : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
30
|
+
const sumUsage = (usages) => {
|
|
31
|
+
const present = usages.filter((usage) => usage !== undefined);
|
|
32
|
+
if (present.length === 0) {
|
|
33
|
+
return undefined;
|
|
34
|
+
}
|
|
35
|
+
const fields = [
|
|
36
|
+
"inputTokens",
|
|
37
|
+
"cachedInputTokens",
|
|
38
|
+
"cacheWriteTokens",
|
|
39
|
+
"outputTokens",
|
|
40
|
+
"reasoningTokens",
|
|
41
|
+
"totalTokens"
|
|
42
|
+
];
|
|
43
|
+
const total = {};
|
|
44
|
+
for (const field of fields) {
|
|
45
|
+
const values = present.map((usage) => usage[field]).filter((value) => value !== undefined);
|
|
46
|
+
if (values.length > 0) {
|
|
47
|
+
total[field] = values.reduce((sum, value) => sum + value, 0);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const speeds = new Set(present.map((usage) => usage.speed).filter((value) => value !== undefined));
|
|
51
|
+
if (speeds.size === 1) {
|
|
52
|
+
total.speed = [...speeds][0];
|
|
53
|
+
}
|
|
54
|
+
return total;
|
|
55
|
+
};
|
|
56
|
+
const catalogPricing = (catalog, candidate) => {
|
|
57
|
+
if (candidate.pricing) {
|
|
58
|
+
return candidate.pricing;
|
|
59
|
+
}
|
|
60
|
+
const entry = catalog?.find(candidate.model.provider, candidate.model.modelId);
|
|
61
|
+
if (!entry) {
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
return pricingFromCatalogEntry(entry, catalog?.metadata.pricing?.currency);
|
|
65
|
+
};
|
|
66
|
+
const pricingFromCatalogEntry = (entry, currency) => ({
|
|
67
|
+
inputCostPer1kTokens: entry.inputCostPer1kTokens,
|
|
68
|
+
cachedInputCostPer1kTokens: entry.cachedInputCostPer1kTokens,
|
|
69
|
+
cacheWriteCostPer1kTokens: entry.cacheWriteCostPer1kTokens,
|
|
70
|
+
outputCostPer1kTokens: entry.outputCostPer1kTokens,
|
|
71
|
+
costPer1kTokens: entry.costPer1kTokens,
|
|
72
|
+
longContextPricing: entry.longContextPricing,
|
|
73
|
+
currency
|
|
74
|
+
});
|
|
75
|
+
const validateSuite = (suite) => {
|
|
76
|
+
if (!Array.isArray(suite.cases) || suite.cases.length === 0) {
|
|
77
|
+
throw new ValidationError("Model evaluation requires at least one case.");
|
|
78
|
+
}
|
|
79
|
+
if (!Array.isArray(suite.candidates) || suite.candidates.length === 0) {
|
|
80
|
+
throw new ValidationError("Model evaluation requires at least one candidate.");
|
|
81
|
+
}
|
|
82
|
+
if (!Array.isArray(suite.scorers) || suite.scorers.length === 0) {
|
|
83
|
+
throw new ValidationError("Model evaluation requires at least one scorer.");
|
|
84
|
+
}
|
|
85
|
+
const caseNames = new Set();
|
|
86
|
+
for (const [index, testCase] of suite.cases.entries()) {
|
|
87
|
+
assertNonEmptyIdentifier(testCase.name, `cases[${index}].name`);
|
|
88
|
+
if (caseNames.has(testCase.name)) {
|
|
89
|
+
throw new ValidationError(`Duplicate model evaluation case name ${JSON.stringify(testCase.name)}.`);
|
|
90
|
+
}
|
|
91
|
+
caseNames.add(testCase.name);
|
|
92
|
+
if ((testCase.prompt === undefined) === (testCase.messages === undefined)) {
|
|
93
|
+
throw new ValidationError(`Case ${JSON.stringify(testCase.name)} must define exactly one of prompt or messages.`);
|
|
94
|
+
}
|
|
95
|
+
if (testCase.messages !== undefined && !Array.isArray(testCase.messages)) {
|
|
96
|
+
throw new ValidationError(`Case ${JSON.stringify(testCase.name)} messages must be an array.`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
const candidateIds = new Set();
|
|
100
|
+
for (const [index, candidate] of suite.candidates.entries()) {
|
|
101
|
+
assertNonEmptyIdentifier(candidate.id, `candidates[${index}].id`);
|
|
102
|
+
if (candidateIds.has(candidate.id)) {
|
|
103
|
+
throw new ValidationError(`Duplicate model evaluation candidate id ${JSON.stringify(candidate.id)}.`);
|
|
104
|
+
}
|
|
105
|
+
candidateIds.add(candidate.id);
|
|
106
|
+
if (!candidate.model || typeof candidate.model.generate !== "function") {
|
|
107
|
+
throw new ValidationError(`Candidate ${JSON.stringify(candidate.id)} must define a language model.`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
const scorerIds = new Set();
|
|
111
|
+
for (const [index, scorer] of suite.scorers.entries()) {
|
|
112
|
+
assertNonEmptyIdentifier(scorer.id, `scorers[${index}].id`);
|
|
113
|
+
if (scorerIds.has(scorer.id)) {
|
|
114
|
+
throw new ValidationError(`Duplicate model evaluation scorer id ${JSON.stringify(scorer.id)}.`);
|
|
115
|
+
}
|
|
116
|
+
scorerIds.add(scorer.id);
|
|
117
|
+
if (typeof scorer.score !== "function") {
|
|
118
|
+
throw new ValidationError(`Scorer ${JSON.stringify(scorer.id)} must define score().`);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
const repetitions = suite.repetitions ?? 1;
|
|
122
|
+
if (!Number.isSafeInteger(repetitions) || repetitions < 1 || repetitions > 100) {
|
|
123
|
+
throw new ValidationError("Model evaluation repetitions must be a safe integer between 1 and 100.");
|
|
124
|
+
}
|
|
125
|
+
const maxConcurrency = suite.maxConcurrency ?? 4;
|
|
126
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 64) {
|
|
127
|
+
throw new ValidationError("Model evaluation maxConcurrency must be a safe integer between 1 and 64.");
|
|
128
|
+
}
|
|
129
|
+
validateThresholds(suite.thresholds);
|
|
130
|
+
return { repetitions, maxConcurrency };
|
|
131
|
+
};
|
|
132
|
+
const validateThresholds = (thresholds) => {
|
|
133
|
+
if (!thresholds) {
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
if (thresholds.minMeanScore !== undefined) {
|
|
137
|
+
assertFiniteRange(thresholds.minMeanScore, "thresholds.minMeanScore", 0, 1);
|
|
138
|
+
}
|
|
139
|
+
if (thresholds.maxErrorRate !== undefined) {
|
|
140
|
+
assertFiniteRange(thresholds.maxErrorRate, "thresholds.maxErrorRate", 0, 1);
|
|
141
|
+
}
|
|
142
|
+
for (const field of ["maxTotalCost", "maxP95LatencyMs"]) {
|
|
143
|
+
const value = thresholds[field];
|
|
144
|
+
if (value !== undefined && (typeof value !== "number" || !Number.isFinite(value) || value < 0)) {
|
|
145
|
+
throw new ValidationError(`thresholds.${field} must be a finite, non-negative number.`);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
const normalizeScorerResult = (scorerId, result) => {
|
|
150
|
+
const normalized = typeof result === "number" ? { score: result } : result;
|
|
151
|
+
assertFiniteRange(normalized.score, `Scorer ${JSON.stringify(scorerId)} score`, 0, 1);
|
|
152
|
+
return {
|
|
153
|
+
scorerId,
|
|
154
|
+
score: normalized.score,
|
|
155
|
+
feedback: normalized.feedback,
|
|
156
|
+
metadata: normalized.metadata
|
|
157
|
+
};
|
|
158
|
+
};
|
|
159
|
+
const evaluateSingleRun = async (testCase, candidate, repetition, suite) => {
|
|
160
|
+
const startedAt = Date.now();
|
|
161
|
+
let outputText;
|
|
162
|
+
let finishReason;
|
|
163
|
+
let usage;
|
|
164
|
+
let toolResults = [];
|
|
165
|
+
let error;
|
|
166
|
+
try {
|
|
167
|
+
const input = testCase.prompt === undefined
|
|
168
|
+
? { messages: testCase.messages }
|
|
169
|
+
: { prompt: testCase.prompt };
|
|
170
|
+
const output = await generateText({
|
|
171
|
+
...candidate.settings,
|
|
172
|
+
...testCase.settings,
|
|
173
|
+
...input,
|
|
174
|
+
model: candidate.model
|
|
175
|
+
});
|
|
176
|
+
outputText = output.text;
|
|
177
|
+
finishReason = output.finishReason;
|
|
178
|
+
usage = output.usage;
|
|
179
|
+
toolResults = output.toolResults;
|
|
180
|
+
}
|
|
181
|
+
catch (cause) {
|
|
182
|
+
error = normalizeError(cause);
|
|
183
|
+
}
|
|
184
|
+
const latencyMs = Math.max(0, Date.now() - startedAt);
|
|
185
|
+
const cost = estimateTokenCost(usage, catalogPricing(suite.catalog, candidate));
|
|
186
|
+
const context = {
|
|
187
|
+
testCase,
|
|
188
|
+
candidate: {
|
|
189
|
+
id: candidate.id,
|
|
190
|
+
provider: candidate.model.provider,
|
|
191
|
+
modelId: candidate.model.modelId,
|
|
192
|
+
metadata: candidate.metadata
|
|
193
|
+
},
|
|
194
|
+
repetition,
|
|
195
|
+
outputText,
|
|
196
|
+
finishReason,
|
|
197
|
+
usage,
|
|
198
|
+
toolResults,
|
|
199
|
+
latencyMs,
|
|
200
|
+
cost,
|
|
201
|
+
error
|
|
202
|
+
};
|
|
203
|
+
const scores = [];
|
|
204
|
+
for (const scorer of suite.scorers) {
|
|
205
|
+
try {
|
|
206
|
+
scores.push(normalizeScorerResult(scorer.id, await scorer.score(context)));
|
|
207
|
+
}
|
|
208
|
+
catch (cause) {
|
|
209
|
+
scores.push({ scorerId: scorer.id, error: normalizeError(cause) });
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
const numericScores = scores.flatMap((score) => score.score === undefined ? [] : [score.score]);
|
|
213
|
+
const scorerFailed = scores.some((score) => score.error !== undefined);
|
|
214
|
+
return {
|
|
215
|
+
caseName: testCase.name,
|
|
216
|
+
candidateId: candidate.id,
|
|
217
|
+
provider: candidate.model.provider,
|
|
218
|
+
modelId: candidate.model.modelId,
|
|
219
|
+
repetition,
|
|
220
|
+
ok: error === undefined && !scorerFailed,
|
|
221
|
+
outputText,
|
|
222
|
+
finishReason,
|
|
223
|
+
usage,
|
|
224
|
+
toolCalls: toolResults.map((result) => result.toolName),
|
|
225
|
+
latencyMs,
|
|
226
|
+
cost: cost.totalCost === undefined && cost.usage === undefined ? undefined : cost,
|
|
227
|
+
scores,
|
|
228
|
+
meanScore: mean(numericScores),
|
|
229
|
+
error,
|
|
230
|
+
caseMetadata: testCase.metadata,
|
|
231
|
+
candidateMetadata: candidate.metadata
|
|
232
|
+
};
|
|
233
|
+
};
|
|
234
|
+
const summarizeCandidate = (candidate, runs) => {
|
|
235
|
+
const successfulRuns = runs.filter((run) => run.ok).length;
|
|
236
|
+
const scores = runs.flatMap((run) => run.meanScore === undefined ? [] : [run.meanScore]);
|
|
237
|
+
const costs = runs.flatMap((run) => run.cost?.totalCost === undefined ? [] : [run.cost.totalCost]);
|
|
238
|
+
const hasCompleteCostEstimate = runs.length > 0 && costs.length === runs.length;
|
|
239
|
+
const currencies = new Set(runs.map((run) => run.cost?.currency).filter((value) => value !== undefined));
|
|
240
|
+
return {
|
|
241
|
+
candidateId: candidate.id,
|
|
242
|
+
provider: candidate.model.provider,
|
|
243
|
+
modelId: candidate.model.modelId,
|
|
244
|
+
runs: runs.length,
|
|
245
|
+
successfulRuns,
|
|
246
|
+
failedRuns: runs.length - successfulRuns,
|
|
247
|
+
errorRate: runs.length === 0 ? 0 : (runs.length - successfulRuns) / runs.length,
|
|
248
|
+
meanScore: mean(scores),
|
|
249
|
+
minScore: scores.length === 0 ? undefined : Math.min(...scores),
|
|
250
|
+
maxScore: scores.length === 0 ? undefined : Math.max(...scores),
|
|
251
|
+
p50LatencyMs: percentile(runs.map((run) => run.latencyMs), 0.5),
|
|
252
|
+
p95LatencyMs: percentile(runs.map((run) => run.latencyMs), 0.95),
|
|
253
|
+
totalCost: hasCompleteCostEstimate ? costs.reduce((sum, value) => sum + value, 0) : undefined,
|
|
254
|
+
meanCost: hasCompleteCostEstimate ? mean(costs) : undefined,
|
|
255
|
+
currency: hasCompleteCostEstimate && currencies.size === 1 ? [...currencies][0] : undefined,
|
|
256
|
+
usage: sumUsage(runs.map((run) => run.usage)),
|
|
257
|
+
metadata: candidate.metadata
|
|
258
|
+
};
|
|
259
|
+
};
|
|
260
|
+
export const evaluateModelEvaluationGate = (candidates, thresholds) => {
|
|
261
|
+
validateThresholds(thresholds);
|
|
262
|
+
const checks = [];
|
|
263
|
+
for (const candidate of candidates) {
|
|
264
|
+
const addCheck = (code, actual, threshold, ok, label) => checks.push({
|
|
265
|
+
candidateId: candidate.candidateId,
|
|
266
|
+
code,
|
|
267
|
+
ok,
|
|
268
|
+
actual,
|
|
269
|
+
threshold,
|
|
270
|
+
message: `${candidate.candidateId} ${label}: ${actual === undefined ? "unavailable" : actual}; threshold ${threshold}.`
|
|
271
|
+
});
|
|
272
|
+
if (thresholds.minMeanScore !== undefined) {
|
|
273
|
+
addCheck("min_mean_score", candidate.meanScore, thresholds.minMeanScore, candidate.meanScore !== undefined && candidate.meanScore >= thresholds.minMeanScore, "mean score");
|
|
274
|
+
}
|
|
275
|
+
if (thresholds.maxTotalCost !== undefined) {
|
|
276
|
+
addCheck("max_total_cost", candidate.totalCost, thresholds.maxTotalCost, candidate.totalCost !== undefined && candidate.totalCost <= thresholds.maxTotalCost, "total cost");
|
|
277
|
+
}
|
|
278
|
+
if (thresholds.maxP95LatencyMs !== undefined) {
|
|
279
|
+
addCheck("max_p95_latency_ms", candidate.p95LatencyMs, thresholds.maxP95LatencyMs, candidate.p95LatencyMs <= thresholds.maxP95LatencyMs, "p95 latency");
|
|
280
|
+
}
|
|
281
|
+
if (thresholds.maxErrorRate !== undefined) {
|
|
282
|
+
addCheck("max_error_rate", candidate.errorRate, thresholds.maxErrorRate, candidate.errorRate <= thresholds.maxErrorRate, "error rate");
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
return {
|
|
286
|
+
ok: checks.every((check) => check.ok),
|
|
287
|
+
thresholds: { ...thresholds },
|
|
288
|
+
checks
|
|
289
|
+
};
|
|
290
|
+
};
|
|
291
|
+
export const runModelEvaluation = async (suite) => {
|
|
292
|
+
const { repetitions, maxConcurrency } = validateSuite(suite);
|
|
293
|
+
const tasks = suite.candidates.flatMap((candidate) => suite.cases.flatMap((testCase) => Array.from({ length: repetitions }, (_, repetition) => ({
|
|
294
|
+
candidate,
|
|
295
|
+
testCase,
|
|
296
|
+
repetition: repetition + 1
|
|
297
|
+
}))));
|
|
298
|
+
const runs = new Array(tasks.length);
|
|
299
|
+
let cursor = 0;
|
|
300
|
+
const workers = Array.from({ length: Math.min(maxConcurrency, tasks.length) }, async () => {
|
|
301
|
+
while (cursor < tasks.length) {
|
|
302
|
+
const index = cursor;
|
|
303
|
+
cursor += 1;
|
|
304
|
+
const task = tasks[index];
|
|
305
|
+
runs[index] = await evaluateSingleRun(task.testCase, task.candidate, task.repetition, suite);
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
await Promise.all(workers);
|
|
309
|
+
const candidates = suite.candidates.map((candidate) => summarizeCandidate(candidate, runs.filter((run) => run.candidateId === candidate.id)));
|
|
310
|
+
const gate = suite.thresholds ? evaluateModelEvaluationGate(candidates, suite.thresholds) : undefined;
|
|
311
|
+
return {
|
|
312
|
+
schemaVersion: MODEL_EVALUATION_REPORT_SCHEMA_VERSION,
|
|
313
|
+
createdAt: new Date().toISOString(),
|
|
314
|
+
ok: runs.every((run) => run.ok) && (gate?.ok ?? true),
|
|
315
|
+
repetitions,
|
|
316
|
+
totalRuns: runs.length,
|
|
317
|
+
cases: suite.cases.map((testCase) => testCase.name),
|
|
318
|
+
candidates,
|
|
319
|
+
runs,
|
|
320
|
+
gate,
|
|
321
|
+
metadata: suite.metadata
|
|
322
|
+
};
|
|
323
|
+
};
|
|
324
|
+
export const compareModelEvaluationReports = (baseline, target) => {
|
|
325
|
+
const ids = new Set([
|
|
326
|
+
...baseline.candidates.map((candidate) => candidate.candidateId),
|
|
327
|
+
...target.candidates.map((candidate) => candidate.candidateId)
|
|
328
|
+
]);
|
|
329
|
+
return {
|
|
330
|
+
baselineCreatedAt: baseline.createdAt,
|
|
331
|
+
targetCreatedAt: target.createdAt,
|
|
332
|
+
candidates: [...ids].sort().map((candidateId) => {
|
|
333
|
+
const base = baseline.candidates.find((candidate) => candidate.candidateId === candidateId);
|
|
334
|
+
const next = target.candidates.find((candidate) => candidate.candidateId === candidateId);
|
|
335
|
+
return {
|
|
336
|
+
candidateId,
|
|
337
|
+
baseline: base,
|
|
338
|
+
target: next,
|
|
339
|
+
scoreDelta: base?.meanScore === undefined || next?.meanScore === undefined
|
|
340
|
+
? undefined
|
|
341
|
+
: next.meanScore - base.meanScore,
|
|
342
|
+
totalCostDelta: base?.totalCost === undefined || next?.totalCost === undefined
|
|
343
|
+
? undefined
|
|
344
|
+
: next.totalCost - base.totalCost,
|
|
345
|
+
p95LatencyDeltaMs: base === undefined || next === undefined
|
|
346
|
+
? undefined
|
|
347
|
+
: next.p95LatencyMs - base.p95LatencyMs,
|
|
348
|
+
errorRateDelta: base === undefined || next === undefined
|
|
349
|
+
? undefined
|
|
350
|
+
: next.errorRate - base.errorRate
|
|
351
|
+
};
|
|
352
|
+
})
|
|
353
|
+
};
|
|
354
|
+
};
|
|
355
|
+
const normalizedText = (value, options) => {
|
|
356
|
+
const trimmed = options.trim === false ? value : value.trim();
|
|
357
|
+
return options.caseSensitive === false ? trimmed.toLocaleLowerCase() : trimmed;
|
|
358
|
+
};
|
|
359
|
+
const canonicalJson = (value) => {
|
|
360
|
+
if (value === null || typeof value !== "object") {
|
|
361
|
+
return JSON.stringify(value);
|
|
362
|
+
}
|
|
363
|
+
if (Array.isArray(value)) {
|
|
364
|
+
return `[${value.map(canonicalJson).join(",")}]`;
|
|
365
|
+
}
|
|
366
|
+
return `{${Object.keys(value)
|
|
367
|
+
.sort()
|
|
368
|
+
.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
|
|
369
|
+
.join(",")}}`;
|
|
370
|
+
};
|
|
371
|
+
export const createExactMatchScorer = (options = {}) => ({
|
|
372
|
+
id: options.id ?? "exact_match",
|
|
373
|
+
score(context) {
|
|
374
|
+
if (typeof context.testCase.reference !== "string") {
|
|
375
|
+
throw new ValidationError(`Case ${JSON.stringify(context.testCase.name)} requires a string reference for exact match.`);
|
|
376
|
+
}
|
|
377
|
+
const actual = normalizedText(context.outputText ?? "", options);
|
|
378
|
+
const expected = normalizedText(context.testCase.reference, options);
|
|
379
|
+
return {
|
|
380
|
+
score: actual === expected ? 1 : 0,
|
|
381
|
+
feedback: actual === expected ? undefined : "Output did not exactly match the reference."
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
});
|
|
385
|
+
export const createContainsScorer = (options = {}) => ({
|
|
386
|
+
id: options.id ?? "contains",
|
|
387
|
+
score(context) {
|
|
388
|
+
if (typeof context.testCase.reference !== "string") {
|
|
389
|
+
throw new ValidationError(`Case ${JSON.stringify(context.testCase.name)} requires a string reference for contains scoring.`);
|
|
390
|
+
}
|
|
391
|
+
const actual = normalizedText(context.outputText ?? "", { caseSensitive: options.caseSensitive, trim: false });
|
|
392
|
+
const expected = normalizedText(context.testCase.reference, { caseSensitive: options.caseSensitive, trim: false });
|
|
393
|
+
return actual.includes(expected) ? 1 : 0;
|
|
394
|
+
}
|
|
395
|
+
});
|
|
396
|
+
export const createRegexScorer = (options) => ({
|
|
397
|
+
id: options.id ?? "regex",
|
|
398
|
+
score(context) {
|
|
399
|
+
const pattern = typeof options.pattern === "function" ? options.pattern(context.testCase) : options.pattern;
|
|
400
|
+
pattern.lastIndex = 0;
|
|
401
|
+
return pattern.test(context.outputText ?? "") ? 1 : 0;
|
|
402
|
+
}
|
|
403
|
+
});
|
|
404
|
+
export const createJsonValueScorer = (options = {}) => ({
|
|
405
|
+
id: options.id ?? "json_value",
|
|
406
|
+
score(context) {
|
|
407
|
+
if (context.testCase.reference === undefined) {
|
|
408
|
+
throw new ValidationError(`Case ${JSON.stringify(context.testCase.name)} requires a JSON reference.`);
|
|
409
|
+
}
|
|
410
|
+
let actual;
|
|
411
|
+
try {
|
|
412
|
+
actual = JSON.parse(context.outputText ?? "");
|
|
413
|
+
}
|
|
414
|
+
catch {
|
|
415
|
+
return { score: 0, feedback: "Output is not valid JSON." };
|
|
416
|
+
}
|
|
417
|
+
return canonicalJson(actual) === canonicalJson(context.testCase.reference)
|
|
418
|
+
? 1
|
|
419
|
+
: { score: 0, feedback: "JSON output did not match the reference." };
|
|
420
|
+
}
|
|
421
|
+
});
|
|
422
|
+
export const createJsonSchemaScorer = (schema, options = {}) => ({
|
|
423
|
+
id: options.id ?? "json_schema",
|
|
424
|
+
score(context) {
|
|
425
|
+
let actual;
|
|
426
|
+
try {
|
|
427
|
+
actual = JSON.parse(context.outputText ?? "");
|
|
428
|
+
}
|
|
429
|
+
catch {
|
|
430
|
+
return { score: 0, feedback: "Output is not valid JSON." };
|
|
431
|
+
}
|
|
432
|
+
const result = schema.safeParse(actual);
|
|
433
|
+
return result.success
|
|
434
|
+
? 1
|
|
435
|
+
: { score: 0, feedback: result.error.issues.map((issue) => issue.message).join("; ") };
|
|
436
|
+
}
|
|
437
|
+
});
|
|
438
|
+
export const createToolCallScorer = (options = {}) => ({
|
|
439
|
+
id: options.id ?? "tool_calls",
|
|
440
|
+
score(context) {
|
|
441
|
+
const expected = context.testCase.expectedToolCalls;
|
|
442
|
+
if (!expected) {
|
|
443
|
+
throw new ValidationError(`Case ${JSON.stringify(context.testCase.name)} requires expectedToolCalls.`);
|
|
444
|
+
}
|
|
445
|
+
const actual = context.toolResults.map((result) => result.toolName);
|
|
446
|
+
const matches = options.requireExactOrder
|
|
447
|
+
? JSON.stringify(actual) === JSON.stringify(expected)
|
|
448
|
+
: (() => {
|
|
449
|
+
if (actual.length !== expected.length) {
|
|
450
|
+
return false;
|
|
451
|
+
}
|
|
452
|
+
const expectedCounts = new Map();
|
|
453
|
+
for (const name of expected) {
|
|
454
|
+
expectedCounts.set(name, (expectedCounts.get(name) ?? 0) + 1);
|
|
455
|
+
}
|
|
456
|
+
for (const name of actual) {
|
|
457
|
+
const remaining = expectedCounts.get(name);
|
|
458
|
+
if (remaining === undefined || remaining === 0) {
|
|
459
|
+
return false;
|
|
460
|
+
}
|
|
461
|
+
expectedCounts.set(name, remaining - 1);
|
|
462
|
+
}
|
|
463
|
+
return [...expectedCounts.values()].every((count) => count === 0);
|
|
464
|
+
})();
|
|
465
|
+
return matches ? 1 : { score: 0, feedback: `Expected tool calls ${JSON.stringify(expected)}, received ${JSON.stringify(actual)}.` };
|
|
466
|
+
}
|
|
467
|
+
});
|
|
468
|
+
export const createEmbeddingSimilarityScorer = (options) => {
|
|
469
|
+
const referenceCache = new Map();
|
|
470
|
+
return {
|
|
471
|
+
id: options.id ?? "embedding_similarity",
|
|
472
|
+
async score(context) {
|
|
473
|
+
if (typeof context.testCase.reference !== "string") {
|
|
474
|
+
throw new ValidationError(`Case ${JSON.stringify(context.testCase.name)} requires a string reference for embedding similarity.`);
|
|
475
|
+
}
|
|
476
|
+
const referenceKey = `${options.model.provider}\u0000${options.model.modelId}\u0000${context.testCase.name}\u0000${context.testCase.reference}`;
|
|
477
|
+
let referenceEmbeddingPromise = referenceCache.get(referenceKey);
|
|
478
|
+
if (!referenceEmbeddingPromise) {
|
|
479
|
+
referenceEmbeddingPromise = embed({ model: options.model, value: context.testCase.reference }).then((result) => {
|
|
480
|
+
const embedding = result.embeddings[0];
|
|
481
|
+
if (!embedding) {
|
|
482
|
+
throw new ValidationError("Embedding model returned no reference embedding.");
|
|
483
|
+
}
|
|
484
|
+
return embedding;
|
|
485
|
+
});
|
|
486
|
+
referenceCache.set(referenceKey, referenceEmbeddingPromise);
|
|
487
|
+
}
|
|
488
|
+
let referenceEmbedding;
|
|
489
|
+
try {
|
|
490
|
+
referenceEmbedding = await referenceEmbeddingPromise;
|
|
491
|
+
}
|
|
492
|
+
catch (cause) {
|
|
493
|
+
if (referenceCache.get(referenceKey) === referenceEmbeddingPromise) {
|
|
494
|
+
referenceCache.delete(referenceKey);
|
|
495
|
+
}
|
|
496
|
+
throw cause;
|
|
497
|
+
}
|
|
498
|
+
const outputEmbedding = (await embed({ model: options.model, value: context.outputText ?? "" })).embeddings[0];
|
|
499
|
+
if (!outputEmbedding) {
|
|
500
|
+
throw new ValidationError("Embedding model returned no output embedding.");
|
|
501
|
+
}
|
|
502
|
+
const similarity = Math.max(0, Math.min(1, cosineSimilarity(referenceEmbedding, outputEmbedding)));
|
|
503
|
+
const minimum = options.minimumSimilarity;
|
|
504
|
+
return {
|
|
505
|
+
score: minimum === undefined || similarity >= minimum ? similarity : 0,
|
|
506
|
+
metadata: { similarity }
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
};
|
|
510
|
+
};
|
|
511
|
+
export const createModelJudgeScorer = (options) => ({
|
|
512
|
+
id: options.id ?? "model_judge",
|
|
513
|
+
async score(context) {
|
|
514
|
+
const response = await generateText({
|
|
515
|
+
model: options.model,
|
|
516
|
+
maxTokens: options.maxTokens ?? 256,
|
|
517
|
+
prompt: [
|
|
518
|
+
"Evaluate the candidate output using the rubric.",
|
|
519
|
+
"Return only JSON with this shape: {\"score\": number between 0 and 1, \"feedback\": string}.",
|
|
520
|
+
`Rubric: ${options.rubric}`,
|
|
521
|
+
`Input: ${context.testCase.prompt ?? JSON.stringify(context.testCase.messages)}`,
|
|
522
|
+
`Reference: ${JSON.stringify(context.testCase.reference)}`,
|
|
523
|
+
`Candidate output: ${context.outputText ?? ""}`
|
|
524
|
+
].join("\n")
|
|
525
|
+
});
|
|
526
|
+
let parsed;
|
|
527
|
+
try {
|
|
528
|
+
parsed = JSON.parse(response.text);
|
|
529
|
+
}
|
|
530
|
+
catch {
|
|
531
|
+
throw new ValidationError("Model judge returned invalid JSON.");
|
|
532
|
+
}
|
|
533
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
534
|
+
throw new ValidationError("Model judge returned an invalid result object.");
|
|
535
|
+
}
|
|
536
|
+
const score = parsed.score;
|
|
537
|
+
assertFiniteRange(score, "Model judge score", 0, 1);
|
|
538
|
+
const feedback = parsed.feedback;
|
|
539
|
+
if (feedback !== undefined && typeof feedback !== "string") {
|
|
540
|
+
throw new ValidationError("Model judge feedback must be a string.");
|
|
541
|
+
}
|
|
542
|
+
return { score, feedback };
|
|
543
|
+
}
|
|
544
|
+
});
|
|
545
|
+
//# sourceMappingURL=model-evaluation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-evaluation.js","sourceRoot":"","sources":["../src/model-evaluation.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAwC,MAAM,kBAAkB,CAAC;AAE3F,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAYlD,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAU,CAAC;AA0KjE,SAAS,wBAAwB,CAAC,KAAc,EAAE,IAAY;IAC5D,IACE,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QACzB,KAAK,KAAK,KAAK,CAAC,IAAI,EAAE;QACtB,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,EACpC,CAAC;QACD,MAAM,IAAI,eAAe,CAAC,GAAG,IAAI,mFAAmF,CAAC,CAAC;IACxH,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAc,EACd,IAAY,EACZ,OAAe,EACf,OAAe;IAEf,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,OAAO,IAAI,KAAK,GAAG,OAAO,EAAE,CAAC;QAC/F,MAAM,IAAI,eAAe,CAAC,GAAG,IAAI,oCAAoC,OAAO,QAAQ,OAAO,GAAG,CAAC,CAAC;IAClG,CAAC;AACH,CAAC;AAED,MAAM,cAAc,GAAG,CAAC,KAAc,EAAU,EAAE,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAE1G,MAAM,UAAU,GAAG,CAAC,MAAgB,EAAE,QAAgB,EAAU,EAAE;IAChE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACnE,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,CAAC,MAAgB,EAAsB,EAAE,CACpD,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAElG,MAAM,QAAQ,GAAG,CAAC,MAAqC,EAA0B,EAAE;IACjF,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAuB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACnF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG;QACb,aAAa;QACb,mBAAmB;QACnB,kBAAkB;QAClB,cAAc;QACd,iBAAiB;QACjB,aAAa;KACL,CAAC;IACX,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;QAC5G,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,KAAK,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAA6C,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC;IAC9I,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,OAAiC,EAAE,SAAmC,EAA4B,EAAE;IAC1H,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC,OAAO,CAAC;IAC3B,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/E,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,uBAAuB,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC7E,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAAC,KAAwB,EAAE,QAAiB,EAAgB,EAAE,CAAC,CAAC;IAC9F,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;IAChD,0BAA0B,EAAE,KAAK,CAAC,0BAA0B;IAC5D,yBAAyB,EAAE,KAAK,CAAC,yBAAyB;IAC1D,qBAAqB,EAAE,KAAK,CAAC,qBAAqB;IAClD,eAAe,EAAE,KAAK,CAAC,eAAe;IACtC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;IAC5C,QAAQ;CACT,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,CAAC,KAA2B,EAAmD,EAAE;IACrG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,eAAe,CAAC,8CAA8C,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,eAAe,CAAC,mDAAmD,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,eAAe,CAAC,gDAAgD,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,wBAAwB,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAC;QAChE,IAAI,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,eAAe,CAAC,wCAAwC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtG,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,CAAC;YAC1E,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QACpH,CAAC;QACD,IAAI,QAAQ,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzE,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5D,wBAAwB,CAAC,SAAS,CAAC,EAAE,EAAE,cAAc,KAAK,MAAM,CAAC,CAAC;QAClE,IAAI,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,eAAe,CAAC,2CAA2C,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QACxG,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,OAAO,SAAS,CAAC,KAAK,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;YACvE,MAAM,IAAI,eAAe,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,gCAAgC,CAAC,CAAC;QACvG,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,wBAAwB,CAAC,MAAM,CAAC,EAAE,EAAE,WAAW,KAAK,MAAM,CAAC,CAAC;QAC5D,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,eAAe,CAAC,wCAAwC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QAClG,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACzB,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YACvC,MAAM,IAAI,eAAe,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,uBAAuB,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;QAC/E,MAAM,IAAI,eAAe,CAAC,wEAAwE,CAAC,CAAC;IACtG,CAAC;IACD,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,IAAI,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,IAAI,cAAc,GAAG,EAAE,EAAE,CAAC;QACvF,MAAM,IAAI,eAAe,CAAC,0EAA0E,CAAC,CAAC;IACxG,CAAC;IACD,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACrC,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC;AACzC,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,UAAiD,EAAQ,EAAE;IACrF,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO;IACT,CAAC;IACD,IAAI,UAAU,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QAC1C,iBAAiB,CAAC,UAAU,CAAC,YAAY,EAAE,yBAAyB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,UAAU,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QAC1C,iBAAiB,CAAC,UAAU,CAAC,YAAY,EAAE,yBAAyB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9E,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,CAAC,cAAc,EAAE,iBAAiB,CAAU,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;YAC/F,MAAM,IAAI,eAAe,CAAC,cAAc,KAAK,yCAAyC,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,QAAgB,EAAE,MAA4C,EAAwB,EAAE;IACrH,MAAM,UAAU,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC3E,iBAAiB,CAAC,UAAU,CAAC,KAAK,EAAE,UAAU,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACtF,OAAO;QACL,QAAQ;QACR,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;KAC9B,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,KAAK,EAC7B,QAA6B,EAC7B,SAAmC,EACnC,UAAkB,EAClB,KAA2B,EACQ,EAAE;IACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,UAA8B,CAAC;IACnC,IAAI,YAAsC,CAAC;IAC3C,IAAI,KAA6B,CAAC;IAClC,IAAI,WAAW,GAA0B,EAAE,CAAC;IAC5C,IAAI,KAAyB,CAAC;IAE9B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,KAAK,SAAS;YACzC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAS,EAAE;YAClC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,GAAG,SAAS,CAAC,QAAQ;YACrB,GAAG,QAAQ,CAAC,QAAQ;YACpB,GAAG,KAAK;YACR,KAAK,EAAE,SAAS,CAAC,KAAK;SACvB,CAAC,CAAC;QACH,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACnC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACrB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;IAChF,MAAM,OAAO,GAAiC;QAC5C,QAAQ;QACR,SAAS,EAAE;YACT,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ;YAClC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO;YAChC,QAAQ,EAAE,SAAS,CAAC,QAAQ;SAC7B;QACD,UAAU;QACV,UAAU;QACV,YAAY;QACZ,KAAK;QACL,WAAW;QACX,SAAS;QACT,IAAI;QACJ,KAAK;KACN,CAAC;IAEF,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAChG,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAEvE,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,IAAI;QACvB,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ;QAClC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO;QAChC,UAAU;QACV,EAAE,EAAE,KAAK,KAAK,SAAS,IAAI,CAAC,YAAY;QACxC,UAAU;QACV,YAAY;QACZ,KAAK;QACL,SAAS,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC;QACvD,SAAS;QACT,IAAI,EAAE,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACjF,MAAM;QACN,SAAS,EAAE,IAAI,CAAC,aAAa,CAAC;QAC9B,KAAK;QACL,YAAY,EAAE,QAAQ,CAAC,QAAQ;QAC/B,iBAAiB,EAAE,SAAS,CAAC,QAAQ;KACtC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,SAAmC,EACnC,IAAgC,EACC,EAAE;IACnC,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;IACzF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IACnG,MAAM,uBAAuB,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC;IAChF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC;IAC1H,OAAO;QACL,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ;QAClC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO;QAChC,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,cAAc;QACd,UAAU,EAAE,IAAI,CAAC,MAAM,GAAG,cAAc;QACxC,SAAS,EAAE,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,GAAG,IAAI,CAAC,MAAM;QAC/E,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;QAC/D,QAAQ,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;QAC/D,YAAY,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC;QAC/D,YAAY,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC;QAChE,SAAS,EAAE,uBAAuB,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QAC7F,QAAQ,EAAE,uBAAuB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QAC3D,QAAQ,EAAE,uBAAuB,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QAC3F,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC7C,QAAQ,EAAE,SAAS,CAAC,QAAQ;KAC7B,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,2BAA2B,GAAG,CACzC,UAA6C,EAC7C,UAAqC,EACV,EAAE;IAC7B,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAC/B,MAAM,MAAM,GAA+B,EAAE,CAAC;IAC9C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,CACf,IAAkC,EAClC,MAA0B,EAC1B,SAAiB,EACjB,EAAW,EACX,KAAa,EACb,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;YACf,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,IAAI;YACJ,EAAE;YACF,MAAM;YACN,SAAS;YACT,OAAO,EAAE,GAAG,SAAS,CAAC,WAAW,IAAI,KAAK,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,eAAe,SAAS,GAAG;SACxH,CAAC,CAAC;QAEH,IAAI,UAAU,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1C,QAAQ,CACN,gBAAgB,EAChB,SAAS,CAAC,SAAS,EACnB,UAAU,CAAC,YAAY,EACvB,SAAS,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,SAAS,IAAI,UAAU,CAAC,YAAY,EACnF,YAAY,CACb,CAAC;QACJ,CAAC;QACD,IAAI,UAAU,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1C,QAAQ,CACN,gBAAgB,EAChB,SAAS,CAAC,SAAS,EACnB,UAAU,CAAC,YAAY,EACvB,SAAS,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,SAAS,IAAI,UAAU,CAAC,YAAY,EACnF,YAAY,CACb,CAAC;QACJ,CAAC;QACD,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAC7C,QAAQ,CACN,oBAAoB,EACpB,SAAS,CAAC,YAAY,EACtB,UAAU,CAAC,eAAe,EAC1B,SAAS,CAAC,YAAY,IAAI,UAAU,CAAC,eAAe,EACpD,aAAa,CACd,CAAC;QACJ,CAAC;QACD,IAAI,UAAU,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1C,QAAQ,CACN,gBAAgB,EAChB,SAAS,CAAC,SAAS,EACnB,UAAU,CAAC,YAAY,EACvB,SAAS,CAAC,SAAS,IAAI,UAAU,CAAC,YAAY,EAC9C,YAAY,CACb,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,UAAU,EAAE,EAAE,GAAG,UAAU,EAAE;QAC7B,MAAM;KACP,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EAAE,KAA2B,EAAkC,EAAE;IACtG,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CACnD,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;QACtD,SAAS;QACT,QAAQ;QACR,UAAU,EAAE,UAAU,GAAG,CAAC;KAC3B,CAAC,CAAC,CACJ,CACF,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,KAAK,CAA2B,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/D,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACxF,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,MAAM,CAAC;YACrB,MAAM,IAAI,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC/F,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3B,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CACpD,kBAAkB,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE,CAAC,CAAC,CACtF,CAAC;IACF,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,2BAA2B,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtG,OAAO;QACL,aAAa,EAAE,sCAAsC;QACrD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,IAAI,CAAC;QACrD,WAAW;QACX,SAAS,EAAE,IAAI,CAAC,MAAM;QACtB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;QACnD,UAAU;QACV,IAAI;QACJ,IAAI;QACJ,QAAQ,EAAE,KAAK,CAAC,QAAQ;KACzB,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAC3C,QAA+B,EAC/B,MAA6B,EACF,EAAE;IAC7B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC;QAClB,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,CAAC;QAChE,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,CAAC;KAC/D,CAAC,CAAC;IACH,OAAO;QACL,iBAAiB,EAAE,QAAQ,CAAC,SAAS;QACrC,eAAe,EAAE,MAAM,CAAC,SAAS;QACjC,UAAU,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;YAC9C,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,KAAK,WAAW,CAAC,CAAC;YAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,KAAK,WAAW,CAAC,CAAC;YAC1F,OAAO;gBACL,WAAW;gBACX,QAAQ,EAAE,IAAI;gBACd,MAAM,EAAE,IAAI;gBACZ,UAAU,EAAE,IAAI,EAAE,SAAS,KAAK,SAAS,IAAI,IAAI,EAAE,SAAS,KAAK,SAAS;oBACxE,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS;gBACnC,cAAc,EAAE,IAAI,EAAE,SAAS,KAAK,SAAS,IAAI,IAAI,EAAE,SAAS,KAAK,SAAS;oBAC5E,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS;gBACnC,iBAAiB,EAAE,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;oBACzD,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY;gBACzC,cAAc,EAAE,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;oBACtD,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS;aACpC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,KAAa,EAAE,OAAoD,EAAU,EAAE;IACrG,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC9D,OAAO,OAAO,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;AACjF,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,KAAgB,EAAU,EAAE;IACjD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACnD,CAAC;IACD,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;SAC1B,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,CAAE,CAAC,EAAE,CAAC;SACpE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,UAInC,EAAE,EAAyB,EAAE,CAAC,CAAC;IACjC,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,aAAa;IAC/B,KAAK,CAAC,OAAO;QACX,IAAI,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAC1H,CAAC;QACD,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACrE,OAAO;YACL,KAAK,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,QAAQ,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,6CAA6C;SAC1F,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,UAGjC,EAAE,EAAyB,EAAE,CAAC,CAAC;IACjC,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,UAAU;IAC5B,KAAK,CAAC,OAAO;QACX,IAAI,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;QAC/H,CAAC;QACD,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,EAAE,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/G,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACnH,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,OAGjC,EAAyB,EAAE,CAAC,CAAC;IAC5B,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,OAAO;IACzB,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,GAAG,OAAO,OAAO,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QAC5G,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QACtB,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,UAA2B,EAAE,EAAyB,EAAE,CAAC,CAAC;IAC9F,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,YAAY;IAC9B,KAAK,CAAC,OAAO;QACX,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAC7C,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QACxG,CAAC;QACD,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,2BAA2B,EAAE,CAAC;QAC7D,CAAC;QACD,OAAO,aAAa,CAAC,MAAmB,CAAC,KAAK,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YACrF,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,0CAA0C,EAAE,CAAC;IACzE,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,MAAkB,EAAE,UAA2B,EAAE,EAAyB,EAAE,CAAC,CAAC;IACnH,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,aAAa;IAC/B,KAAK,CAAC,OAAO;QACX,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,2BAA2B,EAAE,CAAC;QAC7D,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACxC,OAAO,MAAM,CAAC,OAAO;YACnB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC3F,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,UAAwD,EAAE,EAAyB,EAAE,CAAC,CAAC;IAC1H,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,YAAY;IAC9B,KAAK,CAAC,OAAO;QACX,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QACpD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;QACzG,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,OAAO,CAAC,iBAAiB;YACvC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;YACrD,CAAC,CAAC,CAAC,GAAG,EAAE;gBACJ,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACtC,OAAO,KAAK,CAAC;gBACf,CAAC;gBACD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;gBACjD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;oBAC5B,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAChE,CAAC;gBACD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;oBAC1B,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC3C,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;wBAC/C,OAAO,KAAK,CAAC;oBACf,CAAC;oBACD,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;gBAC1C,CAAC;gBACD,OAAO,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;YACpE,CAAC,CAAC,EAAE,CAAC;QACT,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,uBAAuB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IACtI,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,+BAA+B,GAAG,CAAC,OAI/C,EAAyB,EAAE;IAC1B,MAAM,cAAc,GAAG,IAAI,GAAG,EAA6B,CAAC;IAC5D,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,sBAAsB;QACxC,KAAK,CAAC,KAAK,CAAC,OAAO;YACjB,IAAI,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBACnD,MAAM,IAAI,eAAe,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACnI,CAAC;YACD,MAAM,YAAY,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,SAAS,OAAO,CAAC,KAAK,CAAC,OAAO,SAAS,OAAO,CAAC,QAAQ,CAAC,IAAI,SAAS,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAChJ,IAAI,yBAAyB,GAAG,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACjE,IAAI,CAAC,yBAAyB,EAAE,CAAC;gBAC/B,yBAAyB,GAAG,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;oBAC7G,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;oBACvC,IAAI,CAAC,SAAS,EAAE,CAAC;wBACf,MAAM,IAAI,eAAe,CAAC,kDAAkD,CAAC,CAAC;oBAChF,CAAC;oBACD,OAAO,SAAS,CAAC;gBACnB,CAAC,CAAC,CAAC;gBACH,cAAc,CAAC,GAAG,CAAC,YAAY,EAAE,yBAAyB,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,kBAA4B,CAAC;YACjC,IAAI,CAAC;gBACH,kBAAkB,GAAG,MAAM,yBAAyB,CAAC;YACvD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,yBAAyB,EAAE,CAAC;oBACnE,cAAc,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;gBACtC,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,eAAe,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC/G,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,MAAM,IAAI,eAAe,CAAC,+CAA+C,CAAC,CAAC;YAC7E,CAAC;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,gBAAgB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;YACnG,MAAM,OAAO,GAAG,OAAO,CAAC,iBAAiB,CAAC;YAC1C,OAAO;gBACL,KAAK,EAAE,OAAO,KAAK,SAAS,IAAI,UAAU,IAAI,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBACtE,QAAQ,EAAE,EAAE,UAAU,EAAE;aACzB,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,OAKtC,EAAyB,EAAE,CAAC,CAAC;IAC5B,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,aAAa;IAC/B,KAAK,CAAC,KAAK,CAAC,OAAO;QACjB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;YACnC,MAAM,EAAE;gBACN,iDAAiD;gBACjD,8FAA8F;gBAC9F,WAAW,OAAO,CAAC,MAAM,EAAE;gBAC3B,UAAU,OAAO,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;gBAChF,cAAc,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBAC1D,qBAAqB,OAAO,CAAC,UAAU,IAAI,EAAE,EAAE;aAChD,CAAC,IAAI,CAAC,IAAI,CAAC;SACb,CAAC,CAAC;QACH,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,eAAe,CAAC,oCAAoC,CAAC,CAAC;QAClE,CAAC;QACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3E,MAAM,IAAI,eAAe,CAAC,gDAAgD,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,KAAK,GAAI,MAA8B,CAAC,KAAK,CAAC;QACpD,iBAAiB,CAAC,KAAK,EAAE,mBAAmB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAI,MAAiC,CAAC,QAAQ,CAAC;QAC7D,IAAI,QAAQ,KAAK,SAAS,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,eAAe,CAAC,wCAAwC,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IAC7B,CAAC;CACF,CAAC,CAAC"}
|