@prismatic-io/lux 0.0.1

package/lib/orchestrator/grader-experiment.js

@@ -0,0 +1,579 @@
+import { randomUUID } from "node:crypto";
+import { join, relative, resolve, sep } from "node:path";
+import { z } from "zod";
+import { createRubricAssertion, RubricModelAccountingDetailSchema, } from "../assertions/rubric/index.js";
+import { emptyUsage, GradingReportSchema, sumUsage, TokenUsageSchema, totalTokens, valueHash, } from "../core/index.js";
+import { selectGraderAnnotations } from "./annotation-loader.js";
+import { buildRegistryFromConfig } from "./config.js";
+import { readContentBoundJson, retireContentBoundJson, writeContentBoundJson, } from "./content-bound-json.js";
+import { gradeRun } from "./grading.js";
+import { loadFrozenRegradableRun, releaseFrozenRegradableRun } from "./regrade.js";
+import { loadRubricDefinition } from "./rubric-loader.js";
+const GraderJudgmentSchema = z.object({
+    key: z.string(),
+    kind: z.enum(["case-pass", "case-score", "assertion-pass", "assertion-score"]),
+    expected: z.union([z.boolean(), z.number()]),
+    actual: z.union([z.boolean(), z.number()]).nullable(),
+    matched: z.boolean(),
+    absoluteError: z.exactOptional(z.number()),
+});
+const GraderEvidenceItemSchema = z.object({
+    annotationId: z.string(),
+    repetition: z.number().int().positive(),
+    runDir: z.string(),
+    rubricPath: z.string(),
+    feedback: z.exactOptional(z.string()),
+    judgments: z.array(GraderJudgmentSchema),
+    report: GradingReportSchema,
+    durationMs: z.number().nonnegative(),
+    usage: TokenUsageSchema,
+    costUsd: z.number().nonnegative(),
+    unpricedModelCalls: z.number().int().nonnegative(),
+});
+const GraderEvaluationUnitSchema = z
+    .object({
+    schemaVersion: z.literal(1),
+    unitId: z.string(),
+    candidateId: z.string(),
+    profileId: z.string(),
+    split: z.enum(["train", "validation", "test"]),
+    repetitions: z.number().int().positive(),
+    annotationsHash: z.string(),
+    annotationId: z.string(),
+    repetition: z.number().int().positive(),
+    item: GraderEvidenceItemSchema,
+})
+    .strict();
+export const GraderEvaluationEvidenceSchema = z.object({
+    schemaVersion: z.literal(1),
+    candidateId: z.string(),
+    profileId: z.string(),
+    split: z.enum(["train", "validation", "test"]),
+    repetitions: z.number().int().positive(),
+    annotationsHash: z.string(),
+    generatedAt: z.string(),
+    metrics: z.record(z.string(), z.number()),
+    samples: z.record(z.string(), z.array(z.number())),
+    items: z.array(GraderEvidenceItemSchema),
+});
+const assertionOperationalError = (entry) => {
+    if (!entry)
+        return false;
+    if (entry.errorMessage)
+        return true;
+    const detail = RubricModelAccountingDetailSchema.safeParse(entry.detail);
+    return detail.success && detail.data.graderError !== undefined;
+};
+const assertionModelAccounting = (assertion) => {
+    const detail = RubricModelAccountingDetailSchema.safeParse(assertion.detail);
+    if (!detail.success) {
+        return { attempts: 0, costUsd: 0, unpricedModelCalls: 0 };
+    }
+    return {
+        attempts: detail.data.attempts,
+        costUsd: detail.data.costUsd,
+        unpricedModelCalls: detail.data.unpricedAttempts,
+    };
+};
+export const compareGraderAnnotation = (annotation, report, scoreTolerance) => {
+    const judgments = [];
+    const reportFailed = report.assertions.some(assertionOperationalError);
+    if (annotation.expected.casePassed !== undefined) {
+        const actual = reportFailed ? null : report.casePassed;
+        judgments.push({
+            key: "case",
+            kind: "case-pass",
+            expected: annotation.expected.casePassed,
+            actual,
+            matched: actual === annotation.expected.casePassed,
+        });
+    }
+    if (annotation.expected.caseScore !== undefined) {
+        const actual = reportFailed ? null : report.caseScore;
+        const absoluteError = actual === null ? 1 : Math.abs(actual - annotation.expected.caseScore);
+        judgments.push({
+            key: "case",
+            kind: "case-score",
+            expected: annotation.expected.caseScore,
+            actual,
+            matched: actual !== null && absoluteError <= scoreTolerance,
+            absoluteError,
+        });
+    }
+    const entries = new Map(report.assertions.flatMap((entry) => (entry.id ? [[entry.id, entry]] : [])));
+    for (const [assertionId, expected] of Object.entries(annotation.expected.assertions)) {
+        const entry = entries.get(assertionId);
+        const actual = assertionOperationalError(entry) ? undefined : entry;
+        judgments.push({
+            key: assertionId,
+            kind: "assertion-pass",
+            expected: expected.passed,
+            actual: actual?.passed ?? null,
+            matched: actual?.passed === expected.passed,
+        });
+        if (expected.score !== undefined) {
+            const absoluteError = actual ? Math.abs(actual.score - expected.score) : 1;
+            judgments.push({
+                key: assertionId,
+                kind: "assertion-score",
+                expected: expected.score,
+                actual: actual?.score ?? null,
+                matched: actual !== undefined && absoluteError <= scoreTolerance,
+                absoluteError,
+            });
+        }
+    }
+    return judgments;
+};
+const ratio = (numerator, denominator, empty) => denominator === 0 ? empty : numerator / denominator;
+const addSample = (samples, name, value) => {
+    if (!Number.isFinite(value))
+        return;
+    const values = samples[name] ?? [];
+    values.push(value);
+    samples[name] = values;
+};
+const mean = (values, empty = 0) => values.length === 0 ? empty : values.reduce((sum, value) => sum + value, 0) / values.length;
+const collectGraderEvidence = (items) => {
+    const byAnnotation = new Map();
+    const itemsByAnnotation = new Map();
+    const counts = {
+        truePositive: 0,
+        falsePositive: 0,
+        falseNegative: 0,
+        positiveLabels: 0,
+        negativeLabels: 0,
+        abstentions: 0,
+        passJudgments: 0,
+        scoreJudgments: 0,
+    };
+    for (const item of items) {
+        const annotationJudgments = byAnnotation.get(item.annotationId) ?? [];
+        annotationJudgments.push(...item.judgments);
+        byAnnotation.set(item.annotationId, annotationJudgments);
+        const annotationItems = itemsByAnnotation.get(item.annotationId) ?? [];
+        annotationItems.push(item);
+        itemsByAnnotation.set(item.annotationId, annotationItems);
+        for (const judgment of item.judgments) {
+            if (judgment.absoluteError !== undefined)
+                counts.scoreJudgments++;
+            if (!judgment.kind.endsWith("-pass") || typeof judgment.expected !== "boolean")
+                continue;
+            counts.passJudgments++;
+            if (judgment.expected)
+                counts.positiveLabels++;
+            else
+                counts.negativeLabels++;
+            if (judgment.actual === null)
+                counts.abstentions++;
+            else if (judgment.expected && judgment.actual)
+                counts.truePositive++;
+            else if (!judgment.expected && judgment.actual)
+                counts.falsePositive++;
+            else if (judgment.expected && !judgment.actual)
+                counts.falseNegative++;
+        }
+    }
+    return { byAnnotation, itemsByAnnotation, counts };
+};
+export const summarizeGraderEvidence = (items, candidate) => {
+    const samples = {};
+    const { byAnnotation, itemsByAnnotation, counts } = collectGraderEvidence(items);
+    let caseJudgmentCount = 0;
+    let assertionJudgmentCount = 0;
+    for (const [annotationId, judgments] of byAnnotation) {
+        const annotationItems = itemsByAnnotation.get(annotationId) ?? [];
+        addSample(samples, "phase.gradingMs", mean(annotationItems.map((item) => item.durationMs)));
+        addSample(samples, "harness.input", mean(annotationItems.map((item) => item.usage.input)));
+        addSample(samples, "harness.output", mean(annotationItems.map((item) => item.usage.output)));
+        addSample(samples, "harness.cacheRead", mean(annotationItems.map((item) => item.usage.cacheRead)));
+        addSample(samples, "harness.cacheCreation", mean(annotationItems.map((item) => item.usage.cacheCreation)));
+        addSample(samples, "harness.total", mean(annotationItems.map((item) => totalTokens(item.usage))));
+        const allAgreement = mean(judgments.map((judgment) => (judgment.matched ? 1 : 0)));
+        addSample(samples, "quality.passRate", allAgreement);
+        addSample(samples, "grader.agreement", allAgreement);
+        const caseJudgments = judgments.filter((judgment) => judgment.kind.startsWith("case"));
+        if (caseJudgments.length > 0) {
+            caseJudgmentCount += caseJudgments.length;
+            addSample(samples, "grader.caseAgreement", mean(caseJudgments.map((judgment) => (judgment.matched ? 1 : 0))));
+        }
+        const assertionJudgments = judgments.filter((judgment) => judgment.kind.startsWith("assertion"));
+        if (assertionJudgments.length > 0) {
+            assertionJudgmentCount += assertionJudgments.length;
+            addSample(samples, "grader.assertionAgreement", mean(assertionJudgments.map((judgment) => (judgment.matched ? 1 : 0))));
+        }
+        const passes = judgments.filter((judgment) => judgment.kind.endsWith("-pass") && typeof judgment.expected === "boolean");
+        const scoreErrors = judgments.flatMap((judgment) => judgment.absoluteError === undefined ? [] : [judgment.absoluteError]);
+        if (scoreErrors.length > 0) {
+            addSample(samples, "grader.scoreAbsoluteError", mean(scoreErrors));
+            addSample(samples, "grader.scoreMae", mean(scoreErrors));
+        }
+        const positive = passes.filter((judgment) => judgment.expected === true);
+        const negative = passes.filter((judgment) => judgment.expected === false);
+        const coveredPositive = positive.filter((judgment) => judgment.actual !== null);
+        const coveredNegative = negative.filter((judgment) => judgment.actual !== null);
+        if (coveredPositive.length > 0) {
+            addSample(samples, "grader.macroFalseNegativeRate", coveredPositive.filter((judgment) => judgment.actual === false).length /
+                coveredPositive.length);
+        }
+        if (coveredNegative.length > 0) {
+            addSample(samples, "grader.macroFalsePositiveRate", coveredNegative.filter((judgment) => judgment.actual === true).length /
+                coveredNegative.length);
+        }
+        if (passes.length > 0) {
+            addSample(samples, "grader.coverage", passes.filter((judgment) => judgment.actual !== null).length / passes.length);
+            addSample(samples, "grader.abstentionRate", passes.filter((judgment) => judgment.actual === null).length / passes.length);
+        }
+    }
+    const precision = ratio(counts.truePositive, counts.truePositive + counts.falsePositive, Number.NaN);
+    const recall = ratio(counts.truePositive, counts.truePositive + counts.falseNegative, Number.NaN);
+    const metrics = {
+        "quality.passRate": mean(samples["quality.passRate"] ?? []),
+        "grader.agreement": mean(samples["grader.agreement"] ?? []),
+        "grader.labels": byAnnotation.size,
+        "grader.judgments": items.reduce((total, item) => total + item.judgments.length, 0),
+        "grader.passJudgments": counts.passJudgments,
+        "grader.caseJudgments": caseJudgmentCount,
+        "grader.assertionJudgments": assertionJudgmentCount,
+        "grader.scoreJudgments": counts.scoreJudgments,
+        "grader.positiveLabels": counts.positiveLabels,
+        "grader.negativeLabels": counts.negativeLabels,
+        "grader.abstentions": counts.abstentions,
+        "subject.bytes": candidate.sourceBytes,
+    };
+    const addMetric = (name, value) => {
+        if (value !== undefined && Number.isFinite(value))
+            metrics[name] = value;
+    };
+    addMetric("grader.caseAgreement", samples["grader.caseAgreement"] ? mean(samples["grader.caseAgreement"]) : undefined);
+    addMetric("grader.assertionAgreement", samples["grader.assertionAgreement"] ? mean(samples["grader.assertionAgreement"]) : undefined);
+    addMetric("grader.microPrecision", precision);
+    addMetric("grader.microRecall", recall);
+    let microF1;
+    if (Number.isFinite(precision) && Number.isFinite(recall)) {
+        microF1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
+    }
+    addMetric("grader.microF1", microF1);
+    addMetric("grader.macroFalsePositiveRate", samples["grader.macroFalsePositiveRate"]
+        ? mean(samples["grader.macroFalsePositiveRate"])
+        : undefined);
+    addMetric("grader.macroFalseNegativeRate", samples["grader.macroFalseNegativeRate"]
+        ? mean(samples["grader.macroFalseNegativeRate"])
+        : undefined);
+    addMetric("grader.coverage", samples["grader.coverage"] ? mean(samples["grader.coverage"]) : undefined);
+    addMetric("grader.abstentionRate", samples["grader.abstentionRate"] ? mean(samples["grader.abstentionRate"]) : undefined);
+    addMetric("grader.scoreMae", samples["grader.scoreMae"] ? mean(samples["grader.scoreMae"]) : undefined);
+    for (const [name, values] of Object.entries(samples)) {
+        if (!(name in metrics))
+            metrics[name] = mean(values);
+    }
+    return { metrics, samples };
+};
+const graderEvidencePath = (experiment, identityInput) => {
+    const directory = join(experiment.path, "grader-evaluations");
+    const identity = valueHash({
+        candidateId: identityInput.candidateId,
+        profileId: identityInput.profileId,
+        split: identityInput.split,
+        repetitions: identityInput.repetitions,
+        annotationsHash: identityInput.annotationsHash,
+    }).slice(0, 20);
+    return join(directory, `${identityInput.candidateId}-${identity}.json`);
+};
+const graderUnitId = (input) => `grader-${valueHash(input).slice(0, 32)}`;
+const graderUnitPath = (experiment, unitId) => join(experiment.path, "evaluation-units", `${unitId}.json`);
+const itemAccounting = () => ({
+    metricCalls: 1,
+    judgeCalls: 0,
+    usage: emptyUsage(),
+    costUsd: 0,
+    unpricedModelCalls: 0,
+});
+const evaluationFromEvidence = (experiment, evidencePath, evidence) => {
+    const byAnnotation = new Map();
+    for (const item of evidence.items) {
+        const values = byAnnotation.get(item.annotationId) ?? [];
+        values.push(...item.judgments.map((judgment) => (judgment.matched ? 1 : 0)));
+        byAnnotation.set(item.annotationId, values);
+    }
+    const failedCases = evidence.items.flatMap((item) => {
+        const mismatches = item.judgments.filter((judgment) => !judgment.matched);
+        if (mismatches.length === 0)
+            return [];
+        const feedback = mismatches.map((judgment) => `${judgment.key} ${judgment.kind}: expected ${String(judgment.expected)}, got ${String(judgment.actual)}`);
+        if (item.feedback)
+            feedback.unshift(`human feedback: ${item.feedback}`);
+        for (const judgment of mismatches) {
+            if (!judgment.kind.startsWith("assertion"))
+                continue;
+            const comment = item.report.assertions.find((entry) => entry.id === judgment.key)?.comment;
+            if (comment)
+                feedback.push(`grader comment for ${judgment.key}: ${comment}`);
+        }
+        return [
+            {
+                caseId: `${item.annotationId} #${item.repetition}`,
+                score: mean(item.judgments.map((judgment) => (judgment.matched ? 1 : 0))),
+                feedback,
+            },
+        ];
+    });
+    return {
+        profileId: evidence.profileId,
+        split: evidence.split,
+        evidencePath: relative(experiment.path, evidencePath),
+        passRate: evidence.metrics["quality.passRate"] ?? 0,
+        casePassRates: Object.fromEntries([...byAnnotation].map(([id, values]) => [id, mean(values)])),
+        metrics: evidence.metrics,
+        samples: evidence.samples,
+        metricCalls: evidence.items.length,
+        judgeCalls: evidence.items.reduce((total, item) => total +
+            item.report.assertions.reduce((assertionTotal, assertion) => assertionTotal + assertionModelAccounting(assertion).attempts, 0), 0),
+        usage: sumUsage(evidence.items.map((item) => item.usage)),
+        costUsd: evidence.items.reduce((total, item) => total + item.costUsd, 0),
+        unpricedModelCalls: evidence.items.reduce((total, item) => total + item.unpricedModelCalls, 0),
+        failedCases,
+    };
+};
+const candidateRubricPath = (materializedRoot, rubricPath) => {
+    const root = resolve(materializedRoot);
+    const path = resolve(root, rubricPath);
+    if (path !== root && !path.startsWith(`${root}${sep}`)) {
+        throw new Error(`grader rubric path escapes candidate root: ${rubricPath}`);
+    }
+    return path;
+};
+const assertionIds = (rubric) => rubric.assertions.map((assertion) => assertion.id ?? "").sort();
+const assertEvidenceIdentity = (evidence, expected, path) => {
+    if (evidence.candidateId !== expected.candidateId ||
+        evidence.profileId !== expected.profileId ||
+        evidence.split !== expected.split ||
+        evidence.repetitions !== expected.repetitions ||
+        evidence.annotationsHash !== expected.annotationsHash) {
+        throw new Error(`cached grader evidence failed its identity check: ${path}`);
+    }
+};
+const assertUnitIdentity = (unit, expected, path) => {
+    if (unit.unitId !== expected.unitId ||
+        unit.candidateId !== expected.candidateId ||
+        unit.profileId !== expected.profileId ||
+        unit.split !== expected.split ||
+        unit.repetitions !== expected.repetitions ||
+        unit.annotationsHash !== expected.annotationsHash ||
+        unit.annotationId !== expected.annotationId ||
+        unit.repetition !== expected.repetition) {
+        throw new Error(`grader evaluation unit failed its identity check: ${path}`);
+    }
+};
+const throwIfExperimentAborted = (signal) => {
+    if (!signal?.aborted)
+        return;
+    throw signal.reason instanceof Error ? signal.reason : new Error("experiment aborted");
+};
+const validateCandidateRubricIds = (rubricPath, baselineIds, candidateIds, allowedAdditions) => {
+    const missingBaseline = baselineIds.filter((id) => !candidateIds.includes(id));
+    const unlabeledAdditions = candidateIds.filter((id) => !baselineIds.includes(id) && !allowedAdditions.has(id));
+    if (missingBaseline.length === 0 && unlabeledAdditions.length === 0)
+        return;
+    throw new Error(`rubric '${rubricPath}' must preserve baseline IDs and may add only explicitly allowed IDs` +
+        `${missingBaseline.length > 0 ? `; missing: ${missingBaseline.join(", ")}` : ""}` +
+        `${unlabeledAdditions.length > 0 ? `; unlabeled additions: ${unlabeledAdditions.join(", ")}` : ""}`);
+};
+export const evaluateGraderCandidate = async (options) => {
+    const { execution, experiment, candidate, profileId, split, repetitions } = options;
+    const annotations = execution.loaded.graderAnnotations;
+    if (!annotations || execution.loaded.campaign.evaluation.kind !== "grader") {
+        throw new Error("grader experiment is missing its loaded annotation dataset");
+    }
+    const evaluationHash = experiment.manifest.evaluationHash;
+    if (!evaluationHash)
+        throw new Error("grader experiment has no evaluation identity");
+    const selected = selectGraderAnnotations(annotations.annotations, options.selector);
+    if (selected.length === 0) {
+        throw new Error(`experiment '${execution.loaded.campaign.id}' ${split} selector matched no labels`);
+    }
+    const profile = execution.config.profiles?.[profileId];
+    if (!profile)
+        throw new Error(`experiment profile '${profileId}' is not configured`);
+    const expectedEvidencePath = graderEvidencePath(experiment, {
+        candidateId: candidate.id,
+        profileId,
+        split,
+        repetitions,
+        annotationsHash: annotations.hash,
+    });
+    let cachedEvidence = await readContentBoundJson(expectedEvidencePath, GraderEvaluationEvidenceSchema, evaluationHash);
+    if (cachedEvidence) {
+        assertEvidenceIdentity(cachedEvidence, {
+            candidateId: candidate.id,
+            profileId,
+            split,
+            repetitions,
+            annotationsHash: annotations.hash,
+        }, expectedEvidencePath);
+    }
+    const harness = profile.harness ?? execution.config.harness;
+    const registry = buildRegistryFromConfig({
+        ...execution.config,
+        ...(harness ? { harness } : {}),
+    }).withAssertion(createRubricAssertion(harness));
+    const items = [];
+    for (const annotation of selected) {
+        const rubric = await loadRubricDefinition(candidateRubricPath(options.materializedRoot, annotation.rubricPath));
+        const baselineRubric = await loadRubricDefinition(candidateRubricPath(execution.loaded.subjectRoot, annotation.rubricPath));
+        const baselineIds = assertionIds(baselineRubric);
+        const candidateIds = assertionIds(rubric);
+        const allowedAdditions = new Set(execution.loaded.campaign.evaluation.allowedAssertionAdditions[annotation.rubricPath] ?? []);
+        validateCandidateRubricIds(annotation.rubricPath, baselineIds, candidateIds, allowedAdditions);
+        const persistedRun = await loadFrozenRegradableRun(annotation.runDir);
+        try {
+            if (rubric.id !== persistedRun.case.id) {
+                throw new Error(`rubric '${annotation.rubricPath}' id '${rubric.id}' does not match persisted case '${persistedRun.case.id}'`);
+            }
+            for (let repetition = 1; repetition <= repetitions; repetition++) {
+                const unitId = graderUnitId({
+                    candidateId: candidate.id,
+                    profileId,
+                    split,
+                    repetitions,
+                    annotationsHash: annotations.hash,
+                    annotationId: annotation.id,
+                    repetition,
+                });
+                const unitPath = graderUnitPath(experiment, unitId);
+                const existing = await readContentBoundJson(unitPath, GraderEvaluationUnitSchema, evaluationHash);
+                if (existing) {
+                    assertUnitIdentity(existing, {
+                        unitId,
+                        candidateId: candidate.id,
+                        profileId,
+                        split,
+                        repetitions,
+                        annotationsHash: annotations.hash,
+                        annotationId: annotation.id,
+                        repetition,
+                    }, unitPath);
+                    await options.recordUnit(unitId, { ...itemAccounting(), metricCalls: 0 });
+                    throwIfExperimentAborted(execution.abortSignal);
+                    items.push(existing.item);
+                    continue;
+                }
+                if (cachedEvidence) {
+                    await retireContentBoundJson(expectedEvidencePath);
+                    cachedEvidence = null;
+                }
+                options.preflightUnit(rubric.assertions.length * 2);
+                const run = {
+                    ...persistedRun,
+                    case: { ...persistedRun.case, assertions: rubric.assertions },
+                };
+                const startedAt = Date.now();
+                const assertionAttemptId = randomUUID();
+                await options.recordAttempt(`grader-metric-attempt-${unitId}-${assertionAttemptId}`, itemAccounting());
+                let report;
+                try {
+                    report = await gradeRun(run, registry, undefined, execution.abortSignal, async (assertion, index) => {
+                        const accounting = assertionModelAccounting(assertion);
+                        let unpricedModelCalls = accounting.unpricedModelCalls;
+                        if (unpricedModelCalls === 0 && accounting.attempts === 0 && assertion.usage) {
+                            unpricedModelCalls = 1;
+                        }
+                        await options.recordAssertion(`grader-judge-${unitId}-${assertionAttemptId}-${index}`, {
+                            metricCalls: 0,
+                            // Provider-attempt starts journal judgeCalls before spend.
+                            judgeCalls: 0,
+                            usage: assertion.usage ?? emptyUsage(),
+                            costUsd: accounting.costUsd,
+                            unpricedModelCalls,
+                        });
+                    }, {
+                        start: options.recordModelAttemptStart,
+                        complete: options.completeModelAttempts,
+                    });
+                    const failedAssertion = report.assertions.find(assertionOperationalError);
+                    if (report.incomplete?.phase === "grading" || failedAssertion) {
+                        throw new Error(failedAssertion?.errorMessage ??
+                            failedAssertion?.comment ??
+                            "grader assertion could not produce a judgment");
+                    }
+                }
+                catch (error) {
+                    await options.recordUnit(`grader-attempt-${unitId}-${assertionAttemptId}`, {
+                        ...itemAccounting(),
+                        metricCalls: 0,
+                    });
+                    throw error;
+                }
+                const usage = sumUsage(report.assertions.flatMap((assertion) => (assertion.usage ? [assertion.usage] : [])));
+                const costUsd = report.assertions.reduce((total, assertion) => total + assertionModelAccounting(assertion).costUsd, 0);
+                const unpricedModelCalls = report.assertions.reduce((total, assertion) => total + assertionModelAccounting(assertion).unpricedModelCalls, 0);
+                const item = {
+                    annotationId: annotation.id,
+                    repetition,
+                    runDir: annotation.runDir,
+                    rubricPath: annotation.rubricPath,
+                    ...(annotation.feedback ? { feedback: annotation.feedback } : {}),
+                    judgments: compareGraderAnnotation(annotation, report, execution.loaded.campaign.evaluation.scoreTolerance),
+                    report,
+                    durationMs: Date.now() - startedAt,
+                    usage,
+                    costUsd,
+                    unpricedModelCalls,
+                };
+                if (execution.abortSignal?.aborted) {
+                    await options.recordUnit(`grader-attempt-${unitId}-${startedAt}`, {
+                        ...itemAccounting(),
+                        metricCalls: 0,
+                    });
+                    throwIfExperimentAborted(execution.abortSignal);
+                }
+                const unit = {
+                    schemaVersion: 1,
+                    unitId,
+                    candidateId: candidate.id,
+                    profileId,
+                    split,
+                    repetitions,
+                    annotationsHash: annotations.hash,
+                    annotationId: annotation.id,
+                    repetition,
+                    item,
+                };
+                await options.verifyExecutionIdentity();
+                await writeContentBoundJson(unitPath, unit, evaluationHash, async () => {
+                    await options.recordUnit(unitId, { ...itemAccounting(), metricCalls: 0 });
+                    await options.verifyExecutionIdentity();
+                });
+                throwIfExperimentAborted(execution.abortSignal);
+                items.push(item);
+            }
+        }
+        finally {
+            await releaseFrozenRegradableRun(persistedRun);
+        }
+    }
+    const { metrics, samples } = summarizeGraderEvidence(items, candidate);
+    if (cachedEvidence) {
+        const recomputed = { ...cachedEvidence, metrics, samples, items };
+        if (valueHash(cachedEvidence) !== valueHash(recomputed)) {
+            throw new Error("cached grader evidence does not match its verified evaluation units");
+        }
+        return evaluationFromEvidence(experiment, expectedEvidencePath, cachedEvidence);
+    }
+    const evidence = {
+        schemaVersion: 1,
+        candidateId: candidate.id,
+        profileId,
+        split,
+        repetitions,
+        annotationsHash: annotations.hash,
+        generatedAt: new Date().toISOString(),
+        metrics,
+        samples,
+        items,
+    };
+    await options.verifyExecutionIdentity();
+    await writeContentBoundJson(expectedEvidencePath, evidence, evaluationHash, options.verifyExecutionIdentity);
+    return evaluationFromEvidence(experiment, expectedEvidencePath, evidence);
+};
+//# sourceMappingURL=grader-experiment.js.map

package/lib/orchestrator/grader-experiment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grader-experiment.js","sourceRoot":"","sources":["../../src/orchestrator/grader-experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,qBAAqB,EACrB,iCAAiC,GAClC,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAIL,UAAU,EAGV,mBAAmB,EACnB,QAAQ,EACR,gBAAgB,EAChB,WAAW,EACX,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxC,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,CAAC,CAAC;IAC9E,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB,aAAa,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CAC3C,CAAC,CAAC;AAGH,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACvC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,QAAQ,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC;IACxC,MAAM,EAAE,mBAAmB;IAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IACpC,KAAK,EAAE,gBAAgB;IACvB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IACjC,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;CACnD,CAAC,CAAC;AAGH,MAAM,0BAA0B,GAAG,CAAC;KACjC,MAAM,CAAC;IACN,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAC9C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACxC,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACvC,IAAI,EAAE,wBAAwB;CAC/B,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC,MAAM,CAAC;IACrD,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAC9C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACxC,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC;CACzC,CAAC,CAAC;AAGH,MAAM,yBAAyB,GAAG,CAChC,KAAsD,EAC7C,EAAE;IACX,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,IAAI,KAAK,CAAC,YAAY;QAAE,OAAO,IAAI,CAAC;IACpC,MAAM,MAAM,GAAG,iCAAiC,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzE,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC;AACjE,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAC/B,SAA8C,EACqB,EAAE;IACrE,MAAM,MAAM,GAAG,iCAAiC,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC7E,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,CAAC;IAC5D,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO;QAC5B,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,gBAAgB;KACjD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,UAA4B,EAC5B,MAAqB,EACrB,cAAsB,EACJ,EAAE;IACpB,MAAM,SAAS,GAAqB,EAAE,CAAC;IACvC,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACvE,IAAI,UAAU,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;QACvD,SAAS,CAAC,IAAI,CAAC;YACb,GAAG,EAAE,MAAM;YACX,IAAI,EAAE,WAAW;YACjB,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,UAAU;YACxC,MAAM;YACN,OAAO,EAAE,MAAM,KAAK,UAAU,CAAC,QAAQ,CAAC,UAAU;SACnD,CAAC,CAAC;IACL,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;QACtD,MAAM,aAAa,GAAG,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7F,SAAS,CAAC,IAAI,CAAC;YACb,GAAG,EAAE,MAAM;YACX,IAAI,EAAE,YAAY;YAClB,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,SAAS;YACvC,MAAM;YACN,OAAO,EAAE,MAAM,KAAK,IAAI,IAAI,aAAa,IAAI,cAAc;YAC3D,aAAa;SACd,CAAC,CAAC;IACL,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CACrF,CAAC;IACF,KAAK,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QACrF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,yBAAyB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;QACpE,SAAS,CAAC,IAAI,CAAC;YACb,GAAG,EAAE,WAAW;YAChB,IAAI,EAAE,gBAAgB;YACtB,QAAQ,EAAE,QAAQ,CAAC,MAAM;YACzB,MAAM,EAAE,MAAM,EAAE,MAAM,IAAI,IAAI;YAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,KAAK,QAAQ,CAAC,MAAM;SAC5C,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACjC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3E,SAAS,CAAC,IAAI,CAAC;gBACb,GAAG,EAAE,WAAW;gBAChB,IAAI,EAAE,iBAAiB;gBACvB,QAAQ,EAAE,QAAQ,CAAC,KAAK;gBACxB,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,IAAI;gBAC7B,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,aAAa,IAAI,cAAc;gBAChE,aAAa;aACd,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,CAAC,SAAiB,EAAE,WAAmB,EAAE,KAAa,EAAU,EAAE,CAC9E,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC;AAEtD,MAAM,SAAS,GAAG,CAAC,OAAiC,EAAE,IAAY,EAAE,KAAa,EAAQ,EAAE;IACzF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO;IACpC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;AACzB,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,CAAC,MAAyB,EAAE,KAAK,GAAG,CAAC,EAAU,EAAE,CAC5D,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAa9F,MAAM,qBAAqB,GAAG,CAC5B,KAAoC,EAKpC,EAAE;IACF,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkD,CAAC;IAC/E,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAgC,CAAC;IAClE,MAAM,MAAM,GAAyB;QACnC,YAAY,EAAE,CAAC;QACf,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;QACjB,cAAc,EAAE,CAAC;QACjB,WAAW,EAAE,CAAC;QACd,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;KAClB,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,mBAAmB,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACtE,mBAAmB,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5C,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;QACzD,MAAM,eAAe,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACvE,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;QAC1D,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS;gBAAE,MAAM,CAAC,cAAc,EAAE,CAAC;YAClE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,QAAQ,CAAC,QAAQ,KAAK,SAAS;gBAAE,SAAS;YACzF,MAAM,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,QAAQ,CAAC,QAAQ;gBAAE,MAAM,CAAC,cAAc,EAAE,CAAC;;gBAC1C,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,IAAI;gBAAE,MAAM,CAAC,WAAW,EAAE,CAAC;iBAC9C,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM;gBAAE,MAAM,CAAC,YAAY,EAAE,CAAC;iBAChE,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM;gBAAE,MAAM,CAAC,aAAa,EAAE,CAAC;iBAClE,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAAE,MAAM,CAAC,aAAa,EAAE,CAAC;QACzE,CAAC;IACH,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,EAAE,CAAC;AACrD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,KAA0D,EAC1D,SAAoB,EACoD,EAAE;IAC1E,MAAM,OAAO,GAA6B,EAAE,CAAC;IAC7C,MAAM,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IACjF,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,sBAAsB,GAAG,CAAC,CAAC;IAC/B,KAAK,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,IAAI,YAAY,EAAE,CAAC;QACrD,MAAM,eAAe,GAAG,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAClE,SAAS,CAAC,OAAO,EAAE,iBAAiB,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC5F,SAAS,CAAC,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3F,SAAS,CAAC,OAAO,EAAE,gBAAgB,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7F,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAC1D,CAAC;QACF,SAAS,CACP,OAAO,EACP,uBAAuB,EACvB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAC9D,CAAC;QACF,SAAS,CACP,OAAO,EACP,eAAe,EACf,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAC7D,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,SAAS,CAAC,OAAO,EAAE,kBAAkB,EAAE,YAAY,CAAC,CAAC;QACrD,SAAS,CAAC,OAAO,EAAE,kBAAkB,EAAE,YAAY,CAAC,CAAC;QACrD,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACvF,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,iBAAiB,IAAI,aAAa,CAAC,MAAM,CAAC;YAC1C,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAClE,CAAC;QACJ,CAAC;QACD,MAAM,kBAAkB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CACvD,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CACtC,CAAC;QACF,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,sBAAsB,IAAI,kBAAkB,CAAC,MAAM,CAAC;YACpD,SAAS,CACP,OAAO,EACP,2BAA2B,EAC3B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACvE,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAC7B,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,QAAQ,CAAC,QAAQ,KAAK,SAAS,CACxF,CAAC;QACF,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjD,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CACrE,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,SAAS,CAAC,OAAO,EAAE,2BAA2B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;YACnE,SAAS,CAAC,OAAO,EAAE,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC;QAC1E,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChF,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,SAAS,CACP,OAAO,EACP,+BAA+B,EAC/B,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,MAAM;gBACpE,eAAe,CAAC,MAAM,CACzB,CAAC;QACJ,CAAC;QACD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,SAAS,CACP,OAAO,EACP,+BAA+B,EAC/B,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM;gBACnE,eAAe,CAAC,MAAM,CACzB,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,SAAS,CACP,OAAO,EACP,iBAAiB,EACjB,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAC7E,CAAC;YACF,SAAS,CACP,OAAO,EACP,uBAAuB,EACvB,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAC7E,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CACrB,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa,EAC1C,MAAM,CAAC,GAAG,CACX,CAAC;IACF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAClG,MAAM,OAAO,GAA2B;QACtC,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;QAC3D,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;QAC3D,eAAe,EAAE,YAAY,CAAC,IAAI;QAClC,kBAAkB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QACnF,sBAAsB,EAAE,MAAM,CAAC,aAAa;QAC5C,sBAAsB,EAAE,iBAAiB;QACzC,2BAA2B,EAAE,sBAAsB;QACnD,uBAAuB,EAAE,MAAM,CAAC,cAAc;QAC9C,uBAAuB,EAAE,MAAM,CAAC,cAAc;QAC9C,uBAAuB,EAAE,MAAM,CAAC,cAAc;QAC9C,oBAAoB,EAAE,MAAM,CAAC,WAAW;QACxC,eAAe,EAAE,SAAS,CAAC,WAAW;KACvC,CAAC;IACF,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,KAAyB,EAAQ,EAAE;QAClE,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IAC3E,CAAC,CAAC;IACF,SAAS,CACP,sBAAsB,EACtB,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CACpF,CAAC;IACF,SAAS,CACP,2BAA2B,EAC3B,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAC9F,CAAC;IACF,SAAS,CAAC,uBAAuB,EAAE,SAAS,CAAC,CAAC;IAC9C,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;IACxC,IAAI,OAA2B,CAAC;IAChC,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1D,OAAO,GAAG,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IAC3F,CAAC;IACD,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;IACrC,SAAS,CACP,+BAA+B,EAC/B,OAAO,CAAC,+BAA+B,CAAC;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAC;QAChD,CAAC,CAAC,SAAS,CACd,CAAC;IACF,SAAS,CACP,+BAA+B,EAC/B,OAAO,CAAC,+BAA+B,CAAC;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAC;QAChD,CAAC,CAAC,SAAS,CACd,CAAC;IACF,SAAS,CACP,iBAAiB,EACjB,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;IACF,SAAS,CACP,uBAAuB,EACvB,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CACtF,CAAC;IACF,SAAS,CACP,iBAAiB,EACjB,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,UAA+B,EAC/B,aAGC,EACO,EAAE;IACV,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,SAAS,CAAC;QACzB,WAAW,EAAE,aAAa,CAAC,WAAW;QACtC,SAAS,EAAE,aAAa,CAAC,SAAS;QAClC,KAAK,EAAE,aAAa,CAAC,KAAK;QAC1B,WAAW,EAAE,aAAa,CAAC,WAAW;QACtC,eAAe,EAAE,aAAa,CAAC,eAAe;KAC/C,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChB,OAAO,IAAI,CAAC,SAAS,EAAE,GAAG,aAAa,CAAC,WAAW,IAAI,QAAQ,OAAO,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,KAQrB,EAAU,EAAE,CAAC,UAAU,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AAExD,MAAM,cAAc,GAAG,CAAC,UAA+B,EAAE,MAAc,EAAU,EAAE,CACjF,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,kBAAkB,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAE9D,MAAM,cAAc,GAAG,GAGrB,EAAE,CAAC,CAAC;IACJ,WAAW,EAAE,CAAC;IACd,UAAU,EAAE,CAAC;IACb,KAAK,EAAE,UAAU,EAAE;IACnB,OAAO,EAAE,CAAC;IACV,kBAAkB,EAAE,CAAC;CACtB,CAAC,CAAC;AAEH,MAAM,sBAAsB,GAAG,CAC7B,UAA+B,EAC/B,YAAoB,EACpB,QAAkC,EACc,EAAE;IAClD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;IACjD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACzD,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,MAAM,WAAW,GAAuC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACtF,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1E,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAC7B,CAAC,QAAQ,EAAE,EAAE,CACX,GAAG,QAAQ,CAAC,GAAG,IAAI,QAAQ,CAAC,IAAI,cAAc,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAC5G,CAAC;QACF,IAAI,IAAI,CAAC,QAAQ;YAAE,QAAQ,CAAC,OAAO,CAAC,mBAAmB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxE,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;gBAAE,SAAS;YACrD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;YAC3F,IAAI,OAAO;gBAAE,QAAQ,CAAC,IAAI,CAAC,sBAAsB,QAAQ,CAAC,GAAG,KAAK,OAAO,EAAE,CAAC,CAAC;QAC/E,CAAC;QACD,OAAO;YACL;gBACE,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,UAAU,EAAE;gBAClD,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzE,QAAQ;aACT;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO;QACL,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,YAAY,CAAC;QACrD,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC;QACnD,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9F,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM;QAClC,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC/B,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CACd,KAAK;YACL,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAC3B,CAAC,cAAc,EAAE,SAAS,EAAE,EAAE,CAC5B,cAAc,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC,QAAQ,EAC/D,CAAC,CACF,EACH,CAAC,CACF;QACD,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzD,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,kBAAkB,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC9F,WAAW;KACZ,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,gBAAwB,EAAE,UAAkB,EAAU,EAAE;IACnF,MAAM,IAAI,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACvC,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,8CAA8C,UAAU,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,MAAwD,EAAY,EAAE,CAC1F,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAElE,MAAM,sBAAsB,GAAG,CAC7B,QAAkC,EAClC,QAMC,EACD,IAAY,EACN,EAAE;IACR,IACE,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QAC7C,QAAQ,CAAC,SAAS,KAAK,QAAQ,CAAC,SAAS;QACzC,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK;QACjC,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QAC7C,QAAQ,CAAC,eAAe,KAAK,QAAQ,CAAC,eAAe,EACrD,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,IAAI,EAAE,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,IAA0B,EAC1B,QAA8D,EAC9D,IAAY,EACN,EAAE;IACR,IACE,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAC/B,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QACzC,IAAI,CAAC,SAAS,KAAK,QAAQ,CAAC,SAAS;QACrC,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK;QAC7B,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QACzC,IAAI,CAAC,eAAe,KAAK,QAAQ,CAAC,eAAe;QACjD,IAAI,CAAC,YAAY,KAAK,QAAQ,CAAC,YAAY;QAC3C,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,UAAU,EACvC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,IAAI,EAAE,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAAC,MAA+B,EAAQ,EAAE;IACzE,IAAI,CAAC,MAAM,EAAE,OAAO;QAAE,OAAO;IAC7B,MAAM,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;AACzF,CAAC,CAAC;AAEF,MAAM,0BAA0B,GAAG,CACjC,UAAkB,EAClB,WAA8B,EAC9B,YAA+B,EAC/B,gBAAqC,EAC/B,EAAE;IACR,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/E,MAAM,kBAAkB,GAAG,YAAY,CAAC,MAAM,CAC5C,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAC/D,CAAC;IACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC5E,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,sEAAsE;QACzF,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;QACjF,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACtG,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,EAAE,OAkC7C,EAA2D,EAAE;IAC5D,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IACpF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC;IACvD,IAAI,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,CAAC,cAAc,CAAC;IAC1D,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACrF,MAAM,QAAQ,GAAG,uBAAuB,CAAC,WAAW,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,eAAe,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,KAAK,6BAA6B,CACnF,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,SAAS,qBAAqB,CAAC,CAAC;IACrF,MAAM,oBAAoB,GAAG,kBAAkB,CAAC,UAAU,EAAE;QAC1D,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,SAAS;QACT,KAAK;QACL,WAAW;QACX,eAAe,EAAE,WAAW,CAAC,IAAI;KAClC,CAAC,CAAC;IACH,IAAI,cAAc,GAAG,MAAM,oBAAoB,CAC7C,oBAAoB,EACpB,8BAA8B,EAC9B,cAAc,CACf,CAAC;IACF,IAAI,cAAc,EAAE,CAAC;QACnB,sBAAsB,CACpB,cAAc,EACd;YACE,WAAW,EAAE,SAAS,CAAC,EAAE;YACzB,SAAS;YACT,KAAK;YACL,WAAW;YACX,eAAe,EAAE,WAAW,CAAC,IAAI;SAClC,EACD,oBAAoB,CACrB,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;IAC5D,MAAM,QAAQ,GAAG,uBAAuB,CAAC;QACvC,GAAG,SAAS,CAAC,MAAM;QACnB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC,CAAC,aAAa,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC;IACjD,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,KAAK,MAAM,UAAU,IAAI,QAAQ,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,oBAAoB,CACvC,mBAAmB,CAAC,OAAO,CAAC,gBAAgB,EAAE,UAAU,CAAC,UAAU,CAAC,CACrE,CAAC;QACF,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAC/C,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC,CACzE,CAAC;QACF,MAAM,WAAW,GAAG,YAAY,CAAC,cAAc,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAC9B,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,EAAE,CAC5F,CAAC;QACF,0BAA0B,CAAC,UAAU,CAAC,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAC;QAC/F,MAAM,YAAY,GAAG,MAAM,uBAAuB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACtE,IAAI,CAAC;YACH,IAAI,MAAM,CAAC,EAAE,KAAK,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,CAAC,UAAU,SAAS,MAAM,CAAC,EAAE,oCAAoC,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,CAC9G,CAAC;YACJ,CAAC;YACD,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,IAAI,WAAW,EAAE,UAAU,EAAE,EAAE,CAAC;gBACjE,MAAM,MAAM,GAAG,YAAY,CAAC;oBAC1B,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,SAAS;oBACT,KAAK;oBACL,WAAW;oBACX,eAAe,EAAE,WAAW,CAAC,IAAI;oBACjC,YAAY,EAAE,UAAU,CAAC,EAAE;oBAC3B,UAAU;iBACX,CAAC,CAAC;gBACH,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBACpD,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CACzC,QAAQ,EACR,0BAA0B,EAC1B,cAAc,CACf,CAAC;gBACF,IAAI,QAAQ,EAAE,CAAC;oBACb,kBAAkB,CAChB,QAAQ,EACR;wBACE,MAAM;wBACN,WAAW,EAAE,SAAS,CAAC,EAAE;wBACzB,SAAS;wBACT,KAAK;wBACL,WAAW;wBACX,eAAe,EAAE,WAAW,CAAC,IAAI;wBACjC,YAAY,EAAE,UAAU,CAAC,EAAE;wBAC3B,UAAU;qBACX,EACD,QAAQ,CACT,CAAC;oBACF,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,GAAG,cAAc,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC1E,wBAAwB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;oBAChD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;oBAC1B,SAAS;gBACX,CAAC;gBACD,IAAI,cAAc,EAAE,CAAC;oBACnB,MAAM,sBAAsB,CAAC,oBAAoB,CAAC,CAAC;oBACnD,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;gBACD,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACpD,MAAM,GAAG,GAAG;oBACV,GAAG,YAAY;oBACf,IAAI,EAAE,EAAE,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE;iBAC9D,CAAC;gBACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,MAAM,kBAAkB,GAAG,UAAU,EAAE,CAAC;gBACxC,MAAM,OAAO,CAAC,aAAa,CACzB,yBAAyB,MAAM,IAAI,kBAAkB,EAAE,EACvD,cAAc,EAAE,CACjB,CAAC;gBACF,IAAI,MAAqB,CAAC;gBAC1B,IAAI,CAAC;oBACH,MAAM,GAAG,MAAM,QAAQ,CACrB,GAAG,EACH,QAAQ,EACR,SAAS,EACT,SAAS,CAAC,WAAW,EACrB,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;wBACzB,MAAM,UAAU,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;wBACvD,IAAI,kBAAkB,GAAG,UAAU,CAAC,kBAAkB,CAAC;wBACvD,IAAI,kBAAkB,KAAK,CAAC,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;4BAC7E,kBAAkB,GAAG,CAAC,CAAC;wBACzB,CAAC;wBACD,MAAM,OAAO,CAAC,eAAe,CAC3B,gBAAgB,MAAM,IAAI,kBAAkB,IAAI,KAAK,EAAE,EACvD;4BACE,WAAW,EAAE,CAAC;4BACd,2DAA2D;4BAC3D,UAAU,EAAE,CAAC;4BACb,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,UAAU,EAAE;4BACtC,OAAO,EAAE,UAAU,CAAC,OAAO;4BAC3B,kBAAkB;yBACnB,CACF,CAAC;oBACJ,CAAC,EACD;wBACE,KAAK,EAAE,OAAO,CAAC,uBAAuB;wBACtC,QAAQ,EAAE,OAAO,CAAC,qBAAqB;qBACxC,CACF,CAAC;oBACF,MAAM,eAAe,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;oBAC1E,IAAI,MAAM,CAAC,UAAU,EAAE,KAAK,KAAK,SAAS,IAAI,eAAe,EAAE,CAAC;wBAC9D,MAAM,IAAI,KAAK,CACb,eAAe,EAAE,YAAY;4BAC3B,eAAe,EAAE,OAAO;4BACxB,+CAA+C,CAClD,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,OAAO,CAAC,UAAU,CAAC,kBAAkB,MAAM,IAAI,kBAAkB,EAAE,EAAE;wBACzE,GAAG,cAAc,EAAE;wBACnB,WAAW,EAAE,CAAC;qBACf,CAAC,CAAC;oBACH,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,MAAM,KAAK,GAAG,QAAQ,CACpB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CACrF,CAAC;gBACF,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CACtC,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC,OAAO,EACzE,CAAC,CACF,CAAC;gBACF,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CACjD,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC,kBAAkB,EACpF,CAAC,CACF,CAAC;gBACF,MAAM,IAAI,GAAuB;oBAC/B,YAAY,EAAE,UAAU,CAAC,EAAE;oBAC3B,UAAU;oBACV,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,UAAU,EAAE,UAAU,CAAC,UAAU;oBACjC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjE,SAAS,EAAE,uBAAuB,CAChC,UAAU,EACV,MAAM,EACN,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,cAAc,CACpD;oBACD,MAAM;oBACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,KAAK;oBACL,OAAO;oBACP,kBAAkB;iBACnB,CAAC;gBACF,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC;oBACnC,MAAM,OAAO,CAAC,UAAU,CAAC,kBAAkB,MAAM,IAAI,SAAS,EAAE,EAAE;wBAChE,GAAG,cAAc,EAAE;wBACnB,WAAW,EAAE,CAAC;qBACf,CAAC,CAAC;oBACH,wBAAwB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;gBAClD,CAAC;gBACD,MAAM,IAAI,GAAyB;oBACjC,aAAa,EAAE,CAAC;oBAChB,MAAM;oBACN,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,SAAS;oBACT,KAAK;oBACL,WAAW;oBACX,eAAe,EAAE,WAAW,CAAC,IAAI;oBACjC,YAAY,EAAE,UAAU,CAAC,EAAE;oBAC3B,UAAU;oBACV,IAAI;iBACL,CAAC;gBACF,MAAM,OAAO,CAAC,uBAAuB,EAAE,CAAC;gBACxC,MAAM,qBAAqB,CAAC,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,IAAI,EAAE;oBACrE,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,GAAG,cAAc,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC1E,MAAM,OAAO,CAAC,uBAAuB,EAAE,CAAC;gBAC1C,CAAC,CAAC,CAAC;gBACH,wBAAwB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;gBAChD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,0BAA0B,CAAC,YAAY,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,uBAAuB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACvE,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,UAAU,GAAG,EAAE,GAAG,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAClE,IAAI,SAAS,CAAC,cAAc,CAAC,KAAK,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,sBAAsB,CAAC,UAAU,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,QAAQ,GAA6B;QACzC,aAAa,EAAE,CAAC;QAChB,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,SAAS;QACT,KAAK;QACL,WAAW;QACX,eAAe,EAAE,WAAW,CAAC,IAAI;QACjC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,OAAO;QACP,OAAO;QACP,KAAK;KACN,CAAC;IACF,MAAM,OAAO,CAAC,uBAAuB,EAAE,CAAC;IACxC,MAAM,qBAAqB,CACzB,oBAAoB,EACpB,QAAQ,EACR,cAAc,EACd,OAAO,CAAC,uBAAuB,CAChC,CAAC;IACF,OAAO,sBAAsB,CAAC,UAAU,EAAE,oBAAoB,EAAE,QAAQ,CAAC,CAAC;AAC5E,CAAC,CAAC"}