@prismatic-io/lux 0.0.2-preview.3 → 0.0.2-preview.30

package/lib/orchestrator/grader-experiment.js

@@ -1,12 +1,13 @@
 import { randomUUID } from "node:crypto";
 import { join, relative, resolve, sep } from "node:path";
 import { z } from "zod";
-import { createRubricAssertion, RubricModelAccountingDetailSchema, } from "../assertions/rubric/index.js";
+import { RubricModelAccountingDetailSchema } from "../assertions/rubric/index.js";
 import { emptyUsage, GradingReportSchema, sumUsage, TokenUsageSchema, totalTokens, valueHash, } from "../core/index.js";
 import { selectGraderAnnotations } from "./annotation-loader.js";
 import { buildRegistryFromConfig } from "./config.js";
 import { readContentBoundJson, retireContentBoundJson, writeContentBoundJson, } from "./content-bound-json.js";
 import { gradeRun } from "./grading.js";
+import { clusterCaseSamples, groupedCaseMetricSamples } from "./paired-effects.js";
 import { loadFrozenRegradableRun, releaseFrozenRegradableRun } from "./regrade.js";
 import { loadRubricDefinition } from "./rubric-loader.js";
 const GraderJudgmentSchema = z.object({
@@ -16,8 +17,10 @@ const GraderJudgmentSchema = z.object({
     actual: z.union([z.boolean(), z.number()]).nullable(),
     matched: z.boolean(),
     absoluteError: z.exactOptional(z.number()),
+    unavailableReason: z.exactOptional(z.enum(["insufficient-evidence", "operational-error"])),
 });
 const GraderEvidenceItemSchema = z.object({
+    clusterId: z.exactOptional(z.string().min(1)),
     annotationId: z.string(),
     repetition: z.number().int().positive(),
     runDir: z.string(),
@@ -64,6 +67,7 @@ const assertionOperationalError = (entry) => {
     const detail = RubricModelAccountingDetailSchema.safeParse(entry.detail);
     return detail.success && detail.data.graderError !== undefined;
 };
+const assertionUnavailableReason = (entry) => assertionOperationalError(entry) ? "operational-error" : entry?.abstention;
 const assertionModelAccounting = (assertion) => {
     const detail = RubricModelAccountingDetailSchema.safeParse(assertion.detail);
     if (!detail.success) {
@@ -77,19 +81,22 @@ const assertionModelAccounting = (assertion) => {
 };
 export const compareGraderAnnotation = (annotation, report, scoreTolerance) => {
     const judgments = [];
-    const reportFailed = report.assertions.some(assertionOperationalError);
+    const reportUnavailableReason = report.assertions.some(assertionOperationalError)
+        ? "operational-error"
+        : report.assertions.find((entry) => entry.abstention)?.abstention;
     if (annotation.expected.casePassed !== undefined) {
-        const actual = reportFailed ? null : report.casePassed;
+        const actual = reportUnavailableReason ? null : report.casePassed;
         judgments.push({
             key: "case",
             kind: "case-pass",
             expected: annotation.expected.casePassed,
             actual,
             matched: actual === annotation.expected.casePassed,
+            ...(reportUnavailableReason ? { unavailableReason: reportUnavailableReason } : {}),
         });
     }
     if (annotation.expected.caseScore !== undefined) {
-        const actual = reportFailed ? null : report.caseScore;
+        const actual = reportUnavailableReason ? null : report.caseScore;
         const absoluteError = actual === null ? 1 : Math.abs(actual - annotation.expected.caseScore);
         judgments.push({
             key: "case",
@@ -98,18 +105,21 @@ export const compareGraderAnnotation = (annotation, report, scoreTolerance) => {
             actual,
             matched: actual !== null && absoluteError <= scoreTolerance,
             absoluteError,
+            ...(reportUnavailableReason ? { unavailableReason: reportUnavailableReason } : {}),
         });
     }
     const entries = new Map(report.assertions.flatMap((entry) => (entry.id ? [[entry.id, entry]] : [])));
     for (const [assertionId, expected] of Object.entries(annotation.expected.assertions)) {
         const entry = entries.get(assertionId);
-        const actual = assertionOperationalError(entry) ? undefined : entry;
+        const unavailableReason = assertionUnavailableReason(entry);
+        const actual = unavailableReason ? undefined : entry;
         judgments.push({
             key: assertionId,
             kind: "assertion-pass",
             expected: expected.passed,
             actual: actual?.passed ?? null,
             matched: actual?.passed === expected.passed,
+            ...(unavailableReason ? { unavailableReason } : {}),
         });
         if (expected.score !== undefined) {
             const absoluteError = actual ? Math.abs(actual.score - expected.score) : 1;
@@ -120,6 +130,7 @@ export const compareGraderAnnotation = (annotation, report, scoreTolerance) => {
                 actual: actual?.score ?? null,
                 matched: actual !== undefined && absoluteError <= scoreTolerance,
                 absoluteError,
+                ...(unavailableReason ? { unavailableReason } : {}),
             });
         }
     }
@@ -223,6 +234,10 @@ export const summarizeGraderEvidence = (items, candidate) => {
         if (passes.length > 0) {
             addSample(samples, "grader.coverage", passes.filter((judgment) => judgment.actual !== null).length / passes.length);
             addSample(samples, "grader.abstentionRate", passes.filter((judgment) => judgment.actual === null).length / passes.length);
+            addSample(samples, "grader.insufficientEvidenceRate", passes.filter((judgment) => judgment.unavailableReason === "insufficient-evidence").length /
+                passes.length);
+            addSample(samples, "grader.operationalErrorRate", passes.filter((judgment) => judgment.unavailableReason === "operational-error").length /
+                passes.length);
         }
     }
     const precision = ratio(counts.truePositive, counts.truePositive + counts.falsePositive, Number.NaN);
@@ -290,7 +305,35 @@ const itemAccounting = () => ({
     unpricedModelCalls: 0,
 });
 const evaluationFromEvidence = (experiment, evidencePath, evidence) => {
+    const caseSamples = clusterCaseSamples(evidence.items.map((item) => {
+        const summary = summarizeGraderEvidence([item], {
+            sourceBytes: evidence.metrics["subject.bytes"] ?? 0,
+        });
+        return {
+            [item.annotationId]: {
+                ...(item.clusterId ? { clusterId: item.clusterId } : {}),
+                identityHash: experiment.manifest.evaluationHash
+                    ? valueHash({
+                        evaluationHash: experiment.manifest.evaluationHash,
+                        annotationId: item.annotationId,
+                        profileId: evidence.profileId,
+                        split: evidence.split,
+                    })
+                    : null,
+                metrics: Object.fromEntries(Object.entries(summary.samples).flatMap(([metric, values]) => values.length === 1 ? [[metric, { mean: values[0] ?? 0, n: 1 }]] : [])),
+            },
+        };
+    }));
     const byAnnotation = new Map();
+    const grouped = groupedCaseMetricSamples(caseSamples, evidence.repetitions);
+    const metrics = grouped
+        ? Object.fromEntries(Object.entries(evidence.metrics).flatMap(([name, value]) => {
+            if (!Object.hasOwn(evidence.samples, name))
+                return [[name, value]];
+            const values = grouped[name];
+            return values ? [[name, mean(values)]] : [];
+        }))
+        : evidence.metrics;
     for (const item of evidence.items) {
         const values = byAnnotation.get(item.annotationId) ?? [];
         values.push(...item.judgments.map((judgment) => (judgment.matched ? 1 : 0)));
@@ -322,10 +365,11 @@ const evaluationFromEvidence = (experiment, evidencePath, evidence) => {
         profileId: evidence.profileId,
         split: evidence.split,
         evidencePath: relative(experiment.path, evidencePath),
-        passRate: evidence.metrics["quality.passRate"] ?? 0,
+        passRate: metrics["quality.passRate"] ?? 0,
         casePassRates: Object.fromEntries([...byAnnotation].map(([id, values]) => [id, mean(values)])),
-        metrics: evidence.metrics,
-        samples: evidence.samples,
+        metrics,
+        samples: grouped ?? evidence.samples,
+        ...(caseSamples ? { caseSamples } : {}),
         metricCalls: evidence.items.length,
         judgeCalls: evidence.items.reduce((total, item) => total +
             item.report.assertions.reduce((assertionTotal, assertion) => assertionTotal + assertionModelAccounting(assertion).attempts, 0), 0),
@@ -416,7 +460,7 @@ export const evaluateGraderCandidate = async (options) => {
     const registry = buildRegistryFromConfig({
         ...execution.config,
         ...(harness ? { harness } : {}),
-    }).withAssertion(createRubricAssertion(harness));
+    });
     const items = [];
     for (const annotation of selected) {
         const rubric = await loadRubricDefinition(candidateRubricPath(options.materializedRoot, annotation.rubricPath));
@@ -509,6 +553,7 @@ export const evaluateGraderCandidate = async (options) => {
                 const unpricedModelCalls = report.assertions.reduce((total, assertion) => total + assertionModelAccounting(assertion).unpricedModelCalls, 0);
                 const item = {
                     annotationId: annotation.id,
+                    ...(annotation.meta?.cluster ? { clusterId: annotation.meta.cluster } : {}),
                     repetition,
                     runDir: annotation.runDir,
                     rubricPath: annotation.rubricPath,

package/lib/orchestrator/grader-experiment.js.map

@@ -1 +1 @@
-{"version":3,"file":"grader-experiment.js","sourceRoot":"","sources":["../../src/orchestrator/grader-experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,qBAAqB,EACrB,iCAAiC,GAClC,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAIL,UAAU,EAGV,mBAAmB,EACnB,QAAQ,EACR,gBAAgB,EAChB,WAAW,EACX,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxC,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,CAAC,CAAC;IAC9E,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB,aAAa,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CAC3C,CAAC,CAAC;AAGH,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACvC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,QAAQ,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC;IACxC,MAAM,EAAE,mBAAmB;IAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IACpC,KAAK,EAAE,gBAAgB;IACvB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IACjC,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;CACnD,CAAC,CAAC;AAGH,MAAM,0BAA0B,GAAG,CAAC;KACjC,MAAM,CAAC;IACN,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAC9C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACxC,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACvC,IAAI,EAAE,wBAAwB;CAC/B,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC,MAAM,CAAC;IACrD,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAC9C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACxC,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC;CACzC,CAAC,CAAC;AAGH,MAAM,yBAAyB,GAAG,CAChC,KAAsD,EAC7C,EAAE;IACX,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,IAAI,KAAK,CAAC,YAAY;QAAE,OAAO,IAAI,CAAC;IACpC,MAAM,MAAM,GAAG,iCAAiC,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzE,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC;AACjE,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAC/B,SAA8C,EACqB,EAAE;IACrE,MAAM,MAAM,GAAG,iCAAiC,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC7E,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,CAAC;IAC5D,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO;QAC5B,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,gBAAgB;KACjD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,UAA4B,EAC5B,MAAqB,EACrB,cAAsB,EACJ,EAAE;IACpB,MAAM,SAAS,GAAqB,EAAE,CAAC;IACvC,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACvE,IAAI,UAAU,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;QACvD,SAAS,CAAC,IAAI,CAAC;YACb,GAAG,EAAE,MAAM;YACX,IAAI,EAAE,WAAW;YACjB,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,UAAU;YACxC,MAAM;YACN,OAAO,EAAE,MAAM,KAAK,UAAU,CAAC,QAAQ,CAAC,UAAU;SACnD,CAAC,CAAC;IACL,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;QACtD,MAAM,aAAa,GAAG,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7F,SAAS,CAAC,IAAI,CAAC;YACb,GAAG,EAAE,MAAM;YACX,IAAI,EAAE,YAAY;YAClB,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,SAAS;YACvC,MAAM;YACN,OAAO,EAAE,MAAM,KAAK,IAAI,IAAI,aAAa,IAAI,cAAc;YAC3D,aAAa;SACd,CAAC,CAAC;IACL,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CACrF,CAAC;IACF,KAAK,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QACrF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,yBAAyB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;QACpE,SAAS,CAAC,IAAI,CAAC;YACb,GAAG,EAAE,WAAW;YAChB,IAAI,EAAE,gBAAgB;YACtB,QAAQ,EAAE,QAAQ,CAAC,MAAM;YACzB,MAAM,EAAE,MAAM,EAAE,MAAM,IAAI,IAAI;YAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,KAAK,QAAQ,CAAC,MAAM;SAC5C,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACjC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3E,SAAS,CAAC,IAAI,CAAC;gBACb,GAAG,EAAE,WAAW;gBAChB,IAAI,EAAE,iBAAiB;gBACvB,QAAQ,EAAE,QAAQ,CAAC,KAAK;gBACxB,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,IAAI;gBAC7B,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,aAAa,IAAI,cAAc;gBAChE,aAAa;aACd,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,CAAC,SAAiB,EAAE,WAAmB,EAAE,KAAa,EAAU,EAAE,CAC9E,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC;AAEtD,MAAM,SAAS,GAAG,CAAC,OAAiC,EAAE,IAAY,EAAE,KAAa,EAAQ,EAAE;IACzF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO;IACpC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;AACzB,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,CAAC,MAAyB,EAAE,KAAK,GAAG,CAAC,EAAU,EAAE,CAC5D,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAa9F,MAAM,qBAAqB,GAAG,CAC5B,KAAoC,EAKpC,EAAE;IACF,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkD,CAAC;IAC/E,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAgC,CAAC;IAClE,MAAM,MAAM,GAAyB;QACnC,YAAY,EAAE,CAAC;QACf,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;QACjB,cAAc,EAAE,CAAC;QACjB,WAAW,EAAE,CAAC;QACd,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;KAClB,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,mBAAmB,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACtE,mBAAmB,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5C,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;QACzD,MAAM,eAAe,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACvE,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;QAC1D,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS;gBAAE,MAAM,CAAC,cAAc,EAAE,CAAC;YAClE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,QAAQ,CAAC,QAAQ,KAAK,SAAS;gBAAE,SAAS;YACzF,MAAM,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,QAAQ,CAAC,QAAQ;gBAAE,MAAM,CAAC,cAAc,EAAE,CAAC;;gBAC1C,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,IAAI;gBAAE,MAAM,CAAC,WAAW,EAAE,CAAC;iBAC9C,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM;gBAAE,MAAM,CAAC,YAAY,EAAE,CAAC;iBAChE,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM;gBAAE,MAAM,CAAC,aAAa,EAAE,CAAC;iBAClE,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAAE,MAAM,CAAC,aAAa,EAAE,CAAC;QACzE,CAAC;IACH,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,EAAE,CAAC;AACrD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,KAA0D,EAC1D,SAAoB,EACoD,EAAE;IAC1E,MAAM,OAAO,GAA6B,EAAE,CAAC;IAC7C,MAAM,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IACjF,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,sBAAsB,GAAG,CAAC,CAAC;IAC/B,KAAK,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,IAAI,YAAY,EAAE,CAAC;QACrD,MAAM,eAAe,GAAG,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAClE,SAAS,CAAC,OAAO,EAAE,iBAAiB,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC5F,SAAS,CAAC,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3F,SAAS,CAAC,OAAO,EAAE,gBAAgB,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7F,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAC1D,CAAC;QACF,SAAS,CACP,OAAO,EACP,uBAAuB,EACvB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAC9D,CAAC;QACF,SAAS,CACP,OAAO,EACP,eAAe,EACf,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAC7D,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,SAAS,CAAC,OAAO,EAAE,kBAAkB,EAAE,YAAY,CAAC,CAAC;QACrD,SAAS,CAAC,OAAO,EAAE,kBAAkB,EAAE,YAAY,CAAC,CAAC;QACrD,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACvF,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,iBAAiB,IAAI,aAAa,CAAC,MAAM,CAAC;YAC1C,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAClE,CAAC;QACJ,CAAC;QACD,MAAM,kBAAkB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CACvD,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CACtC,CAAC;QACF,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,sBAAsB,IAAI,kBAAkB,CAAC,MAAM,CAAC;YACpD,SAAS,CACP,OAAO,EACP,2BAA2B,EAC3B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACvE,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAC7B,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,QAAQ,CAAC,QAAQ,KAAK,SAAS,CACxF,CAAC;QACF,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjD,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CACrE,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,SAAS,CAAC,OAAO,EAAE,2BAA2B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;YACnE,SAAS,CAAC,OAAO,EAAE,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC;QAC1E,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChF,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,SAAS,CACP,OAAO,EACP,+BAA+B,EAC/B,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,MAAM;gBACpE,eAAe,CAAC,MAAM,CACzB,CAAC;QACJ,CAAC;QACD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,SAAS,CACP,OAAO,EACP,+BAA+B,EAC/B,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM;gBACnE,eAAe,CAAC,MAAM,CACzB,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,SAAS,CACP,OAAO,EACP,iBAAiB,EACjB,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAC7E,CAAC;YACF,SAAS,CACP,OAAO,EACP,uBAAuB,EACvB,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAC7E,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CACrB,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa,EAC1C,MAAM,CAAC,GAAG,CACX,CAAC;IACF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAClG,MAAM,OAAO,GAA2B;QACtC,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;QAC3D,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;QAC3D,eAAe,EAAE,YAAY,CAAC,IAAI;QAClC,kBAAkB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QACnF,sBAAsB,EAAE,MAAM,CAAC,aAAa;QAC5C,sBAAsB,EAAE,iBAAiB;QACzC,2BAA2B,EAAE,sBAAsB;QACnD,uBAAuB,EAAE,MAAM,CAAC,cAAc;QAC9C,uBAAuB,EAAE,MAAM,CAAC,cAAc;QAC9C,uBAAuB,EAAE,MAAM,CAAC,cAAc;QAC9C,oBAAoB,EAAE,MAAM,CAAC,WAAW;QACxC,eAAe,EAAE,SAAS,CAAC,WAAW;KACvC,CAAC;IACF,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,KAAyB,EAAQ,EAAE;QAClE,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IAC3E,CAAC,CAAC;IACF,SAAS,CACP,sBAAsB,EACtB,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CACpF,CAAC;IACF,SAAS,CACP,2BAA2B,EAC3B,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAC9F,CAAC;IACF,SAAS,CAAC,uBAAuB,EAAE,SAAS,CAAC,CAAC;IAC9C,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;IACxC,IAAI,OAA2B,CAAC;IAChC,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1D,OAAO,GAAG,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IAC3F,CAAC;IACD,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;IACrC,SAAS,CACP,+BAA+B,EAC/B,OAAO,CAAC,+BAA+B,CAAC;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAC;QAChD,CAAC,CAAC,SAAS,CACd,CAAC;IACF,SAAS,CACP,+BAA+B,EAC/B,OAAO,CAAC,+BAA+B,CAAC;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAC;QAChD,CAAC,CAAC,SAAS,CACd,CAAC;IACF,SAAS,CACP,iBAAiB,EACjB,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;IACF,SAAS,CACP,uBAAuB,EACvB,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CACtF,CAAC;IACF,SAAS,CACP,iBAAiB,EACjB,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,UAA+B,EAC/B,aAGC,EACO,EAAE;IACV,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,SAAS,CAAC;QACzB,WAAW,EAAE,aAAa,CAAC,WAAW;QACtC,SAAS,EAAE,aAAa,CAAC,SAAS;QAClC,KAAK,EAAE,aAAa,CAAC,KAAK;QAC1B,WAAW,EAAE,aAAa,CAAC,WAAW;QACtC,eAAe,EAAE,aAAa,CAAC,eAAe;KAC/C,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChB,OAAO,IAAI,CAAC,SAAS,EAAE,GAAG,aAAa,CAAC,WAAW,IAAI,QAAQ,OAAO,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,KAQrB,EAAU,EAAE,CAAC,UAAU,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AAExD,MAAM,cAAc,GAAG,CAAC,UAA+B,EAAE,MAAc,EAAU,EAAE,CACjF,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,kBAAkB,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAE9D,MAAM,cAAc,GAAG,GAGrB,EAAE,CAAC,CAAC;IACJ,WAAW,EAAE,CAAC;IACd,UAAU,EAAE,CAAC;IACb,KAAK,EAAE,UAAU,EAAE;IACnB,OAAO,EAAE,CAAC;IACV,kBAAkB,EAAE,CAAC;CACtB,CAAC,CAAC;AAEH,MAAM,sBAAsB,GAAG,CAC7B,UAA+B,EAC/B,YAAoB,EACpB,QAAkC,EACc,EAAE;IAClD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;IACjD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACzD,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,MAAM,WAAW,GAAuC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACtF,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1E,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAC7B,CAAC,QAAQ,EAAE,EAAE,CACX,GAAG,QAAQ,CAAC,GAAG,IAAI,QAAQ,CAAC,IAAI,cAAc,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAC5G,CAAC;QACF,IAAI,IAAI,CAAC,QAAQ;YAAE,QAAQ,CAAC,OAAO,CAAC,mBAAmB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxE,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;gBAAE,SAAS;YACrD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;YAC3F,IAAI,OAAO;gBAAE,QAAQ,CAAC,IAAI,CAAC,sBAAsB,QAAQ,CAAC,GAAG,KAAK,OAAO,EAAE,CAAC,CAAC;QAC/E,CAAC;QACD,OAAO;YACL;gBACE,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,UAAU,EAAE;gBAClD,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzE,QAAQ;aACT;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO;QACL,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,YAAY,CAAC;QACrD,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC;QACnD,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9F,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM;QAClC,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC/B,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CACd,KAAK;YACL,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAC3B,CAAC,cAAc,EAAE,SAAS,EAAE,EAAE,CAC5B,cAAc,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC,QAAQ,EAC/D,CAAC,CACF,EACH,CAAC,CACF;QACD,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzD,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,kBAAkB,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC9F,WAAW;KACZ,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,gBAAwB,EAAE,UAAkB,EAAU,EAAE;IACnF,MAAM,IAAI,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACvC,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,8CAA8C,UAAU,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,MAAwD,EAAY,EAAE,CAC1F,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAElE,MAAM,sBAAsB,GAAG,CAC7B,QAAkC,EAClC,QAMC,EACD,IAAY,EACN,EAAE;IACR,IACE,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QAC7C,QAAQ,CAAC,SAAS,KAAK,QAAQ,CAAC,SAAS;QACzC,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK;QACjC,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QAC7C,QAAQ,CAAC,eAAe,KAAK,QAAQ,CAAC,eAAe,EACrD,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,IAAI,EAAE,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,IAA0B,EAC1B,QAA8D,EAC9D,IAAY,EACN,EAAE;IACR,IACE,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAC/B,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QACzC,IAAI,CAAC,SAAS,KAAK,QAAQ,CAAC,SAAS;QACrC,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK;QAC7B,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QACzC,IAAI,CAAC,eAAe,KAAK,QAAQ,CAAC,eAAe;QACjD,IAAI,CAAC,YAAY,KAAK,QAAQ,CAAC,YAAY;QAC3C,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,UAAU,EACvC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,IAAI,EAAE,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAAC,MAA+B,EAAQ,EAAE;IACzE,IAAI,CAAC,MAAM,EAAE,OAAO;QAAE,OAAO;IAC7B,MAAM,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;AACzF,CAAC,CAAC;AAEF,MAAM,0BAA0B,GAAG,CACjC,UAAkB,EAClB,WAA8B,EAC9B,YAA+B,EAC/B,gBAAqC,EAC/B,EAAE;IACR,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/E,MAAM,kBAAkB,GAAG,YAAY,CAAC,MAAM,CAC5C,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAC/D,CAAC;IACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC5E,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,sEAAsE;QACzF,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;QACjF,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACtG,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,EAAE,OAkC7C,EAA2D,EAAE;IAC5D,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IACpF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC;IACvD,IAAI,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,CAAC,cAAc,CAAC;IAC1D,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACrF,MAAM,QAAQ,GAAG,uBAAuB,CAAC,WAAW,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,eAAe,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,KAAK,6BAA6B,CACnF,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,SAAS,qBAAqB,CAAC,CAAC;IACrF,MAAM,oBAAoB,GAAG,kBAAkB,CAAC,UAAU,EAAE;QAC1D,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,SAAS;QACT,KAAK;QACL,WAAW;QACX,eAAe,EAAE,WAAW,CAAC,IAAI;KAClC,CAAC,CAAC;IACH,IAAI,cAAc,GAAG,MAAM,oBAAoB,CAC7C,oBAAoB,EACpB,8BAA8B,EAC9B,cAAc,CACf,CAAC;IACF,IAAI,cAAc,EAAE,CAAC;QACnB,sBAAsB,CACpB,cAAc,EACd;YACE,WAAW,EAAE,SAAS,CAAC,EAAE;YACzB,SAAS;YACT,KAAK;YACL,WAAW;YACX,eAAe,EAAE,WAAW,CAAC,IAAI;SAClC,EACD,oBAAoB,CACrB,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;IAC5D,MAAM,QAAQ,GAAG,uBAAuB,CAAC;QACvC,GAAG,SAAS,CAAC,MAAM;QACnB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC,CAAC,aAAa,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC;IACjD,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,KAAK,MAAM,UAAU,IAAI,QAAQ,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,oBAAoB,CACvC,mBAAmB,CAAC,OAAO,CAAC,gBAAgB,EAAE,UAAU,CAAC,UAAU,CAAC,CACrE,CAAC;QACF,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAC/C,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC,CACzE,CAAC;QACF,MAAM,WAAW,GAAG,YAAY,CAAC,cAAc,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAC9B,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,EAAE,CAC5F,CAAC;QACF,0BAA0B,CAAC,UAAU,CAAC,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAC;QAC/F,MAAM,YAAY,GAAG,MAAM,uBAAuB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACtE,IAAI,CAAC;YACH,IAAI,MAAM,CAAC,EAAE,KAAK,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,CAAC,UAAU,SAAS,MAAM,CAAC,EAAE,oCAAoC,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,CAC9G,CAAC;YACJ,CAAC;YACD,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,IAAI,WAAW,EAAE,UAAU,EAAE,EAAE,CAAC;gBACjE,MAAM,MAAM,GAAG,YAAY,CAAC;oBAC1B,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,SAAS;oBACT,KAAK;oBACL,WAAW;oBACX,eAAe,EAAE,WAAW,CAAC,IAAI;oBACjC,YAAY,EAAE,UAAU,CAAC,EAAE;oBAC3B,UAAU;iBACX,CAAC,CAAC;gBACH,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBACpD,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CACzC,QAAQ,EACR,0BAA0B,EAC1B,cAAc,CACf,CAAC;gBACF,IAAI,QAAQ,EAAE,CAAC;oBACb,kBAAkB,CAChB,QAAQ,EACR;wBACE,MAAM;wBACN,WAAW,EAAE,SAAS,CAAC,EAAE;wBACzB,SAAS;wBACT,KAAK;wBACL,WAAW;wBACX,eAAe,EAAE,WAAW,CAAC,IAAI;wBACjC,YAAY,EAAE,UAAU,CAAC,EAAE;wBAC3B,UAAU;qBACX,EACD,QAAQ,CACT,CAAC;oBACF,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,GAAG,cAAc,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC1E,wBAAwB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;oBAChD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;oBAC1B,SAAS;gBACX,CAAC;gBACD,IAAI,cAAc,EAAE,CAAC;oBACnB,MAAM,sBAAsB,CAAC,oBAAoB,CAAC,CAAC;oBACnD,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;gBACD,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACpD,MAAM,GAAG,GAAG;oBACV,GAAG,YAAY;oBACf,IAAI,EAAE,EAAE,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE;iBAC9D,CAAC;gBACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,MAAM,kBAAkB,GAAG,UAAU,EAAE,CAAC;gBACxC,MAAM,OAAO,CAAC,aAAa,CACzB,yBAAyB,MAAM,IAAI,kBAAkB,EAAE,EACvD,cAAc,EAAE,CACjB,CAAC;gBACF,IAAI,MAAqB,CAAC;gBAC1B,IAAI,CAAC;oBACH,MAAM,GAAG,MAAM,QAAQ,CACrB,GAAG,EACH,QAAQ,EACR,SAAS,EACT,SAAS,CAAC,WAAW,EACrB,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;wBACzB,MAAM,UAAU,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;wBACvD,IAAI,kBAAkB,GAAG,UAAU,CAAC,kBAAkB,CAAC;wBACvD,IAAI,kBAAkB,KAAK,CAAC,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;4BAC7E,kBAAkB,GAAG,CAAC,CAAC;wBACzB,CAAC;wBACD,MAAM,OAAO,CAAC,eAAe,CAC3B,gBAAgB,MAAM,IAAI,kBAAkB,IAAI,KAAK,EAAE,EACvD;4BACE,WAAW,EAAE,CAAC;4BACd,2DAA2D;4BAC3D,UAAU,EAAE,CAAC;4BACb,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,UAAU,EAAE;4BACtC,OAAO,EAAE,UAAU,CAAC,OAAO;4BAC3B,kBAAkB;yBACnB,CACF,CAAC;oBACJ,CAAC,EACD;wBACE,KAAK,EAAE,OAAO,CAAC,uBAAuB;wBACtC,QAAQ,EAAE,OAAO,CAAC,qBAAqB;qBACxC,CACF,CAAC;oBACF,MAAM,eAAe,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;oBAC1E,IAAI,MAAM,CAAC,UAAU,EAAE,KAAK,KAAK,SAAS,IAAI,eAAe,EAAE,CAAC;wBAC9D,MAAM,IAAI,KAAK,CACb,eAAe,EAAE,YAAY;4BAC3B,eAAe,EAAE,OAAO;4BACxB,+CAA+C,CAClD,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,OAAO,CAAC,UAAU,CAAC,kBAAkB,MAAM,IAAI,kBAAkB,EAAE,EAAE;wBACzE,GAAG,cAAc,EAAE;wBACnB,WAAW,EAAE,CAAC;qBACf,CAAC,CAAC;oBACH,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,MAAM,KAAK,GAAG,QAAQ,CACpB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CACrF,CAAC;gBACF,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CACtC,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC,OAAO,EACzE,CAAC,CACF,CAAC;gBACF,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CACjD,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC,kBAAkB,EACpF,CAAC,CACF,CAAC;gBACF,MAAM,IAAI,GAAuB;oBAC/B,YAAY,EAAE,UAAU,CAAC,EAAE;oBAC3B,UAAU;oBACV,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,UAAU,EAAE,UAAU,CAAC,UAAU;oBACjC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjE,SAAS,EAAE,uBAAuB,CAChC,UAAU,EACV,MAAM,EACN,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,cAAc,CACpD;oBACD,MAAM;oBACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,KAAK;oBACL,OAAO;oBACP,kBAAkB;iBACnB,CAAC;gBACF,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC;oBACnC,MAAM,OAAO,CAAC,UAAU,CAAC,kBAAkB,MAAM,IAAI,SAAS,EAAE,EAAE;wBAChE,GAAG,cAAc,EAAE;wBACnB,WAAW,EAAE,CAAC;qBACf,CAAC,CAAC;oBACH,wBAAwB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;gBAClD,CAAC;gBACD,MAAM,IAAI,GAAyB;oBACjC,aAAa,EAAE,CAAC;oBAChB,MAAM;oBACN,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,SAAS;oBACT,KAAK;oBACL,WAAW;oBACX,eAAe,EAAE,WAAW,CAAC,IAAI;oBACjC,YAAY,EAAE,UAAU,CAAC,EAAE;oBAC3B,UAAU;oBACV,IAAI;iBACL,CAAC;gBACF,MAAM,OAAO,CAAC,uBAAuB,EAAE,CAAC;gBACxC,MAAM,qBAAqB,CAAC,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,IAAI,EAAE;oBACrE,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,GAAG,cAAc,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC1E,MAAM,OAAO,CAAC,uBAAuB,EAAE,CAAC;gBAC1C,CAAC,CAAC,CAAC;gBACH,wBAAwB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;gBAChD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,0BAA0B,CAAC,YAAY,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,uBAAuB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACvE,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,UAAU,GAAG,EAAE,GAAG,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAClE,IAAI,SAAS,CAAC,cAAc,CAAC,KAAK,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,sBAAsB,CAAC,UAAU,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,QAAQ,GAA6B;QACzC,aAAa,EAAE,CAAC;QAChB,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,SAAS;QACT,KAAK;QACL,WAAW;QACX,eAAe,EAAE,WAAW,CAAC,IAAI;QACjC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,OAAO;QACP,OAAO;QACP,KAAK;KACN,CAAC;IACF,MAAM,OAAO,CAAC,uBAAuB,EAAE,CAAC;IACxC,MAAM,qBAAqB,CACzB,oBAAoB,EACpB,QAAQ,EACR,cAAc,EACd,OAAO,CAAC,uBAAuB,CAChC,CAAC;IACF,OAAO,sBAAsB,CAAC,UAAU,EAAE,oBAAoB,EAAE,QAAQ,CAAC,CAAC;AAC5E,CAAC,CAAC"}
+{"version":3,"file":"grader-experiment.js","sourceRoot":"","sources":["../../src/orchestrator/grader-experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,iCAAiC,EAAE,MAAM,+BAA+B,CAAC;AAClF,OAAO,EAIL,UAAU,EAGV,mBAAmB,EACnB,QAAQ,EACR,gBAAgB,EAChB,WAAW,EACX,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACnF,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,CAAC,CAAC;IAC9E,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB,aAAa,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC1C,iBAAiB,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,uBAAuB,EAAE,mBAAmB,CAAC,CAAC,CAAC;CAC3F,CAAC,CAAC;AAGH,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7C,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACvC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,QAAQ,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC;IACxC,MAAM,EAAE,mBAAmB;IAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IACpC,KAAK,EAAE,gBAAgB;IACvB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IACjC,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;CACnD,CAAC,CAAC;AAGH,MAAM,0BAA0B,GAAG,CAAC;KACjC,MAAM,CAAC;IACN,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAC9C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACxC,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACvC,IAAI,EAAE,wBAAwB;CAC/B,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC,MAAM,CAAC;IACrD,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAC9C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACxC,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC;CACzC,CAAC,CAAC;AAGH,MAAM,yBAAyB,GAAG,CAChC,KAAsD,EAC7C,EAAE;IACX,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,IAAI,KAAK,CAAC,YAAY;QAAE,OAAO,IAAI,CAAC;IACpC,MAAM,MAAM,GAAG,iCAAiC,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzE,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC;AACjE,CAAC,CAAC;AAEF,MAAM,0BAA0B,GAAG,CACjC,KAAsD,EACjB,EAAE,CACvC,yBAAyB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC;AAE7E,MAAM,wBAAwB,GAAG,CAC/B,SAA8C,EACqB,EAAE;IACrE,MAAM,MAAM,GAAG,iCAAiC,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC7E,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,CAAC;IAC5D,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO;QAC5B,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,gBAAgB;KACjD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,UAA4B,EAC5B,MAAqB,EACrB,cAAsB,EACJ,EAAE;IACpB,MAAM,SAAS,GAAqB,EAAE,CAAC;IACvC,MAAM,uBAAuB,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC/E,CAAC,CAAC,mBAAmB;QACrB,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACpE,IAAI,UAAU,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,uBAAuB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;QAClE,SAAS,CAAC,IAAI,CAAC;YACb,GAAG,EAAE,MAAM;YACX,IAAI,EAAE,WAAW;YACjB,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,UAAU;YACxC,MAAM;YACN,OAAO,EAAE,MAAM,KAAK,UAAU,CAAC,QAAQ,CAAC,UAAU;YAClD,GAAG,CAAC,uBAAuB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnF,CAAC,CAAC;IACL,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,uBAAuB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;QACjE,MAAM,aAAa,GAAG,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7F,SAAS,CAAC,IAAI,CAAC;YACb,GAAG,EAAE,MAAM;YACX,IAAI,EAAE,YAAY;YAClB,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,SAAS;YACvC,MAAM;YACN,OAAO,EAAE,MAAM,KAAK,IAAI,IAAI,aAAa,IAAI,cAAc;YAC3D,aAAa;YACb,GAAG,CAAC,uBAAuB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnF,CAAC,CAAC;IACL,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CACrF,CAAC;IACF,KAAK,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QACrF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACvC,MAAM,iBAAiB,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;QACrD,SAAS,CAAC,IAAI,CAAC;YACb,GAAG,EAAE,WAAW;YAChB,IAAI,EAAE,gBAAgB;YACtB,QAAQ,EAAE,QAAQ,CAAC,MAAM;YACzB,MAAM,EAAE,MAAM,EAAE,MAAM,IAAI,IAAI;YAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,KAAK,QAAQ,CAAC,MAAM;YAC3C,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACjC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3E,SAAS,CAAC,IAAI,CAAC;gBACb,GAAG,EAAE,WAAW;gBAChB,IAAI,EAAE,iBAAiB;gBACvB,QAAQ,EAAE,QAAQ,CAAC,KAAK;gBACxB,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,IAAI;gBAC7B,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,aAAa,IAAI,cAAc;gBAChE,aAAa;gBACb,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACpD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,CAAC,SAAiB,EAAE,WAAmB,EAAE,KAAa,EAAU,EAAE,CAC9E,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC;AAEtD,MAAM,SAAS,GAAG,CAAC,OAAiC,EAAE,IAAY,EAAE,KAAa,EAAQ,EAAE;IACzF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO;IACpC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;AACzB,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,CAAC,MAAyB,EAAE,KAAK,GAAG,CAAC,EAAU,EAAE,CAC5D,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAa9F,MAAM,qBAAqB,GAAG,CAC5B,KAAoC,EAKpC,EAAE;IACF,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkD,CAAC;IAC/E,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAgC,CAAC;IAClE,MAAM,MAAM,GAAyB;QACnC,YAAY,EAAE,CAAC;QACf,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;QACjB,cAAc,EAAE,CAAC;QACjB,WAAW,EAAE,CAAC;QACd,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;KAClB,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,mBAAmB,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACtE,mBAAmB,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5C,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;QACzD,MAAM,eAAe,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACvE,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;QAC1D,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS;gBAAE,MAAM,CAAC,cAAc,EAAE,CAAC;YAClE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,QAAQ,CAAC,QAAQ,KAAK,SAAS;gBAAE,SAAS;YACzF,MAAM,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,QAAQ,CAAC,QAAQ;gBAAE,MAAM,CAAC,cAAc,EAAE,CAAC;;gBAC1C,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,IAAI;gBAAE,MAAM,CAAC,WAAW,EAAE,CAAC;iBAC9C,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM;gBAAE,MAAM,CAAC,YAAY,EAAE,CAAC;iBAChE,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM;gBAAE,MAAM,CAAC,aAAa,EAAE,CAAC;iBAClE,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAAE,MAAM,CAAC,aAAa,EAAE,CAAC;QACzE,CAAC;IACH,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,EAAE,CAAC;AACrD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,KAA0D,EAC1D,SAAyC,EAC+B,EAAE;IAC1E,MAAM,OAAO,GAA6B,EAAE,CAAC;IAC7C,MAAM,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IACjF,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,sBAAsB,GAAG,CAAC,CAAC;IAC/B,KAAK,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,IAAI,YAAY,EAAE,CAAC;QACrD,MAAM,eAAe,GAAG,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAClE,SAAS,CAAC,OAAO,EAAE,iBAAiB,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC5F,SAAS,CAAC,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3F,SAAS,CAAC,OAAO,EAAE,gBAAgB,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7F,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAC1D,CAAC;QACF,SAAS,CACP,OAAO,EACP,uBAAuB,EACvB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAC9D,CAAC;QACF,SAAS,CACP,OAAO,EACP,eAAe,EACf,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAC7D,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,SAAS,CAAC,OAAO,EAAE,kBAAkB,EAAE,YAAY,CAAC,CAAC;QACrD,SAAS,CAAC,OAAO,EAAE,kBAAkB,EAAE,YAAY,CAAC,CAAC;QACrD,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACvF,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,iBAAiB,IAAI,aAAa,CAAC,MAAM,CAAC;YAC1C,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAClE,CAAC;QACJ,CAAC;QACD,MAAM,kBAAkB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CACvD,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CACtC,CAAC;QACF,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,sBAAsB,IAAI,kBAAkB,CAAC,MAAM,CAAC;YACpD,SAAS,CACP,OAAO,EACP,2BAA2B,EAC3B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACvE,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAC7B,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,QAAQ,CAAC,QAAQ,KAAK,SAAS,CACxF,CAAC;QACF,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjD,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CACrE,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,SAAS,CAAC,OAAO,EAAE,2BAA2B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;YACnE,SAAS,CAAC,OAAO,EAAE,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC;QAC1E,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChF,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,SAAS,CACP,OAAO,EACP,+BAA+B,EAC/B,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,MAAM;gBACpE,eAAe,CAAC,MAAM,CACzB,CAAC;QACJ,CAAC;QACD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,SAAS,CACP,OAAO,EACP,+BAA+B,EAC/B,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM;gBACnE,eAAe,CAAC,MAAM,CACzB,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,SAAS,CACP,OAAO,EACP,iBAAiB,EACjB,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAC7E,CAAC;YACF,SAAS,CACP,OAAO,EACP,uBAAuB,EACvB,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAC7E,CAAC;YACF,SAAS,CACP,OAAO,EACP,iCAAiC,EACjC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,iBAAiB,KAAK,uBAAuB,CAAC,CAAC,MAAM;gBACxF,MAAM,CAAC,MAAM,CAChB,CAAC;YACF,SAAS,CACP,OAAO,EACP,6BAA6B,EAC7B,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,iBAAiB,KAAK,mBAAmB,CAAC,CAAC,MAAM;gBACpF,MAAM,CAAC,MAAM,CAChB,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CACrB,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa,EAC1C,MAAM,CAAC,GAAG,CACX,CAAC;IACF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAClG,MAAM,OAAO,GAA2B;QACtC,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;QAC3D,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;QAC3D,eAAe,EAAE,YAAY,CAAC,IAAI;QAClC,kBAAkB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QACnF,sBAAsB,EAAE,MAAM,CAAC,aAAa;QAC5C,sBAAsB,EAAE,iBAAiB;QACzC,2BAA2B,EAAE,sBAAsB;QACnD,uBAAuB,EAAE,MAAM,CAAC,cAAc;QAC9C,uBAAuB,EAAE,MAAM,CAAC,cAAc;QAC9C,uBAAuB,EAAE,MAAM,CAAC,cAAc;QAC9C,oBAAoB,EAAE,MAAM,CAAC,WAAW;QACxC,eAAe,EAAE,SAAS,CAAC,WAAW;KACvC,CAAC;IACF,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,KAAyB,EAAQ,EAAE;QAClE,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IAC3E,CAAC,CAAC;IACF,SAAS,CACP,sBAAsB,EACtB,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CACpF,CAAC;IACF,SAAS,CACP,2BAA2B,EAC3B,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAC9F,CAAC;IACF,SAAS,CAAC,uBAAuB,EAAE,SAAS,CAAC,CAAC;IAC9C,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;IACxC,IAAI,OAA2B,CAAC;IAChC,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1D,OAAO,GAAG,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IAC3F,CAAC;IACD,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;IACrC,SAAS,CACP,+BAA+B,EAC/B,OAAO,CAAC,+BAA+B,CAAC;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAC;QAChD,CAAC,CAAC,SAAS,CACd,CAAC;IACF,SAAS,CACP,+BAA+B,EAC/B,OAAO,CAAC,+BAA+B,CAAC;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAC;QAChD,CAAC,CAAC,SAAS,CACd,CAAC;IACF,SAAS,CACP,iBAAiB,EACjB,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;IACF,SAAS,CACP,uBAAuB,EACvB,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CACtF,CAAC;IACF,SAAS,CACP,iBAAiB,EACjB,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAC1E,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,UAA+B,EAC/B,aAGC,EACO,EAAE;IACV,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,SAAS,CAAC;QACzB,WAAW,EAAE,aAAa,CAAC,WAAW;QACtC,SAAS,EAAE,aAAa,CAAC,SAAS;QAClC,KAAK,EAAE,aAAa,CAAC,KAAK;QAC1B,WAAW,EAAE,aAAa,CAAC,WAAW;QACtC,eAAe,EAAE,aAAa,CAAC,eAAe;KAC/C,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChB,OAAO,IAAI,CAAC,SAAS,EAAE,GAAG,aAAa,CAAC,WAAW,IAAI,QAAQ,OAAO,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,KAQrB,EAAU,EAAE,CAAC,UAAU,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AAExD,MAAM,cAAc,GAAG,CAAC,UAA+B,EAAE,MAAc,EAAU,EAAE,CACjF,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,kBAAkB,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAE9D,MAAM,cAAc,GAAG,GAGrB,EAAE,CAAC,CAAC;IACJ,WAAW,EAAE,CAAC;IACd,UAAU,EAAE,CAAC;IACb,KAAK,EAAE,UAAU,EAAE;IACnB,OAAO,EAAE,CAAC;IACV,kBAAkB,EAAE,CAAC;CACtB,CAAC,CAAC;AAEH,MAAM,sBAAsB,GAAG,CAC7B,UAA+B,EAC/B,YAAoB,EACpB,QAAkC,EACc,EAAE;IAClD,MAAM,WAAW,GAAG,kBAAkB,CACpC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1B,MAAM,OAAO,GAAG,uBAAuB,CAAC,CAAC,IAAI,CAAC,EAAE;YAC9C,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC;SACpD,CAAC,CAAC;QACH,OAAO;YACL,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE;gBACnB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxD,YAAY,EAAE,UAAU,CAAC,QAAQ,CAAC,cAAc;oBAC9C,CAAC,CAAC,SAAS,CAAC;wBACR,cAAc,EAAE,UAAU,CAAC,QAAQ,CAAC,cAAc;wBAClD,YAAY,EAAE,IAAI,CAAC,YAAY;wBAC/B,SAAS,EAAE,QAAQ,CAAC,SAAS;wBAC7B,KAAK,EAAE,QAAQ,CAAC,KAAK;qBACtB,CAAC;oBACJ,CAAC,CAAC,IAAI;gBACR,OAAO,EAAE,MAAM,CAAC,WAAW,CACzB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,CAC3D,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CACtE,CACF;aACF;SACF,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IACF,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;IACjD,MAAM,OAAO,GAAG,wBAAwB,CAAC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAG,OAAO;QACrB,CAAC,CAAC,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE;YACzD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC;gBAAE,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;YACnE,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9C,CAAC,CAAC,CACH;QACH,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;IACrB,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACzD,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,MAAM,WAAW,GAAuC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACtF,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1E,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAC7B,CAAC,QAAQ,EAAE,EAAE,CACX,GAAG,QAAQ,CAAC,GAAG,IAAI,QAAQ,CAAC,IAAI,cAAc,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAC5G,CAAC;QACF,IAAI,IAAI,CAAC,QAAQ;YAAE,QAAQ,CAAC,OAAO,CAAC,mBAAmB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxE,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;gBAAE,SAAS;YACrD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;YAC3F,IAAI,OAAO;gBAAE,QAAQ,CAAC,IAAI,CAAC,sBAAsB,QAAQ,CAAC,GAAG,KAAK,OAAO,EAAE,CAAC,CAAC;QAC/E,CAAC;QACD,OAAO;YACL;gBACE,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,UAAU,EAAE;gBAClD,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzE,QAAQ;aACT;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO;QACL,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,YAAY,CAAC;QACrD,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC;QAC1C,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9F,OAAO;QACP,OAAO,EAAE,OAAO,IAAI,QAAQ,CAAC,OAAO;QACpC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM;QAClC,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC/B,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CACd,KAAK;YACL,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAC3B,CAAC,cAAc,EAAE,SAAS,EAAE,EAAE,CAC5B,cAAc,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC,QAAQ,EAC/D,CAAC,CACF,EACH,CAAC,CACF;QACD,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzD,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,kBAAkB,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC9F,WAAW;KACZ,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,gBAAwB,EAAE,UAAkB,EAAU,EAAE;IACnF,MAAM,IAAI,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACvC,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,8CAA8C,UAAU,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,MAAwD,EAAY,EAAE,CAC1F,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAElE,MAAM,sBAAsB,GAAG,CAC7B,QAAkC,EAClC,QAMC,EACD,IAAY,EACN,EAAE;IACR,IACE,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QAC7C,QAAQ,CAAC,SAAS,KAAK,QAAQ,CAAC,SAAS;QACzC,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK;QACjC,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QAC7C,QAAQ,CAAC,eAAe,KAAK,QAAQ,CAAC,eAAe,EACrD,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,IAAI,EAAE,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,IAA0B,EAC1B,QAA8D,EAC9D,IAAY,EACN,EAAE;IACR,IACE,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAC/B,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QACzC,IAAI,CAAC,SAAS,KAAK,QAAQ,CAAC,SAAS;QACrC,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK;QAC7B,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QACzC,IAAI,CAAC,eAAe,KAAK,QAAQ,CAAC,eAAe;QACjD,IAAI,CAAC,YAAY,KAAK,QAAQ,CAAC,YAAY;QAC3C,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,UAAU,EACvC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,IAAI,EAAE,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAAC,MAA+B,EAAQ,EAAE;IACzE,IAAI,CAAC,MAAM,EAAE,OAAO;QAAE,OAAO;IAC7B,MAAM,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;AACzF,CAAC,CAAC;AAEF,MAAM,0BAA0B,GAAG,CACjC,UAAkB,EAClB,WAA8B,EAC9B,YAA+B,EAC/B,gBAAqC,EAC/B,EAAE;IACR,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/E,MAAM,kBAAkB,GAAG,YAAY,CAAC,MAAM,CAC5C,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAC/D,CAAC;IACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC5E,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,sEAAsE;QACzF,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;QACjF,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACtG,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,EAAE,OAkC7C,EAA2D,EAAE;IAC5D,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IACpF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC;IACvD,IAAI,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,CAAC,cAAc,CAAC;IAC1D,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACrF,MAAM,QAAQ,GAAG,uBAAuB,CAAC,WAAW,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,eAAe,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,KAAK,6BAA6B,CACnF,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,SAAS,qBAAqB,CAAC,CAAC;IACrF,MAAM,oBAAoB,GAAG,kBAAkB,CAAC,UAAU,EAAE;QAC1D,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,SAAS;QACT,KAAK;QACL,WAAW;QACX,eAAe,EAAE,WAAW,CAAC,IAAI;KAClC,CAAC,CAAC;IACH,IAAI,cAAc,GAAG,MAAM,oBAAoB,CAC7C,oBAAoB,EACpB,8BAA8B,EAC9B,cAAc,CACf,CAAC;IACF,IAAI,cAAc,EAAE,CAAC;QACnB,sBAAsB,CACpB,cAAc,EACd;YACE,WAAW,EAAE,SAAS,CAAC,EAAE;YACzB,SAAS;YACT,KAAK;YACL,WAAW;YACX,eAAe,EAAE,WAAW,CAAC,IAAI;SAClC,EACD,oBAAoB,CACrB,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;IAC5D,MAAM,QAAQ,GAAG,uBAAuB,CAAC;QACvC,GAAG,SAAS,CAAC,MAAM;QACnB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC,CAAC;IACH,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,KAAK,MAAM,UAAU,IAAI,QAAQ,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,oBAAoB,CACvC,mBAAmB,CAAC,OAAO,CAAC,gBAAgB,EAAE,UAAU,CAAC,UAAU,CAAC,CACrE,CAAC;QACF,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAC/C,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC,CACzE,CAAC;QACF,MAAM,WAAW,GAAG,YAAY,CAAC,cAAc,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAC9B,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,EAAE,CAC5F,CAAC;QACF,0BAA0B,CAAC,UAAU,CAAC,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAC;QAC/F,MAAM,YAAY,GAAG,MAAM,uBAAuB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACtE,IAAI,CAAC;YACH,IAAI,MAAM,CAAC,EAAE,KAAK,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,CAAC,UAAU,SAAS,MAAM,CAAC,EAAE,oCAAoC,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,CAC9G,CAAC;YACJ,CAAC;YACD,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,IAAI,WAAW,EAAE,UAAU,EAAE,EAAE,CAAC;gBACjE,MAAM,MAAM,GAAG,YAAY,CAAC;oBAC1B,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,SAAS;oBACT,KAAK;oBACL,WAAW;oBACX,eAAe,EAAE,WAAW,CAAC,IAAI;oBACjC,YAAY,EAAE,UAAU,CAAC,EAAE;oBAC3B,UAAU;iBACX,CAAC,CAAC;gBACH,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBACpD,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CACzC,QAAQ,EACR,0BAA0B,EAC1B,cAAc,CACf,CAAC;gBACF,IAAI,QAAQ,EAAE,CAAC;oBACb,kBAAkB,CAChB,QAAQ,EACR;wBACE,MAAM;wBACN,WAAW,EAAE,SAAS,CAAC,EAAE;wBACzB,SAAS;wBACT,KAAK;wBACL,WAAW;wBACX,eAAe,EAAE,WAAW,CAAC,IAAI;wBACjC,YAAY,EAAE,UAAU,CAAC,EAAE;wBAC3B,UAAU;qBACX,EACD,QAAQ,CACT,CAAC;oBACF,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,GAAG,cAAc,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC1E,wBAAwB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;oBAChD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;oBAC1B,SAAS;gBACX,CAAC;gBACD,IAAI,cAAc,EAAE,CAAC;oBACnB,MAAM,sBAAsB,CAAC,oBAAoB,CAAC,CAAC;oBACnD,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;gBACD,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACpD,MAAM,GAAG,GAAG;oBACV,GAAG,YAAY;oBACf,IAAI,EAAE,EAAE,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE;iBAC9D,CAAC;gBACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,MAAM,kBAAkB,GAAG,UAAU,EAAE,CAAC;gBACxC,MAAM,OAAO,CAAC,aAAa,CACzB,yBAAyB,MAAM,IAAI,kBAAkB,EAAE,EACvD,cAAc,EAAE,CACjB,CAAC;gBACF,IAAI,MAAqB,CAAC;gBAC1B,IAAI,CAAC;oBACH,MAAM,GAAG,MAAM,QAAQ,CACrB,GAAG,EACH,QAAQ,EACR,SAAS,EACT,SAAS,CAAC,WAAW,EACrB,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;wBACzB,MAAM,UAAU,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;wBACvD,IAAI,kBAAkB,GAAG,UAAU,CAAC,kBAAkB,CAAC;wBACvD,IAAI,kBAAkB,KAAK,CAAC,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;4BAC7E,kBAAkB,GAAG,CAAC,CAAC;wBACzB,CAAC;wBACD,MAAM,OAAO,CAAC,eAAe,CAC3B,gBAAgB,MAAM,IAAI,kBAAkB,IAAI,KAAK,EAAE,EACvD;4BACE,WAAW,EAAE,CAAC;4BACd,2DAA2D;4BAC3D,UAAU,EAAE,CAAC;4BACb,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,UAAU,EAAE;4BACtC,OAAO,EAAE,UAAU,CAAC,OAAO;4BAC3B,kBAAkB;yBACnB,CACF,CAAC;oBACJ,CAAC,EACD;wBACE,KAAK,EAAE,OAAO,CAAC,uBAAuB;wBACtC,QAAQ,EAAE,OAAO,CAAC,qBAAqB;qBACxC,CACF,CAAC;oBACF,MAAM,eAAe,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;oBAC1E,IAAI,MAAM,CAAC,UAAU,EAAE,KAAK,KAAK,SAAS,IAAI,eAAe,EAAE,CAAC;wBAC9D,MAAM,IAAI,KAAK,CACb,eAAe,EAAE,YAAY;4BAC3B,eAAe,EAAE,OAAO;4BACxB,+CAA+C,CAClD,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,OAAO,CAAC,UAAU,CAAC,kBAAkB,MAAM,IAAI,kBAAkB,EAAE,EAAE;wBACzE,GAAG,cAAc,EAAE;wBACnB,WAAW,EAAE,CAAC;qBACf,CAAC,CAAC;oBACH,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,MAAM,KAAK,GAAG,QAAQ,CACpB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CACrF,CAAC;gBACF,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CACtC,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC,OAAO,EACzE,CAAC,CACF,CAAC;gBACF,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CACjD,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,KAAK,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC,kBAAkB,EACpF,CAAC,CACF,CAAC;gBACF,MAAM,IAAI,GAAuB;oBAC/B,YAAY,EAAE,UAAU,CAAC,EAAE;oBAC3B,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC3E,UAAU;oBACV,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,UAAU,EAAE,UAAU,CAAC,UAAU;oBACjC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjE,SAAS,EAAE,uBAAuB,CAChC,UAAU,EACV,MAAM,EACN,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,cAAc,CACpD;oBACD,MAAM;oBACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,KAAK;oBACL,OAAO;oBACP,kBAAkB;iBACnB,CAAC;gBACF,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC;oBACnC,MAAM,OAAO,CAAC,UAAU,CAAC,kBAAkB,MAAM,IAAI,SAAS,EAAE,EAAE;wBAChE,GAAG,cAAc,EAAE;wBACnB,WAAW,EAAE,CAAC;qBACf,CAAC,CAAC;oBACH,wBAAwB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;gBAClD,CAAC;gBACD,MAAM,IAAI,GAAyB;oBACjC,aAAa,EAAE,CAAC;oBAChB,MAAM;oBACN,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,SAAS;oBACT,KAAK;oBACL,WAAW;oBACX,eAAe,EAAE,WAAW,CAAC,IAAI;oBACjC,YAAY,EAAE,UAAU,CAAC,EAAE;oBAC3B,UAAU;oBACV,IAAI;iBACL,CAAC;gBACF,MAAM,OAAO,CAAC,uBAAuB,EAAE,CAAC;gBACxC,MAAM,qBAAqB,CAAC,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,IAAI,EAAE;oBACrE,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,GAAG,cAAc,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC1E,MAAM,OAAO,CAAC,uBAAuB,EAAE,CAAC;gBAC1C,CAAC,CAAC,CAAC;gBACH,wBAAwB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;gBAChD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,0BAA0B,CAAC,YAAY,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,uBAAuB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACvE,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,UAAU,GAAG,EAAE,GAAG,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAClE,IAAI,SAAS,CAAC,cAAc,CAAC,KAAK,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,sBAAsB,CAAC,UAAU,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,QAAQ,GAA6B;QACzC,aAAa,EAAE,CAAC;QAChB,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,SAAS;QACT,KAAK;QACL,WAAW;QACX,eAAe,EAAE,WAAW,CAAC,IAAI;QACjC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,OAAO;QACP,OAAO;QACP,KAAK;KACN,CAAC;IACF,MAAM,OAAO,CAAC,uBAAuB,EAAE,CAAC;IACxC,MAAM,qBAAqB,CACzB,oBAAoB,EACpB,QAAQ,EACR,cAAc,EACd,OAAO,CAAC,uBAAuB,CAChC,CAAC;IACF,OAAO,sBAAsB,CAAC,UAAU,EAAE,oBAAoB,EAAE,QAAQ,CAAC,CAAC;AAC5E,CAAC,CAAC"}

package/lib/orchestrator/grading.d.ts

@@ -22,10 +22,11 @@ export declare const graderConfigHash: (input: {
     assertions: readonly AssertionEvaluatorIdentity[];
     validatedAssertionsHash: string;
     graderContext?: unknown;
+    skipJudge?: boolean;
 }) => string;
 export declare const validateAssertionPlan: (assertions: readonly Assertion[], registry: Registry) => Promise<unknown[]>;
 export declare const gradeRun: (run: Run, registry: Registry, observer?: RunObserver, abortSignal?: AbortSignal, recordAssertion?: (entry: GradingAssertionEntry, index: number) => void | Promise<void>, modelAttempts?: {
     start: (kind: "judge") => string | Promise<string>;
     complete: (attemptIds: readonly string[]) => void | Promise<void>;
-}, validatedConfigs?: readonly unknown[]) => Promise<GradingReport>;
+}, validatedConfigs?: readonly unknown[], skipJudge?: boolean) => Promise<GradingReport>;
 //# sourceMappingURL=grading.d.ts.map

package/lib/orchestrator/grading.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"grading.d.ts","sourceRoot":"","sources":["../../src/orchestrator/grading.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAC7B,OAAO,EACL,KAAK,SAAS,EACd,KAAK,aAAa,EAGlB,KAAK,qBAAqB,EAC1B,KAAK,aAAa,EAElB,KAAK,QAAQ,EACb,KAAK,GAAG,EACR,KAAK,WAAW,EAEjB,MAAM,kBAAkB,CAAC;AAU1B;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,MAAO,SAAS,UAAU,CAAC,CAAC,UAAU,KAAG,OAM7E,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG,uBAAuB,GAAG;IACjE,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,4BAA4B,eAC3B,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,0BAA0B,EAYzB,CAAC;AAEL,eAAO,MAAM,yBAAyB,eACxB,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,uBAAuB,EAIrB,CAAC;AAEN,eAAO,MAAM,gBAAgB,UAAW;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,SAAS,0BAA0B,EAAE,CAAC;IAClD,uBAAuB,EAAE,MAAM,CAAC;IAChC,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,KAAG,MAOA,CAAC;AAEL,eAAO,MAAM,qBAAqB,eACpB,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,OAAO,CAAC,OAAO,EAAE,CAuBnB,CAAC;AAWF,eAAO,MAAM,QAAQ,QACd,GAAG,YACE,QAAQ,aACP,WAAW,gBACR,WAAW,oBACP,CAAC,KAAK,EAAE,qBAAqB,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,kBACvE;IACd,KAAK,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,QAAQ,EAAE,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACnE,qBACkB,SAAS,OAAO,EAAE,KACpC,OAAO,CAAC,aAAa,CA4HvB,CAAC"}
+{"version":3,"file":"grading.d.ts","sourceRoot":"","sources":["../../src/orchestrator/grading.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAC7B,OAAO,EACL,KAAK,SAAS,EACd,KAAK,aAAa,EAIlB,KAAK,qBAAqB,EAC1B,KAAK,aAAa,EAElB,KAAK,QAAQ,EACb,KAAK,GAAG,EACR,KAAK,WAAW,EAEjB,MAAM,kBAAkB,CAAC;AAU1B;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,MAAO,SAAS,UAAU,CAAC,CAAC,UAAU,KAAG,OAM7E,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG,uBAAuB,GAAG;IACjE,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,4BAA4B,eAC3B,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,0BAA0B,EAYzB,CAAC;AAEL,eAAO,MAAM,yBAAyB,eACxB,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,uBAAuB,EAIrB,CAAC;AAEN,eAAO,MAAM,gBAAgB,UAAW;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,SAAS,0BAA0B,EAAE,CAAC;IAClD,uBAAuB,EAAE,MAAM,CAAC;IAChC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,KAAG,MAQA,CAAC;AAEL,eAAO,MAAM,qBAAqB,eACpB,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,OAAO,CAAC,OAAO,EAAE,CAuBnB,CAAC;AAiBF,eAAO,MAAM,QAAQ,QACd,GAAG,YACE,QAAQ,aACP,WAAW,gBACR,WAAW,oBACP,CAAC,KAAK,EAAE,qBAAqB,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,kBACvE;IACd,KAAK,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,QAAQ,EAAE,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACnE,qBACkB,SAAS,OAAO,EAAE,0BAEpC,OAAO,CAAC,aAAa,CA8IvB,CAAC"}

package/lib/orchestrator/grading.js

@@ -39,6 +39,7 @@ export const graderConfigHash = (input) => valueHash({
     assertions: input.assertions,
     validatedAssertionsHash: input.validatedAssertionsHash,
     graderContext: input.graderContext ?? null,
+    skipJudge: input.skipJudge ?? false,
 });
 export const validateAssertionPlan = async (assertions, registry) => {
     const validated = [];
@@ -65,18 +66,33 @@ const describeAssertion = (a) => {
         .join(" ");
     return fields ? `${a.type} ${fields}` : a.type;
 };
-export const gradeRun = async (run, registry, observer, abortSignal, recordAssertion, modelAttempts, validatedConfigs) => {
+const assertionPassed = (result, threshold) => result.errorMessage === undefined &&
+    result.abstention === undefined &&
+    result.passed &&
+    result.score >= threshold;
+export const gradeRun = async (run, registry, observer, abortSignal, recordAssertion, modelAttempts, validatedConfigs, skipJudge = false) => {
     const guardedObserver = guardRunObserver(observer);
     const assertions = run.case.assertions ?? [];
     if (assertions.length === 0) {
         return { schemaVersion: 1, casePassed: null, caseScore: null, assertions: [] };
     }
     const entries = [];
+    const skippedAssertions = [];
     let weightedScoreSum = 0;
     let weightSum = 0;
     let allPassed = true;
     let gradingIncomplete = false;
+    let gradingAbstained = false;
     for (const [index, a] of assertions.entries()) {
+        if (skipJudge && a.type === "rubric") {
+            skippedAssertions.push({
+                id: a.id ?? `${a.type}:${index + 1}`,
+                type: a.type,
+                name: describeAssertion(a),
+                reason: "judge-disabled",
+            });
+            continue;
+        }
         // Grading is the last cancellation point — an LLM judge can run for
         // minutes per assertion. An aborted grade can't claim everything passed.
         if (abortSignal?.aborted) {
@@ -113,7 +129,7 @@ export const gradeRun = async (run, registry, observer, abortSignal, recordAsser
             if (validatedConfigs && index >= validatedConfigs.length) {
                 throw new Error(`validated assertion plan is missing assertions[${index}]`);
             }
-            // Pass the validated config so schema defaults (e.g. timeoutMs)
+            // Pass the validated config so schema defaults (e.g. timeout)
             // reach evaluate(); the raw assertion object has none applied.
             const result = AssertionResultSchema.parse(await impl.evaluate(validated, run, {
                 ...(abortSignal ? { abortSignal } : {}),
@@ -125,7 +141,7 @@ export const gradeRun = async (run, registry, observer, abortSignal, recordAsser
                     }
                     : {}),
             }));
-            const passed = result.errorMessage === undefined && result.passed && result.score >= threshold;
+            const passed = assertionPassed(result, threshold);
             entry = {
                 id,
                 type: a.type,
@@ -137,12 +153,16 @@ export const gradeRun = async (run, registry, observer, abortSignal, recordAsser
                 durationMs: Date.now() - startedAt,
                 ...(result.comment !== undefined ? { comment: result.comment } : {}),
                 detail: result.detail,
+                ...(result.abstention !== undefined ? { abstention: result.abstention } : {}),
                 ...(result.errorMessage !== undefined ? { errorMessage: result.errorMessage } : {}),
                 ...(result.usage ? { usage: result.usage } : {}),
             };
             if (result.errorMessage !== undefined) {
                 gradingIncomplete = true;
             }
+            else if (result.abstention !== undefined) {
+                gradingAbstained = true;
+            }
             else {
                 if (!passed)
                     allPassed = false;
@@ -170,14 +190,13 @@ export const gradeRun = async (run, registry, observer, abortSignal, recordAsser
         await recordAssertion?.(entry, index);
         await modelAttempts?.complete(assertionAttemptIds);
     }
-    let caseScore = null;
-    if (!gradingIncomplete)
-        caseScore = weightSum > 0 ? weightedScoreSum / weightSum : 0;
+    const hasVerdict = !gradingIncomplete && !gradingAbstained && weightSum > 0;
     return {
         schemaVersion: 1,
-        casePassed: gradingIncomplete ? null : allPassed,
-        caseScore,
+        casePassed: hasVerdict ? allPassed : null,
+        caseScore: hasVerdict ? weightedScoreSum / weightSum : null,
         assertions: entries,
+        ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
         ...(gradingIncomplete
             ? { incomplete: { exitReason: "error", phase: "grading" } }
             : {}),

package/lib/orchestrator/grading.js.map

@@ -1 +1 @@
-{"version":3,"file":"grading.js","sourceRoot":"","sources":["../../src/orchestrator/grading.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,qBAAqB,EAGrB,gBAAgB,EAIhB,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AAExE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;AAE9E,MAAM,OAAO,GAAG,CAAC,MAAoB,EAAkC,EAAE;IACvE,MAAM,KAAK,GAAI,MAA8B,CAAC,KAAK,CAAC;IACpD,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,IAAI,CAAC;AACxF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAY,EAAE,MAAoB,EAAW,EAAE;IACtF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,KAAK,CAAC,CAC9E,CAAC;AACJ,CAAC,CAAC;AAWF,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAC1C,UAAgC,EAChC,QAAkB,EACY,EAAE,CAChC,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;IAC9E,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO;QACL,IAAI;QACJ,aAAa,EACX,eAAe,IAAI,cAAc,IAAI,OAAO,cAAc,CAAC,aAAa,KAAK,QAAQ;YACnF,CAAC,CAAC,cAAc,CAAC,aAAa;YAC9B,CAAC,CAAC,IAAI;QACV,QAAQ,EAAE,cAAc,CAAC,QAAQ;KAClC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,yBAAyB,GAAG,CACvC,UAAgC,EAChC,QAAkB,EACS,EAAE,CAC7B,4BAA4B,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;IACnF,IAAI;IACJ,aAAa;CACd,CAAC,CAAC,CAAC;AAEN,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,KAMhC,EAAU,EAAE,CACX,SAAS,CAAC;IACR,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;IAC5C,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;IACtD,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;CAC3C,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EACxC,UAAgC,EAChC,QAAkB,EACE,EAAE;IACtB,MAAM,SAAS,GAAc,EAAE,CAAC;IAChC,qEAAqE;IACrE,uEAAuE;IACvE,6DAA6D;IAC7D,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACb,wCAAwC,KAAK,uDAAuD,SAAS,CAAC,IAAI,GAAG,CACtH,CAAC;QACJ,CAAC;QACD,SAAS,CAAC,IAAI,CACZ,MAAM,yBAAyB,CAC7B,cAAc,CAAC,YAAY,EAC3B,cAAc,CAAC,YAAY;YACzB,CAAC,CAAC,wBAAwB,CAAC,SAAS,EAAE,cAAc,CAAC,YAAY,CAAC;YAClE,CAAC,CAAC,SAAS,EACb,cAAc,SAAS,CAAC,IAAI,mBAAmB,KAAK,GAAG,CACxD,CACF,CAAC;IACJ,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAY,EAAU,EAAE;IACjD,IAAI,CAAC,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;SAC7B,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,UAAU,CAAC;SACrE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,GAAQ,EACR,QAAkB,EAClB,QAAsB,EACtB,WAAyB,EACzB,eAAuF,EACvF,aAGC,EACD,gBAAqC,EACb,EAAE;IAC1B,MAAM,eAAe,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;IAC7C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;IACjF,CAAC;IAED,MAAM,OAAO,GAA4B,EAAE,CAAC;IAC5C,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAE9B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,oEAAoE;QACpE,yEAAyE;QACzE,IAAI,WAAW,EAAE,OAAO,EAAE,CAAC;YACzB,iBAAiB,GAAG,IAAI,CAAC;YACzB,MAAM;QACR,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;QAC7B,qEAAqE;QACrE,yEAAyE;QACzE,mEAAmE;QACnE,mEAAmE;QACnE,+DAA+D;QAC/D,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAsB;YAClC,KAAK;YACL,KAAK,EAAE,UAAU,CAAC,MAAM;YACxB,EAAE;YACF,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI;SACL,CAAC;QACF,eAAe,EAAE,cAAc,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,KAA4B,CAAC;QACjC,MAAM,mBAAmB,GAAa,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,gBAAgB;gBAChC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC;gBACzB,CAAC,CAAC,MAAM,yBAAyB,CAC7B,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,EACtE,cAAc,CAAC,CAAC,IAAI,GAAG,CACxB,CAAC;YACN,IAAI,gBAAgB,IAAI,KAAK,IAAI,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBACzD,MAAM,IAAI,KAAK,CAAC,kDAAkD,KAAK,GAAG,CAAC,CAAC;YAC9E,CAAC;YACD,gEAAgE;YAChE,+DAA+D;YAC/D,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CACxC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAqB,EAAE,GAAG,EAAE;gBAC9C,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvC,GAAG,CAAC,aAAa;oBACf,CAAC,CAAC;wBACE,uBAAuB,EAAE,KAAK,EAAE,KAAc,EAAE,EAAE;4BAChD,mBAAmB,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC/D,CAAC;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACR,CAAC,CACH,CAAC;YACF,MAAM,MAAM,GACV,MAAM,CAAC,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;YAClF,KAAK,GAAG;gBACN,EAAE;gBACF,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI;gBACJ,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM;gBACN,SAAS;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,GAAG,CAAC,MAAM,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACjD,CAAC;YACF,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;gBACtC,iBAAiB,GAAG,IAAI,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM;oBAAE,SAAS,GAAG,KAAK,CAAC;gBAC/B,gBAAgB,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC1C,SAAS,IAAI,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,KAAK,GAAG;gBACN,EAAE;gBACF,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI;gBACJ,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM;gBACN,SAAS;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,YAAY,EAAE,OAAO;aACtB,CAAC;YACF,iBAAiB,GAAG,IAAI,CAAC;QAC3B,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,eAAe,EAAE,YAAY,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACjD,MAAM,eAAe,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,aAAa,EAAE,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,SAAS,GAAkB,IAAI,CAAC;IACpC,IAAI,CAAC,iBAAiB;QAAE,SAAS,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;QAChD,SAAS;QACT,UAAU,EAAE,OAAO;QACnB,GAAG,CAAC,iBAAiB;YACnB,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,OAAgB,EAAE,KAAK,EAAE,SAAkB,EAAE,EAAE;YAC7E,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC,CAAC"}
+{"version":3,"file":"grading.js","sourceRoot":"","sources":["../../src/orchestrator/grading.ts"],"names":[],"mappings":"AACA,OAAO,EAKL,qBAAqB,EAGrB,gBAAgB,EAIhB,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AAExE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;AAE9E,MAAM,OAAO,GAAG,CAAC,MAAoB,EAAkC,EAAE;IACvE,MAAM,KAAK,GAAI,MAA8B,CAAC,KAAK,CAAC;IACpD,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,IAAI,CAAC;AACxF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAY,EAAE,MAAoB,EAAW,EAAE;IACtF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,KAAK,CAAC,CAC9E,CAAC;AACJ,CAAC,CAAC;AAWF,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAC1C,UAAgC,EAChC,QAAkB,EACY,EAAE,CAChC,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;IAC9E,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO;QACL,IAAI;QACJ,aAAa,EACX,eAAe,IAAI,cAAc,IAAI,OAAO,cAAc,CAAC,aAAa,KAAK,QAAQ;YACnF,CAAC,CAAC,cAAc,CAAC,aAAa;YAC9B,CAAC,CAAC,IAAI;QACV,QAAQ,EAAE,cAAc,CAAC,QAAQ;KAClC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,yBAAyB,GAAG,CACvC,UAAgC,EAChC,QAAkB,EACS,EAAE,CAC7B,4BAA4B,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;IACnF,IAAI;IACJ,aAAa;CACd,CAAC,CAAC,CAAC;AAEN,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,KAOhC,EAAU,EAAE,CACX,SAAS,CAAC;IACR,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;IAC5C,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;IACtD,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;IAC1C,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,KAAK;CACpC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EACxC,UAAgC,EAChC,QAAkB,EACE,EAAE;IACtB,MAAM,SAAS,GAAc,EAAE,CAAC;IAChC,qEAAqE;IACrE,uEAAuE;IACvE,6DAA6D;IAC7D,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACb,wCAAwC,KAAK,uDAAuD,SAAS,CAAC,IAAI,GAAG,CACtH,CAAC;QACJ,CAAC;QACD,SAAS,CAAC,IAAI,CACZ,MAAM,yBAAyB,CAC7B,cAAc,CAAC,YAAY,EAC3B,cAAc,CAAC,YAAY;YACzB,CAAC,CAAC,wBAAwB,CAAC,SAAS,EAAE,cAAc,CAAC,YAAY,CAAC;YAClE,CAAC,CAAC,SAAS,EACb,cAAc,SAAS,CAAC,IAAI,mBAAmB,KAAK,GAAG,CACxD,CACF,CAAC;IACJ,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAY,EAAU,EAAE;IACjD,IAAI,CAAC,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;SAC7B,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,UAAU,CAAC;SACrE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,MAAuB,EAAE,SAAiB,EAAW,EAAE,CAC9E,MAAM,CAAC,YAAY,KAAK,SAAS;IACjC,MAAM,CAAC,UAAU,KAAK,SAAS;IAC/B,MAAM,CAAC,MAAM;IACb,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;AAE5B,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,GAAQ,EACR,QAAkB,EAClB,QAAsB,EACtB,WAAyB,EACzB,eAAuF,EACvF,aAGC,EACD,gBAAqC,EACrC,SAAS,GAAG,KAAK,EACO,EAAE;IAC1B,MAAM,eAAe,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;IAC7C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;IACjF,CAAC;IAED,MAAM,OAAO,GAA4B,EAAE,CAAC;IAC5C,MAAM,iBAAiB,GAKlB,EAAE,CAAC;IACR,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAE7B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrC,iBAAiB,CAAC,IAAI,CAAC;gBACrB,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE;gBACpC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,iBAAiB,CAAC,CAAC,CAAC;gBAC1B,MAAM,EAAE,gBAAgB;aACzB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,oEAAoE;QACpE,yEAAyE;QACzE,IAAI,WAAW,EAAE,OAAO,EAAE,CAAC;YACzB,iBAAiB,GAAG,IAAI,CAAC;YACzB,MAAM;QACR,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;QAC7B,qEAAqE;QACrE,yEAAyE;QACzE,mEAAmE;QACnE,mEAAmE;QACnE,+DAA+D;QAC/D,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAsB;YAClC,KAAK;YACL,KAAK,EAAE,UAAU,CAAC,MAAM;YACxB,EAAE;YACF,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI;SACL,CAAC;QACF,eAAe,EAAE,cAAc,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,KAA4B,CAAC;QACjC,MAAM,mBAAmB,GAAa,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,gBAAgB;gBAChC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC;gBACzB,CAAC,CAAC,MAAM,yBAAyB,CAC7B,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,EACtE,cAAc,CAAC,CAAC,IAAI,GAAG,CACxB,CAAC;YACN,IAAI,gBAAgB,IAAI,KAAK,IAAI,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBACzD,MAAM,IAAI,KAAK,CAAC,kDAAkD,KAAK,GAAG,CAAC,CAAC;YAC9E,CAAC;YACD,8DAA8D;YAC9D,+DAA+D;YAC/D,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CACxC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAqB,EAAE,GAAG,EAAE;gBAC9C,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvC,GAAG,CAAC,aAAa;oBACf,CAAC,CAAC;wBACE,uBAAuB,EAAE,KAAK,EAAE,KAAc,EAAE,EAAE;4BAChD,mBAAmB,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC/D,CAAC;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACR,CAAC,CACH,CAAC;YACF,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAClD,KAAK,GAAG;gBACN,EAAE;gBACF,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI;gBACJ,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM;gBACN,SAAS;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7E,GAAG,CAAC,MAAM,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACjD,CAAC;YACF,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;gBACtC,iBAAiB,GAAG,IAAI,CAAC;YAC3B,CAAC;iBAAM,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC3C,gBAAgB,GAAG,IAAI,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM;oBAAE,SAAS,GAAG,KAAK,CAAC;gBAC/B,gBAAgB,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC1C,SAAS,IAAI,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,KAAK,GAAG;gBACN,EAAE;gBACF,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI;gBACJ,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM;gBACN,SAAS;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,YAAY,EAAE,OAAO;aACtB,CAAC;YACF,iBAAiB,GAAG,IAAI,CAAC;QAC3B,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,eAAe,EAAE,YAAY,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACjD,MAAM,eAAe,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,aAAa,EAAE,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,iBAAiB,IAAI,CAAC,gBAAgB,IAAI,SAAS,GAAG,CAAC,CAAC;IAC5E,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACzC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI;QAC3D,UAAU,EAAE,OAAO;QACnB,GAAG,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,GAAG,CAAC,iBAAiB;YACnB,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,OAAgB,EAAE,KAAK,EAAE,SAAkB,EAAE,EAAE;YAC7E,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC,CAAC"}

package/lib/orchestrator/harness-catalog-drift.d.ts

@@ -0,0 +1,22 @@
+import { type ClosedHarnessCatalog, type ReasoningEffort } from "../core/index.js";
+export type LiveHarnessModel = {
+    id: string;
+    efforts?: readonly ReasoningEffort[] | undefined;
+};
+export type HarnessCatalogDrift = {
+    unlisted: string[];
+    mismatched: string[];
+    unavailable: string[];
+};
+export declare const harnessCatalogDrift: (catalog: ClosedHarnessCatalog, live: readonly LiveHarnessModel[]) => HarnessCatalogDrift;
+export declare const hasHarnessCatalogDrift: (drift: HarnessCatalogDrift) => boolean;
+export declare const describeHarnessCatalogDrift: (drift: HarnessCatalogDrift) => string;
+export declare const parseCodexModelsCache: (text: string) => LiveHarnessModel[] | undefined;
+export declare const parseCopilotModelValues: (helpText: string) => LiveHarnessModel[] | undefined;
+export type HarnessCatalogProbe = {
+    catalog: ClosedHarnessCatalog;
+    source: string;
+    read: (command: string | undefined) => Promise<LiveHarnessModel[] | undefined>;
+};
+export declare const harnessCatalogProbes: readonly HarnessCatalogProbe[];
+//# sourceMappingURL=harness-catalog-drift.d.ts.map

package/lib/orchestrator/harness-catalog-drift.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness-catalog-drift.d.ts","sourceRoot":"","sources":["../../src/orchestrator/harness-catalog-drift.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,oBAAoB,EAIzB,KAAK,eAAe,EAErB,MAAM,kBAAkB,CAAC;AAE1B,MAAM,MAAM,gBAAgB,GAAG;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,CAAC,EAAE,SAAS,eAAe,EAAE,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,CAAC;AAEF,eAAO,MAAM,mBAAmB,YACrB,oBAAoB,QACvB,SAAS,gBAAgB,EAAE,KAChC,mBAcF,CAAC;AAEF,eAAO,MAAM,sBAAsB,UAAW,mBAAmB,KAAG,OACV,CAAC;AAE3D,eAAO,MAAM,2BAA2B,UAAW,mBAAmB,KAAG,MAO1D,CAAC;AAehB,eAAO,MAAM,qBAAqB,SAAU,MAAM,KAAG,gBAAgB,EAAE,GAAG,SASzE,CAAC;AAYF,eAAO,MAAM,uBAAuB,aAAc,MAAM,KAAG,gBAAgB,EAAE,GAAG,SAW/E,CAAC;AAcF,MAAM,MAAM,mBAAmB,GAAG;IAChC,OAAO,EAAE,oBAAoB,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,gBAAgB,EAAE,GAAG,SAAS,CAAC,CAAC;CAChF,CAAC;AAEF,eAAO,MAAM,oBAAoB,EAAE,SAAS,mBAAmB,EAW9D,CAAC"}

package/lib/orchestrator/harness-catalog-drift.js

@@ -0,0 +1,90 @@
+import { readFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { z } from "zod";
+import { codexCatalog, copilotCatalog, exec, tryParseJson, } from "../core/index.js";
+export const harnessCatalogDrift = (catalog, live) => {
+    const liveIds = new Set(live.map((model) => model.id));
+    return {
+        unlisted: live.filter((model) => !(model.id in catalog.models)).map((model) => model.id),
+        mismatched: live
+            .filter((model) => model.efforts !== undefined &&
+            model.id in catalog.models &&
+            model.efforts.join(",") !== catalog.models[model.id]?.join(","))
+            .map((model) => `${model.id} (${model.efforts?.join(", ") || "no effort"})`),
+        unavailable: Object.keys(catalog.models).filter((id) => !liveIds.has(id)),
+    };
+};
+export const hasHarnessCatalogDrift = (drift) => drift.unlisted.length > 0 || drift.mismatched.length > 0;
+export const describeHarnessCatalogDrift = (drift) => [
+    drift.unlisted.length > 0 ? `not in catalog: ${drift.unlisted.join(", ")}` : undefined,
+    drift.mismatched.length > 0 ? `efforts differ: ${drift.mismatched.join("; ")}` : undefined,
+    drift.unavailable.length > 0 ? `not offered here: ${drift.unavailable.join(", ")}` : undefined,
+]
+    .filter((part) => part !== undefined)
+    .join(". ");
+const CodexModelsCacheSchema = z.object({
+    models: z.array(z.looseObject({
+        slug: z.string(),
+        visibility: z.string().optional(),
+        supported_reasoning_levels: z.array(z.looseObject({ effort: z.string() })).default([]),
+    })),
+});
+const codexModelsCachePath = (env = process.env) => join(env.CODEX_HOME ?? join(homedir(), ".codex"), "models_cache.json");
+export const parseCodexModelsCache = (text) => {
+    const parsed = CodexModelsCacheSchema.safeParse(tryParseJson(text));
+    if (!parsed.success)
+        return undefined;
+    return parsed.data.models
+        .filter((model) => model.visibility !== "hide")
+        .map((model) => ({
+        id: model.slug,
+        efforts: model.supported_reasoning_levels.map((level) => level.effort),
+    }));
+};
+const readCodexLiveModels = async (path = codexModelsCachePath()) => {
+    try {
+        return parseCodexModelsCache(await readFile(path, "utf8"));
+    }
+    catch {
+        return undefined;
+    }
+};
+export const parseCopilotModelValues = (helpText) => {
+    const lines = helpText.split("\n");
+    const start = lines.findIndex((line) => /^\s*`model`:/.test(line));
+    if (start < 0)
+        return undefined;
+    const models = [];
+    for (const line of lines.slice(start + 1)) {
+        const match = /^\s*-\s*"([^"]+)"\s*$/.exec(line);
+        if (!match?.[1])
+            break;
+        models.push({ id: match[1] });
+    }
+    return models.length > 0 ? models : undefined;
+};
+const readCopilotLiveModels = async (command = "copilot") => {
+    try {
+        const result = await exec(command, ["help", "config"], { timeoutMs: 15_000 });
+        if (result.exitCode !== 0 || result.timedOut)
+            return undefined;
+        return parseCopilotModelValues(result.stdout);
+    }
+    catch {
+        return undefined;
+    }
+};
+export const harnessCatalogProbes = [
+    {
+        catalog: codexCatalog,
+        source: "models_cache.json",
+        read: () => readCodexLiveModels(),
+    },
+    {
+        catalog: copilotCatalog,
+        source: "help config",
+        read: (command) => readCopilotLiveModels(command),
+    },
+];
+//# sourceMappingURL=harness-catalog-drift.js.map

package/lib/orchestrator/harness-catalog-drift.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness-catalog-drift.js","sourceRoot":"","sources":["../../src/orchestrator/harness-catalog-drift.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAEL,YAAY,EACZ,cAAc,EACd,IAAI,EAEJ,YAAY,GACb,MAAM,kBAAkB,CAAC;AAa1B,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,OAA6B,EAC7B,IAAiC,EACZ,EAAE;IACvB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;QACxF,UAAU,EAAE,IAAI;aACb,MAAM,CACL,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,OAAO,KAAK,SAAS;YAC3B,KAAK,CAAC,EAAE,IAAI,OAAO,CAAC,MAAM;YAC1B,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAClE;aACA,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,WAAW,GAAG,CAAC;QAC9E,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;KAC1E,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,KAA0B,EAAW,EAAE,CAC5E,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;AAE3D,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,KAA0B,EAAU,EAAE,CAChF;IACE,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,mBAAmB,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;IACtF,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,mBAAmB,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;IAC1F,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;CAC/F;KACE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC;KACpC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEhB,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,MAAM,EAAE,CAAC,CAAC,KAAK,CACb,CAAC,CAAC,WAAW,CAAC;QACZ,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QACjC,0BAA0B,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;KACvF,CAAC,CACH;CACF,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,CAAC,GAAG,GAAsB,OAAO,CAAC,GAAG,EAAU,EAAE,CAC5E,IAAI,CAAC,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,mBAAmB,CAAC,CAAC;AAEzE,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAkC,EAAE;IACpF,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IACpE,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IACtC,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM;SACtB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,KAAK,MAAM,CAAC;SAC9C,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACf,EAAE,EAAE,KAAK,CAAC,IAAI;QACd,OAAO,EAAE,KAAK,CAAC,0BAA0B,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAyB,CAAC;KAC1F,CAAC,CAAC,CAAC;AACR,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,KAAK,EAC/B,IAAI,GAAG,oBAAoB,EAAE,EACY,EAAE;IAC3C,IAAI,CAAC;QACH,OAAO,qBAAqB,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,QAAgB,EAAkC,EAAE;IAC1F,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACnE,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAChC,MAAM,MAAM,GAAuB,EAAE,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAAE,MAAM;QACvB,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,KAAK,EACjC,OAAO,GAAG,SAAS,EACsB,EAAE;IAC3C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9E,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ;YAAE,OAAO,SAAS,CAAC;QAC/D,OAAO,uBAAuB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC,CAAC;AAQF,MAAM,CAAC,MAAM,oBAAoB,GAAmC;IAClE;QACE,OAAO,EAAE,YAAY;QACrB,MAAM,EAAE,mBAAmB;QAC3B,IAAI,EAAE,GAAG,EAAE,CAAC,mBAAmB,EAAE;KAClC;IACD;QACE,OAAO,EAAE,cAAc;QACvB,MAAM,EAAE,aAAa;QACrB,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,qBAAqB,CAAC,OAAO,CAAC;KAClD;CACF,CAAC"}

package/lib/orchestrator/harness-registry.d.ts

@@ -0,0 +1,4 @@
+import type { HarnessModelConfig, Registry } from "../core/index.js";
+import type { ResolvedLuxConfig } from "./config.js";
+export declare const registryWithHarness: (registry: Registry, config: Pick<ResolvedLuxConfig, "assertions">, harness: HarnessModelConfig | undefined) => Registry;
+//# sourceMappingURL=harness-registry.d.ts.map

package/lib/orchestrator/harness-registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness-registry.d.ts","sourceRoot":"","sources":["../../src/orchestrator/harness-registry.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErD,eAAO,MAAM,mBAAmB,aACpB,QAAQ,UACV,IAAI,CAAC,iBAAiB,EAAE,YAAY,CAAC,WACpC,kBAAkB,GAAG,SAAS,KACtC,QAGyD,CAAC"}

package/lib/orchestrator/harness-registry.js

@@ -0,0 +1,5 @@
+import { createRubricAssertion } from "../assertions/rubric/index.js";
+export const registryWithHarness = (registry, config, harness) => config.assertions?.some((assertion) => assertion.type === "rubric")
+    ? registry
+    : registry.withAssertion(createRubricAssertion(harness));
+//# sourceMappingURL=harness-registry.js.map

package/lib/orchestrator/harness-registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness-registry.js","sourceRoot":"","sources":["../../src/orchestrator/harness-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAItE,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,QAAkB,EAClB,MAA6C,EAC7C,OAAuC,EAC7B,EAAE,CACZ,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC;IACjE,CAAC,CAAC,QAAQ;IACV,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC"}