@prismatic-io/lux 0.0.2-preview.30 → 0.0.2-preview.33

package/lib/orchestrator/experiment-selection.js

@@ -154,42 +154,88 @@ const normalizedConstraintViolation = (campaign, candidate, constraint, experime
     }
     return worst;
 };
-export const rankCandidates = (campaign, candidates, experiment, constraintSplits = ["train", "validation", "test"]) => [...candidates].sort((a, b) => {
-    const activeConstraints = campaign.objectives.constraints.filter((constraint) => constraintSplits.includes(constraint.split));
-    const aEligible = activeConstraints.every((constraint) => satisfiesConstraint(campaign, a, constraint, experiment));
-    const bEligible = activeConstraints.every((constraint) => satisfiesConstraint(campaign, b, constraint, experiment));
-    if (aEligible !== bEligible)
-        return aEligible ? -1 : 1;
-    if (!aEligible) {
-        const aViolation = Math.max(0, ...activeConstraints.map((constraint) => normalizedConstraintViolation(campaign, a, constraint, experiment)));
-        const bViolation = Math.max(0, ...activeConstraints.map((constraint) => normalizedConstraintViolation(campaign, b, constraint, experiment)));
-        if (aViolation !== bViolation)
-            return aViolation < bViolation ? -1 : 1;
-    }
-    for (const objective of campaign.objectives.minimize.filter((entry) => constraintSplits.includes(entry.split))) {
-        const aValues = metricValues(a, objective.metric, objective.profile, objective.split, experiment);
-        const bValues = metricValues(b, objective.metric, objective.profile, objective.split, experiment);
-        if (aValues.length === 0 && bValues.length > 0)
-            return 1;
-        if (bValues.length === 0 && aValues.length > 0)
-            return -1;
-        if (aValues.length > 0 && bValues.length > 0) {
-            const delta = average(aValues) - average(bValues);
-            if (delta !== 0)
-                return delta;
+const hasScoreEvidence = (experiment, split) => [...new Set(experiment.evaluations.map((entry) => entry.candidateId))]
+    .flatMap((id) => candidateEvaluations(experiment, id))
+    .some((entry) => entry.split === split &&
+    (entry.metricDefinitions?.score !== undefined ||
+        entry.metrics.score !== undefined ||
+        Object.values(entry.caseSamples ?? {}).some((sample) => sample.metrics.score !== undefined)));
+const completeScoreEvidence = (entry) => entry.metrics.score !== undefined &&
+    (!entry.caseSamples ||
+        Object.keys(entry.casePassRates).every((id) => {
+            const sample = entry.caseSamples?.[id];
+            return sample?.metrics.score?.n === entry.repetitions && Boolean(sample.scoreBasis);
+        }));
+const comparableScoreBasis = (entry, reference) => {
+    if (!reference || !completeScoreEvidence(reference))
+        return false;
+    if (!entry.caseSamples || !reference.caseSamples)
+        return !entry.caseSamples && !reference.caseSamples;
+    const ids = new Set([
+        ...Object.keys(entry.casePassRates),
+        ...Object.keys(reference.casePassRates),
+    ]);
+    return [...ids].every((id) => {
+        const basis = entry.caseSamples?.[id]?.scoreBasis;
+        return Boolean(basis) && basis === reference.caseSamples?.[id]?.scoreBasis;
+    });
+};
+export const rankCandidates = (campaign, candidates, experiment, constraintSplits = ["train", "validation", "test"]) => {
+    const validationEvaluations = (id) => candidateEvaluations(experiment, id).filter((entry) => entry.split === "validation");
+    const referenceId = ["baseline", ...candidates.map((candidate) => candidate.id).sort()].find((id) => {
+        const entries = validationEvaluations(id);
+        return campaign.profiles.every((profile) => entries.some((entry) => entry.profileId === profile && completeScoreEvidence(entry)));
+    });
+    const reference = referenceId ? validationEvaluations(referenceId) : [];
+    return [...candidates].sort((a, b) => {
+        const activeConstraints = campaign.objectives.constraints.filter((constraint) => constraintSplits.includes(constraint.split));
+        const aEligible = activeConstraints.every((constraint) => satisfiesConstraint(campaign, a, constraint, experiment));
+        const bEligible = activeConstraints.every((constraint) => satisfiesConstraint(campaign, b, constraint, experiment));
+        if (aEligible !== bEligible)
+            return aEligible ? -1 : 1;
+        if (!aEligible) {
+            const aViolation = Math.max(0, ...activeConstraints.map((constraint) => normalizedConstraintViolation(campaign, a, constraint, experiment)));
+            const bViolation = Math.max(0, ...activeConstraints.map((constraint) => normalizedConstraintViolation(campaign, b, constraint, experiment)));
+            if (aViolation !== bViolation)
+                return aViolation < bViolation ? -1 : 1;
         }
-    }
-    const quality = (candidate) => {
-        const values = metricValues(candidate, "quality.passRate", undefined, "validation", experiment);
-        return values.length > 0 ? Math.min(...values) : -1;
-    };
-    const qualityDelta = quality(b) - quality(a);
-    if (qualityDelta !== 0)
-        return qualityDelta;
-    if (a.sourceBytes !== b.sourceBytes)
-        return a.sourceBytes - b.sourceBytes;
-    return a.id.localeCompare(b.id);
-});
+        for (const objective of campaign.objectives.minimize.filter((entry) => constraintSplits.includes(entry.split))) {
+            const aValues = metricValues(a, objective.metric, objective.profile, objective.split, experiment);
+            const bValues = metricValues(b, objective.metric, objective.profile, objective.split, experiment);
+            if (aValues.length === 0 && bValues.length > 0)
+                return 1;
+            if (bValues.length === 0 && aValues.length > 0)
+                return -1;
+            if (aValues.length > 0 && bValues.length > 0) {
+                const delta = average(aValues) - average(bValues);
+                if (delta !== 0)
+                    return delta;
+            }
+        }
+        const quality = (candidate) => {
+            if (!hasScoreEvidence(experiment, "validation")) {
+                const values = metricValues(candidate, "quality.passRate", undefined, "validation", experiment);
+                return values.length > 0 ? Math.min(...values) : -1;
+            }
+            const evaluations = candidateEvaluations(experiment, candidate.id);
+            const scores = campaign.profiles.map((profile) => {
+                const entry = evaluations.find((value) => value.profileId === profile && value.split === "validation");
+                if (!entry ||
+                    !completeScoreEvidence(entry) ||
+                    !comparableScoreBasis(entry, reference.find((value) => value.profileId === profile)))
+                    return -1;
+                return entry.metrics.score ?? -1;
+            });
+            return scores.length > 0 ? Math.min(...scores) : -1;
+        };
+        const qualityDelta = quality(b) - quality(a);
+        if (qualityDelta !== 0)
+            return qualityDelta;
+        if (a.sourceBytes !== b.sourceBytes)
+            return a.sourceBytes - b.sourceBytes;
+        return a.id.localeCompare(b.id);
+    });
+};
 const candidateCaseVector = (candidate, experiment, preferredSplit) => {
     const evaluations = candidateEvaluations(experiment, candidate.id);
     const split = evaluations.some((entry) => entry.split === preferredSplit)
@@ -197,10 +243,15 @@ const candidateCaseVector = (candidate, experiment, preferredSplit) => {
         : "validation";
     return Object.fromEntries(evaluations
         .filter((entry) => entry.split === split)
-        .flatMap((entry) => Object.entries(entry.casePassRates).map(([caseId, passRate]) => [
-        `${entry.profileId}:${caseId}`,
-        passRate,
-    ])));
+        .flatMap((entry) => Object.entries(entry.casePassRates).flatMap(([caseId, passRate]) => {
+        if (!hasScoreEvidence(experiment, split))
+            return [[`${entry.profileId}:${caseId}`, passRate]];
+        const sample = entry.caseSamples?.[caseId];
+        const score = sample?.metrics.score;
+        return score?.n === entry.repetitions && sample?.scoreBasis
+            ? [[JSON.stringify([entry.profileId, caseId, sample.scoreBasis]), score.mean]]
+            : [];
+    })));
 };
 const dominates = (a, b) => {
     const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
@@ -208,8 +259,10 @@ const dominates = (a, b) => {
         return false;
     let strictlyBetter = false;
     for (const key of keys) {
-        const av = a[key] ?? 0;
-        const bv = b[key] ?? 0;
+        const av = a[key];
+        const bv = b[key];
+        if (av === undefined || bv === undefined)
+            return false;
         if (av < bv)
             return false;
         if (av > bv)
@@ -241,8 +294,11 @@ const frontierCoverage = (candidate, frontier, experiment) => {
     const candidateVector = candidateCaseVector(candidate, experiment, "train");
     const keys = new Set(vectors.flatMap((vector) => Object.keys(vector)));
     return [...keys].filter((key) => {
-        const best = Math.max(...vectors.map((vector) => vector[key] ?? 0));
-        return (candidateVector[key] ?? 0) === best;
+        const values = vectors.flatMap((vector) => (vector[key] === undefined ? [] : [vector[key]]));
+        const best = Math.max(...values);
+        return ((frontier.length === 1 || values.length > 1) &&
+            candidateVector[key] !== undefined &&
+            candidateVector[key] === best);
     }).length;
 };
 export const searchFrontier = (campaign, candidates, experiment) => {

package/lib/orchestrator/experiment-selection.js.map

@@ -1 +1 @@
-{"version":3,"file":"experiment-selection.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-selection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,+BAA+B,EAE/B,iBAAiB,EACjB,eAAe,GAGhB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAGL,yBAAyB,GAE1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,mBAAmB,EAAqB,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAI3F,MAAM,CAAC,MAAM,iCAAiC,GAAG,eAAe,CAAC,IAAI,CAAC;IACpE,EAAE,EAAE,IAAI;IACR,WAAW,EAAE,IAAI;CAClB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,UAA2C,EAC3C,WAAmB,EACK,EAAE;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAgC,CAAC;IACzD,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,WAAW,CAAC,MAAM,CACpD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,WAAW,CAC7C,EAAE,CAAC;QACF,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,SAAS,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QAC1D,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC9F,CAAC;IACD,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CACtB,SAAsC,EACtC,OAA2B,EAC3B,KAAsB,EACtB,UAA2C,EACnC,EAAE;IACV,IAAI,SAAS,CAAC,EAAE,KAAK,UAAU;QAAE,OAAO,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,MAAM,CAClE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC,CAC9E,CAAC;IACF,MAAM,QAAQ,GAAG,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IAChE,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CACzB,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,IAAI,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS,CACzE,CAAC;QACF,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;YACtE,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,QAAQ;gBAAE,WAAW,EAAE,CAAC;QAC7E,CAAC;IACH,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CACnB,SAAsC,EACtC,MAAc,EACd,OAA2B,EAC3B,KAAsB,EACtB,UAA2C,EACjC,EAAE;IACZ,IAAI,MAAM,KAAK,yBAAyB,EAAE,CAAC;QACzC,OAAO,CAAC,eAAe,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,MAAM,KAAK,eAAe;QAAE,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAC/D,OAAO,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC;SAClD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC;SACrF,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QACjB,MAAM,KAAK,GAAG,MAAM,KAAK,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrF,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,SAAsC,EACtC,MAAc,EACd,OAA2B,EAC3B,KAAsB,EACtB,UAA2C,EAC/B,EAAE;IACd,IAAI,MAAM,KAAK,yBAAyB,IAAI,MAAM,KAAK,eAAe,EAAE,CAAC;QACvE,OAAO,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7F,CAAC;IACD,OAAO,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC;SAClD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC;SACrF,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QACjB,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7C,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAC1B,QAA4B,EAC5B,SAAsC,EACtC,UAAgC,EAChC,UAA2C,EAClC,EAAE;IACX,IAAI,UAAU,CAAC,UAAU,IAAI,SAAS,CAAC,EAAE,KAAK,UAAU;QAAE,OAAO,IAAI,CAAC;IACtE,IAAI,QAAgC,CAAC;IACrC,IAAI,UAAU,CAAC,MAAM,KAAK,eAAe,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;QAChE,QAAQ,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;SAAM,CAAC;QACN,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IAC/B,CAAC;IACD,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE;QAChC,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;YACpF,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAC;YAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC;YACzD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC;YACzD,OAAO,CACL,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC;gBACzD,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,CAC1D,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CACzB,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACtC,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YAC1B,MAAM,YAAY,GAAG,kBAAkB,CACrC,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC;YACF,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,CAAC,MAAM,qDAAqD;oBAC/E,GAAG,OAAO,CAAC,CAAC,CAAC,gBAAgB,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACjD,CAAC;YACJ,CAAC;YACD,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE;gBACpC,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,+BAA+B,CAAC,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;oBACjF,IAAI,mBAAmB,CAAC,QAAQ,CAAC;wBAAE,OAAO,KAAK,CAAC;oBAChD,OAAO,CACL,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,QAAQ,CAAC,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC;wBAClE,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,QAAQ,CAAC,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,CACnE,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAC;wBAAE,OAAO,KAAK,CAAC;oBAC1F,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC,KAAK,CACjB,CAAC,KAAK,EAAE,EAAE,CACR,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC;YACzD,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,CAC5D,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAyB,EAAU,EAAE,CAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AAE7E,MAAM,wBAAwB,GAAG,CAC/B,SAAsC,EACtC,UAAgC,EAChC,OAA2B,EAC3B,UAA2C,EAC7B,EAAE;IAChB,MAAM,KAAK,GAAG,CAAC,KAA2B,EAAW,EAAE,CACrD,KAAK,CAAC,SAAS,KAAK,OAAO,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,CAAC,KAAK,CAAC;IAClE,OAAO,YAAY,CACjB,oBAAoB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EACxD,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAC1D,UAAU,CAAC,MAAM,EACjB,UAAU,CAAC,UAAU,CACtB,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,6BAA6B,GAAG,CACpC,QAA4B,EAC5B,SAAsC,EACtC,UAAgC,EAChC,UAA2C,EACnC,EAAE;IACV,IAAI,UAAU,CAAC,UAAU,IAAI,SAAS,CAAC,EAAE,KAAK,UAAU;QAAE,OAAO,CAAC,CAAC;IACnE,IAAI,QAAQ,GAA8B,QAAQ,CAAC,QAAQ,CAAC;IAC5D,IAAI,UAAU,CAAC,MAAM,KAAK,eAAe,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;QAChE,QAAQ,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa,EAAU,EAAE,CACzD,IAAI,CAAC,GAAG,CACN,UAAU,CAAC,GAAG,KAAK,SAAS;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAC/E,UAAU,CAAC,GAAG,KAAK,SAAS;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAChF,CAAC;IACJ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;YACpF,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI;gBAAE,OAAO,MAAM,CAAC,iBAAiB,CAAC;YAC9E,KAAK,GAAG,IAAI,CAAC,GAAG,CACd,KAAK,EACL,SAAS,CACP,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,MAAM,CAAC,SAAS,EAC1C,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,MAAM,CAAC,SAAS,CAC3C,CACF,CAAC;YACF,SAAS;QACX,CAAC;QACD,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,kBAAkB,CAC/B,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC;YACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,MAAM,CAAC,iBAAiB,CAAC;YACzD,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,+BAA+B,CAAC,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;oBACjF,IAAI,mBAAmB,CAAC,QAAQ,CAAC;wBAAE,OAAO,MAAM,CAAC,iBAAiB,CAAC;oBACnE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;gBACrE,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,MAAM,CAAC,iBAAiB,CAAC;gBAClC,CAAC;YACH,CAAC;YACD,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CACzB,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,iBAAiB,CAAC;QACzD,KAAK,MAAM,KAAK,IAAI,MAAM;YAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,QAA4B,EAC5B,UAAwB,EACxB,UAA2C,EAC3C,gBAAgB,GAA+B,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,EACzE,EAAE,CACP,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;IAC5B,MAAM,iBAAiB,GAAG,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAC9E,gBAAgB,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAC5C,CAAC;IACF,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CACvD,mBAAmB,CAAC,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CACzD,CAAC;IACF,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CACvD,mBAAmB,CAAC,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CACzD,CAAC;IACF,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,CAAC,EACD,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACtC,6BAA6B,CAAC,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CACnE,CACF,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,CAAC,EACD,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACtC,6BAA6B,CAAC,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CACnE,CACF,CAAC;QACF,IAAI,UAAU,KAAK,UAAU;YAAE,OAAO,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CACpE,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CACvC,EAAE,CAAC;QACF,MAAM,OAAO,GAAG,YAAY,CAC1B,CAAC,EACD,SAAS,CAAC,MAAM,EAChB,SAAS,CAAC,OAAO,EACjB,SAAS,CAAC,KAAK,EACf,UAAU,CACX,CAAC;QACF,MAAM,OAAO,GAAG,YAAY,CAC1B,CAAC,EACD,SAAS,CAAC,MAAM,EAChB,SAAS,CAAC,OAAO,EACjB,SAAS,CAAC,KAAK,EACf,UAAU,CACX,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,CAAC;QACzD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC;QAC1D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;QAChC,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,SAAsC,EAAU,EAAE;QACjE,MAAM,MAAM,GAAG,YAAY,CACzB,SAAS,EACT,kBAAkB,EAClB,SAAS,EACT,YAAY,EACZ,UAAU,CACX,CAAC;QACF,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC;IACF,MAAM,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,YAAY,KAAK,CAAC;QAAE,OAAO,YAAY,CAAC;IAC5C,IAAI,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;QAAE,OAAO,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC;IAC1E,OAAO,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAClC,CAAC,CAAC,CAAC;AAEL,MAAM,mBAAmB,GAAG,CAC1B,SAAoB,EACpB,UAA2C,EAC3C,cAA+B,EACP,EAAE;IAC1B,MAAM,WAAW,GAAG,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,cAAc,CAAC;QACvE,CAAC,CAAC,cAAc;QAChB,CAAC,CAAC,YAAY,CAAC;IACjB,OAAO,MAAM,CAAC,WAAW,CACvB,WAAW;SACR,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC;SACxC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACjB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;QAC9D,GAAG,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE;QAC9B,QAAQ;KACT,CAAC,CACH,CACJ,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,CAAyB,EAAE,CAAyB,EAAW,EAAE;IAClF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAClC,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvB,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,EAAE,GAAG,EAAE;YAAE,OAAO,KAAK,CAAC;QAC1B,IAAI,EAAE,GAAG,EAAE;YAAE,cAAc,GAAG,IAAI,CAAC;IACrC,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,QAA4B,EAC5B,UAAgC,EAChC,UAA2C,EAC3C,KAAK,GAAoB,OAAO,EAChC,gBAAgB,GAA+B,CAAC,OAAO,EAAE,YAAY,CAAC,EACzD,EAAE;IACf,MAAM,iBAAiB,GAAG,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAC9E,gBAAgB,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAC5C,CAAC;IACF,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAC/C,iBAAiB,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CACrC,mBAAmB,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CACjE,CACF,CAAC;IACF,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC;QAC5B,SAAS,CAAC,EAAE;QACZ,mBAAmB,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC;KAClD,CAAC,CACH,CAAC;IACF,OAAO,UAAU;SACd,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE;QACpB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/C,OAAO,CAAC,UAAU,CAAC,IAAI,CACrB,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,SAAS,CAAC,EAAE,IAAI,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CACvF,CAAC;IACJ,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CACvB,SAAoB,EACpB,QAA8B,EAC9B,UAA2C,EACnC,EAAE;IACV,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IACzF,MAAM,eAAe,GAAG,mBAAmB,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IAC5E,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpE,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC;IAC9C,CAAC,CAAC,CAAC,MAAM,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,QAA4B,EAC5B,UAAgC,EAChC,UAA2C,EAC9B,EAAE;IACf,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IAClE,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,QAAQ,GACZ,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,GAAG,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QACxF,OAAO,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAC/B,QAA4B,EAC5B,SAAsC,EACtC,UAAgC,EAChC,OAA2B,EAC3B,UAA2C,EACC,EAAE;IAC9C,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACpF,MAAM,SAAS,GAAG,SAAS,CAAC,EAAE,KAAK,UAAU,CAAC;IAC9C,IAAI,MAAM,GAAG,MAAM,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACjE,IAAI,SAAS;QAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,kDAAkD,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;IAC1F,OAAO;QACL,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,UAAU,EAAE,UAAU;QACtB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/B,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,MAAM;QACN,MAAM,EAAE,mBAAmB,CACzB,QAAQ,EACR,SAAS,EACT,EAAE,GAAG,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAClD,UAAU,CACX;QACD,GAAG,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1B,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,yBAAyB,GAAG,CACvC,QAA4B,EAC5B,UAAkD,EAClD,UAA2C,EAC3C,UAAU,GAA+D,EAAE,EACtD,EAAE;IACvB,MAAM,OAAO,GAAG,0CAA0C,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7E,MAAM,kBAAkB,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CACtD,iCAAiC,CAAC,KAAK,CAAC,SAAS,CAAC,CACnD,CAAC;IACF,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,EAAE,kBAAkB,EAAE,UAAU,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IACnC,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CACrD,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CACnD,mBAAmB,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CACjE,CACF,CAAC;IACF,MAAM,QAAQ,GACZ,QAAQ,KAAK,IAAI,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC5F,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CACjE,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE;QACrD,MAAM,QAAQ,GACZ,UAAU,CAAC,OAAO,IAAI,UAAU,CAAC,MAAM,KAAK,eAAe;YACzD,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC;YACtB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACxB,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC9B,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC1B,OAAO,wBAAwB,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;YACxF,CAAC;YACD,MAAM,MAAM,GAAG,YAAY,CACzB,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC;YACF,MAAM,MAAM,GAAG,EAAE,GAAG,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YAClE,IAAI,QAAwE,CAAC;YAC7E,IAAI,IAAwB,CAAC;YAC7B,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC1B,MAAM,OAAO,GAAG,kBAAkB,CAChC,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC,CAAC,CAAC,CAAC;gBACL,IAAI,OAAO,EAAE,CAAC;oBACZ,IAAI,CAAC;wBACH,QAAQ,GAAG,+BAA+B,CAAC,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;wBAC3E,IAAI,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;oBACvC,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAChE,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,IAAI,GAAG,+BAA+B,CAAC;gBACzC,CAAC;YACH,CAAC;YACD,OAAO;gBACL,WAAW,EAAE,SAAS,CAAC,EAAE;gBACzB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/B,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,MAAM;gBACN,MAAM,EAAE,mBAAmB,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC;gBACpE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1B,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CACH,CAAC;IACF,OAAO,yBAAyB,CAAC,KAAK,CAAC;QACrC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACxD,oBAAoB,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,6BAA6B;YACvC,CAAC,CAAC,EAAE,6BAA6B,EAAE,OAAO,CAAC,6BAA6B,EAAE;YAC1E,CAAC,CAAC,EAAE,CAAC;QACP,kBAAkB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3D,oBAAoB,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,WAAW,EAAE,iBAAiB;KAC/B,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,0CAA0C,GAAG,CAAC;KACxD,MAAM,CAAC;IACN,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,6BAA6B,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,QAAQ,EAAE;CACrE,CAAC;KACD,MAAM,EAAE,CAAC"}
+{"version":3,"file":"experiment-selection.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-selection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,+BAA+B,EAE/B,iBAAiB,EACjB,eAAe,GAGhB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAGL,yBAAyB,GAE1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,mBAAmB,EAAqB,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAI3F,MAAM,CAAC,MAAM,iCAAiC,GAAG,eAAe,CAAC,IAAI,CAAC;IACpE,EAAE,EAAE,IAAI;IACR,WAAW,EAAE,IAAI;CAClB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,UAA2C,EAC3C,WAAmB,EACK,EAAE;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAgC,CAAC;IACzD,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,WAAW,CAAC,MAAM,CACpD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,WAAW,CAC7C,EAAE,CAAC;QACF,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,SAAS,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QAC1D,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC9F,CAAC;IACD,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CACtB,SAAsC,EACtC,OAA2B,EAC3B,KAAsB,EACtB,UAA2C,EACnC,EAAE;IACV,IAAI,SAAS,CAAC,EAAE,KAAK,UAAU;QAAE,OAAO,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,MAAM,CAClE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC,CAC9E,CAAC;IACF,MAAM,QAAQ,GAAG,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IAChE,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CACzB,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,IAAI,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS,CACzE,CAAC;QACF,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;YACtE,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,QAAQ;gBAAE,WAAW,EAAE,CAAC;QAC7E,CAAC;IACH,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CACnB,SAAsC,EACtC,MAAc,EACd,OAA2B,EAC3B,KAAsB,EACtB,UAA2C,EACjC,EAAE;IACZ,IAAI,MAAM,KAAK,yBAAyB,EAAE,CAAC;QACzC,OAAO,CAAC,eAAe,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,MAAM,KAAK,eAAe;QAAE,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAC/D,OAAO,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC;SAClD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC;SACrF,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QACjB,MAAM,KAAK,GAAG,MAAM,KAAK,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrF,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,SAAsC,EACtC,MAAc,EACd,OAA2B,EAC3B,KAAsB,EACtB,UAA2C,EAC/B,EAAE;IACd,IAAI,MAAM,KAAK,yBAAyB,IAAI,MAAM,KAAK,eAAe,EAAE,CAAC;QACvE,OAAO,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7F,CAAC;IACD,OAAO,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC;SAClD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC;SACrF,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QACjB,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7C,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAC1B,QAA4B,EAC5B,SAAsC,EACtC,UAAgC,EAChC,UAA2C,EAClC,EAAE;IACX,IAAI,UAAU,CAAC,UAAU,IAAI,SAAS,CAAC,EAAE,KAAK,UAAU;QAAE,OAAO,IAAI,CAAC;IACtE,IAAI,QAAgC,CAAC;IACrC,IAAI,UAAU,CAAC,MAAM,KAAK,eAAe,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;QAChE,QAAQ,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;SAAM,CAAC;QACN,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IAC/B,CAAC;IACD,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE;QAChC,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;YACpF,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAC;YAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC;YACzD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC;YACzD,OAAO,CACL,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC;gBACzD,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,CAC1D,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CACzB,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACtC,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YAC1B,MAAM,YAAY,GAAG,kBAAkB,CACrC,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC;YACF,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,CAAC,MAAM,qDAAqD;oBAC/E,GAAG,OAAO,CAAC,CAAC,CAAC,gBAAgB,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACjD,CAAC;YACJ,CAAC;YACD,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE;gBACpC,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,+BAA+B,CAAC,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;oBACjF,IAAI,mBAAmB,CAAC,QAAQ,CAAC;wBAAE,OAAO,KAAK,CAAC;oBAChD,OAAO,CACL,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,QAAQ,CAAC,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC;wBAClE,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,QAAQ,CAAC,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,CACnE,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAC;wBAAE,OAAO,KAAK,CAAC;oBAC1F,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC,KAAK,CACjB,CAAC,KAAK,EAAE,EAAE,CACR,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC;YACzD,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,CAC5D,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAyB,EAAU,EAAE,CAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AAE7E,MAAM,wBAAwB,GAAG,CAC/B,SAAsC,EACtC,UAAgC,EAChC,OAA2B,EAC3B,UAA2C,EAC7B,EAAE;IAChB,MAAM,KAAK,GAAG,CAAC,KAA2B,EAAW,EAAE,CACrD,KAAK,CAAC,SAAS,KAAK,OAAO,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,CAAC,KAAK,CAAC;IAClE,OAAO,YAAY,CACjB,oBAAoB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EACxD,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAC1D,UAAU,CAAC,MAAM,EACjB,UAAU,CAAC,UAAU,CACtB,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,6BAA6B,GAAG,CACpC,QAA4B,EAC5B,SAAsC,EACtC,UAAgC,EAChC,UAA2C,EACnC,EAAE;IACV,IAAI,UAAU,CAAC,UAAU,IAAI,SAAS,CAAC,EAAE,KAAK,UAAU;QAAE,OAAO,CAAC,CAAC;IACnE,IAAI,QAAQ,GAA8B,QAAQ,CAAC,QAAQ,CAAC;IAC5D,IAAI,UAAU,CAAC,MAAM,KAAK,eAAe,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;QAChE,QAAQ,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa,EAAU,EAAE,CACzD,IAAI,CAAC,GAAG,CACN,UAAU,CAAC,GAAG,KAAK,SAAS;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAC/E,UAAU,CAAC,GAAG,KAAK,SAAS;QAC1B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAChF,CAAC;IACJ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;YACpF,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI;gBAAE,OAAO,MAAM,CAAC,iBAAiB,CAAC;YAC9E,KAAK,GAAG,IAAI,CAAC,GAAG,CACd,KAAK,EACL,SAAS,CACP,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,MAAM,CAAC,SAAS,EAC1C,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,MAAM,CAAC,SAAS,CAC3C,CACF,CAAC;YACF,SAAS;QACX,CAAC;QACD,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,kBAAkB,CAC/B,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC;YACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,MAAM,CAAC,iBAAiB,CAAC;YACzD,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,+BAA+B,CAAC,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;oBACjF,IAAI,mBAAmB,CAAC,QAAQ,CAAC;wBAAE,OAAO,MAAM,CAAC,iBAAiB,CAAC;oBACnE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;gBACrE,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,MAAM,CAAC,iBAAiB,CAAC;gBAClC,CAAC;YACH,CAAC;YACD,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CACzB,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,iBAAiB,CAAC;QACzD,KAAK,MAAM,KAAK,IAAI,MAAM;YAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CACvB,UAA2C,EAC3C,KAAsB,EACb,EAAE,CACX,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;KACnE,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,oBAAoB,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;KACrD,IAAI,CACH,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,KAAK,KAAK,KAAK;IACrB,CAAC,KAAK,CAAC,iBAAiB,EAAE,KAAK,KAAK,SAAS;QAC3C,KAAK,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS;QACjC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,CACzC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAC/C,CAAC,CACP,CAAC;AAEN,MAAM,qBAAqB,GAAG,CAAC,KAA2B,EAAW,EAAE,CACrE,KAAK,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS;IACjC,CAAC,CAAC,KAAK,CAAC,WAAW;QACjB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE;YAC5C,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;YACvC,OAAO,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,KAAK,CAAC,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC,CAAC;AAER,MAAM,oBAAoB,GAAG,CAC3B,KAA2B,EAC3B,SAA2C,EAClC,EAAE;IACX,IAAI,CAAC,SAAS,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,WAAW;QAC9C,OAAO,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;IACtD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC;QAClB,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;QACnC,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;KACxC,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE;QAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC;QAClD,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,SAAS,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC;IAC7E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,QAA4B,EAC5B,UAAwB,EACxB,UAA2C,EAC3C,gBAAgB,GAA+B,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,EACzE,EAAE;IACP,MAAM,qBAAqB,GAAG,CAAC,EAAU,EAA0B,EAAE,CACnE,oBAAoB,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,YAAY,CAAC,CAAC;IACvF,MAAM,WAAW,GAAG,CAAC,UAAU,EAAE,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAC1F,CAAC,EAAE,EAAE,EAAE;QACL,MAAM,OAAO,GAAG,qBAAqB,CAAC,EAAE,CAAC,CAAC;QAC1C,OAAO,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CACzC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,OAAO,IAAI,qBAAqB,CAAC,KAAK,CAAC,CAAC,CACrF,CAAC;IACJ,CAAC,CACF,CAAC;IACF,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxE,OAAO,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACnC,MAAM,iBAAiB,GAAG,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAC9E,gBAAgB,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAC5C,CAAC;QACF,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CACvD,mBAAmB,CAAC,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CACzD,CAAC;QACF,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CACvD,mBAAmB,CAAC,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CACzD,CAAC;QACF,IAAI,SAAS,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,CAAC,EACD,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACtC,6BAA6B,CAAC,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CACnE,CACF,CAAC;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,CAAC,EACD,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACtC,6BAA6B,CAAC,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,CACnE,CACF,CAAC;YACF,IAAI,UAAU,KAAK,UAAU;gBAAE,OAAO,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CACpE,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CACvC,EAAE,CAAC;YACF,MAAM,OAAO,GAAG,YAAY,CAC1B,CAAC,EACD,SAAS,CAAC,MAAM,EAChB,SAAS,CAAC,OAAO,EACjB,SAAS,CAAC,KAAK,EACf,UAAU,CACX,CAAC;YACF,MAAM,OAAO,GAAG,YAAY,CAC1B,CAAC,EACD,SAAS,CAAC,MAAM,EAChB,SAAS,CAAC,OAAO,EACjB,SAAS,CAAC,KAAK,EACf,UAAU,CACX,CAAC;YACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,CAAC;YACzD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,CAAC,CAAC;YAC1D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;gBAClD,IAAI,KAAK,KAAK,CAAC;oBAAE,OAAO,KAAK,CAAC;YAChC,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,SAAsC,EAAU,EAAE;YACjE,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,EAAE,CAAC;gBAChD,MAAM,MAAM,GAAG,YAAY,CACzB,SAAS,EACT,kBAAkB,EAClB,SAAS,EACT,YAAY,EACZ,UAAU,CACX,CAAC;gBACF,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,MAAM,WAAW,GAAG,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;YACnE,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC/C,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAC5B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,OAAO,IAAI,KAAK,CAAC,KAAK,KAAK,YAAY,CACvE,CAAC;gBACF,IACE,CAAC,KAAK;oBACN,CAAC,qBAAqB,CAAC,KAAK,CAAC;oBAC7B,CAAC,oBAAoB,CACnB,KAAK,EACL,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC,CACvD;oBAED,OAAO,CAAC,CAAC,CAAC;gBACZ,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;YACH,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC,CAAC;QACF,MAAM,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7C,IAAI,YAAY,KAAK,CAAC;YAAE,OAAO,YAAY,CAAC;QAC5C,IAAI,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;YAAE,OAAO,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC;QAC1E,OAAO,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAC1B,SAAoB,EACpB,UAA2C,EAC3C,cAA+B,EACP,EAAE;IAC1B,MAAM,WAAW,GAAG,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,cAAc,CAAC;QACvE,CAAC,CAAC,cAAc;QAChB,CAAC,CAAC,YAAY,CAAC;IACjB,OAAO,MAAM,CAAC,WAAW,CACvB,WAAW;SACR,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC;SACxC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACjB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE;QACjE,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC;YACtC,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC;QACpC,OAAO,KAAK,EAAE,CAAC,KAAK,KAAK,CAAC,WAAW,IAAI,MAAM,EAAE,UAAU;YACzD,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9E,CAAC,CAAC,EAAE,CAAC;IACT,CAAC,CAAC,CACH,CACJ,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,CAAyB,EAAE,CAAyB,EAAW,EAAE;IAClF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAClC,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAClB,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,EAAE,KAAK,SAAS,IAAI,EAAE,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACvD,IAAI,EAAE,GAAG,EAAE;YAAE,OAAO,KAAK,CAAC;QAC1B,IAAI,EAAE,GAAG,EAAE;YAAE,cAAc,GAAG,IAAI,CAAC;IACrC,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,QAA4B,EAC5B,UAAgC,EAChC,UAA2C,EAC3C,KAAK,GAAoB,OAAO,EAChC,gBAAgB,GAA+B,CAAC,OAAO,EAAE,YAAY,CAAC,EACzD,EAAE;IACf,MAAM,iBAAiB,GAAG,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAC9E,gBAAgB,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAC5C,CAAC;IACF,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAC/C,iBAAiB,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CACrC,mBAAmB,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CACjE,CACF,CAAC;IACF,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC;QAC5B,SAAS,CAAC,EAAE;QACZ,mBAAmB,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC;KAClD,CAAC,CACH,CAAC;IACF,OAAO,UAAU;SACd,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE;QACpB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/C,OAAO,CAAC,UAAU,CAAC,IAAI,CACrB,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,SAAS,CAAC,EAAE,IAAI,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CACvF,CAAC;IACJ,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CACvB,SAAoB,EACpB,QAA8B,EAC9B,UAA2C,EACnC,EAAE;IACV,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IACzF,MAAM,eAAe,GAAG,mBAAmB,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IAC5E,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;QACjC,OAAO,CACL,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YAC5C,eAAe,CAAC,GAAG,CAAC,KAAK,SAAS;YAClC,eAAe,CAAC,GAAG,CAAC,KAAK,IAAI,CAC9B,CAAC;IACJ,CAAC,CAAC,CAAC,MAAM,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,QAA4B,EAC5B,UAAgC,EAChC,UAA2C,EAC9B,EAAE;IACf,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IAClE,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,QAAQ,GACZ,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,GAAG,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QACxF,OAAO,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAC/B,QAA4B,EAC5B,SAAsC,EACtC,UAAgC,EAChC,OAA2B,EAC3B,UAA2C,EACC,EAAE;IAC9C,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACpF,MAAM,SAAS,GAAG,SAAS,CAAC,EAAE,KAAK,UAAU,CAAC;IAC9C,IAAI,MAAM,GAAG,MAAM,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACjE,IAAI,SAAS;QAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,kDAAkD,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;IAC1F,OAAO;QACL,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,UAAU,EAAE,UAAU;QACtB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/B,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,MAAM;QACN,MAAM,EAAE,mBAAmB,CACzB,QAAQ,EACR,SAAS,EACT,EAAE,GAAG,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAClD,UAAU,CACX;QACD,GAAG,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1B,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,yBAAyB,GAAG,CACvC,QAA4B,EAC5B,UAAkD,EAClD,UAA2C,EAC3C,UAAU,GAA+D,EAAE,EACtD,EAAE;IACvB,MAAM,OAAO,GAAG,0CAA0C,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7E,MAAM,kBAAkB,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CACtD,iCAAiC,CAAC,KAAK,CAAC,SAAS,CAAC,CACnD,CAAC;IACF,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,EAAE,kBAAkB,EAAE,UAAU,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IACnC,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CACrD,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CACnD,mBAAmB,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CACjE,CACF,CAAC;IACF,MAAM,QAAQ,GACZ,QAAQ,KAAK,IAAI,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC5F,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CACjE,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE;QACrD,MAAM,QAAQ,GACZ,UAAU,CAAC,OAAO,IAAI,UAAU,CAAC,MAAM,KAAK,eAAe;YACzD,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC;YACtB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACxB,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC9B,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC1B,OAAO,wBAAwB,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;YACxF,CAAC;YACD,MAAM,MAAM,GAAG,YAAY,CACzB,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC;YACF,MAAM,MAAM,GAAG,EAAE,GAAG,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YAClE,IAAI,QAAwE,CAAC;YAC7E,IAAI,IAAwB,CAAC;YAC7B,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC1B,MAAM,OAAO,GAAG,kBAAkB,CAChC,SAAS,EACT,UAAU,CAAC,MAAM,EACjB,OAAO,EACP,UAAU,CAAC,KAAK,EAChB,UAAU,CACX,CAAC,CAAC,CAAC,CAAC;gBACL,IAAI,OAAO,EAAE,CAAC;oBACZ,IAAI,CAAC;wBACH,QAAQ,GAAG,+BAA+B,CAAC,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;wBAC3E,IAAI,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;oBACvC,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAChE,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,IAAI,GAAG,+BAA+B,CAAC;gBACzC,CAAC;YACH,CAAC;YACD,OAAO;gBACL,WAAW,EAAE,SAAS,CAAC,EAAE;gBACzB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/B,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,MAAM;gBACN,MAAM,EAAE,mBAAmB,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC;gBACpE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1B,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CACH,CAAC;IACF,OAAO,yBAAyB,CAAC,KAAK,CAAC;QACrC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACxD,oBAAoB,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,6BAA6B;YACvC,CAAC,CAAC,EAAE,6BAA6B,EAAE,OAAO,CAAC,6BAA6B,EAAE;YAC1E,CAAC,CAAC,EAAE,CAAC;QACP,kBAAkB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3D,oBAAoB,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,WAAW,EAAE,iBAAiB;KAC/B,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,0CAA0C,GAAG,CAAC;KACxD,MAAM,CAAC;IACN,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,6BAA6B,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,QAAQ,EAAE;CACrE,CAAC;KACD,MAAM,EAAE,CAAC"}

package/lib/orchestrator/grader-experiment.d.ts

@@ -52,7 +52,8 @@ declare const GraderEvidenceItemSchema: z.ZodObject<{
             id: z.ZodExactOptional<z.ZodString>;
             type: z.ZodString;
             name: z.ZodString;
-            passed: z.ZodBoolean;
+            passed: z.ZodNullable<z.ZodBoolean>;
+            informational: z.ZodExactOptional<z.ZodBoolean>;
             score: z.ZodNumber;
             weight: z.ZodNumber;
             threshold: z.ZodNumber;
@@ -143,7 +144,8 @@ export declare const GraderEvaluationEvidenceSchema: z.ZodObject<{
                 id: z.ZodExactOptional<z.ZodString>;
                 type: z.ZodString;
                 name: z.ZodString;
-                passed: z.ZodBoolean;
+                passed: z.ZodNullable<z.ZodBoolean>;
+                informational: z.ZodExactOptional<z.ZodBoolean>;
                 score: z.ZodNumber;
                 weight: z.ZodNumber;
                 threshold: z.ZodNumber;

package/lib/orchestrator/grader-experiment.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"grader-experiment.d.ts","sourceRoot":"","sources":["../../src/orchestrator/grader-experiment.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EACL,KAAK,SAAS,EACd,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EAEvB,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAMnB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAMhE,QAAA,MAAM,oBAAoB;;;;;;;;;;;;;;;;iBAQxB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,QAAA,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAa5B,CAAC;AAmBH,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAWzC,CAAC;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AA8BtF,eAAO,MAAM,uBAAuB,eACtB,gBAAgB,UACpB,aAAa,kBACL,MAAM,KACrB,cAAc,EA0DhB,CAAC;AAmEF,eAAO,MAAM,uBAAuB,UAC3B,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,EAAE,aAC/C,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,KACxC;IAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;CAyKtE,CAAC;AAsNF,eAAO,MAAM,uBAAuB,YAAmB;IACrD,SAAS,EAAE,wBAAwB,CAAC;IACpC,UAAU,EAAE,mBAAmB,CAAC;IAChC,SAAS,EAAE,SAAS,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC;IACvC,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,CAAC,iBAAiB,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,UAAU,EAAE,CACV,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,IAAI,CACd,mBAAmB,EACnB,aAAa,GAAG,YAAY,GAAG,oBAAoB,GAAG,OAAO,GAAG,SAAS,CAC1E,KACE,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,aAAa,EAAE,CACb,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,IAAI,CACd,mBAAmB,EACnB,aAAa,GAAG,YAAY,GAAG,oBAAoB,GAAG,OAAO,GAAG,SAAS,CAC1E,KACE,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,eAAe,EAAE,CACf,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,IAAI,CACd,mBAAmB,EACnB,aAAa,GAAG,YAAY,GAAG,oBAAoB,GAAG,OAAO,GAAG,SAAS,CAC1E,KACE,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,uBAAuB,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,qBAAqB,EAAE,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/E,uBAAuB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C,KAAG,OAAO,CAAC,mBAAmB,GAAG;IAAE,YAAY,EAAE,MAAM,CAAA;CAAE,CA2PzD,CAAC"}
+{"version":3,"file":"grader-experiment.d.ts","sourceRoot":"","sources":["../../src/orchestrator/grader-experiment.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EACL,KAAK,SAAS,EACd,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EAEvB,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAMnB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAMhE,QAAA,MAAM,oBAAoB;;;;;;;;;;;;;;;;iBAQxB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,QAAA,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAa5B,CAAC;AAmBH,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAWzC,CAAC;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AA8BtF,eAAO,MAAM,uBAAuB,eACtB,gBAAgB,UACpB,aAAa,kBACL,MAAM,KACrB,cAAc,EA0DhB,CAAC;AAmEF,eAAO,MAAM,uBAAuB,UAC3B,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,EAAE,aAC/C,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,KACxC;IAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;CAyKtE,CAAC;AAsNF,eAAO,MAAM,uBAAuB,YAAmB;IACrD,SAAS,EAAE,wBAAwB,CAAC;IACpC,UAAU,EAAE,mBAAmB,CAAC;IAChC,SAAS,EAAE,SAAS,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC;IACvC,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,CAAC,iBAAiB,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,UAAU,EAAE,CACV,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,IAAI,CACd,mBAAmB,EACnB,aAAa,GAAG,YAAY,GAAG,oBAAoB,GAAG,OAAO,GAAG,SAAS,CAC1E,KACE,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,aAAa,EAAE,CACb,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,IAAI,CACd,mBAAmB,EACnB,aAAa,GAAG,YAAY,GAAG,oBAAoB,GAAG,OAAO,GAAG,SAAS,CAC1E,KACE,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,eAAe,EAAE,CACf,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,IAAI,CACd,mBAAmB,EACnB,aAAa,GAAG,YAAY,GAAG,oBAAoB,GAAG,OAAO,GAAG,SAAS,CAC1E,KACE,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,uBAAuB,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,qBAAqB,EAAE,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/E,uBAAuB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C,KAAG,OAAO,CAAC,mBAAmB,GAAG;IAAE,YAAY,EAAE,MAAM,CAAA;CAAE,CA2PzD,CAAC"}

package/lib/orchestrator/grading.d.ts

@@ -25,6 +25,7 @@ export declare const graderConfigHash: (input: {
     skipJudge?: boolean;
 }) => string;
 export declare const validateAssertionPlan: (assertions: readonly Assertion[], registry: Registry) => Promise<unknown[]>;
+export declare const gradingScoreBasis: (report: GradingReport | null) => string | null;
 export declare const gradeRun: (run: Run, registry: Registry, observer?: RunObserver, abortSignal?: AbortSignal, recordAssertion?: (entry: GradingAssertionEntry, index: number) => void | Promise<void>, modelAttempts?: {
     start: (kind: "judge") => string | Promise<string>;
     complete: (attemptIds: readonly string[]) => void | Promise<void>;

package/lib/orchestrator/grading.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"grading.d.ts","sourceRoot":"","sources":["../../src/orchestrator/grading.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAC7B,OAAO,EACL,KAAK,SAAS,EACd,KAAK,aAAa,EAIlB,KAAK,qBAAqB,EAC1B,KAAK,aAAa,EAElB,KAAK,QAAQ,EACb,KAAK,GAAG,EACR,KAAK,WAAW,EAEjB,MAAM,kBAAkB,CAAC;AAU1B;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,MAAO,SAAS,UAAU,CAAC,CAAC,UAAU,KAAG,OAM7E,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG,uBAAuB,GAAG;IACjE,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,4BAA4B,eAC3B,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,0BAA0B,EAYzB,CAAC;AAEL,eAAO,MAAM,yBAAyB,eACxB,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,uBAAuB,EAIrB,CAAC;AAEN,eAAO,MAAM,gBAAgB,UAAW;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,SAAS,0BAA0B,EAAE,CAAC;IAClD,uBAAuB,EAAE,MAAM,CAAC;IAChC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,KAAG,MAQA,CAAC;AAEL,eAAO,MAAM,qBAAqB,eACpB,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,OAAO,CAAC,OAAO,EAAE,CAuBnB,CAAC;AAiBF,eAAO,MAAM,QAAQ,QACd,GAAG,YACE,QAAQ,aACP,WAAW,gBACR,WAAW,oBACP,CAAC,KAAK,EAAE,qBAAqB,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,kBACvE;IACd,KAAK,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,QAAQ,EAAE,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACnE,qBACkB,SAAS,OAAO,EAAE,0BAEpC,OAAO,CAAC,aAAa,CA8IvB,CAAC"}
+{"version":3,"file":"grading.d.ts","sourceRoot":"","sources":["../../src/orchestrator/grading.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAC7B,OAAO,EACL,KAAK,SAAS,EACd,KAAK,aAAa,EAIlB,KAAK,qBAAqB,EAC1B,KAAK,aAAa,EAElB,KAAK,QAAQ,EACb,KAAK,GAAG,EACR,KAAK,WAAW,EAEjB,MAAM,kBAAkB,CAAC;AAU1B;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,MAAO,SAAS,UAAU,CAAC,CAAC,UAAU,KAAG,OAM7E,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG,uBAAuB,GAAG;IACjE,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,4BAA4B,eAC3B,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,0BAA0B,EAYzB,CAAC;AAEL,eAAO,MAAM,yBAAyB,eACxB,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,uBAAuB,EAIrB,CAAC;AAEN,eAAO,MAAM,gBAAgB,UAAW;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,SAAS,0BAA0B,EAAE,CAAC;IAClD,uBAAuB,EAAE,MAAM,CAAC;IAChC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,KAAG,MAQA,CAAC;AAEL,eAAO,MAAM,qBAAqB,eACpB,SAAS,SAAS,EAAE,YACtB,QAAQ,KACjB,OAAO,CAAC,OAAO,EAAE,CAuBnB,CAAC;AAiBF,eAAO,MAAM,iBAAiB,WAAY,aAAa,GAAG,IAAI,KAAG,MAAM,GAAG,IASnE,CAAC;AAmCR,eAAO,MAAM,QAAQ,QACd,GAAG,YACE,QAAQ,aACP,WAAW,gBACR,WAAW,oBACP,CAAC,KAAK,EAAE,qBAAqB,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,kBACvE;IACd,KAAK,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,QAAQ,EAAE,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACnE,qBACkB,SAAS,OAAO,EAAE,0BAEpC,OAAO,CAAC,aAAa,CAwHvB,CAAC"}

package/lib/orchestrator/grading.js

@@ -1,6 +1,6 @@
 import { AssertionResultSchema, guardRunObserver, valueHash, } from "../core/index.js";
 import { validatePluginConfigAsync } from "./validate-plugin-config.js";
-const FRAMEWORK_KEYS = new Set(["id", "type", "name", "weight", "threshold"]);
+const FRAMEWORK_KEYS = new Set(["id", "type", "name", "weight", "threshold", "informational"]);
 const shapeOf = (schema) => {
     const shape = schema.shape;
     return shape && typeof shape === "object" ? shape : null;
@@ -70,6 +70,34 @@ const assertionPassed = (result, threshold) => result.errorMessage === undefined
     result.abstention === undefined &&
     result.passed &&
     result.score >= threshold;
+export const gradingScoreBasis = (report) => report?.caseScore === null || !report
+    ? null
+    : valueHash(report.assertions
+        .map((entry, index) => ({ entry, id: entry.id ?? `${entry.type}:${index + 1}` }))
+        .filter(({ entry }) => entry.errorMessage === undefined && entry.abstention === undefined)
+        .map(({ entry, id }) => ({ id, weight: entry.weight }))
+        .sort((a, b) => a.id.localeCompare(b.id)));
+const aggregateGrading = (entries, skippedAssertions, gradingIncomplete) => {
+    const requiredEntries = entries.filter((entry) => entry.passed !== null);
+    const incomplete = gradingIncomplete || requiredEntries.some((entry) => entry.errorMessage !== undefined);
+    const gradingAbstained = requiredEntries.some((entry) => entry.abstention !== undefined);
+    const scoredEntries = entries.filter((entry) => entry.errorMessage === undefined && entry.abstention === undefined);
+    const weightSum = scoredEntries.reduce((sum, entry) => sum + entry.weight, 0);
+    const weightedScoreSum = scoredEntries.reduce((sum, entry) => sum + entry.weight * entry.score, 0);
+    const hasVerdict = !incomplete && !gradingAbstained && weightSum > 0;
+    return {
+        schemaVersion: 1,
+        casePassed: hasVerdict && requiredEntries.length > 0
+            ? requiredEntries.every((entry) => entry.passed)
+            : null,
+        caseScore: hasVerdict ? weightedScoreSum / weightSum : null,
+        assertions: entries,
+        ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
+        ...(incomplete
+            ? { incomplete: { exitReason: "error", phase: "grading" } }
+            : {}),
+    };
+};
 export const gradeRun = async (run, registry, observer, abortSignal, recordAssertion, modelAttempts, validatedConfigs, skipJudge = false) => {
     const guardedObserver = guardRunObserver(observer);
     const assertions = run.case.assertions ?? [];
@@ -78,11 +106,7 @@ export const gradeRun = async (run, registry, observer, abortSignal, recordAsser
     }
     const entries = [];
     const skippedAssertions = [];
-    let weightedScoreSum = 0;
-    let weightSum = 0;
-    let allPassed = true;
     let gradingIncomplete = false;
-    let gradingAbstained = false;
     for (const [index, a] of assertions.entries()) {
         if (skipJudge && a.type === "rubric") {
             skippedAssertions.push({
@@ -146,7 +170,8 @@ export const gradeRun = async (run, registry, observer, abortSignal, recordAsser
                 id,
                 type: a.type,
                 name,
-                passed,
+                passed: a.informational ? null : passed,
+                ...(a.informational ? { informational: true } : {}),
                 score: result.score,
                 weight,
                 threshold,
@@ -157,18 +182,6 @@ export const gradeRun = async (run, registry, observer, abortSignal, recordAsser
                 ...(result.errorMessage !== undefined ? { errorMessage: result.errorMessage } : {}),
                 ...(result.usage ? { usage: result.usage } : {}),
             };
-            if (result.errorMessage !== undefined) {
-                gradingIncomplete = true;
-            }
-            else if (result.abstention !== undefined) {
-                gradingAbstained = true;
-            }
-            else {
-                if (!passed)
-                    allPassed = false;
-                weightedScoreSum += weight * result.score;
-                weightSum += weight;
-            }
         }
         catch (err) {
             const message = err instanceof Error ? err.message : String(err);
@@ -176,30 +189,20 @@ export const gradeRun = async (run, registry, observer, abortSignal, recordAsser
                 id,
                 type: a.type,
                 name,
-                passed: false,
+                passed: a.informational ? null : false,
+                ...(a.informational ? { informational: true } : {}),
                 score: 0,
                 weight,
                 threshold,
                 durationMs: Date.now() - startedAt,
                 errorMessage: message,
             };
-            gradingIncomplete = true;
         }
         entries.push(entry);
         guardedObserver?.assertionEnd?.(progress, entry);
         await recordAssertion?.(entry, index);
         await modelAttempts?.complete(assertionAttemptIds);
     }
-    const hasVerdict = !gradingIncomplete && !gradingAbstained && weightSum > 0;
-    return {
-        schemaVersion: 1,
-        casePassed: hasVerdict ? allPassed : null,
-        caseScore: hasVerdict ? weightedScoreSum / weightSum : null,
-        assertions: entries,
-        ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
-        ...(gradingIncomplete
-            ? { incomplete: { exitReason: "error", phase: "grading" } }
-            : {}),
-    };
+    return aggregateGrading(entries, skippedAssertions, gradingIncomplete);
 };
 //# sourceMappingURL=grading.js.map

package/lib/orchestrator/grading.js.map

@@ -1 +1 @@
-{"version":3,"file":"grading.js","sourceRoot":"","sources":["../../src/orchestrator/grading.ts"],"names":[],"mappings":"AACA,OAAO,EAKL,qBAAqB,EAGrB,gBAAgB,EAIhB,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AAExE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;AAE9E,MAAM,OAAO,GAAG,CAAC,MAAoB,EAAkC,EAAE;IACvE,MAAM,KAAK,GAAI,MAA8B,CAAC,KAAK,CAAC;IACpD,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,IAAI,CAAC;AACxF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAY,EAAE,MAAoB,EAAW,EAAE;IACtF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,KAAK,CAAC,CAC9E,CAAC;AACJ,CAAC,CAAC;AAWF,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAC1C,UAAgC,EAChC,QAAkB,EACY,EAAE,CAChC,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;IAC9E,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO;QACL,IAAI;QACJ,aAAa,EACX,eAAe,IAAI,cAAc,IAAI,OAAO,cAAc,CAAC,aAAa,KAAK,QAAQ;YACnF,CAAC,CAAC,cAAc,CAAC,aAAa;YAC9B,CAAC,CAAC,IAAI;QACV,QAAQ,EAAE,cAAc,CAAC,QAAQ;KAClC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,yBAAyB,GAAG,CACvC,UAAgC,EAChC,QAAkB,EACS,EAAE,CAC7B,4BAA4B,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;IACnF,IAAI;IACJ,aAAa;CACd,CAAC,CAAC,CAAC;AAEN,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,KAOhC,EAAU,EAAE,CACX,SAAS,CAAC;IACR,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;IAC5C,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;IACtD,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;IAC1C,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,KAAK;CACpC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EACxC,UAAgC,EAChC,QAAkB,EACE,EAAE;IACtB,MAAM,SAAS,GAAc,EAAE,CAAC;IAChC,qEAAqE;IACrE,uEAAuE;IACvE,6DAA6D;IAC7D,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACb,wCAAwC,KAAK,uDAAuD,SAAS,CAAC,IAAI,GAAG,CACtH,CAAC;QACJ,CAAC;QACD,SAAS,CAAC,IAAI,CACZ,MAAM,yBAAyB,CAC7B,cAAc,CAAC,YAAY,EAC3B,cAAc,CAAC,YAAY;YACzB,CAAC,CAAC,wBAAwB,CAAC,SAAS,EAAE,cAAc,CAAC,YAAY,CAAC;YAClE,CAAC,CAAC,SAAS,EACb,cAAc,SAAS,CAAC,IAAI,mBAAmB,KAAK,GAAG,CACxD,CACF,CAAC;IACJ,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAY,EAAU,EAAE;IACjD,IAAI,CAAC,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;SAC7B,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,UAAU,CAAC;SACrE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,MAAuB,EAAE,SAAiB,EAAW,EAAE,CAC9E,MAAM,CAAC,YAAY,KAAK,SAAS;IACjC,MAAM,CAAC,UAAU,KAAK,SAAS;IAC/B,MAAM,CAAC,MAAM;IACb,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;AAE5B,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,GAAQ,EACR,QAAkB,EAClB,QAAsB,EACtB,WAAyB,EACzB,eAAuF,EACvF,aAGC,EACD,gBAAqC,EACrC,SAAS,GAAG,KAAK,EACO,EAAE;IAC1B,MAAM,eAAe,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;IAC7C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;IACjF,CAAC;IAED,MAAM,OAAO,GAA4B,EAAE,CAAC;IAC5C,MAAM,iBAAiB,GAKlB,EAAE,CAAC;IACR,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAE7B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrC,iBAAiB,CAAC,IAAI,CAAC;gBACrB,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE;gBACpC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,iBAAiB,CAAC,CAAC,CAAC;gBAC1B,MAAM,EAAE,gBAAgB;aACzB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,oEAAoE;QACpE,yEAAyE;QACzE,IAAI,WAAW,EAAE,OAAO,EAAE,CAAC;YACzB,iBAAiB,GAAG,IAAI,CAAC;YACzB,MAAM;QACR,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;QAC7B,qEAAqE;QACrE,yEAAyE;QACzE,mEAAmE;QACnE,mEAAmE;QACnE,+DAA+D;QAC/D,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAsB;YAClC,KAAK;YACL,KAAK,EAAE,UAAU,CAAC,MAAM;YACxB,EAAE;YACF,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI;SACL,CAAC;QACF,eAAe,EAAE,cAAc,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,KAA4B,CAAC;QACjC,MAAM,mBAAmB,GAAa,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,gBAAgB;gBAChC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC;gBACzB,CAAC,CAAC,MAAM,yBAAyB,CAC7B,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,EACtE,cAAc,CAAC,CAAC,IAAI,GAAG,CACxB,CAAC;YACN,IAAI,gBAAgB,IAAI,KAAK,IAAI,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBACzD,MAAM,IAAI,KAAK,CAAC,kDAAkD,KAAK,GAAG,CAAC,CAAC;YAC9E,CAAC;YACD,8DAA8D;YAC9D,+DAA+D;YAC/D,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CACxC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAqB,EAAE,GAAG,EAAE;gBAC9C,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvC,GAAG,CAAC,aAAa;oBACf,CAAC,CAAC;wBACE,uBAAuB,EAAE,KAAK,EAAE,KAAc,EAAE,EAAE;4BAChD,mBAAmB,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC/D,CAAC;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACR,CAAC,CACH,CAAC;YACF,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAClD,KAAK,GAAG;gBACN,EAAE;gBACF,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI;gBACJ,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM;gBACN,SAAS;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7E,GAAG,CAAC,MAAM,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACjD,CAAC;YACF,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;gBACtC,iBAAiB,GAAG,IAAI,CAAC;YAC3B,CAAC;iBAAM,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC3C,gBAAgB,GAAG,IAAI,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM;oBAAE,SAAS,GAAG,KAAK,CAAC;gBAC/B,gBAAgB,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC1C,SAAS,IAAI,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,KAAK,GAAG;gBACN,EAAE;gBACF,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI;gBACJ,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM;gBACN,SAAS;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,YAAY,EAAE,OAAO;aACtB,CAAC;YACF,iBAAiB,GAAG,IAAI,CAAC;QAC3B,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,eAAe,EAAE,YAAY,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACjD,MAAM,eAAe,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,aAAa,EAAE,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,iBAAiB,IAAI,CAAC,gBAAgB,IAAI,SAAS,GAAG,CAAC,CAAC;IAC5E,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACzC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI;QAC3D,UAAU,EAAE,OAAO;QACnB,GAAG,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,GAAG,CAAC,iBAAiB;YACnB,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,OAAgB,EAAE,KAAK,EAAE,SAAkB,EAAE,EAAE;YAC7E,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC,CAAC"}
+{"version":3,"file":"grading.js","sourceRoot":"","sources":["../../src/orchestrator/grading.ts"],"names":[],"mappings":"AACA,OAAO,EAKL,qBAAqB,EAGrB,gBAAgB,EAIhB,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AAExE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,CAAC,CAAC,CAAC;AAE/F,MAAM,OAAO,GAAG,CAAC,MAAoB,EAAkC,EAAE;IACvE,MAAM,KAAK,GAAI,MAA8B,CAAC,KAAK,CAAC;IACpD,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,IAAI,CAAC;AACxF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAY,EAAE,MAAoB,EAAW,EAAE;IACtF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,KAAK,CAAC,CAC9E,CAAC;AACJ,CAAC,CAAC;AAWF,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAC1C,UAAgC,EAChC,QAAkB,EACY,EAAE,CAChC,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;IAC9E,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO;QACL,IAAI;QACJ,aAAa,EACX,eAAe,IAAI,cAAc,IAAI,OAAO,cAAc,CAAC,aAAa,KAAK,QAAQ;YACnF,CAAC,CAAC,cAAc,CAAC,aAAa;YAC9B,CAAC,CAAC,IAAI;QACV,QAAQ,EAAE,cAAc,CAAC,QAAQ;KAClC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,yBAAyB,GAAG,CACvC,UAAgC,EAChC,QAAkB,EACS,EAAE,CAC7B,4BAA4B,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;IACnF,IAAI;IACJ,aAAa;CACd,CAAC,CAAC,CAAC;AAEN,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,KAOhC,EAAU,EAAE,CACX,SAAS,CAAC;IACR,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;IAC5C,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;IACtD,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;IAC1C,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,KAAK;CACpC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EACxC,UAAgC,EAChC,QAAkB,EACE,EAAE;IACtB,MAAM,SAAS,GAAc,EAAE,CAAC;IAChC,qEAAqE;IACrE,uEAAuE;IACvE,6DAA6D;IAC7D,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACb,wCAAwC,KAAK,uDAAuD,SAAS,CAAC,IAAI,GAAG,CACtH,CAAC;QACJ,CAAC;QACD,SAAS,CAAC,IAAI,CACZ,MAAM,yBAAyB,CAC7B,cAAc,CAAC,YAAY,EAC3B,cAAc,CAAC,YAAY;YACzB,CAAC,CAAC,wBAAwB,CAAC,SAAS,EAAE,cAAc,CAAC,YAAY,CAAC;YAClE,CAAC,CAAC,SAAS,EACb,cAAc,SAAS,CAAC,IAAI,mBAAmB,KAAK,GAAG,CACxD,CACF,CAAC;IACJ,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAY,EAAU,EAAE;IACjD,IAAI,CAAC,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;SAC7B,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,UAAU,CAAC;SACrE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,MAAuB,EAAE,SAAiB,EAAW,EAAE,CAC9E,MAAM,CAAC,YAAY,KAAK,SAAS;IACjC,MAAM,CAAC,UAAU,KAAK,SAAS;IAC/B,MAAM,CAAC,MAAM;IACb,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;AAE5B,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,MAA4B,EAAiB,EAAE,CAC/E,MAAM,EAAE,SAAS,KAAK,IAAI,IAAI,CAAC,MAAM;IACnC,CAAC,CAAC,IAAI;IACN,CAAC,CAAC,SAAS,CACP,MAAM,CAAC,UAAU;SACd,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;SAChF,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC;SACzF,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;SACtD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAC5C,CAAC;AAER,MAAM,gBAAgB,GAAG,CACvB,OAAgC,EAChC,iBAAkE,EAClE,iBAA0B,EACX,EAAE;IACjB,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;IACzE,MAAM,UAAU,GACd,iBAAiB,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC;IACzF,MAAM,gBAAgB,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC;IACzF,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAClC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,CAC9E,CAAC;IACF,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9E,MAAM,gBAAgB,GAAG,aAAa,CAAC,MAAM,CAC3C,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,EAChD,CAAC,CACF,CAAC;IACF,MAAM,UAAU,GAAG,CAAC,UAAU,IAAI,CAAC,gBAAgB,IAAI,SAAS,GAAG,CAAC,CAAC;IACrE,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,UAAU,EACR,UAAU,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC;YACtC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;YAChD,CAAC,CAAC,IAAI;QACV,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI;QAC3D,UAAU,EAAE,OAAO;QACnB,GAAG,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,GAAG,CAAC,UAAU;YACZ,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,OAAgB,EAAE,KAAK,EAAE,SAAkB,EAAE,EAAE;YAC7E,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,GAAQ,EACR,QAAkB,EAClB,QAAsB,EACtB,WAAyB,EACzB,eAAuF,EACvF,aAGC,EACD,gBAAqC,EACrC,SAAS,GAAG,KAAK,EACO,EAAE;IAC1B,MAAM,eAAe,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;IAC7C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;IACjF,CAAC;IAED,MAAM,OAAO,GAA4B,EAAE,CAAC;IAC5C,MAAM,iBAAiB,GAKlB,EAAE,CAAC;IACR,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAE9B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrC,iBAAiB,CAAC,IAAI,CAAC;gBACrB,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE;gBACpC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,iBAAiB,CAAC,CAAC,CAAC;gBAC1B,MAAM,EAAE,gBAAgB;aACzB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,oEAAoE;QACpE,yEAAyE;QACzE,IAAI,WAAW,EAAE,OAAO,EAAE,CAAC;YACzB,iBAAiB,GAAG,IAAI,CAAC;YACzB,MAAM;QACR,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;QAC7B,qEAAqE;QACrE,yEAAyE;QACzE,mEAAmE;QACnE,mEAAmE;QACnE,+DAA+D;QAC/D,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAsB;YAClC,KAAK;YACL,KAAK,EAAE,UAAU,CAAC,MAAM;YACxB,EAAE;YACF,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI;SACL,CAAC;QACF,eAAe,EAAE,cAAc,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,KAA4B,CAAC;QACjC,MAAM,mBAAmB,GAAa,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,gBAAgB;gBAChC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC;gBACzB,CAAC,CAAC,MAAM,yBAAyB,CAC7B,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,EACtE,cAAc,CAAC,CAAC,IAAI,GAAG,CACxB,CAAC;YACN,IAAI,gBAAgB,IAAI,KAAK,IAAI,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBACzD,MAAM,IAAI,KAAK,CAAC,kDAAkD,KAAK,GAAG,CAAC,CAAC;YAC9E,CAAC;YACD,8DAA8D;YAC9D,+DAA+D;YAC/D,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CACxC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAqB,EAAE,GAAG,EAAE;gBAC9C,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvC,GAAG,CAAC,aAAa;oBACf,CAAC,CAAC;wBACE,uBAAuB,EAAE,KAAK,EAAE,KAAc,EAAE,EAAE;4BAChD,mBAAmB,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC/D,CAAC;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACR,CAAC,CACH,CAAC;YACF,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAClD,KAAK,GAAG;gBACN,EAAE;gBACF,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI;gBACJ,MAAM,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;gBACvC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnD,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM;gBACN,SAAS;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7E,GAAG,CAAC,MAAM,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,KAAK,GAAG;gBACN,EAAE;gBACF,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI;gBACJ,MAAM,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;gBACtC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnD,KAAK,EAAE,CAAC;gBACR,MAAM;gBACN,SAAS;gBACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,YAAY,EAAE,OAAO;aACtB,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,eAAe,EAAE,YAAY,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACjD,MAAM,eAAe,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,aAAa,EAAE,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;AACzE,CAAC,CAAC"}

package/lib/orchestrator/list-runs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"list-runs.d.ts","sourceRoot":"","sources":["../../src/orchestrator/list-runs.ts"],"names":[],"mappings":"AAGA,OAAO,EAIL,KAAK,QAAQ,EAEb,KAAK,aAAa,EAGlB,KAAK,aAAa,EAElB,KAAK,QAAQ,EAEb,KAAK,WAAW,EAKjB,MAAM,kBAAkB,CAAC;AAQ1B,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,WAAW,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC;IAC7C,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,4BAA4B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,aAAa,EAAE,WAAW,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,CAAC;IACzD,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,OAAO,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB,CAAC;AA8FF,eAAO,MAAM,QAAQ,aAAoB,MAAM,KAAG,OAAO,CAAC,SAAS,EAAE,CASpE,CAAC;AAEF,eAAO,MAAM,eAAe,SAAU,SAAS,EAAE,KAAG,MAyBnD,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,aAAa,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,WAAW,GAAG,IAAI,CAAC;IAC7B,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB,CAAC;AAEF,eAAO,MAAM,gBAAgB,WAAkB,MAAM,KAAG,OAAO,CAAC,QAAQ,EAAE,CACU,CAAC;AAiDrF,eAAO,MAAM,aAAa,WAChB,MAAM,YACL;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAE,KAClF,OAAO,CAAC,SAAS,CAiBnB,CAAC;AA+IF,eAAO,MAAM,eAAe,WAAY,SAAS,KAAG,MAQnD,CAAC"}
+{"version":3,"file":"list-runs.d.ts","sourceRoot":"","sources":["../../src/orchestrator/list-runs.ts"],"names":[],"mappings":"AAGA,OAAO,EAIL,KAAK,QAAQ,EAEb,KAAK,aAAa,EAGlB,KAAK,aAAa,EAElB,KAAK,QAAQ,EAEb,KAAK,WAAW,EAKjB,MAAM,kBAAkB,CAAC;AAQ1B,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,WAAW,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC;IAC7C,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,4BAA4B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,aAAa,EAAE,WAAW,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,CAAC;IACzD,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,OAAO,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB,CAAC;AA8FF,eAAO,MAAM,QAAQ,aAAoB,MAAM,KAAG,OAAO,CAAC,SAAS,EAAE,CASpE,CAAC;AAEF,eAAO,MAAM,eAAe,SAAU,SAAS,EAAE,KAAG,MAuBnD,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,aAAa,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,WAAW,GAAG,IAAI,CAAC;IAC7B,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB,CAAC;AAEF,eAAO,MAAM,gBAAgB,WAAkB,MAAM,KAAG,OAAO,CAAC,QAAQ,EAAE,CACU,CAAC;AAiDrF,eAAO,MAAM,aAAa,WAChB,MAAM,YACL;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAE,KAClF,OAAO,CAAC,SAAS,CAiBnB,CAAC;AAiJF,eAAO,MAAM,eAAe,WAAY,SAAS,KAAG,MAQnD,CAAC"}

package/lib/orchestrator/list-runs.js

@@ -43,7 +43,7 @@ const summarizeRun = async (caseSlug, driverName, ts, runDir) => {
         casePassed: grading?.casePassed ?? null,
         caseScore: grading?.caseScore ?? null,
         assertionPassed: passing,
-        assertionTotal: grading?.assertions.length ?? 0,
+        assertionTotal: grading?.assertions.filter((a) => a.passed !== null).length ?? 0,
         ...(readErrors.length > 0 ? { readErrors } : {}),
     };
 };
@@ -108,9 +108,7 @@ export const formatRunsTable = (runs) => {
                 status = "PASS";
             else if (r.casePassed === false)
                 status = "FAIL";
-            const score = r.caseScore !== null
-                ? `${(r.caseScore * 100).toFixed(0)}% (${r.assertionPassed}/${r.assertionTotal})`
-                : "—";
+            const score = r.caseScore !== null ? `${r.caseScore} (${r.assertionPassed}/${r.assertionTotal})` : "—";
             const duration = r.durationMs !== null ? `${(r.durationMs / 1000).toFixed(1)}s` : "—";
             const agents = r.subagentCount > 0 ? `1+${r.subagentCount}` : "1";
             return [status, r.caseSlug, r.driverName, agents, r.ts, r.exitReason ?? "?", score, duration];
@@ -270,15 +268,17 @@ const appendGradingDetail = (lines, detail) => {
             status = "PASS";
         else if (detail.grading.casePassed === false)
             status = "FAIL";
-        const score = detail.grading.caseScore !== null ? `${(detail.grading.caseScore * 100).toFixed(0)}%` : "—";
-        lines.push(`grading: ${status} (${score}, ${detail.grading.assertions.filter((a) => a.passed).length}/${detail.grading.assertions.length})`);
+        const score = detail.grading.caseScore !== null ? String(detail.grading.caseScore) : "—";
+        lines.push(`grading: ${status} (${score}, ${detail.grading.assertions.filter((a) => a.passed).length}/${detail.grading.assertions.filter((a) => a.passed !== null).length})`);
         for (const a of detail.grading.assertions) {
             let mark = a.passed ? "✓" : "✗";
+            if (a.passed === null)
+                mark = "ℹ informational";
             if (a.abstention)
                 mark = "?";
             const took = ` (${(a.durationMs / 1000).toFixed(1)}s)`;
             const abstention = a.abstention ? " — insufficient evidence" : "";
-            lines.push(`  ${mark} ${a.name}${took}${abstention}${a.comment ? ` — ${a.comment}` : ""}`);
+            lines.push(`  ${mark} ${a.name} [score ${a.errorMessage !== undefined || a.abstention ? "unavailable" : a.score}]${took}${abstention}${a.comment ? ` — ${a.comment}` : ""}`);
         }
     }
 };