@bastani/atomic 0.9.22 → 0.9.23-alpha.1

package/dist/core/model-routing-candidates.js

@@ -1,4 +1,4 @@
-import { candidateEvidenceRows, modelEvidenceTokens } from "./model-routing-evals.js";
+import { candidateEvidenceRows, catalogModelIdentity, modelEvidenceTokens, } from "./model-routing-evals.js";
 import { taskDemand } from "./model-routing-needs.js";
 const percent = (key, label) => ({ key, label, unit: "%" });
 const points = (key, label) => ({ key, label, unit: "" });
@@ -55,7 +55,7 @@ const WORK_METRICS = {
 };
 /** When speed matters, a fast route edges out its standard route for the model's single slot. */
 const FAST_ROUTE_BONUS = 0.02;
-/** A standing is stated only when enough eligible models share the measurement. */
+/** A standing is stated only when enough benchmarked models share the measurement. */
 const MIN_RANKED_MODELS = 4;
 function sectionKind(intro) {
     const heading = intro.find((line) => line.startsWith("## ")) ?? "";
@@ -93,15 +93,14 @@ function nameCondition(name) {
     const setting = /\(([^)]+)\)\s*$/u.exec(name ?? "")?.[1];
     return setting && /^(?:minimal|low|medium|high|xhigh|max)$/u.test(setting) ? `${setting} effort` : setting;
 }
-/**
- * Best value per metric across every evals row describing the model, with the
- * conditions it was measured under, plus the model's newest release date.
- */
-function measure(catalog, model) {
+const readingsByCatalog = new WeakMap();
+/** Every measured value in the catalog, read once per parsed catalog. */
+function catalogReadings(catalog) {
+    const cached = readingsByCatalog.get(catalog);
+    if (cached)
+        return cached;
     const kinds = catalog.sections.map((section) => sectionKind(section.intro));
     const headers = catalog.sections.map((section) => cells(section.intro.find((line) => line.startsWith("| slug |")) ?? ""));
-    const values = new Map();
-    const conditions = new Map();
     const modelColumn = headers.map((header) => header.indexOf("Model"));
     const modelWords = catalog.sections.map((_, section) => {
         const words = new Set();
@@ -111,41 +110,72 @@ function measure(catalog, model) {
                     words.add(word);
         return words;
     });
-    let released;
-    const record = (key, raw, condition) => {
-        const value = Number.parseFloat(raw ?? "");
-        if (!Number.isFinite(value) || value <= (values.get(key) ?? Number.NEGATIVE_INFINITY))
-            return;
-        values.set(key, value);
-        if (condition)
-            conditions.set(key, condition);
-        else
-            conditions.delete(key);
-    };
-    for (const row of candidateEvidenceRows(catalog, model)) {
-        const kind = kinds[row.section];
+    const rows = catalog.rows.map((row) => {
         const header = headers[row.section];
         const rowCells = cells(row.line);
         const column = (name) => (header.includes(name) ? rowCells[header.indexOf(name)] : undefined);
-        if (kind === "pub") {
+        const readings = [];
+        const read = (key, raw, condition) => {
+            const value = Number.parseFloat(raw ?? "");
+            if (Number.isFinite(value))
+                readings.push({ key, value, ...(condition ? { condition } : {}) });
+        };
+        if (kinds[row.section] === "pub") {
             const reporter = column("Source");
             const setting = ownSetting(column("Setting") ?? "", column("Model") ?? "", modelWords[row.section]);
             const condition = [setting, reporter ? `reported by ${reporter}` : undefined].filter(Boolean).join(", ");
             // Each reporter's results form their own cohort: a vendor harness and an
             // official leaderboard are not ranked against each other.
-            record(`pub:${column("Benchmark")}@${reporter ?? ""}`, column("Score"), condition);
+            read(`pub:${column("Benchmark")}@${reporter ?? ""}`, column("Score"), condition);
         }
         else {
             const effort = column("Effort");
             const condition = effort ? `${effort} effort` : nameCondition(column("Model"));
             for (const [index, name] of header.entries())
                 if (index > 1 && name !== "Effort" && name !== "Release date")
-                    record(`${kind}:${name}`, rowCells[index], condition);
+                    read(`${kinds[row.section]}:${name}`, rowCells[index], condition);
+        }
+        return readings;
+    });
+    const cohorts = new Map();
+    for (const row of catalog.rows) {
+        const identity = catalogModelIdentity(row.tokens);
+        for (const { key, value } of rows[row.order]) {
+            const cohort = cohorts.get(key) ?? new Map();
+            cohort.set(identity, Math.max(cohort.get(identity) ?? value, value));
+            cohorts.set(key, cohort);
+        }
+    }
+    const readings = { rows, cohorts };
+    readingsByCatalog.set(catalog, readings);
+    return readings;
+}
+/**
+ * Best value per metric across every evals row describing the model, with the
+ * conditions it was measured under, the catalog models those rows belong to,
+ * and the model's newest release date.
+ */
+function measure(catalog, model) {
+    const readings = catalogReadings(catalog);
+    const values = new Map();
+    const conditions = new Map();
+    const identities = new Set();
+    let released;
+    for (const row of candidateEvidenceRows(catalog, model)) {
+        identities.add(catalogModelIdentity(row.tokens));
+        for (const { key, value, condition } of readings.rows[row.order]) {
+            if (value <= (values.get(key) ?? Number.NEGATIVE_INFINITY))
+                continue;
+            values.set(key, value);
+            if (condition)
+                conditions.set(key, condition);
+            else
+                conditions.delete(key);
         }
         if (row.releaseDate && (!released || row.releaseDate > released))
             released = row.releaseDate;
     }
-    return { values, conditions, ...(released ? { released } : {}) };
+    return { values, conditions, identities, ...(released ? { released } : {}) };
 }
 /** Value keys measuring `metric`: the column itself, or one per reporter for published results. */
 const cohortKeys = (values, metric) => [...values.keys()].filter((key) => key === metric.key || key.startsWith(`${metric.key}@`));
@@ -153,33 +183,29 @@ const baseKey = (candidate) => modelEvidenceTokens(candidate.fastRouteOf ?? cand
 const blended = (candidate) => (3 * candidate.cost.input + candidate.cost.output) / 4;
 /**
  * Rank every eligible model for the task in code. Quality is the model's
- * standing on the benchmarks for this kind of work (overall intelligence when it
- * has none), weighed against price by how demanding the task is, with a small
- * preference for newer releases.
+ * standing among every benchmarked model in the catalog on the benchmarks for
+ * this kind of work (overall intelligence when it has none), so it does not
+ * depend on which other models are eligible. It is weighed against price by how
+ * demanding the task is, with a small preference for newer releases.
  */
 export function rankCandidates(catalog, candidates, needs) {
+    const { cohorts } = catalogReadings(catalog);
     const measured = candidates.map((candidate) => ({
         candidate,
         base: baseKey(candidate),
         ...measure(catalog, candidate.model),
     }));
-    // One observation per base model: provider copies and fast routes of one model
-    // neither move other models' standings nor count toward the minimum.
-    const rankOf = (key, value) => {
-        const byBase = new Map();
-        for (const entry of measured) {
-            const observed = entry.values.get(key);
-            if (observed !== undefined)
-                byBase.set(entry.base, Math.max(byBase.get(entry.base) ?? observed, observed));
-        }
-        if (byBase.size < MIN_RANKED_MODELS)
+    const rankOf = (key, value, own) => {
+        const cohort = cohorts.get(key);
+        if (cohort === undefined || cohort.size < MIN_RANKED_MODELS)
             return undefined;
-        return [...byBase.values()].filter((other) => other < value).length / (byBase.size - 1);
+        const others = [...cohort].filter(([identity]) => !own.has(identity)).map(([, other]) => other);
+        return others.length ? others.filter((other) => other < value).length / others.length : undefined;
     };
-    const standingOf = (values, metrics) => {
+    const standingOf = (values, own, metrics) => {
         const ranks = metrics
             .flatMap((metric) => cohortKeys(values, metric))
-            .map((key) => rankOf(key, values.get(key)))
+            .map((key) => rankOf(key, values.get(key), own))
             .filter((rank) => rank !== undefined);
         return ranks.length ? ranks.reduce((a, b) => a + b, 0) / ranks.length : undefined;
     };
@@ -196,9 +222,9 @@ export function rankCandidates(catalog, candidates, needs) {
     const qualityWeight = 0.25 + 0.65 * demand;
     const priceWeight = 0.65 - 0.65 * demand;
     return measured
-        .map(({ candidate, base, values, conditions, released }) => {
-        const workStanding = standingOf(values, WORK_METRICS[needs.work]);
-        const overallStanding = standingOf(values, [OVERALL]);
+        .map(({ candidate, base, values, conditions, identities, released }) => {
+        const workStanding = standingOf(values, identities, WORK_METRICS[needs.work]);
+        const overallStanding = standingOf(values, identities, [OVERALL]);
         const quality = workStanding ?? (overallStanding !== undefined ? 0.8 * overallStanding : 0.2);
         const price = blended(candidate);
         const cheapness = price > 0 && high > low ? 1 - (Math.log(price) - low) / (high - low) : 1;
@@ -251,7 +277,7 @@ const money = (value) => `$${Number(value.toPrecision(3))}`;
 /**
  * One shortlisted option described with its own evidence, so the router compares
  * options directly: release date, price tier, image input, and results for this
- * kind of work and overall, with standings relative to every eligible model.
+ * kind of work and overall, with standings among catalog models measured on the same benchmark by the same source.
  */
 export function describeOption(option, needs, eligible) {
     const prices = eligible.map(blended).sort((a, b) => a - b);
@@ -278,7 +304,7 @@ export function describeOption(option, needs, eligible) {
             return `${metric.label} ${option.values.get(key)}${metric.unit}${condition ? ` measured with ${condition}` : ""}`;
         })
             .join("; ");
-        return `${standing === undefined ? "measured" : standingLabel(standing)} (${results})`;
+        return `${standing === undefined ? "measured" : `${standingLabel(standing)} of models with the same benchmark and source`} (${results})`;
     };
     let released = "unknown";
     if (option.released) {

package/dist/core/model-routing-candidates.js.map

@@ -1 +1 @@
-{"version":3,"file":"model-routing-candidates.js","sourceRoot":"","sources":["../../src/core/model-routing-candidates.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAqB,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACzG,OAAO,EAA0B,UAAU,EAAiB,MAAM,0BAA0B,CAAC;AAmB7F,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,KAAa,EAAU,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;AACpF,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,KAAa,EAAU,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;AAElF,MAAM,OAAO,GAAW,MAAM,CAAC,QAAQ,EAAE,uBAAuB,CAAC,CAAC;AAElE,wFAAwF;AACxF,MAAM,YAAY,GAAwC;IACzD,YAAY,EAAE;QACb,OAAO,CAAC,UAAU,EAAE,aAAa,CAAC;QAClC,OAAO,CAAC,SAAS,EAAE,gBAAgB,CAAC;QACpC,OAAO,CAAC,SAAS,EAAE,mBAAmB,CAAC;QACvC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC;KAC9B;IACD,MAAM,EAAE;QACP,OAAO,CAAC,QAAQ,EAAE,oBAAoB,CAAC;QACvC,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC;QACjC,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC;QAClC,OAAO,CAAC,SAAS,EAAE,oBAAoB,CAAC;KACxC;IACD,WAAW,EAAE;QACZ,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC;QAClC,OAAO,CAAC,QAAQ,EAAE,oBAAoB,CAAC;QACvC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC;QACjC,OAAO,CAAC,YAAY,EAAE,eAAe,CAAC;KACtC;IACD,eAAe,EAAE;QAChB,OAAO,CAAC,QAAQ,EAAE,oBAAoB,CAAC;QACvC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC;QAC9B,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC;KAC3B;IACD,QAAQ,EAAE;QACT,OAAO,CAAC,QAAQ,EAAE,sBAAsB,CAAC;QACzC,OAAO,CAAC,WAAW,EAAE,YAAY,CAAC;QAClC,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC;QAClC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC;KAC3B;IACD,iBAAiB,EAAE;QAClB,OAAO,CAAC,SAAS,EAAE,oBAAoB,CAAC;QACxC,OAAO,CAAC,YAAY,EAAE,iBAAiB,CAAC;QACxC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC;QAC9B,OAAO,CAAC,QAAQ,EAAE,mBAAmB,CAAC;KACtC;IACD,YAAY,EAAE;QACb,OAAO,CAAC,UAAU,EAAE,qBAAqB,CAAC;QAC1C,OAAO,CAAC,WAAW,EAAE,wBAAwB,CAAC;QAC9C,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC;QAC5B,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC;KAC5B;IACD,OAAO,EAAE;QACR,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC;QAC3B,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC;QACzB,MAAM,CAAC,UAAU,EAAE,cAAc,CAAC;QAClC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC;KAC3B;CACD,CAAC;AAEF,iGAAiG;AACjG,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,mFAAmF;AACnF,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAc5B,SAAS,WAAW,CAAC,KAAwB;IAC5C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACnE,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IAC5C,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/C,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7C,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAC9B,IAAI;KACF,KAAK,CAAC,GAAG,CAAC;KACV,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;KACZ,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AAE9B,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,CAAC;AAErF;;;;GAIG;AACH,SAAS,UAAU,CAAC,OAAe,EAAE,QAAgB,EAAE,UAA+B;IACrF,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAChC,OAAO,OAAO;SACZ,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;SAC9B,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE;QAClB,MAAM,KAAK,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5E,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAClE,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,CAAC;AAED,wGAAwG;AACxG,SAAS,aAAa,CAAC,IAAwB;IAC9C,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzD,OAAO,OAAO,IAAI,0CAA0C,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;AAC5G,CAAC;AAED;;;GAGG;AACH,SAAS,OAAO,CACf,OAAqB,EACrB,KAAa;IAEb,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAChD,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,CACtE,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QACtD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI;YAC7B,IAAI,GAAG,CAAC,OAAO,KAAK,OAAO;gBAC1B,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,OAAO,CAAE,CAAC,IAAI,EAAE,CAAC;oBAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9F,OAAO,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IACH,IAAI,QAA4B,CAAC;IACjC,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,GAAuB,EAAE,SAA6B,EAAE,EAAE;QACtF,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC;YAAE,OAAO;QAC9F,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACvB,IAAI,SAAS;YAAE,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;;YACzC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,qBAAqB,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACtG,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;YAClC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,UAAU,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,CAAC;YACrG,MAAM,SAAS,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzG,yEAAyE;YACzE,0DAA0D;YAC1D,MAAM,CAAC,OAAO,MAAM,CAAC,WAAW,CAAC,IAAI,QAAQ,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC;QACpF,CAAC;aAAM,CAAC;YACP,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;YAChC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/E,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE;gBAC3C,IAAI,KAAK,GAAG,CAAC,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,cAAc;oBAC5D,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,QAAQ,IAAI,GAAG,CAAC,WAAW,GAAG,QAAQ,CAAC;YAAE,QAAQ,GAAG,GAAG,CAAC,WAAW,CAAC;IAC9F,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AAClE,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,GAAG,CAAC,MAAmC,EAAE,MAAc,EAAE,EAAE,CAC1E,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;AAE5F,MAAM,OAAO,GAAG,CAAC,SAAyB,EAAE,EAAE,CAAC,mBAAmB,CAAC,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEvH,MAAM,OAAO,GAAG,CAAC,SAAyB,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAEtG;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAC7B,OAAqB,EACrB,UAAqC,EACrC,KAAwB;IAExB,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC/C,SAAS;QACT,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC;QACxB,GAAG,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC;KACpC,CAAC,CAAC,CAAC;IACJ,+EAA+E;IAC/E,qEAAqE;IACrE,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,KAAa,EAAsB,EAAE;QACjE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,QAAQ,KAAK,SAAS;gBAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC5G,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,GAAG,iBAAiB;YAAE,OAAO,SAAS,CAAC;QACtD,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACzF,CAAC,CAAC;IACF,MAAM,UAAU,GAAG,CAAC,MAAmC,EAAE,OAA0B,EAAE,EAAE;QACtF,MAAM,KAAK,GAAG,OAAO;aACnB,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;aAC/C,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,CAAC;aAC3C,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;QACvD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACnF,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,QAAQ;SACxB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;SACxC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC;SAC5B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,QAAQ;SACrB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;SACpC,IAAI,EAAE;SACN,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACT,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,aAAa,GAAG,IAAI,GAAG,IAAI,GAAG,MAAM,CAAC;IAC3C,MAAM,WAAW,GAAG,IAAI,GAAG,IAAI,GAAG,MAAM,CAAC;IAEzC,OAAO,QAAQ;SACb,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE;QAC1D,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,MAAM,eAAe,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,YAAY,IAAI,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC9F,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QACjC,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,IAAI,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3F,MAAM,OAAO,GAAG,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC;QACpG,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,GAAG,CAAC,CAAC;QAC/C,OAAO;YACN,GAAG,SAAS;YACZ,OAAO,EAAE,IAAI;YACb,KAAK,EACJ,aAAa,GAAG,OAAO;gBACvB,WAAW,GAAG,SAAS;gBACvB,GAAG,GAAG,OAAO;gBACb,CAAC,KAAK,CAAC,gBAAgB,IAAI,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,MAAM;YACN,UAAU;YACV,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,GAAG,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7D,CAAC;IACH,CAAC,CAAC;SACD,IAAI,CACJ,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACR,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;QACjB,MAAM,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC;QACzE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAC/B,CAAC;AACJ,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,WAAW,CAAC,MAAkC,EAAE,IAAY;IAC3E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAChC,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC;YAAE,SAAS;QAC1C,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC5B,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpB,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI;YAAE,MAAM;IAChC,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC;AAED,SAAS,aAAa,CAAC,QAAgB;IACtC,IAAI,QAAQ,IAAI,GAAG;QAAE,OAAO,SAAS,CAAC;IACtC,IAAI,QAAQ,IAAI,IAAI;QAAE,OAAO,aAAa,CAAC;IAC3C,IAAI,QAAQ,IAAI,GAAG;QAAE,OAAO,cAAc,CAAC;IAC3C,IAAI,QAAQ,IAAI,IAAI;QAAE,OAAO,cAAc,CAAC;IAC5C,OAAO,gBAAgB,CAAC;AACzB,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAEpE;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAC7B,MAAuB,EACvB,KAAwB,EACxB,QAAoC;IAEpC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3G,MAAM,IAAI,GAAG,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1E,MAAM,MAAM,GAAG,QAAQ;SACrB,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,IAAI,EAAE,CAAC;SAC5C,IAAI,EAAE;SACN,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACT,MAAM,KAAK,GAAG,CAAC,OAA0B,EAAE,QAA4B,EAAE,EAAE;QAC1E,MAAM,OAAO,GAAG,OAAO;aACrB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACf,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YACnF,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;QAC/D,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;aACtC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACd,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,sBAAsB,CAAC;QACxD,MAAM,OAAO,GAAG,OAAO;aACrB,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE;YACxB,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7C,OAAO,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,kBAAkB,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACnH,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,OAAO,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,OAAO,GAAG,CAAC;IACxF,CAAC,CAAC;IACF,IAAI,QAAQ,GAAG,SAAS,CAAC;IACzB,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClH,QAAQ;YACP,MAAM,IAAI,CAAC;gBACV,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,oBAAoB;gBACxC,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,KAAK,MAAM,+BAA+B,CAAC;IAClE,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW;QAClC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,KAAK,MAAM,CAAC,WAAW,CAAC;QACtE,CAAC,CAAC,SAAS,CAAC;IACb,OAAO,IAAI,CAAC,SAAS,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC,IAAI;QAClB,EAAE,EAAE,MAAM,CAAC,KAAK;QAChB,QAAQ;QACR,KAAK,EAAE,GAAG,IAAI,KAAK,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,qBAAqB;QAC/F,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC5C,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC;QAClE,OAAO,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,eAAe,CAAC;QACjD,GAAG,CAAC,MAAM,CAAC,WAAW;YACrB,CAAC,CAAC;gBACA,KAAK,EAAE,mBAAmB,QAAQ,EAAE,IAAI,IAAI,MAAM,CAAC,WAAW,wDAAwD;aACtH;YACF,CAAC,CAAC,EAAE,CAAC;KACN,CAAC,CAAC;AACJ,CAAC","sourcesContent":["import { candidateEvidenceRows, type EvalsCatalog, modelEvidenceTokens } from \"./model-routing-evals.js\";\nimport { type ResolvedTaskNeeds, taskDemand, type WorkKind } from \"./model-routing-needs.js\";\n\n/** What ranking needs to know about one eligible model. */\nexport interface CandidateModel {\n\t/** `provider/id`, as used in router selections. */\n\treadonly model: string;\n\treadonly name: string;\n\treadonly cost: { readonly input: number; readonly output: number };\n\treadonly input: readonly string[];\n\t/** `provider/id` of the base model when this entry is its derived fast route (`fastRoute` metadata). */\n\treadonly fastRouteOf?: string;\n}\n\ninterface Metric {\n\treadonly key: string;\n\treadonly label: string;\n\treadonly unit: \"%\" | \"\";\n}\n\nconst percent = (key: string, label: string): Metric => ({ key, label, unit: \"%\" });\nconst points = (key: string, label: string): Metric => ({ key, label, unit: \"\" });\n\nconst OVERALL: Metric = points(\"aa:idx\", \"AA Intelligence Index\");\n\n/** evals.md columns that measure each kind of work, in the order results are quoted. */\nconst WORK_METRICS: Record<WorkKind, readonly Metric[]> = {\n\tcomputer_use: [\n\t\tpercent(\"pub:OSW2\", \"OSWorld 2.0\"),\n\t\tpercent(\"pub:SSP\", \"ScreenSpot-Pro\"),\n\t\tpercent(\"pub:ALE\", \"Agents' Last Exam\"),\n\t\tpercent(\"aa:MMMU\", \"MMMU-Pro\"),\n\t],\n\tcoding: [\n\t\tpercent(\"aa:TB4\", \"Terminal-Bench 4.0\"),\n\t\tpercent(\"dswe:Pass@1\", \"DeepSWE\"),\n\t\tpercent(\"fc:Main\", \"FrontierCode\"),\n\t\tpercent(\"aa:TB21\", \"Terminal-Bench 2.1\"),\n\t],\n\tcode_review: [\n\t\tpercent(\"fc:Main\", \"FrontierCode\"),\n\t\tpercent(\"aa:TB4\", \"Terminal-Bench 4.0\"),\n\t\tpercent(\"pub:XGym\", \"ExploitGym\"),\n\t\tpercent(\"pub:SECPro\", \"SEC-Bench Pro\"),\n\t],\n\tcodebase_lookup: [\n\t\tpercent(\"aa:TB4\", \"Terminal-Bench 4.0\"),\n\t\tpercent(\"aa:tau2\", \"τ²-Bench\"),\n\t\tpercent(\"aa:IF\", \"IFBench\"),\n\t],\n\tresearch: [\n\t\tpercent(\"aa:HLE\", \"Humanity's Last Exam\"),\n\t\tpercent(\"pub:BComp\", \"BrowseComp\"),\n\t\tpercent(\"aa:GPQA\", \"GPQA Diamond\"),\n\t\tpercent(\"aa:LCR\", \"AA-LCR\"),\n\t],\n\tbusiness_workflow: [\n\t\tpercent(\"aa:Auto\", \"AutomationBench-AA\"),\n\t\tpercent(\"pub:ABench\", \"AutomationBench\"),\n\t\tpercent(\"aa:tau2\", \"τ²-Bench\"),\n\t\tpercent(\"aa:Ent\", \"EnterpriseOps-Gym\"),\n\t],\n\tmath_science: [\n\t\tpercent(\"pub:FMT4\", \"FrontierMath Tier 4\"),\n\t\tpercent(\"pub:TBSci\", \"Terminal-Bench-Science\"),\n\t\tpercent(\"aa:Crit\", \"CritPt\"),\n\t\tpercent(\"aa:Sci\", \"SciCode\"),\n\t],\n\twriting: [\n\t\tpercent(\"aa:IF\", \"IFBench\"),\n\t\tpoints(\"aa:Gn\", \"GDPval\"),\n\t\tpoints(\"aa:Brief\", \"AA-Briefcase\"),\n\t\tpercent(\"aa:LCR\", \"AA-LCR\"),\n\t],\n};\n\n/** When speed matters, a fast route edges out its standard route for the model's single slot. */\nconst FAST_ROUTE_BONUS = 0.02;\n\n/** A standing is stated only when enough eligible models share the measurement. */\nconst MIN_RANKED_MODELS = 4;\n\nexport interface RankedCandidate extends CandidateModel {\n\t/** Identity shared by one model's providers and its derived fast route, which collapse onto it. */\n\treadonly baseKey: string;\n\treadonly score: number;\n\treadonly values: ReadonlyMap<string, number>;\n\t/** How each quoted value was measured, for example its effort, harness or reporter. */\n\treadonly conditions: ReadonlyMap<string, string>;\n\treadonly released?: string;\n\treadonly workStanding?: number;\n\treadonly overallStanding?: number;\n}\n\nfunction sectionKind(intro: readonly string[]): \"aa\" | \"dswe\" | \"fc\" | \"pub\" {\n\tconst heading = intro.find((line) => line.startsWith(\"## \")) ?? \"\";\n\tif (/DeepSWE/u.test(heading)) return \"dswe\";\n\tif (/FrontierCode/u.test(heading)) return \"fc\";\n\tif (/Published/u.test(heading)) return \"pub\";\n\treturn \"aa\";\n}\n\nconst cells = (line: string) =>\n\tline\n\t\t.split(\"|\")\n\t\t.slice(1, -1)\n\t\t.map((cell) => cell.trim());\n\nconst nameWords = (text: string) => new Set(text.match(/\\p{Lu}[\\p{L}]{2,}/gu) ?? []);\n\n/**\n * The parts of a published result's setting that apply to this row: a clause\n * naming other models (for example \"Fable values are Mythos\" on a GPT row) is a\n * footnote for them and is dropped.\n */\nfunction ownSetting(setting: string, rowModel: string, modelWords: ReadonlySet<string>): string {\n\tconst own = nameWords(rowModel);\n\treturn setting\n\t\t.split(\";\")\n\t\t.map((clause) => clause.trim())\n\t\t.filter((clause) => {\n\t\t\tconst named = [...nameWords(clause)].filter((word) => modelWords.has(word));\n\t\t\treturn named.length === 0 || named.some((word) => own.has(word));\n\t\t})\n\t\t.join(\", \");\n}\n\n/** Parenthetical measurement setting in an Artificial Analysis model name, for example \"Max Effort\". */\nfunction nameCondition(name: string | undefined): string | undefined {\n\tconst setting = /\\(([^)]+)\\)\\s*$/u.exec(name ?? \"\")?.[1];\n\treturn setting && /^(?:minimal|low|medium|high|xhigh|max)$/u.test(setting) ? `${setting} effort` : setting;\n}\n\n/**\n * Best value per metric across every evals row describing the model, with the\n * conditions it was measured under, plus the model's newest release date.\n */\nfunction measure(\n\tcatalog: EvalsCatalog,\n\tmodel: string,\n): { values: Map<string, number>; conditions: Map<string, string>; released?: string } {\n\tconst kinds = catalog.sections.map((section) => sectionKind(section.intro));\n\tconst headers = catalog.sections.map((section) =>\n\t\tcells(section.intro.find((line) => line.startsWith(\"| slug |\")) ?? \"\"),\n\t);\n\tconst values = new Map<string, number>();\n\tconst conditions = new Map<string, string>();\n\tconst modelColumn = headers.map((header) => header.indexOf(\"Model\"));\n\tconst modelWords = catalog.sections.map((_, section) => {\n\t\tconst words = new Set<string>();\n\t\tfor (const row of catalog.rows)\n\t\t\tif (row.section === section)\n\t\t\t\tfor (const word of nameWords(cells(row.line)[modelColumn[section]!] ?? \"\")) words.add(word);\n\t\treturn words;\n\t});\n\tlet released: string | undefined;\n\tconst record = (key: string, raw: string | undefined, condition: string | undefined) => {\n\t\tconst value = Number.parseFloat(raw ?? \"\");\n\t\tif (!Number.isFinite(value) || value <= (values.get(key) ?? Number.NEGATIVE_INFINITY)) return;\n\t\tvalues.set(key, value);\n\t\tif (condition) conditions.set(key, condition);\n\t\telse conditions.delete(key);\n\t};\n\tfor (const row of candidateEvidenceRows(catalog, model)) {\n\t\tconst kind = kinds[row.section]!;\n\t\tconst header = headers[row.section]!;\n\t\tconst rowCells = cells(row.line);\n\t\tconst column = (name: string) => (header.includes(name) ? rowCells[header.indexOf(name)] : undefined);\n\t\tif (kind === \"pub\") {\n\t\t\tconst reporter = column(\"Source\");\n\t\t\tconst setting = ownSetting(column(\"Setting\") ?? \"\", column(\"Model\") ?? \"\", modelWords[row.section]!);\n\t\t\tconst condition = [setting, reporter ? `reported by ${reporter}` : undefined].filter(Boolean).join(\", \");\n\t\t\t// Each reporter's results form their own cohort: a vendor harness and an\n\t\t\t// official leaderboard are not ranked against each other.\n\t\t\trecord(`pub:${column(\"Benchmark\")}@${reporter ?? \"\"}`, column(\"Score\"), condition);\n\t\t} else {\n\t\t\tconst effort = column(\"Effort\");\n\t\t\tconst condition = effort ? `${effort} effort` : nameCondition(column(\"Model\"));\n\t\t\tfor (const [index, name] of header.entries())\n\t\t\t\tif (index > 1 && name !== \"Effort\" && name !== \"Release date\")\n\t\t\t\t\trecord(`${kind}:${name}`, rowCells[index], condition);\n\t\t}\n\t\tif (row.releaseDate && (!released || row.releaseDate > released)) released = row.releaseDate;\n\t}\n\treturn { values, conditions, ...(released ? { released } : {}) };\n}\n\n/** Value keys measuring `metric`: the column itself, or one per reporter for published results. */\nconst cohortKeys = (values: ReadonlyMap<string, number>, metric: Metric) =>\n\t[...values.keys()].filter((key) => key === metric.key || key.startsWith(`${metric.key}@`));\n\nconst baseKey = (candidate: CandidateModel) => modelEvidenceTokens(candidate.fastRouteOf ?? candidate.model).join(\"-\");\n\nconst blended = (candidate: CandidateModel) => (3 * candidate.cost.input + candidate.cost.output) / 4;\n\n/**\n * Rank every eligible model for the task in code. Quality is the model's\n * standing on the benchmarks for this kind of work (overall intelligence when it\n * has none), weighed against price by how demanding the task is, with a small\n * preference for newer releases.\n */\nexport function rankCandidates(\n\tcatalog: EvalsCatalog,\n\tcandidates: readonly CandidateModel[],\n\tneeds: ResolvedTaskNeeds,\n): RankedCandidate[] {\n\tconst measured = candidates.map((candidate) => ({\n\t\tcandidate,\n\t\tbase: baseKey(candidate),\n\t\t...measure(catalog, candidate.model),\n\t}));\n\t// One observation per base model: provider copies and fast routes of one model\n\t// neither move other models' standings nor count toward the minimum.\n\tconst rankOf = (key: string, value: number): number | undefined => {\n\t\tconst byBase = new Map<string, number>();\n\t\tfor (const entry of measured) {\n\t\t\tconst observed = entry.values.get(key);\n\t\t\tif (observed !== undefined) byBase.set(entry.base, Math.max(byBase.get(entry.base) ?? observed, observed));\n\t\t}\n\t\tif (byBase.size < MIN_RANKED_MODELS) return undefined;\n\t\treturn [...byBase.values()].filter((other) => other < value).length / (byBase.size - 1);\n\t};\n\tconst standingOf = (values: ReadonlyMap<string, number>, metrics: readonly Metric[]) => {\n\t\tconst ranks = metrics\n\t\t\t.flatMap((metric) => cohortKeys(values, metric))\n\t\t\t.map((key) => rankOf(key, values.get(key)!))\n\t\t\t.filter((rank): rank is number => rank !== undefined);\n\t\treturn ranks.length ? ranks.reduce((a, b) => a + b, 0) / ranks.length : undefined;\n\t};\n\tconst logPrices = measured\n\t\t.map((entry) => blended(entry.candidate))\n\t\t.filter((price) => price > 0)\n\t\t.map(Math.log);\n\tconst [low, high] = [Math.min(...logPrices), Math.max(...logPrices)];\n\tconst newest = measured\n\t\t.map((entry) => entry.released ?? \"\")\n\t\t.sort()\n\t\t.at(-1);\n\tconst demand = taskDemand(needs);\n\tconst qualityWeight = 0.25 + 0.65 * demand;\n\tconst priceWeight = 0.65 - 0.65 * demand;\n\n\treturn measured\n\t\t.map(({ candidate, base, values, conditions, released }) => {\n\t\t\tconst workStanding = standingOf(values, WORK_METRICS[needs.work]);\n\t\t\tconst overallStanding = standingOf(values, [OVERALL]);\n\t\t\tconst quality = workStanding ?? (overallStanding !== undefined ? 0.8 * overallStanding : 0.2);\n\t\t\tconst price = blended(candidate);\n\t\t\tconst cheapness = price > 0 && high > low ? 1 - (Math.log(price) - low) / (high - low) : 1;\n\t\t\tconst ageDays = released && newest ? (Date.parse(newest) - Date.parse(released)) / 86_400_000 : 365;\n\t\t\tconst recency = 1 - Math.min(1, ageDays / 365);\n\t\t\treturn {\n\t\t\t\t...candidate,\n\t\t\t\tbaseKey: base,\n\t\t\t\tscore:\n\t\t\t\t\tqualityWeight * quality +\n\t\t\t\t\tpriceWeight * cheapness +\n\t\t\t\t\t0.1 * recency +\n\t\t\t\t\t(needs.latencySensitive && candidate.fastRouteOf ? FAST_ROUTE_BONUS : 0),\n\t\t\t\tvalues,\n\t\t\t\tconditions,\n\t\t\t\t...(released ? { released } : {}),\n\t\t\t\t...(workStanding !== undefined ? { workStanding } : {}),\n\t\t\t\t...(overallStanding !== undefined ? { overallStanding } : {}),\n\t\t\t};\n\t\t})\n\t\t.sort(\n\t\t\t(a, b) =>\n\t\t\t\tb.score - a.score ||\n\t\t\t\tNumber(a.fastRouteOf !== undefined) - Number(b.fastRouteOf !== undefined) ||\n\t\t\t\ta.model.localeCompare(b.model),\n\t\t);\n}\n\n/** The best `size` candidates with at most one per base model. */\nexport function distinctTop(ranked: readonly RankedCandidate[], size: number): RankedCandidate[] {\n\tconst seen = new Set<string>();\n\tconst top: RankedCandidate[] = [];\n\tfor (const candidate of ranked) {\n\t\tif (seen.has(candidate.baseKey)) continue;\n\t\tseen.add(candidate.baseKey);\n\t\ttop.push(candidate);\n\t\tif (top.length === size) break;\n\t}\n\treturn top;\n}\n\nfunction standingLabel(fraction: number): string {\n\tif (fraction >= 0.9) return \"top 10%\";\n\tif (fraction >= 0.75) return \"top quarter\";\n\tif (fraction >= 0.5) return \"above median\";\n\tif (fraction >= 0.25) return \"below median\";\n\treturn \"bottom quarter\";\n}\n\nconst money = (value: number) => `$${Number(value.toPrecision(3))}`;\n\n/**\n * One shortlisted option described with its own evidence, so the router compares\n * options directly: release date, price tier, image input, and results for this\n * kind of work and overall, with standings relative to every eligible model.\n */\nexport function describeOption(\n\toption: RankedCandidate,\n\tneeds: ResolvedTaskNeeds,\n\teligible: readonly RankedCandidate[],\n): string {\n\tconst prices = eligible.map(blended).sort((a, b) => a - b);\n\tconst position = prices.filter((other) => other < blended(option)).length / Math.max(1, prices.length - 1);\n\tconst tier = position < 1 / 3 ? \"low\" : position < 2 / 3 ? \"mid\" : \"high\";\n\tconst newest = eligible\n\t\t.map((candidate) => candidate.released ?? \"\")\n\t\t.sort()\n\t\t.at(-1);\n\tconst quote = (metrics: readonly Metric[], standing: number | undefined) => {\n\t\tconst present = metrics\n\t\t\t.map((metric) => {\n\t\t\t\tconst keys = cohortKeys(option.values, metric);\n\t\t\t\tconst best = keys.sort((a, b) => option.values.get(b)! - option.values.get(a)!)[0];\n\t\t\t\treturn best === undefined ? undefined : { metric, key: best };\n\t\t\t})\n\t\t\t.filter((entry) => entry !== undefined)\n\t\t\t.slice(0, 2);\n\t\tif (present.length === 0) return \"no published results\";\n\t\tconst results = present\n\t\t\t.map(({ metric, key }) => {\n\t\t\t\tconst condition = option.conditions.get(key);\n\t\t\t\treturn `${metric.label} ${option.values.get(key)}${metric.unit}${condition ? ` measured with ${condition}` : \"\"}`;\n\t\t\t})\n\t\t\t.join(\"; \");\n\t\treturn `${standing === undefined ? \"measured\" : standingLabel(standing)} (${results})`;\n\t};\n\tlet released = \"unknown\";\n\tif (option.released) {\n\t\tconst months = newest ? Math.round((Date.parse(newest) - Date.parse(option.released)) / (30.44 * 86_400_000)) : 0;\n\t\treleased =\n\t\t\tmonths <= 1\n\t\t\t\t? `${option.released}, among the newest`\n\t\t\t\t: `${option.released}, ${months} months older than the newest`;\n\t}\n\tconst fastBase = option.fastRouteOf\n\t\t? eligible.find((candidate) => candidate.model === option.fastRouteOf)\n\t\t: undefined;\n\treturn JSON.stringify({\n\t\tmodel: option.name,\n\t\tid: option.model,\n\t\treleased,\n\t\tprice: `${tier}: ${money(option.cost.input)} / ${money(option.cost.output)} per million tokens`,\n\t\treads_images: option.input.includes(\"image\"),\n\t\t[needs.work]: quote(WORK_METRICS[needs.work], option.workStanding),\n\t\toverall: quote([OVERALL], option.overallStanding),\n\t\t...(option.fastRouteOf\n\t\t\t? {\n\t\t\t\t\troute: `faster route of ${fastBase?.name ?? option.fastRouteOf} with the same results; billed above the listed prices`,\n\t\t\t\t}\n\t\t\t: {}),\n\t});\n}\n"]}
+{"version":3,"file":"model-routing-candidates.js","sourceRoot":"","sources":["../../src/core/model-routing-candidates.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,qBAAqB,EACrB,oBAAoB,EAEpB,mBAAmB,GACnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAA0B,UAAU,EAAiB,MAAM,0BAA0B,CAAC;AAmB7F,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,KAAa,EAAU,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;AACpF,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,KAAa,EAAU,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;AAElF,MAAM,OAAO,GAAW,MAAM,CAAC,QAAQ,EAAE,uBAAuB,CAAC,CAAC;AAElE,wFAAwF;AACxF,MAAM,YAAY,GAAwC;IACzD,YAAY,EAAE;QACb,OAAO,CAAC,UAAU,EAAE,aAAa,CAAC;QAClC,OAAO,CAAC,SAAS,EAAE,gBAAgB,CAAC;QACpC,OAAO,CAAC,SAAS,EAAE,mBAAmB,CAAC;QACvC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC;KAC9B;IACD,MAAM,EAAE;QACP,OAAO,CAAC,QAAQ,EAAE,oBAAoB,CAAC;QACvC,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC;QACjC,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC;QAClC,OAAO,CAAC,SAAS,EAAE,oBAAoB,CAAC;KACxC;IACD,WAAW,EAAE;QACZ,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC;QAClC,OAAO,CAAC,QAAQ,EAAE,oBAAoB,CAAC;QACvC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC;QACjC,OAAO,CAAC,YAAY,EAAE,eAAe,CAAC;KACtC;IACD,eAAe,EAAE;QAChB,OAAO,CAAC,QAAQ,EAAE,oBAAoB,CAAC;QACvC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC;QAC9B,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC;KAC3B;IACD,QAAQ,EAAE;QACT,OAAO,CAAC,QAAQ,EAAE,sBAAsB,CAAC;QACzC,OAAO,CAAC,WAAW,EAAE,YAAY,CAAC;QAClC,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC;QAClC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC;KAC3B;IACD,iBAAiB,EAAE;QAClB,OAAO,CAAC,SAAS,EAAE,oBAAoB,CAAC;QACxC,OAAO,CAAC,YAAY,EAAE,iBAAiB,CAAC;QACxC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC;QAC9B,OAAO,CAAC,QAAQ,EAAE,mBAAmB,CAAC;KACtC;IACD,YAAY,EAAE;QACb,OAAO,CAAC,UAAU,EAAE,qBAAqB,CAAC;QAC1C,OAAO,CAAC,WAAW,EAAE,wBAAwB,CAAC;QAC9C,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC;QAC5B,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC;KAC5B;IACD,OAAO,EAAE;QACR,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC;QAC3B,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC;QACzB,MAAM,CAAC,UAAU,EAAE,cAAc,CAAC;QAClC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC;KAC3B;CACD,CAAC;AAEF,iGAAiG;AACjG,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,sFAAsF;AACtF,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAc5B,SAAS,WAAW,CAAC,KAAwB;IAC5C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACnE,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IAC5C,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/C,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7C,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAC9B,IAAI;KACF,KAAK,CAAC,GAAG,CAAC;KACV,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;KACZ,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AAE9B,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,CAAC;AAErF;;;;GAIG;AACH,SAAS,UAAU,CAAC,OAAe,EAAE,QAAgB,EAAE,UAA+B;IACrF,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAChC,OAAO,OAAO;SACZ,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;SAC9B,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE;QAClB,MAAM,KAAK,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5E,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAClE,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,CAAC;AAED,wGAAwG;AACxG,SAAS,aAAa,CAAC,IAAwB;IAC9C,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzD,OAAO,OAAO,IAAI,0CAA0C,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;AAC5G,CAAC;AAeD,MAAM,iBAAiB,GAAG,IAAI,OAAO,EAAiC,CAAC;AAEvE,yEAAyE;AACzE,SAAS,eAAe,CAAC,OAAqB;IAC7C,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC9C,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAChD,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,CACtE,CAAC;IACF,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QACtD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI;YAC7B,IAAI,GAAG,CAAC,OAAO,KAAK,OAAO;gBAC1B,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,OAAO,CAAE,CAAC,IAAI,EAAE,CAAC;oBAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9F,OAAO,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACtG,MAAM,QAAQ,GAAc,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,GAAuB,EAAE,SAA6B,EAAE,EAAE;YACpF,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;YAC3C,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAChG,CAAC,CAAC;QACF,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,EAAE,CAAC;YAClC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;YAClC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,UAAU,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,CAAC;YACrG,MAAM,SAAS,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzG,yEAAyE;YACzE,0DAA0D;YAC1D,IAAI,CAAC,OAAO,MAAM,CAAC,WAAW,CAAC,IAAI,QAAQ,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC;QAClF,CAAC;aAAM,CAAC;YACP,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;YAChC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/E,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE;gBAC3C,IAAI,KAAK,GAAG,CAAC,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,cAAc;oBAC5D,IAAI,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;QACrE,CAAC;QACD,OAAO,QAAQ,CAAC;IACjB,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,GAAG,EAA+B,CAAC;IACvD,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAClD,KAAK,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAE,EAAE,CAAC;YAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;YAC7D,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YACrE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC;IACF,CAAC;IACD,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IACnC,iBAAiB,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACzC,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CACf,OAAqB,EACrB,KAAa;IAEb,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,IAAI,QAA4B,CAAC;IACjC,KAAK,MAAM,GAAG,IAAI,qBAAqB,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC;QACzD,UAAU,CAAC,GAAG,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACjD,KAAK,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAE,EAAE,CAAC;YACnE,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC;gBAAE,SAAS;YACrE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACvB,IAAI,SAAS;gBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;;gBACzC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QACD,IAAI,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,QAAQ,IAAI,GAAG,CAAC,WAAW,GAAG,QAAQ,CAAC;YAAE,QAAQ,GAAG,GAAG,CAAC,WAAW,CAAC;IAC9F,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AAC9E,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,GAAG,CAAC,MAAmC,EAAE,MAAc,EAAE,EAAE,CAC1E,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;AAE5F,MAAM,OAAO,GAAG,CAAC,SAAyB,EAAE,EAAE,CAAC,mBAAmB,CAAC,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEvH,MAAM,OAAO,GAAG,CAAC,SAAyB,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAEtG;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC7B,OAAqB,EACrB,UAAqC,EACrC,KAAwB;IAExB,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC/C,SAAS;QACT,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC;QACxB,GAAG,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC;KACpC,CAAC,CAAC,CAAC;IACJ,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,KAAa,EAAE,GAAwB,EAAsB,EAAE;QAC3F,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,GAAG,iBAAiB;YAAE,OAAO,SAAS,CAAC;QAC9E,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;QAChG,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACnG,CAAC,CAAC;IACF,MAAM,UAAU,GAAG,CAAC,MAAmC,EAAE,GAAwB,EAAE,OAA0B,EAAE,EAAE;QAChH,MAAM,KAAK,GAAG,OAAO;aACnB,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;aAC/C,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,EAAE,GAAG,CAAC,CAAC;aAChD,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;QACvD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACnF,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,QAAQ;SACxB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;SACxC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC;SAC5B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,QAAQ;SACrB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;SACpC,IAAI,EAAE;SACN,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACT,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,aAAa,GAAG,IAAI,GAAG,IAAI,GAAG,MAAM,CAAC;IAC3C,MAAM,WAAW,GAAG,IAAI,GAAG,IAAI,GAAG,MAAM,CAAC;IAEzC,OAAO,QAAQ;SACb,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE;QACtE,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,EAAE,UAAU,EAAE,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9E,MAAM,eAAe,GAAG,UAAU,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,YAAY,IAAI,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC9F,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QACjC,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,IAAI,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3F,MAAM,OAAO,GAAG,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC;QACpG,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,GAAG,CAAC,CAAC;QAC/C,OAAO;YACN,GAAG,SAAS;YACZ,OAAO,EAAE,IAAI;YACb,KAAK,EACJ,aAAa,GAAG,OAAO;gBACvB,WAAW,GAAG,SAAS;gBACvB,GAAG,GAAG,OAAO;gBACb,CAAC,KAAK,CAAC,gBAAgB,IAAI,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,MAAM;YACN,UAAU;YACV,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,GAAG,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7D,CAAC;IACH,CAAC,CAAC;SACD,IAAI,CACJ,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACR,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;QACjB,MAAM,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC;QACzE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAC/B,CAAC;AACJ,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,WAAW,CAAC,MAAkC,EAAE,IAAY;IAC3E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAChC,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC;YAAE,SAAS;QAC1C,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC5B,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpB,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI;YAAE,MAAM;IAChC,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC;AAED,SAAS,aAAa,CAAC,QAAgB;IACtC,IAAI,QAAQ,IAAI,GAAG;QAAE,OAAO,SAAS,CAAC;IACtC,IAAI,QAAQ,IAAI,IAAI;QAAE,OAAO,aAAa,CAAC;IAC3C,IAAI,QAAQ,IAAI,GAAG;QAAE,OAAO,cAAc,CAAC;IAC3C,IAAI,QAAQ,IAAI,IAAI;QAAE,OAAO,cAAc,CAAC;IAC5C,OAAO,gBAAgB,CAAC;AACzB,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAEpE;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAC7B,MAAuB,EACvB,KAAwB,EACxB,QAAoC;IAEpC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3G,MAAM,IAAI,GAAG,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1E,MAAM,MAAM,GAAG,QAAQ;SACrB,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,IAAI,EAAE,CAAC;SAC5C,IAAI,EAAE;SACN,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACT,MAAM,KAAK,GAAG,CAAC,OAA0B,EAAE,QAA4B,EAAE,EAAE;QAC1E,MAAM,OAAO,GAAG,OAAO;aACrB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACf,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YACnF,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;QAC/D,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;aACtC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACd,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,sBAAsB,CAAC;QACxD,MAAM,OAAO,GAAG,OAAO;aACrB,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE;YACxB,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7C,OAAO,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,kBAAkB,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACnH,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,OAAO,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,QAAQ,CAAC,+CAA+C,KAAK,OAAO,GAAG,CAAC;IAC1I,CAAC,CAAC;IACF,IAAI,QAAQ,GAAG,SAAS,CAAC;IACzB,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClH,QAAQ;YACP,MAAM,IAAI,CAAC;gBACV,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,oBAAoB;gBACxC,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,KAAK,MAAM,+BAA+B,CAAC;IAClE,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW;QAClC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,KAAK,MAAM,CAAC,WAAW,CAAC;QACtE,CAAC,CAAC,SAAS,CAAC;IACb,OAAO,IAAI,CAAC,SAAS,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC,IAAI;QAClB,EAAE,EAAE,MAAM,CAAC,KAAK;QAChB,QAAQ;QACR,KAAK,EAAE,GAAG,IAAI,KAAK,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,qBAAqB;QAC/F,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC5C,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC;QAClE,OAAO,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,eAAe,CAAC;QACjD,GAAG,CAAC,MAAM,CAAC,WAAW;YACrB,CAAC,CAAC;gBACA,KAAK,EAAE,mBAAmB,QAAQ,EAAE,IAAI,IAAI,MAAM,CAAC,WAAW,wDAAwD;aACtH;YACF,CAAC,CAAC,EAAE,CAAC;KACN,CAAC,CAAC;AACJ,CAAC","sourcesContent":["import {\n\tcandidateEvidenceRows,\n\tcatalogModelIdentity,\n\ttype EvalsCatalog,\n\tmodelEvidenceTokens,\n} from \"./model-routing-evals.js\";\nimport { type ResolvedTaskNeeds, taskDemand, type WorkKind } from \"./model-routing-needs.js\";\n\n/** What ranking needs to know about one eligible model. */\nexport interface CandidateModel {\n\t/** `provider/id`, as used in router selections. */\n\treadonly model: string;\n\treadonly name: string;\n\treadonly cost: { readonly input: number; readonly output: number };\n\treadonly input: readonly string[];\n\t/** `provider/id` of the base model when this entry is its derived fast route (`fastRoute` metadata). */\n\treadonly fastRouteOf?: string;\n}\n\ninterface Metric {\n\treadonly key: string;\n\treadonly label: string;\n\treadonly unit: \"%\" | \"\";\n}\n\nconst percent = (key: string, label: string): Metric => ({ key, label, unit: \"%\" });\nconst points = (key: string, label: string): Metric => ({ key, label, unit: \"\" });\n\nconst OVERALL: Metric = points(\"aa:idx\", \"AA Intelligence Index\");\n\n/** evals.md columns that measure each kind of work, in the order results are quoted. */\nconst WORK_METRICS: Record<WorkKind, readonly Metric[]> = {\n\tcomputer_use: [\n\t\tpercent(\"pub:OSW2\", \"OSWorld 2.0\"),\n\t\tpercent(\"pub:SSP\", \"ScreenSpot-Pro\"),\n\t\tpercent(\"pub:ALE\", \"Agents' Last Exam\"),\n\t\tpercent(\"aa:MMMU\", \"MMMU-Pro\"),\n\t],\n\tcoding: [\n\t\tpercent(\"aa:TB4\", \"Terminal-Bench 4.0\"),\n\t\tpercent(\"dswe:Pass@1\", \"DeepSWE\"),\n\t\tpercent(\"fc:Main\", \"FrontierCode\"),\n\t\tpercent(\"aa:TB21\", \"Terminal-Bench 2.1\"),\n\t],\n\tcode_review: [\n\t\tpercent(\"fc:Main\", \"FrontierCode\"),\n\t\tpercent(\"aa:TB4\", \"Terminal-Bench 4.0\"),\n\t\tpercent(\"pub:XGym\", \"ExploitGym\"),\n\t\tpercent(\"pub:SECPro\", \"SEC-Bench Pro\"),\n\t],\n\tcodebase_lookup: [\n\t\tpercent(\"aa:TB4\", \"Terminal-Bench 4.0\"),\n\t\tpercent(\"aa:tau2\", \"τ²-Bench\"),\n\t\tpercent(\"aa:IF\", \"IFBench\"),\n\t],\n\tresearch: [\n\t\tpercent(\"aa:HLE\", \"Humanity's Last Exam\"),\n\t\tpercent(\"pub:BComp\", \"BrowseComp\"),\n\t\tpercent(\"aa:GPQA\", \"GPQA Diamond\"),\n\t\tpercent(\"aa:LCR\", \"AA-LCR\"),\n\t],\n\tbusiness_workflow: [\n\t\tpercent(\"aa:Auto\", \"AutomationBench-AA\"),\n\t\tpercent(\"pub:ABench\", \"AutomationBench\"),\n\t\tpercent(\"aa:tau2\", \"τ²-Bench\"),\n\t\tpercent(\"aa:Ent\", \"EnterpriseOps-Gym\"),\n\t],\n\tmath_science: [\n\t\tpercent(\"pub:FMT4\", \"FrontierMath Tier 4\"),\n\t\tpercent(\"pub:TBSci\", \"Terminal-Bench-Science\"),\n\t\tpercent(\"aa:Crit\", \"CritPt\"),\n\t\tpercent(\"aa:Sci\", \"SciCode\"),\n\t],\n\twriting: [\n\t\tpercent(\"aa:IF\", \"IFBench\"),\n\t\tpoints(\"aa:Gn\", \"GDPval\"),\n\t\tpoints(\"aa:Brief\", \"AA-Briefcase\"),\n\t\tpercent(\"aa:LCR\", \"AA-LCR\"),\n\t],\n};\n\n/** When speed matters, a fast route edges out its standard route for the model's single slot. */\nconst FAST_ROUTE_BONUS = 0.02;\n\n/** A standing is stated only when enough benchmarked models share the measurement. */\nconst MIN_RANKED_MODELS = 4;\n\nexport interface RankedCandidate extends CandidateModel {\n\t/** Identity shared by one model's providers and its derived fast route, which collapse onto it. */\n\treadonly baseKey: string;\n\treadonly score: number;\n\treadonly values: ReadonlyMap<string, number>;\n\t/** How each quoted value was measured, for example its effort, harness or reporter. */\n\treadonly conditions: ReadonlyMap<string, string>;\n\treadonly released?: string;\n\treadonly workStanding?: number;\n\treadonly overallStanding?: number;\n}\n\nfunction sectionKind(intro: readonly string[]): \"aa\" | \"dswe\" | \"fc\" | \"pub\" {\n\tconst heading = intro.find((line) => line.startsWith(\"## \")) ?? \"\";\n\tif (/DeepSWE/u.test(heading)) return \"dswe\";\n\tif (/FrontierCode/u.test(heading)) return \"fc\";\n\tif (/Published/u.test(heading)) return \"pub\";\n\treturn \"aa\";\n}\n\nconst cells = (line: string) =>\n\tline\n\t\t.split(\"|\")\n\t\t.slice(1, -1)\n\t\t.map((cell) => cell.trim());\n\nconst nameWords = (text: string) => new Set(text.match(/\\p{Lu}[\\p{L}]{2,}/gu) ?? []);\n\n/**\n * The parts of a published result's setting that apply to this row: a clause\n * naming other models (for example \"Fable values are Mythos\" on a GPT row) is a\n * footnote for them and is dropped.\n */\nfunction ownSetting(setting: string, rowModel: string, modelWords: ReadonlySet<string>): string {\n\tconst own = nameWords(rowModel);\n\treturn setting\n\t\t.split(\";\")\n\t\t.map((clause) => clause.trim())\n\t\t.filter((clause) => {\n\t\t\tconst named = [...nameWords(clause)].filter((word) => modelWords.has(word));\n\t\t\treturn named.length === 0 || named.some((word) => own.has(word));\n\t\t})\n\t\t.join(\", \");\n}\n\n/** Parenthetical measurement setting in an Artificial Analysis model name, for example \"Max Effort\". */\nfunction nameCondition(name: string | undefined): string | undefined {\n\tconst setting = /\\(([^)]+)\\)\\s*$/u.exec(name ?? \"\")?.[1];\n\treturn setting && /^(?:minimal|low|medium|high|xhigh|max)$/u.test(setting) ? `${setting} effort` : setting;\n}\n\ninterface Reading {\n\treadonly key: string;\n\treadonly value: number;\n\treadonly condition?: string;\n}\n\ninterface CatalogReadings {\n\t/** Readings of each catalog row, indexed by the row's `order`. */\n\treadonly rows: readonly (readonly Reading[])[];\n\t/** Per measurement, the best value of each catalog model. */\n\treadonly cohorts: ReadonlyMap<string, ReadonlyMap<string, number>>;\n}\n\nconst readingsByCatalog = new WeakMap<EvalsCatalog, CatalogReadings>();\n\n/** Every measured value in the catalog, read once per parsed catalog. */\nfunction catalogReadings(catalog: EvalsCatalog): CatalogReadings {\n\tconst cached = readingsByCatalog.get(catalog);\n\tif (cached) return cached;\n\tconst kinds = catalog.sections.map((section) => sectionKind(section.intro));\n\tconst headers = catalog.sections.map((section) =>\n\t\tcells(section.intro.find((line) => line.startsWith(\"| slug |\")) ?? \"\"),\n\t);\n\tconst modelColumn = headers.map((header) => header.indexOf(\"Model\"));\n\tconst modelWords = catalog.sections.map((_, section) => {\n\t\tconst words = new Set<string>();\n\t\tfor (const row of catalog.rows)\n\t\t\tif (row.section === section)\n\t\t\t\tfor (const word of nameWords(cells(row.line)[modelColumn[section]!] ?? \"\")) words.add(word);\n\t\treturn words;\n\t});\n\tconst rows = catalog.rows.map((row) => {\n\t\tconst header = headers[row.section]!;\n\t\tconst rowCells = cells(row.line);\n\t\tconst column = (name: string) => (header.includes(name) ? rowCells[header.indexOf(name)] : undefined);\n\t\tconst readings: Reading[] = [];\n\t\tconst read = (key: string, raw: string | undefined, condition: string | undefined) => {\n\t\t\tconst value = Number.parseFloat(raw ?? \"\");\n\t\t\tif (Number.isFinite(value)) readings.push({ key, value, ...(condition ? { condition } : {}) });\n\t\t};\n\t\tif (kinds[row.section] === \"pub\") {\n\t\t\tconst reporter = column(\"Source\");\n\t\t\tconst setting = ownSetting(column(\"Setting\") ?? \"\", column(\"Model\") ?? \"\", modelWords[row.section]!);\n\t\t\tconst condition = [setting, reporter ? `reported by ${reporter}` : undefined].filter(Boolean).join(\", \");\n\t\t\t// Each reporter's results form their own cohort: a vendor harness and an\n\t\t\t// official leaderboard are not ranked against each other.\n\t\t\tread(`pub:${column(\"Benchmark\")}@${reporter ?? \"\"}`, column(\"Score\"), condition);\n\t\t} else {\n\t\t\tconst effort = column(\"Effort\");\n\t\t\tconst condition = effort ? `${effort} effort` : nameCondition(column(\"Model\"));\n\t\t\tfor (const [index, name] of header.entries())\n\t\t\t\tif (index > 1 && name !== \"Effort\" && name !== \"Release date\")\n\t\t\t\t\tread(`${kinds[row.section]}:${name}`, rowCells[index], condition);\n\t\t}\n\t\treturn readings;\n\t});\n\tconst cohorts = new Map<string, Map<string, number>>();\n\tfor (const row of catalog.rows) {\n\t\tconst identity = catalogModelIdentity(row.tokens);\n\t\tfor (const { key, value } of rows[row.order]!) {\n\t\t\tconst cohort = cohorts.get(key) ?? new Map<string, number>();\n\t\t\tcohort.set(identity, Math.max(cohort.get(identity) ?? value, value));\n\t\t\tcohorts.set(key, cohort);\n\t\t}\n\t}\n\tconst readings = { rows, cohorts };\n\treadingsByCatalog.set(catalog, readings);\n\treturn readings;\n}\n\n/**\n * Best value per metric across every evals row describing the model, with the\n * conditions it was measured under, the catalog models those rows belong to,\n * and the model's newest release date.\n */\nfunction measure(\n\tcatalog: EvalsCatalog,\n\tmodel: string,\n): { values: Map<string, number>; conditions: Map<string, string>; identities: Set<string>; released?: string } {\n\tconst readings = catalogReadings(catalog);\n\tconst values = new Map<string, number>();\n\tconst conditions = new Map<string, string>();\n\tconst identities = new Set<string>();\n\tlet released: string | undefined;\n\tfor (const row of candidateEvidenceRows(catalog, model)) {\n\t\tidentities.add(catalogModelIdentity(row.tokens));\n\t\tfor (const { key, value, condition } of readings.rows[row.order]!) {\n\t\t\tif (value <= (values.get(key) ?? Number.NEGATIVE_INFINITY)) continue;\n\t\t\tvalues.set(key, value);\n\t\t\tif (condition) conditions.set(key, condition);\n\t\t\telse conditions.delete(key);\n\t\t}\n\t\tif (row.releaseDate && (!released || row.releaseDate > released)) released = row.releaseDate;\n\t}\n\treturn { values, conditions, identities, ...(released ? { released } : {}) };\n}\n\n/** Value keys measuring `metric`: the column itself, or one per reporter for published results. */\nconst cohortKeys = (values: ReadonlyMap<string, number>, metric: Metric) =>\n\t[...values.keys()].filter((key) => key === metric.key || key.startsWith(`${metric.key}@`));\n\nconst baseKey = (candidate: CandidateModel) => modelEvidenceTokens(candidate.fastRouteOf ?? candidate.model).join(\"-\");\n\nconst blended = (candidate: CandidateModel) => (3 * candidate.cost.input + candidate.cost.output) / 4;\n\n/**\n * Rank every eligible model for the task in code. Quality is the model's\n * standing among every benchmarked model in the catalog on the benchmarks for\n * this kind of work (overall intelligence when it has none), so it does not\n * depend on which other models are eligible. It is weighed against price by how\n * demanding the task is, with a small preference for newer releases.\n */\nexport function rankCandidates(\n\tcatalog: EvalsCatalog,\n\tcandidates: readonly CandidateModel[],\n\tneeds: ResolvedTaskNeeds,\n): RankedCandidate[] {\n\tconst { cohorts } = catalogReadings(catalog);\n\tconst measured = candidates.map((candidate) => ({\n\t\tcandidate,\n\t\tbase: baseKey(candidate),\n\t\t...measure(catalog, candidate.model),\n\t}));\n\tconst rankOf = (key: string, value: number, own: ReadonlySet<string>): number | undefined => {\n\t\tconst cohort = cohorts.get(key);\n\t\tif (cohort === undefined || cohort.size < MIN_RANKED_MODELS) return undefined;\n\t\tconst others = [...cohort].filter(([identity]) => !own.has(identity)).map(([, other]) => other);\n\t\treturn others.length ? others.filter((other) => other < value).length / others.length : undefined;\n\t};\n\tconst standingOf = (values: ReadonlyMap<string, number>, own: ReadonlySet<string>, metrics: readonly Metric[]) => {\n\t\tconst ranks = metrics\n\t\t\t.flatMap((metric) => cohortKeys(values, metric))\n\t\t\t.map((key) => rankOf(key, values.get(key)!, own))\n\t\t\t.filter((rank): rank is number => rank !== undefined);\n\t\treturn ranks.length ? ranks.reduce((a, b) => a + b, 0) / ranks.length : undefined;\n\t};\n\tconst logPrices = measured\n\t\t.map((entry) => blended(entry.candidate))\n\t\t.filter((price) => price > 0)\n\t\t.map(Math.log);\n\tconst [low, high] = [Math.min(...logPrices), Math.max(...logPrices)];\n\tconst newest = measured\n\t\t.map((entry) => entry.released ?? \"\")\n\t\t.sort()\n\t\t.at(-1);\n\tconst demand = taskDemand(needs);\n\tconst qualityWeight = 0.25 + 0.65 * demand;\n\tconst priceWeight = 0.65 - 0.65 * demand;\n\n\treturn measured\n\t\t.map(({ candidate, base, values, conditions, identities, released }) => {\n\t\t\tconst workStanding = standingOf(values, identities, WORK_METRICS[needs.work]);\n\t\t\tconst overallStanding = standingOf(values, identities, [OVERALL]);\n\t\t\tconst quality = workStanding ?? (overallStanding !== undefined ? 0.8 * overallStanding : 0.2);\n\t\t\tconst price = blended(candidate);\n\t\t\tconst cheapness = price > 0 && high > low ? 1 - (Math.log(price) - low) / (high - low) : 1;\n\t\t\tconst ageDays = released && newest ? (Date.parse(newest) - Date.parse(released)) / 86_400_000 : 365;\n\t\t\tconst recency = 1 - Math.min(1, ageDays / 365);\n\t\t\treturn {\n\t\t\t\t...candidate,\n\t\t\t\tbaseKey: base,\n\t\t\t\tscore:\n\t\t\t\t\tqualityWeight * quality +\n\t\t\t\t\tpriceWeight * cheapness +\n\t\t\t\t\t0.1 * recency +\n\t\t\t\t\t(needs.latencySensitive && candidate.fastRouteOf ? FAST_ROUTE_BONUS : 0),\n\t\t\t\tvalues,\n\t\t\t\tconditions,\n\t\t\t\t...(released ? { released } : {}),\n\t\t\t\t...(workStanding !== undefined ? { workStanding } : {}),\n\t\t\t\t...(overallStanding !== undefined ? { overallStanding } : {}),\n\t\t\t};\n\t\t})\n\t\t.sort(\n\t\t\t(a, b) =>\n\t\t\t\tb.score - a.score ||\n\t\t\t\tNumber(a.fastRouteOf !== undefined) - Number(b.fastRouteOf !== undefined) ||\n\t\t\t\ta.model.localeCompare(b.model),\n\t\t);\n}\n\n/** The best `size` candidates with at most one per base model. */\nexport function distinctTop(ranked: readonly RankedCandidate[], size: number): RankedCandidate[] {\n\tconst seen = new Set<string>();\n\tconst top: RankedCandidate[] = [];\n\tfor (const candidate of ranked) {\n\t\tif (seen.has(candidate.baseKey)) continue;\n\t\tseen.add(candidate.baseKey);\n\t\ttop.push(candidate);\n\t\tif (top.length === size) break;\n\t}\n\treturn top;\n}\n\nfunction standingLabel(fraction: number): string {\n\tif (fraction >= 0.9) return \"top 10%\";\n\tif (fraction >= 0.75) return \"top quarter\";\n\tif (fraction >= 0.5) return \"above median\";\n\tif (fraction >= 0.25) return \"below median\";\n\treturn \"bottom quarter\";\n}\n\nconst money = (value: number) => `$${Number(value.toPrecision(3))}`;\n\n/**\n * One shortlisted option described with its own evidence, so the router compares\n * options directly: release date, price tier, image input, and results for this\n * kind of work and overall, with standings among catalog models measured on the same benchmark by the same source.\n */\nexport function describeOption(\n\toption: RankedCandidate,\n\tneeds: ResolvedTaskNeeds,\n\teligible: readonly RankedCandidate[],\n): string {\n\tconst prices = eligible.map(blended).sort((a, b) => a - b);\n\tconst position = prices.filter((other) => other < blended(option)).length / Math.max(1, prices.length - 1);\n\tconst tier = position < 1 / 3 ? \"low\" : position < 2 / 3 ? \"mid\" : \"high\";\n\tconst newest = eligible\n\t\t.map((candidate) => candidate.released ?? \"\")\n\t\t.sort()\n\t\t.at(-1);\n\tconst quote = (metrics: readonly Metric[], standing: number | undefined) => {\n\t\tconst present = metrics\n\t\t\t.map((metric) => {\n\t\t\t\tconst keys = cohortKeys(option.values, metric);\n\t\t\t\tconst best = keys.sort((a, b) => option.values.get(b)! - option.values.get(a)!)[0];\n\t\t\t\treturn best === undefined ? undefined : { metric, key: best };\n\t\t\t})\n\t\t\t.filter((entry) => entry !== undefined)\n\t\t\t.slice(0, 2);\n\t\tif (present.length === 0) return \"no published results\";\n\t\tconst results = present\n\t\t\t.map(({ metric, key }) => {\n\t\t\t\tconst condition = option.conditions.get(key);\n\t\t\t\treturn `${metric.label} ${option.values.get(key)}${metric.unit}${condition ? ` measured with ${condition}` : \"\"}`;\n\t\t\t})\n\t\t\t.join(\"; \");\n\t\treturn `${standing === undefined ? \"measured\" : `${standingLabel(standing)} of models with the same benchmark and source`} (${results})`;\n\t};\n\tlet released = \"unknown\";\n\tif (option.released) {\n\t\tconst months = newest ? Math.round((Date.parse(newest) - Date.parse(option.released)) / (30.44 * 86_400_000)) : 0;\n\t\treleased =\n\t\t\tmonths <= 1\n\t\t\t\t? `${option.released}, among the newest`\n\t\t\t\t: `${option.released}, ${months} months older than the newest`;\n\t}\n\tconst fastBase = option.fastRouteOf\n\t\t? eligible.find((candidate) => candidate.model === option.fastRouteOf)\n\t\t: undefined;\n\treturn JSON.stringify({\n\t\tmodel: option.name,\n\t\tid: option.model,\n\t\treleased,\n\t\tprice: `${tier}: ${money(option.cost.input)} / ${money(option.cost.output)} per million tokens`,\n\t\treads_images: option.input.includes(\"image\"),\n\t\t[needs.work]: quote(WORK_METRICS[needs.work], option.workStanding),\n\t\toverall: quote([OVERALL], option.overallStanding),\n\t\t...(option.fastRouteOf\n\t\t\t? {\n\t\t\t\t\troute: `faster route of ${fastBase?.name ?? option.fastRouteOf} with the same results; billed above the listed prices`,\n\t\t\t\t}\n\t\t\t: {}),\n\t});\n}\n"]}

package/dist/core/model-routing-evals.d.ts

@@ -2,6 +2,8 @@
 export declare const MODEL_SELECTION_EVALS_JSON_BYTES = 14200;
 /** Provider-independent model tokens: `us.anthropic.claude-opus-4-6-v1` and `anthropic/claude-opus-4.6` agree. */
 export declare function modelEvidenceTokens(id: string): string[];
+/** One catalog model across its effort, edition and snapshot rows: `astra-high` and `astra-0902` are `astra`. */
+export declare function catalogModelIdentity(slug: readonly string[]): string;
 interface CatalogRow {
     readonly line: string;
     readonly section: number;

package/dist/core/model-routing-evals.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"model-routing-evals.d.ts","sourceRoot":"","sources":["../../src/core/model-routing-evals.ts"],"names":[],"mappings":"AAEA,oEAAoE;AACpE,eAAO,MAAM,gCAAgC,QAAS,CAAC;AA2CvD,kHAAkH;AAClH,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,CAexD;AA6BD,UAAU,UAAU;IACnB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,gFAAgF;AAChF,UAAU,cAAc;IACvB,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AAED,wFAAwF;AACxF,MAAM,WAAW,YAAY;IAC5B,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;IACrC,wDAAwD;IACxD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,CAAC,CAAC;CACrD;AAID,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,CAmC7D;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,UAAU,EAAE,CAcrG;AAED,qFAAqF;AACrF,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGjG;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC9B,OAAO,EAAE,YAAY,EACrB,UAAU,EAAE,SAAS,MAAM,EAAE,EAC7B,QAAQ,SAA2B,GACjC,MAAM,CAeR;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAE9F"}
+{"version":3,"file":"model-routing-evals.d.ts","sourceRoot":"","sources":["../../src/core/model-routing-evals.ts"],"names":[],"mappings":"AAEA,oEAAoE;AACpE,eAAO,MAAM,gCAAgC,QAAS,CAAC;AA2CvD,kHAAkH;AAClH,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,CAexD;AAWD,iHAAiH;AACjH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAIpE;AAqBD,UAAU,UAAU;IACnB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,gFAAgF;AAChF,UAAU,cAAc;IACvB,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AAED,wFAAwF;AACxF,MAAM,WAAW,YAAY;IAC5B,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;IACrC,wDAAwD;IACxD,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,CAAC,CAAC;CACrD;AAID,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,CAmC7D;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,UAAU,EAAE,CAcrG;AAED,qFAAqF;AACrF,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGjG;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC9B,OAAO,EAAE,YAAY,EACrB,UAAU,EAAE,SAAS,MAAM,EAAE,EAC7B,QAAQ,SAA2B,GACjC,MAAM,CAeR;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAE9F"}

package/dist/core/model-routing-evals.js

@@ -59,13 +59,19 @@ export function modelEvidenceTokens(id) {
         .map((token) => (token === "thinking" ? "reasoning" : token));
     return canonicalClaudeOrder(tokens);
 }
+const isVariantSuffix = (token) => VARIANT_TOKENS.has(token) || ROW_EDITION_TOKENS.has(token) || SNAPSHOT_TOKEN.test(token);
 /** True when `slug` is the candidate model or one of its effort, edition or snapshot variants. */
 function slugMatchesCandidate(slug, candidate) {
     if (slug.length < candidate.length || candidate.some((token, index) => slug[index] !== token))
         return false;
-    return slug
-        .slice(candidate.length)
-        .every((token) => VARIANT_TOKENS.has(token) || ROW_EDITION_TOKENS.has(token) || SNAPSHOT_TOKEN.test(token));
+    return slug.slice(candidate.length).every(isVariantSuffix);
+}
+/** One catalog model across its effort, edition and snapshot rows: `astra-high` and `astra-0902` are `astra`. */
+export function catalogModelIdentity(slug) {
+    let end = slug.length;
+    while (end > 1 && isVariantSuffix(slug[end - 1]))
+        end--;
+    return slug.slice(0, end).join("-");
 }
 /**
  * The candidate's own tokens first, then the same model without trailing

package/dist/core/model-routing-evals.js.map

@@ -1 +1 @@
-{"version":3,"file":"model-routing-evals.js","sourceRoot":"","sources":["../../src/core/model-routing-evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAEtE,oEAAoE;AACpE,MAAM,CAAC,MAAM,gCAAgC,GAAG,MAAM,CAAC;AAEvD,MAAM,uBAAuB,GAAG,uBAAuB,CAAC;AACxD,MAAM,cAAc,GAAG,UAAU,CAAC;AAClC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IACjC,MAAM;IACN,WAAW;IACX,YAAY;IACZ,WAAW;IACX,MAAM;IACN,KAAK;IACL,MAAM;IACN,IAAI;IACJ,UAAU;IACV,aAAa;CACb,CAAC,CAAC;AACH,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;AAClE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC9B,KAAK;IACL,SAAS;IACT,KAAK;IACL,QAAQ;IACR,MAAM;IACN,OAAO;IACP,KAAK;IACL,UAAU;IACV,UAAU;IACV,WAAW;IACX,KAAK;IACL,QAAQ;IACR,SAAS;CACT,CAAC,CAAC;AACH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;AAEtE,2GAA2G;AAC3G,SAAS,oBAAoB,CAAC,MAAgB;IAC7C,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAAE,OAAO,MAAM,CAAC;IACnF,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1F,MAAM,GAAG,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;IACxD,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAC7B,OAAO,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED,kHAAkH;AAClH,MAAM,UAAU,mBAAmB,CAAC,EAAU;IAC7C,MAAM,UAAU,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,UAAU;SACtB,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SACtC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC;SACpC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC;SACnC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;SACtB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC1B,MAAM,MAAM,GAAG,KAAK;SAClB,KAAK,CAAC,QAAQ,CAAC;SACf,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/D,OAAO,oBAAoB,CAAC,MAAM,CAAC,CAAC;AACrC,CAAC;AAED,kGAAkG;AAClG,SAAS,oBAAoB,CAAC,IAAuB,EAAE,SAA4B;IAClF,IAAI,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5G,OAAO,IAAI;SACT,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC;SACvB,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,MAAyB;IAChD,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;IAC5B,IAAI,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC1B,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;QAC1C,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM;QACnG,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AA2BD,MAAM,YAAY,GAAG,kBAAkB,CAAC;AAExC,MAAM,UAAU,iBAAiB,CAAC,KAAa;IAC9C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IAC1G,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,MAAM,IAAI,GAAiB,EAAE,CAAC;IAC9B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,KAAK,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,KAAK,GAAG,WAAW,CAAC;QACxB,KAAK,IAAI,KAAK,GAAG,WAAW,GAAG,CAAC,EAAE,KAAK,IAAI,gBAAgB,EAAE,KAAK,EAAE;YACnE,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBACrC,KAAK,GAAG,KAAK,CAAC;gBACd,MAAM;YACP,CAAC;QACF,IAAI,OAAO,KAAK,CAAC;YAAE,WAAW,GAAG,KAAK,CAAC;QACvC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACzE,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACrD,IAAI,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC;QAC5B,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YACvE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAClE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAAE,SAAS;YACxB,MAAM,OAAO,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACrE,IAAI,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,KAAK,CAAC,KAAK,CAAE;gBACnB,OAAO;gBACP,KAAK,EAAE,IAAI,CAAC,MAAM;gBAClB,MAAM,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrC,GAAG,CAAC,OAAO,IAAI,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACpF,CAAC,CAAC;QACJ,CAAC;QACD,gBAAgB,GAAG,KAAK,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;AACtF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAqB,EAAE,SAAiB;IAC7E,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC9C,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QACvD,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC;QAC1E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;YAClF,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;QACpC,CAAC;QACD,OAAO,EAAE,CAAC;IACX,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACxC,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,oBAAoB,CAAC,OAAqB,EAAE,SAAiB;IAC5E,MAAM,KAAK,GAAG,qBAAqB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrH,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACvD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAC9B,OAAqB,EACrB,UAA6B,EAC7B,QAAQ,GAAG,MAAM,CAAC,iBAAiB;IAEnC,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS;QAAE,OAAO,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,SAAS,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC;QAC1C,KAAK,MAAM,GAAG,IAAI,qBAAqB,CAAC,OAAO,EAAE,SAAS,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACtF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpC,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3D,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC;QAChE,IAAI,KAAK,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACpD,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,OAAO,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;AACzF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CAAC,KAAa,EAAE,UAA6B;IACrF,OAAO,eAAe,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,gCAAgC,CAAC,CAAC;AAChG,CAAC","sourcesContent":["import { jsonBytes, truncateToBytes } from \"./model-routing-bytes.js\";\n\n/** Bounds the evaluation evidence sent with one routing request. */\nexport const MODEL_SELECTION_EVALS_JSON_BYTES = 14_200;\n\nconst VENDOR_OR_REGION_PREFIX = /^(?:[a-z][a-z-]*\\.)+/u;\nconst SNAPSHOT_TOKEN = /^\\d{4}$/u;\nconst DEPLOYMENT_TOKENS = new Set([\n\t\"fast\",\n\t\"highspeed\",\n\t\"ultraspeed\",\n\t\"lightning\",\n\t\"beta\",\n\t\"exp\",\n\t\"free\",\n\t\"it\",\n\t\"instruct\",\n\t\"contributor\",\n]);\nconst ROW_EDITION_TOKENS = new Set([\"preview\", \"instruct\", \"it\"]);\nconst VARIANT_TOKENS = new Set([\n\t\"off\",\n\t\"minimal\",\n\t\"low\",\n\t\"medium\",\n\t\"high\",\n\t\"xhigh\",\n\t\"max\",\n\t\"adaptive\",\n\t\"thinking\",\n\t\"reasoning\",\n\t\"non\",\n\t\"effort\",\n\t\"preview\",\n]);\nconst CLAUDE_FAMILIES = new Set([\"opus\", \"sonnet\", \"haiku\", \"fable\"]);\n\n/** Artificial Analysis names older Claude models `claude-4-5-sonnet`; catalogs use `claude-sonnet-4-5`. */\nfunction canonicalClaudeOrder(tokens: string[]): string[] {\n\tif (tokens[0] !== \"claude\" || !CLAUDE_FAMILIES.has(tokens[1] ?? \"\")) return tokens;\n\tconst versionEnd = tokens.findIndex((token, index) => index > 1 && !/^\\d+$/u.test(token));\n\tconst end = versionEnd < 0 ? tokens.length : versionEnd;\n\tif (end === 2) return tokens;\n\treturn [\"claude\", ...tokens.slice(2, end), tokens[1]!, ...tokens.slice(end)];\n}\n\n/** Provider-independent model tokens: `us.anthropic.claude-opus-4-6-v1` and `anthropic/claude-opus-4.6` agree. */\nexport function modelEvidenceTokens(id: string): string[] {\n\tconst normalized = id.trim().toLowerCase();\n\tconst model = normalized\n\t\t.slice(normalized.lastIndexOf(\"/\") + 1)\n\t\t.replace(/^~/u, \"\")\n\t\t.replace(VENDOR_OR_REGION_PREFIX, \"\")\n\t\t.replace(/(?<=[a-z])-\\d+:\\d+$/u, \"\")\n\t\t.replace(/:[a-z0-9]+$/u, \"\")\n\t\t.replace(/-v\\d+$/u, \"\")\n\t\t.replace(/-\\d{8}$/u, \"\");\n\tconst tokens = model\n\t\t.split(/[-._]/u)\n\t\t.filter(Boolean)\n\t\t.map((token) => (token === \"thinking\" ? \"reasoning\" : token));\n\treturn canonicalClaudeOrder(tokens);\n}\n\n/** True when `slug` is the candidate model or one of its effort, edition or snapshot variants. */\nfunction slugMatchesCandidate(slug: readonly string[], candidate: readonly string[]): boolean {\n\tif (slug.length < candidate.length || candidate.some((token, index) => slug[index] !== token)) return false;\n\treturn slug\n\t\t.slice(candidate.length)\n\t\t.every((token) => VARIANT_TOKENS.has(token) || ROW_EDITION_TOKENS.has(token) || SNAPSHOT_TOKEN.test(token));\n}\n\n/**\n * The candidate's own tokens first, then the same model without trailing\n * deployment, reasoning-mode or snapshot suffixes (`gpt-5.4-fast`,\n * `grok-4.20-reasoning`, `qwen3.8-max-0902`). A fallback applies only when the\n * more specific form matched no row, so a model that is itself named `-fast`\n * keeps its own evidence.\n */\nfunction candidateForms(tokens: readonly string[]): string[][] {\n\tconst forms = [[...tokens]];\n\tlet current = [...tokens];\n\twhile (current.length > 1) {\n\t\tconst last = current[current.length - 1]!;\n\t\tif (!DEPLOYMENT_TOKENS.has(last) && !VARIANT_TOKENS.has(last) && !SNAPSHOT_TOKEN.test(last)) break;\n\t\tcurrent = current.slice(0, -1);\n\t\tforms.push(current);\n\t}\n\treturn forms;\n}\n\ninterface CatalogRow {\n\treadonly line: string;\n\treadonly section: number;\n\treadonly order: number;\n\treadonly tokens: readonly string[];\n\treadonly releaseDate?: string;\n}\n\n/** One `## ` section of `evals.md` that holds a model table keyed by `slug`. */\ninterface CatalogSection {\n\t/** The section heading, its notes and key, the table header and separator. */\n\treadonly intro: readonly string[];\n}\n\n/** `evals.md` parsed once so batch packing can query evidence per candidate cheaply. */\nexport interface EvalsCatalog {\n\t/** Text before the first table section: title, access date and the shared key. */\n\treadonly preamble: readonly string[];\n\treadonly sections: readonly CatalogSection[];\n\treadonly rows: readonly CatalogRow[];\n\t/** The unparsed document when it has no model table. */\n\treadonly raw?: string;\n\treadonly matches: Map<string, readonly CatalogRow[]>;\n}\n\nconst TABLE_HEADER = /^\\|\\s*slug\\s*\\|/u;\n\nexport function parseEvalsCatalog(evals: string): EvalsCatalog {\n\tconst lines = evals.split(\"\\n\");\n\tconst headers = lines.flatMap((line, index) => (TABLE_HEADER.test(line) ? [index] : []));\n\tif (headers.length === 0) return { preamble: [], sections: [], rows: [], raw: evals, matches: new Map() };\n\tconst sections: CatalogSection[] = [];\n\tconst rows: CatalogRow[] = [];\n\tlet preambleEnd = 0;\n\tlet previousTableEnd = 0;\n\tfor (const [section, headerIndex] of headers.entries()) {\n\t\tlet start = headerIndex;\n\t\tfor (let index = headerIndex - 1; index >= previousTableEnd; index--)\n\t\t\tif (lines[index]!.startsWith(\"## \")) {\n\t\t\t\tstart = index;\n\t\t\t\tbreak;\n\t\t\t}\n\t\tif (section === 0) preambleEnd = start;\n\t\tsections.push({ intro: lines.slice(start, headerIndex + 2) });\n\t\tconst header = lines[headerIndex]!.split(\"|\").map((cell) => cell.trim());\n\t\tconst releaseColumn = header.indexOf(\"Release date\");\n\t\tlet index = headerIndex + 2;\n\t\tfor (; index < lines.length && lines[index]!.startsWith(\"|\"); index++) {\n\t\t\tconst cells = lines[index]!.split(\"|\").map((cell) => cell.trim());\n\t\t\tif (!cells[1]) continue;\n\t\t\tconst release = releaseColumn > 0 ? cells[releaseColumn] : undefined;\n\t\t\trows.push({\n\t\t\t\tline: lines[index]!,\n\t\t\t\tsection,\n\t\t\t\torder: rows.length,\n\t\t\t\ttokens: modelEvidenceTokens(cells[1]),\n\t\t\t\t...(release && /^\\d{4}-\\d{2}-\\d{2}$/u.test(release) ? { releaseDate: release } : {}),\n\t\t\t});\n\t\t}\n\t\tpreviousTableEnd = index;\n\t}\n\treturn { preamble: lines.slice(0, preambleEnd), sections, rows, matches: new Map() };\n}\n\n/**\n * Rows describing `candidate`: in each section, its own model and variants, or\n * its base model when that section has none. Sections resolve independently, so\n * an exact snapshot row in one table does not hide the base model's rows in another.\n */\nexport function candidateEvidenceRows(catalog: EvalsCatalog, candidate: string): readonly CatalogRow[] {\n\tconst cached = catalog.matches.get(candidate);\n\tif (cached) return cached;\n\tconst forms = candidateForms(modelEvidenceTokens(candidate));\n\tconst matches = catalog.sections.flatMap((_, section) => {\n\t\tconst sectionRows = catalog.rows.filter((row) => row.section === section);\n\t\tfor (const form of forms) {\n\t\t\tconst found = sectionRows.filter((row) => slugMatchesCandidate(row.tokens, form));\n\t\t\tif (found.length > 0) return found;\n\t\t}\n\t\treturn [];\n\t});\n\tcatalog.matches.set(candidate, matches);\n\treturn matches;\n}\n\n/** Latest release date among the candidate's evidence rows, when any row has one. */\nexport function candidateReleaseDate(catalog: EvalsCatalog, candidate: string): string | undefined {\n\tconst dates = candidateEvidenceRows(catalog, candidate).flatMap((row) => (row.releaseDate ? [row.releaseDate] : []));\n\treturn dates.length ? dates.sort().at(-1) : undefined;\n}\n\n/**\n * The catalog preamble plus, for every section with rows for `candidates`, that\n * section's heading, key and table header followed by only those rows.\n * Sections without a matching row are left out. `maxBytes` bounds the\n * JSON-encoded result; batched routing omits it because its request budget\n * already decides how many candidates, and so how many rows, one batch holds.\n */\nexport function catalogEvidence(\n\tcatalog: EvalsCatalog,\n\tcandidates: readonly string[],\n\tmaxBytes = Number.POSITIVE_INFINITY,\n): string {\n\tif (catalog.raw !== undefined) return truncateToBytes(catalog.raw, maxBytes);\n\tconst selected = new Set<number>();\n\tfor (const candidate of new Set(candidates))\n\t\tfor (const row of candidateEvidenceRows(catalog, candidate)) selected.add(row.order);\n\tconst kept = catalog.rows.filter((row) => selected.has(row.order));\n\tconst parts = [...catalog.preamble];\n\tfor (const [index, section] of catalog.sections.entries()) {\n\t\tconst sectionRows = kept.filter((row) => row.section === index);\n\t\tif (index > 0 && sectionRows.length === 0) continue;\n\t\tif (index > 0 && parts.at(-1) !== \"\") parts.push(\"\");\n\t\tparts.push(...section.intro, ...sectionRows.map((row) => row.line));\n\t}\n\tconst filtered = parts.join(\"\\n\");\n\treturn jsonBytes(filtered) <= maxBytes ? filtered : truncateToBytes(filtered, maxBytes);\n}\n\n/**\n * Keep the catalog preamble and only the table rows that describe an eligible\n * candidate model, bounded to the routing evidence budget.\n */\nexport function filterModelSelectionEvals(evals: string, candidates: readonly string[]): string {\n\treturn catalogEvidence(parseEvalsCatalog(evals), candidates, MODEL_SELECTION_EVALS_JSON_BYTES);\n}\n"]}
+{"version":3,"file":"model-routing-evals.js","sourceRoot":"","sources":["../../src/core/model-routing-evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAEtE,oEAAoE;AACpE,MAAM,CAAC,MAAM,gCAAgC,GAAG,MAAM,CAAC;AAEvD,MAAM,uBAAuB,GAAG,uBAAuB,CAAC;AACxD,MAAM,cAAc,GAAG,UAAU,CAAC;AAClC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IACjC,MAAM;IACN,WAAW;IACX,YAAY;IACZ,WAAW;IACX,MAAM;IACN,KAAK;IACL,MAAM;IACN,IAAI;IACJ,UAAU;IACV,aAAa;CACb,CAAC,CAAC;AACH,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;AAClE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC9B,KAAK;IACL,SAAS;IACT,KAAK;IACL,QAAQ;IACR,MAAM;IACN,OAAO;IACP,KAAK;IACL,UAAU;IACV,UAAU;IACV,WAAW;IACX,KAAK;IACL,QAAQ;IACR,SAAS;CACT,CAAC,CAAC;AACH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;AAEtE,2GAA2G;AAC3G,SAAS,oBAAoB,CAAC,MAAgB;IAC7C,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAAE,OAAO,MAAM,CAAC;IACnF,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1F,MAAM,GAAG,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;IACxD,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAC7B,OAAO,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED,kHAAkH;AAClH,MAAM,UAAU,mBAAmB,CAAC,EAAU;IAC7C,MAAM,UAAU,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,UAAU;SACtB,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SACtC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC;SACpC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC;SACnC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;SACtB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC1B,MAAM,MAAM,GAAG,KAAK;SAClB,KAAK,CAAC,QAAQ,CAAC;SACf,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/D,OAAO,oBAAoB,CAAC,MAAM,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,eAAe,GAAG,CAAC,KAAa,EAAE,EAAE,CACzC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAE1F,kGAAkG;AAClG,SAAS,oBAAoB,CAAC,IAAuB,EAAE,SAA4B;IAClF,IAAI,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5G,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;AAC5D,CAAC;AAED,iHAAiH;AACjH,MAAM,UAAU,oBAAoB,CAAC,IAAuB;IAC3D,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,OAAO,GAAG,GAAG,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC;QAAE,GAAG,EAAE,CAAC;IACzD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACrC,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,MAAyB;IAChD,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;IAC5B,IAAI,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC1B,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;QAC1C,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM;QACnG,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AA2BD,MAAM,YAAY,GAAG,kBAAkB,CAAC;AAExC,MAAM,UAAU,iBAAiB,CAAC,KAAa;IAC9C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;IAC1G,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,MAAM,IAAI,GAAiB,EAAE,CAAC;IAC9B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,KAAK,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,KAAK,GAAG,WAAW,CAAC;QACxB,KAAK,IAAI,KAAK,GAAG,WAAW,GAAG,CAAC,EAAE,KAAK,IAAI,gBAAgB,EAAE,KAAK,EAAE;YACnE,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBACrC,KAAK,GAAG,KAAK,CAAC;gBACd,MAAM;YACP,CAAC;QACF,IAAI,OAAO,KAAK,CAAC;YAAE,WAAW,GAAG,KAAK,CAAC;QACvC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACzE,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACrD,IAAI,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC;QAC5B,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YACvE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAClE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAAE,SAAS;YACxB,MAAM,OAAO,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACrE,IAAI,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,KAAK,CAAC,KAAK,CAAE;gBACnB,OAAO;gBACP,KAAK,EAAE,IAAI,CAAC,MAAM;gBAClB,MAAM,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrC,GAAG,CAAC,OAAO,IAAI,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACpF,CAAC,CAAC;QACJ,CAAC;QACD,gBAAgB,GAAG,KAAK,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;AACtF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAqB,EAAE,SAAiB;IAC7E,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC9C,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QACvD,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC;QAC1E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;YAClF,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;QACpC,CAAC;QACD,OAAO,EAAE,CAAC;IACX,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACxC,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,oBAAoB,CAAC,OAAqB,EAAE,SAAiB;IAC5E,MAAM,KAAK,GAAG,qBAAqB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrH,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACvD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAC9B,OAAqB,EACrB,UAA6B,EAC7B,QAAQ,GAAG,MAAM,CAAC,iBAAiB;IAEnC,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS;QAAE,OAAO,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,SAAS,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC;QAC1C,KAAK,MAAM,GAAG,IAAI,qBAAqB,CAAC,OAAO,EAAE,SAAS,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACtF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpC,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3D,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC;QAChE,IAAI,KAAK,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACpD,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,OAAO,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;AACzF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CAAC,KAAa,EAAE,UAA6B;IACrF,OAAO,eAAe,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,gCAAgC,CAAC,CAAC;AAChG,CAAC","sourcesContent":["import { jsonBytes, truncateToBytes } from \"./model-routing-bytes.js\";\n\n/** Bounds the evaluation evidence sent with one routing request. */\nexport const MODEL_SELECTION_EVALS_JSON_BYTES = 14_200;\n\nconst VENDOR_OR_REGION_PREFIX = /^(?:[a-z][a-z-]*\\.)+/u;\nconst SNAPSHOT_TOKEN = /^\\d{4}$/u;\nconst DEPLOYMENT_TOKENS = new Set([\n\t\"fast\",\n\t\"highspeed\",\n\t\"ultraspeed\",\n\t\"lightning\",\n\t\"beta\",\n\t\"exp\",\n\t\"free\",\n\t\"it\",\n\t\"instruct\",\n\t\"contributor\",\n]);\nconst ROW_EDITION_TOKENS = new Set([\"preview\", \"instruct\", \"it\"]);\nconst VARIANT_TOKENS = new Set([\n\t\"off\",\n\t\"minimal\",\n\t\"low\",\n\t\"medium\",\n\t\"high\",\n\t\"xhigh\",\n\t\"max\",\n\t\"adaptive\",\n\t\"thinking\",\n\t\"reasoning\",\n\t\"non\",\n\t\"effort\",\n\t\"preview\",\n]);\nconst CLAUDE_FAMILIES = new Set([\"opus\", \"sonnet\", \"haiku\", \"fable\"]);\n\n/** Artificial Analysis names older Claude models `claude-4-5-sonnet`; catalogs use `claude-sonnet-4-5`. */\nfunction canonicalClaudeOrder(tokens: string[]): string[] {\n\tif (tokens[0] !== \"claude\" || !CLAUDE_FAMILIES.has(tokens[1] ?? \"\")) return tokens;\n\tconst versionEnd = tokens.findIndex((token, index) => index > 1 && !/^\\d+$/u.test(token));\n\tconst end = versionEnd < 0 ? tokens.length : versionEnd;\n\tif (end === 2) return tokens;\n\treturn [\"claude\", ...tokens.slice(2, end), tokens[1]!, ...tokens.slice(end)];\n}\n\n/** Provider-independent model tokens: `us.anthropic.claude-opus-4-6-v1` and `anthropic/claude-opus-4.6` agree. */\nexport function modelEvidenceTokens(id: string): string[] {\n\tconst normalized = id.trim().toLowerCase();\n\tconst model = normalized\n\t\t.slice(normalized.lastIndexOf(\"/\") + 1)\n\t\t.replace(/^~/u, \"\")\n\t\t.replace(VENDOR_OR_REGION_PREFIX, \"\")\n\t\t.replace(/(?<=[a-z])-\\d+:\\d+$/u, \"\")\n\t\t.replace(/:[a-z0-9]+$/u, \"\")\n\t\t.replace(/-v\\d+$/u, \"\")\n\t\t.replace(/-\\d{8}$/u, \"\");\n\tconst tokens = model\n\t\t.split(/[-._]/u)\n\t\t.filter(Boolean)\n\t\t.map((token) => (token === \"thinking\" ? \"reasoning\" : token));\n\treturn canonicalClaudeOrder(tokens);\n}\n\nconst isVariantSuffix = (token: string) =>\n\tVARIANT_TOKENS.has(token) || ROW_EDITION_TOKENS.has(token) || SNAPSHOT_TOKEN.test(token);\n\n/** True when `slug` is the candidate model or one of its effort, edition or snapshot variants. */\nfunction slugMatchesCandidate(slug: readonly string[], candidate: readonly string[]): boolean {\n\tif (slug.length < candidate.length || candidate.some((token, index) => slug[index] !== token)) return false;\n\treturn slug.slice(candidate.length).every(isVariantSuffix);\n}\n\n/** One catalog model across its effort, edition and snapshot rows: `astra-high` and `astra-0902` are `astra`. */\nexport function catalogModelIdentity(slug: readonly string[]): string {\n\tlet end = slug.length;\n\twhile (end > 1 && isVariantSuffix(slug[end - 1]!)) end--;\n\treturn slug.slice(0, end).join(\"-\");\n}\n\n/**\n * The candidate's own tokens first, then the same model without trailing\n * deployment, reasoning-mode or snapshot suffixes (`gpt-5.4-fast`,\n * `grok-4.20-reasoning`, `qwen3.8-max-0902`). A fallback applies only when the\n * more specific form matched no row, so a model that is itself named `-fast`\n * keeps its own evidence.\n */\nfunction candidateForms(tokens: readonly string[]): string[][] {\n\tconst forms = [[...tokens]];\n\tlet current = [...tokens];\n\twhile (current.length > 1) {\n\t\tconst last = current[current.length - 1]!;\n\t\tif (!DEPLOYMENT_TOKENS.has(last) && !VARIANT_TOKENS.has(last) && !SNAPSHOT_TOKEN.test(last)) break;\n\t\tcurrent = current.slice(0, -1);\n\t\tforms.push(current);\n\t}\n\treturn forms;\n}\n\ninterface CatalogRow {\n\treadonly line: string;\n\treadonly section: number;\n\treadonly order: number;\n\treadonly tokens: readonly string[];\n\treadonly releaseDate?: string;\n}\n\n/** One `## ` section of `evals.md` that holds a model table keyed by `slug`. */\ninterface CatalogSection {\n\t/** The section heading, its notes and key, the table header and separator. */\n\treadonly intro: readonly string[];\n}\n\n/** `evals.md` parsed once so batch packing can query evidence per candidate cheaply. */\nexport interface EvalsCatalog {\n\t/** Text before the first table section: title, access date and the shared key. */\n\treadonly preamble: readonly string[];\n\treadonly sections: readonly CatalogSection[];\n\treadonly rows: readonly CatalogRow[];\n\t/** The unparsed document when it has no model table. */\n\treadonly raw?: string;\n\treadonly matches: Map<string, readonly CatalogRow[]>;\n}\n\nconst TABLE_HEADER = /^\\|\\s*slug\\s*\\|/u;\n\nexport function parseEvalsCatalog(evals: string): EvalsCatalog {\n\tconst lines = evals.split(\"\\n\");\n\tconst headers = lines.flatMap((line, index) => (TABLE_HEADER.test(line) ? [index] : []));\n\tif (headers.length === 0) return { preamble: [], sections: [], rows: [], raw: evals, matches: new Map() };\n\tconst sections: CatalogSection[] = [];\n\tconst rows: CatalogRow[] = [];\n\tlet preambleEnd = 0;\n\tlet previousTableEnd = 0;\n\tfor (const [section, headerIndex] of headers.entries()) {\n\t\tlet start = headerIndex;\n\t\tfor (let index = headerIndex - 1; index >= previousTableEnd; index--)\n\t\t\tif (lines[index]!.startsWith(\"## \")) {\n\t\t\t\tstart = index;\n\t\t\t\tbreak;\n\t\t\t}\n\t\tif (section === 0) preambleEnd = start;\n\t\tsections.push({ intro: lines.slice(start, headerIndex + 2) });\n\t\tconst header = lines[headerIndex]!.split(\"|\").map((cell) => cell.trim());\n\t\tconst releaseColumn = header.indexOf(\"Release date\");\n\t\tlet index = headerIndex + 2;\n\t\tfor (; index < lines.length && lines[index]!.startsWith(\"|\"); index++) {\n\t\t\tconst cells = lines[index]!.split(\"|\").map((cell) => cell.trim());\n\t\t\tif (!cells[1]) continue;\n\t\t\tconst release = releaseColumn > 0 ? cells[releaseColumn] : undefined;\n\t\t\trows.push({\n\t\t\t\tline: lines[index]!,\n\t\t\t\tsection,\n\t\t\t\torder: rows.length,\n\t\t\t\ttokens: modelEvidenceTokens(cells[1]),\n\t\t\t\t...(release && /^\\d{4}-\\d{2}-\\d{2}$/u.test(release) ? { releaseDate: release } : {}),\n\t\t\t});\n\t\t}\n\t\tpreviousTableEnd = index;\n\t}\n\treturn { preamble: lines.slice(0, preambleEnd), sections, rows, matches: new Map() };\n}\n\n/**\n * Rows describing `candidate`: in each section, its own model and variants, or\n * its base model when that section has none. Sections resolve independently, so\n * an exact snapshot row in one table does not hide the base model's rows in another.\n */\nexport function candidateEvidenceRows(catalog: EvalsCatalog, candidate: string): readonly CatalogRow[] {\n\tconst cached = catalog.matches.get(candidate);\n\tif (cached) return cached;\n\tconst forms = candidateForms(modelEvidenceTokens(candidate));\n\tconst matches = catalog.sections.flatMap((_, section) => {\n\t\tconst sectionRows = catalog.rows.filter((row) => row.section === section);\n\t\tfor (const form of forms) {\n\t\t\tconst found = sectionRows.filter((row) => slugMatchesCandidate(row.tokens, form));\n\t\t\tif (found.length > 0) return found;\n\t\t}\n\t\treturn [];\n\t});\n\tcatalog.matches.set(candidate, matches);\n\treturn matches;\n}\n\n/** Latest release date among the candidate's evidence rows, when any row has one. */\nexport function candidateReleaseDate(catalog: EvalsCatalog, candidate: string): string | undefined {\n\tconst dates = candidateEvidenceRows(catalog, candidate).flatMap((row) => (row.releaseDate ? [row.releaseDate] : []));\n\treturn dates.length ? dates.sort().at(-1) : undefined;\n}\n\n/**\n * The catalog preamble plus, for every section with rows for `candidates`, that\n * section's heading, key and table header followed by only those rows.\n * Sections without a matching row are left out. `maxBytes` bounds the\n * JSON-encoded result; batched routing omits it because its request budget\n * already decides how many candidates, and so how many rows, one batch holds.\n */\nexport function catalogEvidence(\n\tcatalog: EvalsCatalog,\n\tcandidates: readonly string[],\n\tmaxBytes = Number.POSITIVE_INFINITY,\n): string {\n\tif (catalog.raw !== undefined) return truncateToBytes(catalog.raw, maxBytes);\n\tconst selected = new Set<number>();\n\tfor (const candidate of new Set(candidates))\n\t\tfor (const row of candidateEvidenceRows(catalog, candidate)) selected.add(row.order);\n\tconst kept = catalog.rows.filter((row) => selected.has(row.order));\n\tconst parts = [...catalog.preamble];\n\tfor (const [index, section] of catalog.sections.entries()) {\n\t\tconst sectionRows = kept.filter((row) => row.section === index);\n\t\tif (index > 0 && sectionRows.length === 0) continue;\n\t\tif (index > 0 && parts.at(-1) !== \"\") parts.push(\"\");\n\t\tparts.push(...section.intro, ...sectionRows.map((row) => row.line));\n\t}\n\tconst filtered = parts.join(\"\\n\");\n\treturn jsonBytes(filtered) <= maxBytes ? filtered : truncateToBytes(filtered, maxBytes);\n}\n\n/**\n * Keep the catalog preamble and only the table rows that describe an eligible\n * candidate model, bounded to the routing evidence budget.\n */\nexport function filterModelSelectionEvals(evals: string, candidates: readonly string[]): string {\n\treturn catalogEvidence(parseEvalsCatalog(evals), candidates, MODEL_SELECTION_EVALS_JSON_BYTES);\n}\n"]}

package/docs/models/model-selection.md

@@ -14,7 +14,7 @@ For specific benchmark records, read [Evals](/models/evals). That page is the da
 Subagent and workflow-stage `model: "auto"` routes before execution starts, in at most two short requests:
 
 1. **Questions about the task, answered by a chat model.** Kind of work, difficulty, mistake cost, image input, very large context, and whether speed matters. The chat model is `routerModel` when that is a chat model, otherwise your current one; a classifier such as Jev never reads the task. A task too large for that model's window is read by the cheapest eligible model that holds it. Anything the caller states in `taskNeeds` is not asked, and when the caller states all six this request is skipped. Computer-use tasks count as needing images unless the caller says otherwise.
-2. **A choice between a shortlist, made by the router without the task.** Atomic narrows the eligible models in code: it drops models that cannot read images when the task needs them and prefers 400k-token windows for long-context tasks, then ranks the rest on the [Evals](/models/evals) results for that kind of work, weighing proven quality against price by how demanding the task is and preferring newer releases. The top six different models form the shortlist; a model's fast route and the same model on another provider share one place. When the caller lists models in `modelConstraints.allowedModels`, those eligible models are the shortlist instead (as many as fit one routing request). Each option carries its own evidence (release date, price tier, image input, and how it was measured on this kind of work and overall), ranked against every model you could route to. With a single option, this request is skipped.
+2. **A choice between a shortlist, made by the router without the task.** Atomic narrows the eligible models in code: it drops models that cannot read images when the task needs them and prefers 400k-token windows for long-context tasks, then ranks the rest on the [Evals](/models/evals) results for that kind of work, weighing proven quality against price by how demanding the task is and preferring newer releases. The top six different models form the shortlist; a model's fast route and the same model on another provider share one place. When the caller lists models in `modelConstraints.allowedModels`, those eligible models are the shortlist instead (as many as fit one routing request). Each option carries its own evidence (release date, price tier, image input, and results on this kind of work and overall), ranked against all Evals models however few you enable. With a single option, this request is skipped.
 
 Effort follows the task's difficulty, one level lower when speed matters, limited to the levels the chosen model supports. The two fallbacks are the next models in Atomic's ranking. The router cannot add candidates, bypass constraints, alter the execution prompt, or change the selected chat model. [`routerModel`](/settings#routermodel) chooses the decision model only: an explicit registered classifier or chat language model. Unset and `auto` use the current chat model, regardless of saved classifier credentials. An image-generation model cannot decide.
 

package/docs/workflows/operations.md

@@ -702,7 +702,7 @@ Validation uses the final retained transcript for a repeated stage replay key, s
 
 After `workflow({ action: "pause", runId, stageId: "tool:<argsHash>" })`, inspect the run. If the author did not catch cancellation, its terminal status is `failed`, its selected frontier is `failedToolNodeId`, and the tool node is `cancelled`. Use `/workflow resume <full-run-uuid>` or `workflow({ action: "resume", runId: "<full-run-uuid>" })` **without `stageId`**. Completed model and tool checkpoints replay without invoking their callbacks; only the unfinished tool and subsequent work execute. This does not roll back external work a cancelled callback performed before abort, so verify that retrying that unfinished operation is safe.
 
-Recovery requires saved checkpoints proving one unfinished tool and its completed predecessors. Missing, ambiguous, corrupt, or cyclic state fails with `insufficient_state`. Chat text and status snapshots cannot substitute for checkpoints. Preserve the original diagnostic and run ID when reporting a failure.
+Recovery requires saved checkpoints proving one unfinished tool and its completed predecessors. When the failed tool ran in parallel (for example inside `Promise.all`), siblings that were cancelled because the run failed, or that threw in the same batch, are run again on resume and may be called before the failed tool. Any other tool reached before the failed tool is rejected before its callback runs, and the failed tool is not run either, so the run stays resumable once the original flow is restored. Missing, ambiguous, corrupt, or cyclic state fails with `insufficient_state`. Chat text and status snapshots cannot substitute for checkpoints. Preserve the original diagnostic and run ID when reporting a failure.
 
 A tool-frontier continuation must reach and consume the exact unfinished tool before reporting `completed`, whether the body returns normally or calls `ctx.exit({ status: "completed" })`. Omitting the call, including by changing control flow, fails with `insufficient_state: replay topology mismatch` and the pending tool's exact ID. A substituted tool is rejected before its callback runs. While that frontier is pending, completed model and child-workflow predecessors can replay, but new model stages, tasks, and child workflows cannot execute in its place. Stage and task worktree setup also waits for live admission. Replaying completed predecessors alone is not proof of completion. Failed replay publishes no successful result; inspect the same ID and restore the matching flow before another supported resume. Quit, kill, caught targeted cancellation, and intentional non-completed exits retain their existing behavior.