agent-nuvira 2.7.2 → 3.0.0

package/dist/learning/auto-router.js

@@ -44,14 +44,28 @@ import { getRouterBandit, DEFAULT_MIN_SAMPLES } from './router-bandit.js';
 import { getRouterPromotion, DEFAULT_MIN_PROMOTION_DECISIONS } from './router-promotion.js';
 import { getMlRouter, DEFAULT_ML_K, DEFAULT_ML_MIN_SAMPLES, DEFAULT_ML_STRENGTH } from './ml-router.js';
 import { buildModelCandidates } from './model-first-router.js';
+import { pickBestModel, topModelCandidates } from './model-scoring.js';
 import { getModelRegistry } from './model-registry.js';
+// R4: the agentic capability floor needs the same "too small to hold a tool
+// loop" judgement the harness uses. model-harness.ts imports nothing, so this
+// adds no cycle risk to the router.
+import { isTinyModel } from './model-harness.js';
 import { estimateTokens } from './cost-tracker.js';
 import { preferredModelsFor, PROVIDER_CONTEXT_WINDOWS } from './model-selection.js';
+import { isNonChatModel } from '../inference/model-catalog.js';
 import { CATALOG_PROVIDER_IDS, getCatalogProvider, getDefaultModel, isCatalogKeyless } from '../inference/provider-catalog.js';
+import { ProviderFactory } from '../inference/factory.js';
 import { logger } from '../utils/logger.js';
 // ─── Constants ──────────────────────────────────────────────────────────────
 /** The special model value that triggers automatic per-task routing. */
 export const AUTO_MODEL = 'auto';
+/**
+ * How many models ONE provider contributes to the failover chain (DEEP
+ * FAILOVER). More than one so a provider's 2nd/3rd-best model is actually
+ * reachable — per-model RPD/TPM limits on free tiers mean a 429 on one model
+ * says nothing about its siblings.
+ */
+export const FALLBACK_MODELS_PER_PROVIDER = 3;
 /** The special provider value stored in active-model state for Auto mode. */
 export const AUTO_PROVIDER = 'auto';
 /** Human labels for each dimension (used in explanations). */
@@ -621,7 +635,7 @@ export class AutoModelRouter {
      */
     getDefaultAllowedProviders(configManager) {
         if (!configManager || typeof configManager.hasRequiredCredentials !== 'function') {
-            return { allowed: DEFAULT_AUTO_PROVIDERS, excluded: [] };
+            return { allowed: DEFAULT_AUTO_PROVIDERS, excluded: [], reserve: [] };
         }
         // ── Registry-aware filtering: prefer providers with VERIFIED, usable
         // models over bare credential checks. A key existing ≠ the models on that
@@ -654,6 +668,16 @@ export class AutoModelRouter {
         }
         const registered = registry.getUsableProviders();
         const base = [...new Set([...CATALOG_PROVIDER_IDS, ...configuredIds])].filter((p) => {
+            // A provider with no adapter can never be resolved: `resolveProvider`
+            // refuses to substitute another provider for it (see router.ts), so a
+            // candidate like this is guaranteed to fail — and leaving it in is how a
+            // credentialed-but-unconstructible id (`bedrock` is in the catalog but
+            // has no adapter) reached the failover walk, silently resolved to a
+            // DIFFERENT provider, and mislabeled that provider's models as its own.
+            if (!ProviderFactory.isConstructible(p)) {
+                excluded.push({ provider: p, reason: 'no adapter in this build' });
+                return false;
+            }
             if (blocked.has(p) || degraded.has(p)) {
                 // Cite the registry's own counts so the explanation proves the data
                 // is working ("openrouter excluded — 0 verified, 6 unavailable"). The
@@ -688,14 +712,19 @@ export class AutoModelRouter {
         if (registered.length > 0) {
             const intersection = base.filter((p) => registered.includes(p));
             if (intersection.length > 0) {
-                return { allowed: intersection, excluded };
+                // Verified providers win the primary ranking, but the credentialed-yet-
+                // unverified rest is NOT thrown away — it becomes the reserve fallback
+                // pool, so a task still reaches those models when every verified
+                // candidate fails ("only reject when nothing is left").
+                const reserve = base.filter((p) => !registered.includes(p));
+                return { allowed: intersection, excluded, reserve };
             }
         }
         // Never return an empty list: if EVERY configured provider is
         // registry-blocked (pathological), fall back to the full built-in list so
         // the caller still gets a decision and surfaces availability instead of
         // crashing on an empty ranking.
-        return { allowed: base.length > 0 ? base : DEFAULT_AUTO_PROVIDERS, excluded };
+        return { allowed: base.length > 0 ? base : DEFAULT_AUTO_PROVIDERS, excluded, reserve: [] };
     }
     /**
      * Resolve the optimal provider/model for a task.
@@ -796,6 +825,7 @@ export class AutoModelRouter {
         // proves the gathered data is driving decisions. Explicit allowedProviders
         // opt out of the registry filter entirely (caller knows best).
         let registryExcluded = [];
+        let reserveProviders = [];
         let allowed;
         if (options.allowedProviders?.length) {
             allowed = options.allowedProviders;
@@ -804,6 +834,7 @@ export class AutoModelRouter {
             const defaultPool = this.getDefaultAllowedProviders(configManager);
             allowed = defaultPool.allowed;
             registryExcluded = defaultPool.excluded;
+            reserveProviders = defaultPool.reserve;
         }
         let escalationApplied = false;
         let allowedProviders = allowed;
@@ -1169,6 +1200,52 @@ export class AutoModelRouter {
                 }
             }
         }
+        // ── Agentic capability floor (R4) ──────────────────────────────────────
+        // Effective capability is `model × (1 − harness tax)`: a multi-step agentic
+        // task needs the served model to hold a tool loop across turns. A model too
+        // small for that is not "a bit worse" here, it is a doomed run — it burns
+        // steps, emits malformed tool calls, and then gets BOOKED as a model failure,
+        // teaching the bandit a lesson about the model when the mismatch was ours
+        // (auto handed an agentic task to a model that cannot do one). This is the
+        // routing-side half of R1: the harness fits the model, and the router stops
+        // feeding the harness a model it cannot fit.
+        //
+        // ORDERING (why this runs AFTER the hard constraints): the constraint slot
+        // above owns explicit user/admin policy (maxCostUsd/minSpeed/minReasoning/
+        // governance) and its benign "eliminated everyone" fallback restores the RAW
+        // ranking. Running this floor first made that fallback restore the floor's
+        // survivors instead, so an impossible minReasoning ask could still be served
+        // a provider the user had just ruled out. Policy decides who is eligible;
+        // this heuristic only narrows the eligible set afterwards.
+        //
+        // Deliberately conservative, exactly like the free/local gate above:
+        //   - only the SERVED model is judged (a provider hosting both a tiny and a
+        //     strong model keeps its strong entry);
+        //   - an UNKNOWN served model is kept (never eliminate on ignorance — the
+        //     JSON-contract transport still gives it a way to act);
+        //   - if the floor would eliminate everyone, the ranking is left alone rather
+        //     than returning no route at all. Auto must never dead-end.
+        const agenticTask = complexity === 'complex' ||
+            complexity === 'critical' ||
+            taskProfile.requiresVerification === true;
+        if (agenticTask) {
+            const capable = scored.filter((s) => {
+                let servedModel;
+                try {
+                    servedModel = this.resolveModel(s.provider, agentType, configManager, taskDescription);
+                }
+                catch {
+                    servedModel = undefined;
+                }
+                return servedModel === undefined || !isTinyModel(servedModel);
+            });
+            if (capable.length > 0) {
+                scored = capable;
+            }
+            else if (options.verbose) {
+                logger.warn('  ⚠️ agentic capability floor would eliminate every provider — keeping the full ranking');
+            }
+        }
         // Rank: circuit-breaker-cooldown providers sink first, then quota-parked
         // ones, then healthy ones; ties broken by score. A quota-parked provider
         // is only selected when every candidate is parked (matching cooldown).
@@ -1372,17 +1449,121 @@ export class AutoModelRouter {
             getRouterPromotion().noteParallelDecision(agentType, taskDescription, this.toParallelPick(heuristicWinner, agentType, configManager), this.toParallelPick(selected, agentType, configManager, model));
         }
         // Build fallback chain (skip in-cooldown providers when alternatives exist)
-        const fallbackChain = scored
-            .filter((s) => s.provider !== provider)
-            .map((s) => ({
-            provider: s.provider,
-            model: this.resolveModel(s.provider, agentType, configManager, taskDescription),
-            estimatedCost: 0,
-            qualityScore: s.score,
-            contextWindowTokens: s.contextWindowTokens,
-            reason: s.inCooldown ? `Fallback (in cooldown): ${s.provider}` : `Fallback: ${s.provider}`,
-        }));
-        const explanation = this.buildExplanation(agentType, complexity, taskType, selected, mode, model, weights, taskProfile) + (routedBy === 'bandit' ? ' | bandit-learned' : '') +
+        //
+        // DEEP FAILOVER: every provider contributes MULTIPLE models, not one. Before
+        // this the chain carried a single resolveModel() pick per provider, so a
+        // provider's 2nd-best model was effectively unreachable — a 429 on the one
+        // listed model skipped the provider entirely even though its other models
+        // were healthy (free tiers meter per-model RPD/TPM, so siblings usually ARE
+        // healthy). Ordering: the PRIMARY provider's alternate models first (stay on
+        // the provider whose latency/health is already known, no provider switch
+        // cost), then the other providers' primary picks, then their alternates, so
+        // the chain both uses every model AND still leaves the provider before
+        // burning through all of its fallbacks.
+        const fallbackChain = [];
+        const seenFallback = new Set();
+        const pushFallback = (s, model, alternate) => {
+            const key = `${s.provider}|${model}`;
+            if (seenFallback.has(key))
+                return;
+            seenFallback.add(key);
+            fallbackChain.push({
+                provider: s.provider,
+                model,
+                estimatedCost: 0,
+                qualityScore: s.score,
+                contextWindowTokens: s.contextWindowTokens,
+                reason: alternate
+                    ? `Fallback (alternate model on ${s.provider})`
+                    : s.inCooldown
+                        ? `Fallback (in cooldown): ${s.provider}`
+                        : `Fallback: ${s.provider}`,
+            });
+        };
+        const modelsFor = (s) => this.fallbackModelsFor(s.provider, agentType, configManager, taskDescription, complexity);
+        // Pass 1: the primary provider's OTHER models (deep failover within the
+        // provider that just failed a call).
+        for (const s of scored) {
+            if (s.provider !== provider)
+                continue;
+            const models = modelsFor(s).slice(1); // [0] is the primary pick already in use
+            for (const m of models)
+                pushFallback(s, m, true);
+        }
+        // Pass 2: every other provider's best model (historical chain order).
+        for (const s of scored) {
+            if (s.provider === provider)
+                continue;
+            const models = modelsFor(s);
+            if (models.length === 0)
+                continue;
+            pushFallback(s, models[0], false);
+        }
+        // Pass 3: those providers' remaining models.
+        for (const s of scored) {
+            if (s.provider === provider)
+                continue;
+            const models = modelsFor(s);
+            for (const m of models.slice(1))
+                pushFallback(s, m, true);
+        }
+        // Last-resort reserve: credentialed providers the registry has not verified
+        // (no data yet). They never enter `ranked`/the primary pick — they are tried
+        // only after every ranked fallback is exhausted, so "use every model we can
+        // actually call, reject only when nothing is left" holds without routing
+        // into unproven models by default.
+        for (const p of reserveProviders) {
+            if (p === provider)
+                continue;
+            if (fallbackChain.some((c) => c.provider === p))
+                continue;
+            fallbackChain.push({
+                provider: p,
+                model: this.resolveModel(p, agentType, configManager, taskDescription),
+                estimatedCost: 0,
+                qualityScore: 0,
+                reason: `Fallback (unverified — last resort): ${p}`,
+            });
+        }
+        // Pass 4 (reserve, deep): the reserve providers' alternate models too, so a
+        // last-resort credential pool is not limited to one model per provider.
+        for (const p of reserveProviders) {
+            if (p === provider)
+                continue;
+            for (const m of this.fallbackModelsFor(p, agentType, configManager, taskDescription, complexity).slice(1)) {
+                const key = `${p}|${m}`;
+                if (seenFallback.has(key))
+                    continue;
+                seenFallback.add(key);
+                fallbackChain.push({
+                    provider: p,
+                    model: m,
+                    estimatedCost: 0,
+                    qualityScore: 0,
+                    reason: `Fallback (unverified, alternate model — last resort): ${p}`,
+                });
+            }
+        }
+        // ── Present the FINAL decision consistently ──────────────────────────────
+        // Model-first routing (and per-model learning) can select a provider/model
+        // that differs from the deterministic `selected`. Before this, the rationale
+        // line named the PRE-override provider, `decision.score` was that loser's
+        // score, and the ✅ marker could sit on a row ranked below #1 — so
+        // `models explain`'s rationale, Decision line and ranked table disagreed
+        // (`Decision: gemini/gemini-3.1-flash-lite` beside a `groq/…` rationale).
+        // Promote the FINAL provider's entry to the head of the ranked list and
+        // present ITS score everywhere the decision is shown. `selected` itself is
+        // left untouched — the promotion gate's A/B record must keep the true
+        // deterministic/bandit pick.
+        let winner = scored.find((s) => s.provider === provider);
+        if (!winner) {
+            // A model-first pick can name a provider with no scored entry only in
+            // pathological cases; synthesize one so presentation never breaks.
+            winner = { ...selected, provider, score: selected.score, reason: `${provider}: model-first pick` };
+        }
+        scored = [winner, ...scored.filter((s) => s.provider !== provider)];
+        const decisionScore = winner.score;
+        const explanation = this.buildExplanation(agentType, complexity, taskType, provider, decisionScore, mode, model, weights, taskProfile) + (routedBy === 'bandit' ? ' | bandit-learned' : '') +
             (banditEscalation ? ' | escalated: winner unlearned' : '') +
             // ISSUE-002 explanation transparency: cite the registry data that
             // excluded providers from the candidate pool, so auto routing proves
@@ -1427,7 +1608,7 @@ export class AutoModelRouter {
             taskType,
             provider,
             model,
-            score: selected.score,
+            score: decisionScore,
             weights,
             ranked: scored,
             fallbackChain,
@@ -1572,7 +1753,6 @@ export class AutoModelRouter {
         // This ensures the bandit learns from the BEST models, not just verified ones.
         if (taskDescription) {
             try {
-                const { topModelCandidates } = require('./model-scoring.js');
                 const scored = topModelCandidates(provider, taskDescription, 10, complexity);
                 for (const s of scored) {
                     if (!candidates.includes(s.model))
@@ -1710,6 +1890,66 @@ export class AutoModelRouter {
         }
         return undefined;
     }
+    /**
+     * The ranked model list a provider can serve this task with, best first.
+     *
+     * Always starts with the model the router would actually pick, then the
+     * provider's other task-scored candidates, then its remaining health-ranked
+     * verified models. This is what makes the failover chain DEEP (several models
+     * per provider) instead of one: a provider with five working models must be
+     * able to serve from all five, not give up after its first rate limit.
+     *
+     * Never returns the `'default'` SENTINEL or a non-chat model (a probe can
+     * verify a classifier/embedding/vision model, which can never answer a turn).
+     * Best-effort — scoring failures degrade to the pick + verified list.
+     */
+    fallbackModelsFor(provider, agentType, configManager, taskDescription, complexity) {
+        const out = [];
+        const push = (model) => {
+            if (!model || model === 'default')
+                return;
+            if (out.includes(model))
+                return;
+            if (isNonChatModel(model))
+                return;
+            out.push(model);
+        };
+        push(this.resolveModel(provider, agentType, configManager, taskDescription));
+        if (taskDescription) {
+            try {
+                for (const c of topModelCandidates(provider, taskDescription, FALLBACK_MODELS_PER_PROVIDER, complexity)) {
+                    push(c.model);
+                }
+            }
+            catch {
+                // Best-effort — model scoring must never break the chain.
+            }
+        }
+        for (const m of preferredModelsFor(provider))
+            push(m);
+        // HEALTHY-FIRST, but never dropped: a model the registry currently deems
+        // unusable (parked on its own quota / marked unavailable) must not outrank
+        // a healthy sibling, yet it stays in the list as the LAST resort — parks
+        // are short and the model may well have recovered by the time the walk
+        // gets to it ("use every model available; reject only when nothing is
+        // left"). Stable partition preserves the scoring order within each group.
+        const usable = [];
+        const unusable = [];
+        try {
+            const registry = getModelRegistry();
+            for (const m of out) {
+                if (registry.getEntry(provider, m) && !registry.isUsable(provider, m))
+                    unusable.push(m);
+                else
+                    usable.push(m);
+            }
+        }
+        catch {
+            // Best-effort — a registry failure must not empty the chain.
+            return out.slice(0, FALLBACK_MODELS_PER_PROVIDER);
+        }
+        return [...usable, ...unusable].slice(0, FALLBACK_MODELS_PER_PROVIDER);
+    }
     /**
      * Resolve the model name to use within a chosen provider.
      * Prefers the provider's configured model; falls back to 'default'.
@@ -1760,10 +2000,26 @@ export class AutoModelRouter {
         // hardcoded default.
         if (taskDescription) {
             try {
-                const { pickBestModel } = require('./model-scoring.js');
                 const bestModel = pickBestModel(provider, taskDescription);
-                if (bestModel)
+                if (bestModel) {
+                    // USABILITY GATE: a task-scored model can still be one the registry
+                    // has parked (its own quota window) or marked unavailable. It must
+                    // never win the PRIMARY pick while a servable sibling exists —
+                    // scoring alone ranks the parked model low, not out. The registry
+                    // re-admits it automatically once its window lapses, so this is the
+                    // per-model quota contract end to end: skip the resting model, serve
+                    // from a sibling, and pick the resting one again when it recovers.
+                    const registry = getModelRegistry();
+                    const entry = registry.getEntry(provider, bestModel);
+                    if (!entry || registry.isUsable(provider, bestModel))
+                        return bestModel;
+                    const usable = preferredModelsFor(provider)[0];
+                    if (usable)
+                        return usable;
+                    // Nothing usable — fall through to the parked pick (the park may have
+                    // lapsed, and a rejected pick is better than no model at all).
                     return bestModel;
+                }
             }
             catch {
                 // Fall through to preferred models
@@ -1872,7 +2128,12 @@ export class AutoModelRouter {
         return parts.length ? `${provider}: ${parts.join(', ')}` : `${provider}: adequate for ${complexity} complexity`;
     }
     /** Build the human-readable decision explanation. */
-    buildExplanation(agentType, complexity, taskType, selected, mode, model, weights, taskProfile) {
+    buildExplanation(agentType, complexity, taskType, 
+    /** The FINAL provider id (post model-first override), so the rationale can
+     *  never name a different provider than the Decision line. */
+    provider, 
+    /** The FINAL decision score (the promoted winner entry's score). */
+    score, mode, model, weights, taskProfile) {
         const complexityLabels = {
             trivial: '🟢 trivial',
             simple: '🔵 simple',
@@ -1884,8 +2145,8 @@ export class AutoModelRouter {
             .reduce((a, b) => (weights[b] > weights[a] ? b : a), 'reasoning');
         const modeStr = mode !== 'balanced' ? ` | ${mode}` : '';
         const profileSuffix = taskProfile.requiresVerification ? ' | verification' : '';
-        return `${agentType} (${complexityLabels[complexity]}, ${taskType}) → ${selected.provider}/${model} ` +
-            `score ${selected.score.toFixed(2)} | dominant: ${DIMENSION_LABELS[dominant]}${modeStr}${profileSuffix}`;
+        return `${agentType} (${complexityLabels[complexity]}, ${taskType}) → ${provider}/${model} ` +
+            `score ${score.toFixed(2)} | dominant: ${DIMENSION_LABELS[dominant]}${modeStr}${profileSuffix}`;
     }
 }
 // ─── Singleton ──────────────────────────────────────────────────────────────