@jsonstudio/llms 0.6.954 → 0.6.1172

package/dist/router/virtual-router/engine-selection.js

@@ -1,3 +1,5 @@
+import { computeContextMultiplier, computeEffectiveSafeWindowTokens, resolveContextWeightedConfig } from './context-weighted.js';
+import { computeHealthWeight, resolveHealthWeightedConfig } from './health-weighted.js';
 import { DEFAULT_ROUTE, ROUTE_PRIORITY, VirtualRouterError, VirtualRouterErrorCode } from './types.js';
 export function selectProviderImpl(requestedRoute, metadata, classification, features, activeState, deps, options = {}) {
     const state = options.routingState ?? activeState;
@@ -294,12 +296,13 @@ function trySelectFromTier(routeName, tier, stickyKey, estimatedTokens, features
     if (excludedKeys.size > 0) {
         targets = targets.filter((key) => !excludedKeys.has(key));
     }
+    const isRecoveryAttempt = excludedKeys.size > 0;
+    const singleCandidateFallback = targets.length === 1 ? targets[0] : undefined;
     if (targets.length > 0) {
-        const cooled = targets.filter((key) => !deps.isProviderCoolingDown(key));
-        // 单 provider 兜底：当一个 tier 只有一个候选 key 时，不因 cooldown 造成路由池为空。
-        if (cooled.length > 0 || targets.length !== 1) {
-            targets = cooled;
-        }
+        // Always respect cooldown signals. If a route/tier is depleted due to cooldown,
+        // routing is expected to fall back to other tiers/routes (e.g. longcontext → default),
+        // rather than repeatedly selecting the cooled-down provider.
+        targets = targets.filter((key) => !deps.isProviderCoolingDown(key));
     }
     if (allowedProviders && allowedProviders.size > 0) {
         targets = targets.filter((key) => {
@@ -399,6 +402,10 @@ function trySelectFromTier(routeName, tier, stickyKey, estimatedTokens, features
     const prioritizedPools = buildContextCandidatePools(contextResult);
     const quotaView = deps.quotaView;
     const now = quotaView ? Date.now() : 0;
+    const healthWeightedCfg = resolveHealthWeightedConfig(deps.loadBalancer.getPolicy().healthWeighted);
+    const contextWeightedCfg = resolveContextWeightedConfig(deps.loadBalancer.getPolicy().contextWeighted);
+    const warnRatio = deps.contextAdvisor.getConfig().warnRatio;
+    const nowForWeights = Date.now();
     const selectFirstAvailable = (candidates) => {
         for (const key of candidates) {
             if (deps.healthManager.isAvailable(key)) {
@@ -407,32 +414,148 @@ function trySelectFromTier(routeName, tier, stickyKey, estimatedTokens, features
         }
         return null;
     };
-    const selectWithQuota = (candidates) => {
+    const resolvePriorityMeta = (orderedTargets) => {
+        // Priority mode semantics (strict group priority + alias-level balancing):
+        // - Targets are interpreted as ordered (providerId, modelId) groups.
+        // - Group base priorities: 100, 90, 80, ... (step=10) by appearance order.
+        // - Within a group (different auth aliases), base scores: 100, 99, 98, ... (step=1).
+        //
+        // Group selection is strict: always use the best group until it is unavailable.
+        // Alias selection is balanced within the chosen group (RR / health-weighted / context-weighted).
+        const meta = new Map();
+        if (!Array.isArray(orderedTargets) || orderedTargets.length === 0) {
+            return meta;
+        }
+        let groupIndex = -1;
+        let aliasOffset = 0;
+        let lastGroupKey = '';
+        for (const key of orderedTargets) {
+            const providerId = extractProviderId(key) ?? '';
+            const modelId = getProviderModelId(key, deps.providerRegistry) ?? '';
+            const groupKey = `${providerId}::${modelId}`;
+            if (groupKey !== lastGroupKey) {
+                groupIndex += 1;
+                aliasOffset = 0;
+                lastGroupKey = groupKey;
+            }
+            const groupBase = 100 - groupIndex * 10;
+            const base = groupBase - aliasOffset;
+            meta.set(key, { groupId: `${providerId}.${modelId}`, groupBase, base });
+            aliasOffset += 1;
+        }
+        return meta;
+    };
+    const pickPriorityGroup = (candidates, orderedTargets, penalties) => {
+        const meta = resolvePriorityMeta(orderedTargets);
+        let bestGroupId = null;
+        let bestScore = Number.NEGATIVE_INFINITY;
+        for (const key of candidates) {
+            if (!deps.healthManager.isAvailable(key))
+                continue;
+            const m = meta.get(key);
+            if (!m)
+                continue;
+            const penalty = penalties ? Math.max(0, Math.floor(penalties[key] ?? 0)) : 0;
+            const score = m.base - penalty;
+            if (score > bestScore) {
+                bestScore = score;
+                bestGroupId = m.groupId;
+            }
+        }
+        if (!bestGroupId)
+            return null;
+        const groupCandidates = candidates.filter((key) => meta.get(key)?.groupId === bestGroupId);
+        return groupCandidates.length ? { groupId: bestGroupId, groupCandidates } : null;
+    };
+    const computeContextWeightMultipliers = (candidates) => {
+        if (!contextWeightedCfg.enabled) {
+            return null;
+        }
+        const eff = {};
+        let ref = 1;
+        for (const key of candidates) {
+            const usage = contextResult.usage?.[key];
+            const limit = usage && typeof usage.limit === 'number' && Number.isFinite(usage.limit) ? Math.floor(usage.limit) : 0;
+            const safeEff = computeEffectiveSafeWindowTokens({
+                modelMaxTokens: Math.max(1, limit),
+                warnRatio,
+                clientCapTokens: contextWeightedCfg.clientCapTokens
+            });
+            eff[key] = safeEff;
+            if (safeEff > ref) {
+                ref = safeEff;
+            }
+        }
+        return { ref, eff };
+    };
+    const selectWithQuota = (candidates, isSafePool) => {
         if (!quotaView) {
             if (tier.mode === 'priority') {
-                const selected = selectFirstAvailable(candidates);
-                if (!selected && candidates.length === 1) {
-                    return candidates[0];
+                if (isRecoveryAttempt) {
+                    return selectFirstAvailable(candidates);
+                }
+                const group = pickPriorityGroup(candidates, tier.targets);
+                if (!group) {
+                    return null;
                 }
-                return selected;
+                const weights = (() => {
+                    if (!isSafePool)
+                        return undefined;
+                    const ctx = computeContextWeightMultipliers(group.groupCandidates);
+                    if (!ctx)
+                        return undefined;
+                    const out = {};
+                    for (const key of group.groupCandidates) {
+                        const m = computeContextMultiplier({
+                            effectiveSafeRefTokens: ctx.ref,
+                            effectiveSafeTokens: ctx.eff[key] ?? 1,
+                            cfg: contextWeightedCfg
+                        });
+                        out[key] = Math.max(1, Math.round(100 * m));
+                    }
+                    return out;
+                })();
+                return deps.loadBalancer.select({
+                    routeName: `${routeName}:${tier.id}:priority:group:${group.groupId}`,
+                    candidates: group.groupCandidates,
+                    stickyKey: options.allowAliasRotation ? undefined : stickyKey,
+                    weights,
+                    availabilityCheck: (key) => deps.healthManager.isAvailable(key)
+                }, 'round-robin');
             }
+            const weights = (() => {
+                if (!isSafePool || !contextWeightedCfg.enabled)
+                    return undefined;
+                const ctx = computeContextWeightMultipliers(candidates);
+                if (!ctx)
+                    return undefined;
+                const out = {};
+                for (const key of candidates) {
+                    const m = computeContextMultiplier({
+                        effectiveSafeRefTokens: ctx.ref,
+                        effectiveSafeTokens: ctx.eff[key] ?? 1,
+                        cfg: contextWeightedCfg
+                    });
+                    out[key] = Math.max(1, Math.round(100 * m));
+                }
+                return out;
+            })();
             const selected = deps.loadBalancer.select({
                 routeName: `${routeName}:${tier.id}`,
                 candidates,
                 stickyKey: options.allowAliasRotation ? undefined : stickyKey,
+                weights,
                 availabilityCheck: (key) => deps.healthManager.isAvailable(key)
             }, tier.mode === 'round-robin' ? 'round-robin' : undefined);
-            if (!selected && candidates.length === 1) {
-                return candidates[0];
-            }
             return selected;
         }
         const buckets = new Map();
+        let order = 0;
         for (const key of candidates) {
             const entry = quotaView(key);
             if (!entry) {
                 const list = buckets.get(100) ?? [];
-                list.push(key);
+                list.push({ key, penalty: 0, order: order++ });
                 buckets.set(100, list);
                 continue;
             }
@@ -448,50 +571,169 @@ function trySelectFromTier(routeName, tier, stickyKey, estimatedTokens, features
             const tierPriority = typeof entry.priorityTier === 'number' && Number.isFinite(entry.priorityTier)
                 ? entry.priorityTier
                 : 100;
+            const penaltyRaw = entry.selectionPenalty;
+            const penalty = typeof penaltyRaw === 'number' && Number.isFinite(penaltyRaw) && penaltyRaw > 0 ? Math.floor(penaltyRaw) : 0;
             const list = buckets.get(tierPriority) ?? [];
-            list.push(key);
+            list.push({ key, penalty, order: order++ });
             buckets.set(tierPriority, list);
         }
         const sortedPriorities = Array.from(buckets.keys()).sort((a, b) => a - b);
         for (const priority of sortedPriorities) {
-            const bucketCandidates = buckets.get(priority) ?? [];
-            if (!bucketCandidates.length) {
+            const bucket = buckets.get(priority) ?? [];
+            if (!bucket.length) {
                 continue;
             }
-            if (tier.mode === 'priority') {
-                const selected = selectFirstAvailable(bucketCandidates);
-                if (selected) {
-                    return selected;
+            bucket.sort((a, b) => (a.penalty - b.penalty) || (a.order - b.order));
+            const bucketCandidates = bucket.map((item) => item.key);
+            // antigravity special: avoid rotating across keys while the current key is healthy.
+            // Rationale: some upstream gateways reject rapid cross-key switching even when quota exists,
+            // causing repeated 429s. We therefore pin a single key per (providerId, modelId) until it is
+            // excluded by quota/cooldown, then fail over to the next available key.
+            //
+            // This is only applied when the request has no session-level sticky key, to avoid breaking
+            // explicit session stickiness.
+            const shouldPinAntigravityModel = (() => {
+                // Only respect explicit session/conversation stickiness. requestId-scoped sticky keys
+                // (used for request-chain pinning) should not prevent global antigravity key pinning.
+                if (typeof stickyKey === 'string' && (stickyKey.startsWith('session:') || stickyKey.startsWith('conversation:'))) {
+                    return false;
+                }
+                if (bucketCandidates.length < 2) {
+                    return false;
+                }
+                let modelId = null;
+                for (const key of bucketCandidates) {
+                    const providerId = extractProviderId(key);
+                    if (providerId !== 'antigravity') {
+                        return false;
+                    }
+                    const candidateModel = getProviderModelId(key, deps.providerRegistry);
+                    if (!candidateModel) {
+                        return false;
+                    }
+                    if (modelId === null) {
+                        modelId = candidateModel;
+                    }
+                    else if (modelId !== candidateModel) {
+                        return false;
+                    }
+                }
+                return Boolean(modelId);
+            })();
+            if (shouldPinAntigravityModel && !isRecoveryAttempt) {
+                const pinned = selectFirstAvailable(bucketCandidates);
+                if (pinned) {
+                    return pinned;
                 }
             }
-            else {
-                const selected = deps.loadBalancer.select({
-                    routeName: `${routeName}:${tier.id}`,
-                    candidates: bucketCandidates,
-                    stickyKey: options.allowAliasRotation ? undefined : stickyKey,
-                    availabilityCheck: (key) => deps.healthManager.isAvailable(key)
-                }, tier.mode === 'round-robin' ? 'round-robin' : undefined);
-                if (selected) {
-                    return selected;
+            const bucketPenaltyMap = {};
+            for (const item of bucket) {
+                bucketPenaltyMap[item.key] = item.penalty;
+            }
+            const bucketWeights = {};
+            const bucketMultipliers = {};
+            for (const item of bucket) {
+                if (healthWeightedCfg.enabled) {
+                    const entry = quotaView(item.key);
+                    const { weight, multiplier } = computeHealthWeight(entry, nowForWeights, healthWeightedCfg);
+                    bucketWeights[item.key] = weight;
+                    bucketMultipliers[item.key] = multiplier;
+                }
+                else {
+                    // Legacy: penalty => lower weight, but never zero (unhealthy should still get a chance).
+                    bucketWeights[item.key] = Math.max(1, Math.floor(100 / (1 + Math.max(0, item.penalty))));
+                    bucketMultipliers[item.key] = 1;
+                }
+            }
+            if (isSafePool && contextWeightedCfg.enabled) {
+                const ctx = computeContextWeightMultipliers(bucketCandidates);
+                if (ctx) {
+                    for (const key of bucketCandidates) {
+                        const m = computeContextMultiplier({
+                            effectiveSafeRefTokens: ctx.ref,
+                            effectiveSafeTokens: ctx.eff[key] ?? 1,
+                            cfg: contextWeightedCfg
+                        });
+                        bucketWeights[key] = Math.max(1, Math.round((bucketWeights[key] ?? 1) * m));
+                    }
                 }
             }
-        }
-        // default 路由永不因 quota gating 而“空池”：
-        // 当 quotaView 过滤后没有任何可用候选时，默认路由允许忽略 quotaView，
-        // 继续按健康/负载均衡选择一个 providerKey（但不覆盖 forced/required 约束）。
-        const quotaBypassAllowed = routeName === DEFAULT_ROUTE && (!requiredProviderKeys || requiredProviderKeys.size === 0);
-        if (quotaBypassAllowed) {
             if (tier.mode === 'priority') {
-                const selected = selectFirstAvailable(candidates);
-                if (selected) {
-                    return selected;
+                if (!isRecoveryAttempt) {
+                    const group = pickPriorityGroup(bucketCandidates, tier.targets, bucketPenaltyMap);
+                    if (!group) {
+                        continue;
+                    }
+                    const groupWeights = {};
+                    for (const key of group.groupCandidates) {
+                        groupWeights[key] = bucketWeights[key] ?? 1;
+                    }
+                    const selected = deps.loadBalancer.select({
+                        routeName: `${routeName}:${tier.id}:priority:${priority}:group:${group.groupId}`,
+                        candidates: group.groupCandidates,
+                        stickyKey: options.allowAliasRotation ? undefined : stickyKey,
+                        weights: groupWeights,
+                        availabilityCheck: (key) => deps.healthManager.isAvailable(key)
+                    }, 'round-robin');
+                    if (selected) {
+                        return selected;
+                    }
+                    continue;
                 }
+                if (isRecoveryAttempt && healthWeightedCfg.enabled && healthWeightedCfg.recoverToBestOnRetry) {
+                    let best = null;
+                    let bestM = Number.NEGATIVE_INFINITY;
+                    for (const key of bucketCandidates) {
+                        if (!deps.healthManager.isAvailable(key))
+                            continue;
+                        const m = bucketMultipliers[key] ?? 1;
+                        if (m > bestM) {
+                            bestM = m;
+                            best = key;
+                        }
+                    }
+                    if (best) {
+                        return best;
+                    }
+                    continue;
+                }
+                else if (isRecoveryAttempt) {
+                    const recovered = selectFirstAvailable(bucketCandidates);
+                    if (recovered)
+                        return recovered;
+                    continue;
+                }
+                // (unreachable) recovery handled above
             }
             else {
+                if (isRecoveryAttempt && healthWeightedCfg.enabled && healthWeightedCfg.recoverToBestOnRetry) {
+                    let best = null;
+                    let bestM = Number.NEGATIVE_INFINITY;
+                    for (const key of bucketCandidates) {
+                        if (!deps.healthManager.isAvailable(key))
+                            continue;
+                        const m = bucketMultipliers[key] ?? 1;
+                        if (m > bestM) {
+                            bestM = m;
+                            best = key;
+                        }
+                    }
+                    if (best) {
+                        return best;
+                    }
+                    continue;
+                }
+                else if (isRecoveryAttempt) {
+                    const recovered = selectFirstAvailable(bucketCandidates);
+                    if (recovered)
+                        return recovered;
+                    continue;
+                }
                 const selected = deps.loadBalancer.select({
-                    routeName: `${routeName}:${tier.id}:quota-bypass`,
-                    candidates,
+                    routeName: `${routeName}:${tier.id}`,
+                    candidates: bucketCandidates,
                     stickyKey: options.allowAliasRotation ? undefined : stickyKey,
+                    weights: bucketWeights,
                     availabilityCheck: (key) => deps.healthManager.isAvailable(key)
                 }, tier.mode === 'round-robin' ? 'round-robin' : undefined);
                 if (selected) {
@@ -499,14 +741,10 @@ function trySelectFromTier(routeName, tier, stickyKey, estimatedTokens, features
                 }
             }
         }
-        // 单 provider 兜底：当只剩一个候选 key 时，不因 quota/blacklist/cooldown 或健康状态过滤导致无 provider。
-        if (candidates.length === 1) {
-            return candidates[0];
-        }
         return null;
     };
     for (const candidatePool of prioritizedPools) {
-        const providerKey = selectWithQuota(candidatePool);
+        const providerKey = selectWithQuota(candidatePool, candidatePool === contextResult.safe);
         if (providerKey) {
             return { providerKey, poolTargets: tier.targets, tierId: tier.id };
         }
@@ -802,7 +1040,6 @@ function buildContextCandidatePools(result) {
     const ordered = [];
     if (result.safe.length) {
         ordered.push(result.safe);
-        return ordered;
     }
     if (result.risky.length) {
         ordered.push(result.risky);

package/dist/router/virtual-router/engine.d.ts

@@ -5,6 +5,7 @@ import type { ProviderQuotaView } from './types.js';
 interface RoutingInstructionStateStore {
     loadSync(key: string): RoutingInstructionState | null;
     saveAsync(key: string, state: RoutingInstructionState | null): void;
+    saveSync?: (key: string, state: RoutingInstructionState | null) => void;
 }
 export declare class VirtualRouterEngine {
     private routing;
@@ -63,9 +64,11 @@ export declare class VirtualRouterEngine {
     private providerHealthConfig;
     private resolveStickyKey;
     private resolveSessionScope;
+    private resolveStopMessageScope;
     private getRoutingInstructionState;
     private buildMetadataInstructions;
     private parseMetadataDisableDescriptor;
+    private parseMetadataForceProviderKey;
     private resolveRoutingMode;
     private resolveInstructionTarget;
     private filterCandidatesByRoutingState;