@jsonstudio/llms 0.6.2979 → 0.6.3238

package/dist/router/virtual-router/engine-legacy/route-finalize.js

@@ -0,0 +1,84 @@
+import { resolveRoutingMode } from '../engine/routing-state/metadata.js';
+import { buildHitReason, createVirtualRouterHitRecord, formatVirtualRouterHit, toVirtualRouterHitEvent } from '../engine-logging.js';
+import { recordAntigravitySessionLease } from '../engine/antigravity/alias-lease.js';
+export function finalizeRoutingDecision(engine, metadata, routingState, metadataInstructions, instructions, selectionResult) {
+    const { selection, classification, requestedRoute, features } = selectionResult;
+    const baseTarget = engine.providerRegistry.buildTarget(selection.providerKey);
+    const forceVision = selection.routeUsed === 'vision' && engine.routeHasForceFlag('vision');
+    const target = {
+        ...baseTarget,
+        ...(engine.webSearchForce ? { forceWebSearch: true } : {}),
+        ...(forceVision ? { forceVision: true } : {})
+    };
+    const instructionProcessMode = engine.resolveInstructionProcessModeForSelection(selection.providerKey, routingState);
+    if (instructionProcessMode) {
+        target.processMode = instructionProcessMode;
+    }
+    recordAntigravitySessionLease({
+        metadata: features.metadata,
+        providerKey: selection.providerKey,
+        sessionKey: engine.resolveSessionScope(features.metadata),
+        providerRegistry: engine.providerRegistry,
+        leaseStore: engine.stickySessionManager.getAllStores().aliasLeaseStore,
+        sessionAliasStore: engine.stickySessionManager.getAllStores().sessionAliasStore,
+        persistence: engine.antigravityLeasePersistence,
+        aliasReuseCooldownMs: engine.stickySessionManager.getAliasReuseCooldownMs(),
+        commitSessionBinding: false,
+        debug: engine.debug
+    });
+    const routingMode = resolveRoutingMode([...metadataInstructions, ...instructions], routingState);
+    const hitReason = buildHitReason(selection.routeUsed, selection.providerKey, classification, features, routingMode, { providerRegistry: engine.providerRegistry, contextRouting: engine.contextRouting });
+    const stickyScope = routingMode !== 'none' ? engine.resolveSessionScope(metadata) : undefined;
+    const routeForLog = routingMode === 'sticky' ? 'sticky' : selection.routeUsed;
+    const hitRecord = createVirtualRouterHitRecord({
+        requestId: metadata.requestId,
+        routeName: routeForLog,
+        poolId: selection.poolId,
+        providerKey: selection.providerKey,
+        modelId: target.modelId || undefined,
+        hitReason,
+        stickyScope,
+        routingState,
+        requestTokens: features.estimatedTokens,
+        selectionPenalty: engine.resolveSelectionPenalty(selection.providerKey)
+    });
+    engine.routeAnalytics.incrementRouteStat(selection.routeUsed, selection.providerKey, hitRecord);
+    try {
+        engine.statsCenter.recordVirtualRouterHit(toVirtualRouterHitEvent(hitRecord, {
+            requestId: metadata.requestId,
+            entryEndpoint: metadata.entryEndpoint || '/v1/chat/completions'
+        }));
+    }
+    catch {
+        // stats must never break routing
+    }
+    const formatted = formatVirtualRouterHit(hitRecord);
+    if (formatted) {
+        engine.debug?.log?.(formatted);
+    }
+    else {
+        engine.debug?.log?.(formatVirtualRouterHit(hitRecord));
+    }
+    const didFallback = selection.routeUsed !== requestedRoute;
+    return {
+        target,
+        decision: {
+            routeName: selection.routeUsed,
+            providerKey: selection.providerKey,
+            pool: selection.pool,
+            poolId: selection.poolId,
+            confidence: classification.confidence,
+            reasoning: classification.reasoning,
+            fallback: didFallback
+        },
+        diagnostics: {
+            routeName: selection.routeUsed,
+            providerKey: selection.providerKey,
+            pool: selection.pool,
+            poolId: selection.poolId,
+            reasoning: classification.reasoning,
+            fallback: didFallback,
+            confidence: classification.confidence
+        }
+    };
+}

package/dist/router/virtual-router/engine-legacy/route-selection.d.ts

@@ -0,0 +1,17 @@
+import type { ProcessedRequest, StandardizedRequest } from '../../../conversion/hub/types/standardized.js';
+import type { ClassificationResult, RoutingFeatures, RouterMetadataInput } from '../types.js';
+import type { RoutingInstructionState } from '../routing-instructions.js';
+import type { VirtualRouterEngine } from '../engine-legacy.js';
+export type RoutingSelectionResult = {
+    routingState: RoutingInstructionState;
+    requestedRoute: string;
+    classification: ClassificationResult;
+    selection: {
+        providerKey: string;
+        routeUsed: string;
+        pool: string[];
+        poolId?: string;
+    };
+    features: RoutingFeatures;
+};
+export declare function selectRoutingTarget(engine: VirtualRouterEngine, request: StandardizedRequest | ProcessedRequest, metadata: RouterMetadataInput, routingState: RoutingInstructionState, stateKey: string): RoutingSelectionResult;

package/dist/router/virtual-router/engine-legacy/route-selection.js

@@ -0,0 +1,205 @@
+import { DEFAULT_ROUTE, VirtualRouterError, VirtualRouterErrorCode } from '../types.js';
+import { buildRoutingFeatures } from '../features.js';
+import { persistRoutingInstructionState } from '../engine/routing-state/store.js';
+import { selectDirectProviderModel, selectFromStickyPool as selectFromStickyPoolImpl } from '../engine/routing-pools/index.js';
+export function selectRoutingTarget(engine, request, metadata, routingState, stateKey) {
+    const features = buildRoutingFeatures(request, metadata);
+    const directProviderModel = engine.parseDirectProviderModel(request?.model);
+    let classification;
+    let requestedRoute;
+    let selection = null;
+    const selectionDeps = {
+        routing: engine.routing,
+        providerRegistry: engine.providerRegistry,
+        healthManager: engine.healthManager,
+        contextAdvisor: engine.contextAdvisor,
+        loadBalancer: engine.loadBalancer,
+        isProviderCoolingDown: (key) => engine.isProviderCoolingDown(key),
+        resolveStickyKey: (m) => engine.resolveStickyKey(m),
+        quotaView: engine.quotaView
+    };
+    if (directProviderModel) {
+        const forceMediaFallback = engine.shouldFallbackDirectModelForMedia(directProviderModel, features);
+        const providerKeys = engine.providerRegistry.listProviderKeys(directProviderModel.providerId);
+        let hasModel = false;
+        for (const key of providerKeys) {
+            try {
+                const profile = engine.providerRegistry.get(key);
+                if (profile?.modelId === directProviderModel.modelId) {
+                    hasModel = true;
+                    break;
+                }
+            }
+            catch {
+                continue;
+            }
+        }
+        if (!hasModel) {
+            throw new VirtualRouterError(`Unknown model ${directProviderModel.modelId} for provider ${directProviderModel.providerId}`, VirtualRouterErrorCode.CONFIG_ERROR, { providerId: directProviderModel.providerId, modelId: directProviderModel.modelId });
+        }
+        if (!forceMediaFallback) {
+            const directSelection = selectDirectProviderModel(directProviderModel.providerId, directProviderModel.modelId, metadata, features, routingState, selectionDeps);
+            if (!directSelection) {
+                throw new VirtualRouterError(`All providers unavailable for model ${directProviderModel.providerId}.${directProviderModel.modelId}`, VirtualRouterErrorCode.PROVIDER_NOT_AVAILABLE, { providerId: directProviderModel.providerId, modelId: directProviderModel.modelId });
+            }
+            classification = {
+                routeName: 'direct',
+                confidence: 1,
+                reasoning: `direct_model:${directProviderModel.providerId}.${directProviderModel.modelId}`,
+                fallback: false,
+                candidates: ['direct']
+            };
+            requestedRoute = 'direct';
+            selection = directSelection;
+        }
+        else {
+            classification = engine.classifier.classify(features);
+            requestedRoute = engine.normalizeRouteAlias(classification.routeName || DEFAULT_ROUTE);
+            selection = engine.selectProvider(requestedRoute, metadata, classification, features, routingState);
+        }
+    }
+    else {
+        // Prefer target (from "<**!provider.model**>") is evaluated before routing classification.
+        const preferTarget = routingState.preferTarget;
+        if (preferTarget && typeof preferTarget.provider === 'string' && preferTarget.provider.trim()) {
+            const providerId = preferTarget.provider.trim();
+            const keyAlias = typeof preferTarget.keyAlias === 'string' ? preferTarget.keyAlias.trim() : '';
+            const modelId = typeof preferTarget.model === 'string' ? preferTarget.model.trim() : '';
+            const keyIndex = typeof preferTarget.keyIndex === 'number' && Number.isFinite(preferTarget.keyIndex)
+                ? Math.floor(preferTarget.keyIndex)
+                : undefined;
+            const candidateKeys = [];
+            if (keyIndex !== undefined && keyIndex > 0) {
+                const runtimeKey = engine.providerRegistry.resolveRuntimeKeyByIndex(providerId, keyIndex);
+                if (runtimeKey) {
+                    candidateKeys.push(runtimeKey);
+                }
+            }
+            else if (modelId) {
+                const allKeys = engine.providerRegistry.listProviderKeys(providerId);
+                for (const key of allKeys) {
+                    if (keyAlias) {
+                        const prefix = `${providerId}.${keyAlias}.`;
+                        if (!key.startsWith(prefix)) {
+                            continue;
+                        }
+                    }
+                    try {
+                        const profile = engine.providerRegistry.get(key);
+                        if (profile?.modelId === modelId) {
+                            candidateKeys.push(key);
+                        }
+                    }
+                    catch {
+                        continue;
+                    }
+                }
+            }
+            const allowAliasRotation = !keyAlias && keyIndex === undefined;
+            const eligibleKeys = (() => {
+                if (candidateKeys.length === 0) {
+                    return [];
+                }
+                const quotaView = selectionDeps.quotaView;
+                const now = quotaView ? Date.now() : 0;
+                return candidateKeys.filter((key) => {
+                    if (!quotaView) {
+                        if (engine.isProviderCoolingDown(key)) {
+                            return false;
+                        }
+                        if (!engine.healthManager.isAvailable(key)) {
+                            return false;
+                        }
+                        return true;
+                    }
+                    const entry = quotaView(key);
+                    if (!entry) {
+                        return true;
+                    }
+                    if (!entry.inPool) {
+                        return false;
+                    }
+                    if (entry.cooldownUntil && entry.cooldownUntil > now) {
+                        return false;
+                    }
+                    if (entry.blacklistUntil && entry.blacklistUntil > now) {
+                        return false;
+                    }
+                    return true;
+                });
+            })();
+            const preferSelection = eligibleKeys.length > 0
+                ? selectFromStickyPoolImpl(new Set(eligibleKeys), metadata, features, routingState, selectionDeps, {
+                    allowAliasRotation
+                })
+                : null;
+            if (preferSelection) {
+                classification = {
+                    routeName: 'prefer',
+                    confidence: 1,
+                    reasoning: keyIndex !== undefined ? `prefer_key:${providerId}.${keyIndex}` : `prefer_model:${providerId}.${modelId}`,
+                    fallback: false,
+                    candidates: ['prefer']
+                };
+                requestedRoute = 'prefer';
+                selection = {
+                    ...preferSelection,
+                    routeUsed: 'prefer',
+                    poolId: 'prefer-primary'
+                };
+            }
+            else if (routingState.preferTarget) {
+                // Auto-clear only when the target becomes invalid or blocked by explicit routing instructions.
+                // Do NOT clear for temporary unavailability (e.g. 429 cooldown, quota cooldown, transient health).
+                const shouldAutoClear = (() => {
+                    if (candidateKeys.length === 0) {
+                        return true;
+                    }
+                    // Prefer selection failed despite eligible keys existing: treat as a hard block (e.g. routing rules).
+                    if (eligibleKeys.length > 0) {
+                        return true;
+                    }
+                    // If quota explicitly marks the preferred target as out-of-pool, clear the prefer instruction so
+                    // the router can fall back to other targets without repeatedly retrying an impossible preference.
+                    if (selectionDeps.quotaView) {
+                        for (const key of candidateKeys) {
+                            const entry = selectionDeps.quotaView(key);
+                            if (entry && entry.inPool === false) {
+                                return true;
+                            }
+                        }
+                    }
+                    return false;
+                })();
+                if (shouldAutoClear) {
+                    routingState = {
+                        ...routingState,
+                        preferTarget: undefined
+                    };
+                    engine.routingInstructionState.set(stateKey, routingState);
+                    persistRoutingInstructionState(stateKey, routingState, engine.routingStateStore);
+                }
+            }
+        }
+        if (!selection) {
+            classification = metadata.routeHint && metadata.routeHint.trim()
+                ? {
+                    routeName: metadata.routeHint.trim(),
+                    confidence: 1,
+                    reasoning: `route_hint:${metadata.routeHint.trim()}`,
+                    fallback: false,
+                    candidates: [metadata.routeHint.trim()]
+                }
+                : engine.classifier.classify(features);
+            requestedRoute = engine.normalizeRouteAlias(classification.routeName || DEFAULT_ROUTE);
+            selection = engine.selectProvider(requestedRoute, metadata, classification, features, routingState);
+        }
+    }
+    return {
+        routingState,
+        requestedRoute,
+        classification: classification,
+        selection: selection,
+        features
+    };
+}

package/dist/router/virtual-router/engine-legacy/route-state-allowlist.d.ts

@@ -0,0 +1,3 @@
+import type { RoutingInstructionState } from '../routing-instructions.js';
+import type { VirtualRouterEngine } from '../engine-legacy.js';
+export declare function enforceAllowlistIntersection(engine: VirtualRouterEngine, routingState: RoutingInstructionState, stateKey: string): RoutingInstructionState;

package/dist/router/virtual-router/engine-legacy/route-state-allowlist.js

@@ -0,0 +1,36 @@
+import { extractProviderId } from '../engine/provider-key/parse.js';
+import { persistRoutingInstructionState } from '../engine/routing-state/store.js';
+export function enforceAllowlistIntersection(engine, routingState, stateKey) {
+    if (routingState.allowedProviders.size === 0) {
+        return routingState;
+    }
+    const providersInRouting = new Set();
+    for (const pools of Object.values(engine.routing)) {
+        if (!Array.isArray(pools))
+            continue;
+        for (const pool of pools) {
+            if (!pool || !Array.isArray(pool.targets))
+                continue;
+            for (const key of pool.targets) {
+                if (typeof key !== 'string' || !key)
+                    continue;
+                const providerId = extractProviderId(key);
+                if (providerId) {
+                    providersInRouting.add(providerId);
+                }
+            }
+        }
+    }
+    const allowed = Array.from(routingState.allowedProviders).filter((provider) => typeof provider === 'string');
+    const hasIntersection = allowed.some((provider) => providersInRouting.has(provider));
+    if (!hasIntersection) {
+        const nextState = {
+            ...routingState,
+            allowedProviders: new Set()
+        };
+        engine.routingInstructionState.set(stateKey, nextState);
+        persistRoutingInstructionState(stateKey, nextState, engine.routingStateStore);
+        return nextState;
+    }
+    return routingState;
+}

package/dist/router/virtual-router/engine-legacy/route-state.d.ts

@@ -0,0 +1,12 @@
+import type { ProcessedRequest, StandardizedRequest } from '../../../conversion/hub/types/standardized.js';
+import type { RouterMetadataInput } from '../types.js';
+import type { RoutingInstruction, RoutingInstructionState } from '../routing-instructions.js';
+import type { VirtualRouterEngine } from '../engine-legacy.js';
+export type RoutingStateResult = {
+    routingState: RoutingInstructionState;
+    stateKey: string;
+    stopMessageScope?: string;
+    metadataInstructions: RoutingInstruction[];
+    instructions: RoutingInstruction[];
+};
+export declare function buildRoutingState(engine: VirtualRouterEngine, request: StandardizedRequest | ProcessedRequest, metadata: RouterMetadataInput): RoutingStateResult;