@juspay/neurolink 9.76.0 → 9.78.0

package/dist/lib/neurolink.js

@@ -55,7 +55,7 @@ import { resolveDynamicArgument } from "./dynamic/dynamicResolver.js";
 import { initializeHippocampus } from "./memory/hippocampusInitializer.js";
 import { createMemoryRetrievalTools } from "./memory/memoryRetrievalTools.js";
 import { getMetricsAggregator, MetricsAggregator, } from "./observability/metricsAggregator.js";
-import { SpanStatus, SpanType, CircuitBreakerOpenError, ConversationMemoryError, AuthenticationError, AuthorizationError, InvalidModelError, ModelAccessDeniedError, } from "./types/index.js";
+import { SpanStatus, SpanType, CircuitBreakerOpenError, ConversationMemoryError, ModelAccessDeniedError, } from "./types/index.js";
 import { SpanSerializer } from "./observability/utils/spanSerializer.js";
 import { flushOpenTelemetry, getLangfuseContext, getLangfuseHealthStatus, initializeOpenTelemetry, isOpenTelemetryInitialized, runWithCurrentLangfuseContext, setLangfuseContext, shutdownOpenTelemetry, stampGuestRescueIdentity, } from "./services/server/ai/observability/instrumentation.js";
 import { TaskManager } from "./tasks/taskManager.js";
@@ -78,7 +78,6 @@ import { resolveModel } from "./utils/modelAliasResolver.js";
 // Import orchestration components
 import { ModelRouter } from "./utils/modelRouter.js";
 import { getBestProvider } from "./utils/providerUtils.js";
-import { NON_RETRYABLE_HTTP_STATUS_CODES, isDeterministicClientErrorMessage, } from "./utils/retryability.js";
 import { isZodSchema } from "./utils/schemaConversion.js";
 import { BinaryTaskClassifier } from "./utils/taskClassifier.js";
 // Tool detection and execution imports
@@ -87,6 +86,8 @@ import { extractToolNames, optimizeToolForCollection, transformAvailableTools, t
 import { isNonNullObject } from "./utils/typeUtils.js";
 import { getWorkflow } from "./workflow/core/workflowRegistry.js";
 import { runWorkflow } from "./workflow/core/workflowRunner.js";
+import { ModelPool, classifyProviderError } from "./routing/index.js";
+import { looksLikeModelAccessDenied as sharedLooksLikeModelAccessDenied, isNonRetryableProviderError as sharedIsNonRetryableProviderError, } from "./utils/providerErrorClassification.js";
 /**
  * NL-002: Classify MCP error messages into categories for AI disambiguation.
  * Returns a human-readable error category based on error message content.
@@ -151,95 +152,12 @@ function mcpCategoryToErrorCategory(mcpCategory) {
  * This prevents wasting tokens and latency on guaranteed-to-fail retries.
  * For example, a NOT_FOUND error for a model causes 6 retries of a 418KB
  * message, wasting ~628,000 tokens and adding 10+ seconds of latency.
+ *
+ * Delegates to the shared utility in utils/providerErrorClassification.ts so
+ * that modelPool.ts (classifyProviderError) and this function stay in sync.
  */
-/**
- * Curator P2-3: detect model-access-denied without requiring the typed
- * ModelAccessDeniedError class to be present (Issue #1 ships that class
- * separately). Matches LiteLLM "team not allowed" / "team can only access
- * models=[...]" plus typed-error markers when present.
- */
-function looksLikeModelAccessDenied(error) {
-    if (!error) {
-        return false;
-    }
-    const e = error;
-    if (e.name === "ModelAccessDeniedError") {
-        return true;
-    }
-    if (e.code === "MODEL_ACCESS_DENIED") {
-        return true;
-    }
-    const msg = typeof e.message === "string"
-        ? e.message
-        : error instanceof Error
-            ? error.message
-            : String(error);
-    if (!msg) {
-        return false;
-    }
-    const lower = msg.toLowerCase();
-    return ((lower.includes("team") && lower.includes("not allowed")) ||
-        lower.includes("team can only access") ||
-        /not\s+allowed\s+to\s+access\s+(this\s+)?model/i.test(msg));
-}
-function isNonRetryableProviderError(error) {
-    // Check for typed error classes from providers
-    if (error instanceof InvalidModelError) {
-        return true;
-    }
-    if (error instanceof AuthenticationError) {
-        return true;
-    }
-    if (error instanceof AuthorizationError) {
-        return true;
-    }
-    // Curator P1-1: model-access-denied is permanent for the (provider, model)
-    // pair until the team whitelist changes. Retrying with the same config
-    // would just waste a second roundtrip. Caller / fallback-orchestrator
-    // should pick a different model.
-    if (error instanceof ModelAccessDeniedError) {
-        return true;
-    }
-    // Note: ContextBudgetExceededError is intentionally NOT non-retryable.
-    // Each provider has its own context window, so a budget rejection on
-    // one provider doesn't preclude another provider's window fitting the
-    // same payload. The directProviderGeneration loop should continue
-    // trying alternate providers; the after-loop rethrow preserves the
-    // typed error when all providers reject (see `directProviderGeneration`).
-    // Check for HTTP status codes on error objects (e.g., from Vercel AI SDK)
-    if (error && typeof error === "object") {
-        const err = error;
-        const status = typeof err.status === "number"
-            ? err.status
-            : typeof err.statusCode === "number"
-                ? err.statusCode
-                : undefined;
-        if (status && NON_RETRYABLE_HTTP_STATUS_CODES.includes(status)) {
-            return true;
-        }
-    }
-    // Check error message for NOT_FOUND patterns (catches wrapped errors)
-    if (error instanceof Error) {
-        const msg = error.message;
-        if (msg.includes("NOT_FOUND") ||
-            msg.includes("Model Not Found") ||
-            msg.includes("model not found") ||
-            msg.includes("PERMISSION_DENIED") ||
-            msg.includes("UNAUTHENTICATED")) {
-            return true;
-        }
-        // A deterministic 400 / malformed-request whose status is only present in
-        // the message string (e.g. Vertex wraps `{"code":400,"status":
-        // "INVALID_ARGUMENT"}` inside the message). The object-level status check
-        // above misses it, so without this the fallback orchestrator retries the
-        // identical bad payload on every other provider — they reject it the same
-        // way. The request itself is malformed, so abort fast.
-        if (isDeterministicClientErrorMessage(msg)) {
-            return true;
-        }
-    }
-    return false;
-}
+const looksLikeModelAccessDenied = sharedLooksLikeModelAccessDenied;
+const isNonRetryableProviderError = sharedIsNonRetryableProviderError;
 /**
  * NeuroLink - Universal AI Development Platform
  *
@@ -445,6 +363,11 @@ export class NeuroLink {
     // Lazy-initialized routing decision cache (ITEM C). Created on first use so
     // instances that don't use routing pay no overhead.
     toolRoutingCacheInstance;
+    // Persisted vector cache for the L2 embedding fast-path (ITEM B). Populated
+    // on the first turn and reused across subsequent turns so tool embedding
+    // vectors are computed once. Cleared when the catalog changes via
+    // setToolRoutingServers() so stale vectors are never reused.
+    toolRoutingVectorCache;
     // Opt-in tool-signature deduplication config.
     toolDedupConfig;
     // Add orchestration property
@@ -458,6 +381,12 @@ export class NeuroLink {
     // Curator P2-3: instance-level fallback policy. Read by
     // runWithFallbackOrchestration on model-access-denied.
     fallbackConfig = {};
+    // ModelPool: opt-in multi-provider failover with per-member cooldown.
+    // Built once from config.modelPool in the constructor; null when not configured.
+    modelPool;
+    // RequestRouter: pluggable pre-call provider/model selector.
+    // Stored directly from config.requestRouter; null when not configured.
+    requestRouter;
     /**
      * Merge instance-level credentials with per-call credentials.
      *
@@ -862,6 +791,10 @@ export class NeuroLink {
         if (config?.toolDedup) {
             this.toolDedupConfig = { ...config.toolDedup };
         }
+        // ModelPool: build one instance from config; null when not configured.
+        this.modelPool = config?.modelPool ? new ModelPool(config.modelPool) : null;
+        // RequestRouter: store the host-supplied function; null when not configured.
+        this.requestRouter = config?.requestRouter ?? null;
         logger.setEventEmitter(this.emitter);
         // Read tool cache duration from environment variables, with a default
         const cacheDurationEnv = process.env.NEUROLINK_TOOL_CACHE_DURATION;
@@ -3172,6 +3105,11 @@ Current user's request: ${currentInput}`;
     async runStandardGenerateRequest(options, originalPrompt, generateSpan) {
         const startTime = Date.now();
         await this.maybeApplyGenerateOrchestration(options);
+        // Pre-call request router: opt-in, only when the caller did not set both
+        // provider+model. Fails open (any router error leaves options unchanged).
+        await this.applyRequestRouter(options, options.input?.text ?? originalPrompt ?? "", !options.disableTools &&
+            (!!(options.tools && Object.keys(options.tools).length > 0) ||
+                this.getCustomTools().size > 0), !!(options.input?.images && options.input.images.length > 0), options.thinkingConfig?.thinkingLevel);
         this.emitter.emit("generation:start", {
             provider: options.provider || "auto",
             timestamp: startTime,
@@ -3280,6 +3218,94 @@ Current user's request: ${currentInput}`;
             });
         }
     }
+    /**
+     * Applies the host-configured `requestRouter` to `options` in place.
+     *
+     * The router is skipped when:
+     *   - no `requestRouter` is configured on this instance, or
+     *   - the caller explicitly set both `options.provider` AND `options.model`
+     *     (we only skip the router if BOTH are set; a caller setting only one
+     *     still lets the router fill in the other field).
+     *
+     * Fails open: any router error is logged at WARN level and the call
+     * continues with the original options unmodified.
+     *
+     * @param options — the mutable options object (generate or stream).
+     * @param promptText — the text prompt used to build the RouterInputContext.
+     * @param hasTools — true when at least one tool is available for this call.
+     * @param requiresVision — true when the call includes image attachments.
+     * @param thinkingLevel — optional thinking level string from the call options.
+     */
+    async applyRequestRouter(options, promptText, hasTools, requiresVision, thinkingLevel) {
+        if (!this.requestRouter) {
+            return;
+        }
+        // Skip if the caller explicitly set both provider AND model — they know
+        // what they want; the router should not override an intentional choice.
+        if (options.provider && options.model) {
+            return;
+        }
+        // Skip when a ModelPool is also configured: the pool unconditionally
+        // overrides provider+model for each member it selects.  Allowing the
+        // router to run first would set options.provider/model to a value the
+        // pool immediately clobbers, creating silent ownership confusion and
+        // making the pool bypass the "user didn't set provider+model" guard above
+        // on subsequent attempts.
+        if (this.modelPool) {
+            logger.debug("[NeuroLink] applyRequestRouter: skipped — modelPool takes precedence");
+            return;
+        }
+        const ctx = {
+            prompt: promptText,
+            // Cheap token estimate: ~4 chars/token.
+            estimatedInputTokens: Math.ceil(promptText.length / 4),
+            hasTools,
+            requiresVision,
+            thinkingLevel,
+        };
+        try {
+            const decision = await this.requestRouter(ctx);
+            // Apply provider/model/region as a coherent set: when the router supplies
+            // a provider (the caller didn't pin one), take its model and region too.
+            // If the caller pinned the provider but not the model, only apply the
+            // router's model when it targets the same provider.
+            if (decision.provider && !options.provider) {
+                options.provider = decision.provider;
+                // Adopt the router's model and region only when they belong to the
+                // provider the router just selected — never mix a router model onto a
+                // caller-pinned different provider.
+                if (decision.model && !options.model) {
+                    options.model = decision.model;
+                }
+                if (decision.region && !options.region) {
+                    options.region = decision.region;
+                }
+            }
+            else if (!decision.provider) {
+                // Router returned a no-op or model-only decision.
+                if (decision.model && !options.model) {
+                    options.model = decision.model;
+                }
+                if (decision.region && !options.region) {
+                    options.region = decision.region;
+                }
+            }
+            // If decision.provider differs from the caller-pinned options.provider,
+            // we skip the router result entirely to avoid an incoherent pairing.
+            if (decision.reason) {
+                logger.debug("[NeuroLink] requestRouter decision", {
+                    reason: decision.reason,
+                    provider: options.provider,
+                    model: options.model,
+                });
+            }
+        }
+        catch (routerErr) {
+            logger.warn("[NeuroLink] requestRouter threw — proceeding unrouted", {
+                error: routerErr instanceof Error ? routerErr.message : String(routerErr),
+            });
+        }
+    }
     async prepareGenerateAugmentations(options) {
         if (options.rag?.files?.length) {
             try {
@@ -4961,6 +4987,149 @@ Current user's request: ${currentInput}`;
             tryProviders,
             allowFallback: !requestedProvider || !!preferredOrchestrated,
         });
+        // ─── ModelPool path ──────────────────────────────────────────────────────
+        // When a ModelPool is configured, source the candidate sequence from the
+        // pool instead of (or in addition to) the static tryProviders list.
+        // Preserves the existing isNonRetryableProviderError short-circuit and
+        // AbortError propagation semantics. Falls through to the standard path
+        // when no pool is configured.
+        if (this.modelPool) {
+            const pool = this.modelPool;
+            const maxPoolAttempts = pool.maxAttempts;
+            const triedKeys = new Set();
+            let poolLastError = null;
+            for (let attempt = 0; attempt < maxPoolAttempts; attempt++) {
+                if (options.abortSignal?.aborted) {
+                    throw new DOMException("The operation was aborted", "AbortError");
+                }
+                const member = pool.selectNext(triedKeys);
+                if (!member) {
+                    // All available members exhausted.
+                    break;
+                }
+                triedKeys.add(pool.memberKey(member));
+                // Temporarily override provider/model/region from the pool member.
+                const savedProvider = options.provider;
+                const savedModel = options.model;
+                const savedRegion = options.region;
+                options.provider = member.provider;
+                // When the member omits model/region, clear inherited values so the
+                // provider's own default is used rather than a mismatched caller value.
+                options.model = member.model ?? undefined;
+                options.region = member.region ?? undefined;
+                const poolProviderName = member.provider;
+                logger.debug(`[${functionTag}] ModelPool: attempting member`, {
+                    provider: poolProviderName,
+                    model: member.model ?? options.model,
+                    attempt,
+                });
+                try {
+                    // Get conversation messages for context (use pre-compacted if provided)
+                    const poolOptionsWithMessages = options;
+                    const poolConversationMessages = poolOptionsWithMessages
+                        .conversationMessages?.length
+                        ? poolOptionsWithMessages.conversationMessages
+                        : await getConversationMessages(this.conversationMemory, options);
+                    const poolProvider = await AIProviderFactory.createProvider(poolProviderName, options.model, !options.disableTools, this, options.region, this.resolveCredentials(options.credentials));
+                    poolProvider.setTraceContext(this._metricsTraceContext);
+                    poolProvider.setupToolExecutor({
+                        customTools: this.getCustomTools(),
+                        executeTool: (toolName, params) => this.executeTool(toolName, params, {
+                            disableToolCache: options.disableToolCache,
+                        }),
+                    }, functionTag);
+                    const poolResult = await poolProvider.generate({
+                        ...options,
+                        conversationMessages: poolConversationMessages,
+                    });
+                    if (!poolResult) {
+                        throw new Error(`ModelPool: provider ${poolProviderName} returned null result`);
+                    }
+                    pool.recordSuccess(member);
+                    const responseTime = Date.now() - startTime;
+                    // Restore original options fields so callers see a clean object.
+                    options.provider = savedProvider;
+                    options.model = savedModel;
+                    options.region = savedRegion;
+                    return {
+                        content: poolResult.content || "",
+                        provider: poolProviderName,
+                        model: poolResult.model,
+                        usage: poolResult.usage,
+                        responseTime,
+                        finishReason: poolResult.finishReason,
+                        toolsUsed: poolResult.toolsUsed || [],
+                        toolExecutions: poolResult.toolExecutions?.map((te) => {
+                            const t = te;
+                            return {
+                                ...te,
+                                toolName: te.name,
+                                executionTime: typeof t.executionTime === "number"
+                                    ? t.executionTime
+                                    : typeof t.duration === "number"
+                                        ? t.duration
+                                        : 0,
+                                success: typeof t.success === "boolean"
+                                    ? t.success
+                                    : t.status === "success",
+                            };
+                        }),
+                        enhancedWithTools: !!poolResult.toolExecutions?.length,
+                        analytics: poolResult.analytics,
+                        evaluation: poolResult.evaluation,
+                        audio: poolResult.audio,
+                        video: poolResult.video,
+                        avatar: poolResult.avatar,
+                        music: poolResult.music,
+                        ppt: poolResult.ppt,
+                        imageOutput: poolResult.imageOutput,
+                        reasoning: poolResult.reasoning,
+                        reasoningTokens: poolResult.reasoningTokens,
+                        _generationEndEmitted: poolResult._generationEndEmitted,
+                    };
+                }
+                catch (poolError) {
+                    // Restore options unconditionally on failure.
+                    options.provider = savedProvider;
+                    options.model = savedModel;
+                    options.region = savedRegion;
+                    if (isAbortError(poolError)) {
+                        throw poolError;
+                    }
+                    if (isNonRetryableProviderError(poolError)) {
+                        const poolErrMsg = poolError instanceof Error
+                            ? poolError.message
+                            : String(poolError);
+                        logger.warn(`[${functionTag}] ModelPool: non-retryable error from ${poolProviderName}, stopping pool`, { error: poolErrMsg });
+                        pool.recordFailure(member, classifyProviderError(poolError));
+                        // Wrap in a plain Error so runWithFallbackOrchestration does not
+                        // mistake this for a ModelAccessDeniedError and start a second
+                        // retry layer on top of the already-exhausted pool.
+                        throw new Error(`[ModelPool] non-retryable: ${poolErrMsg}`, {
+                            cause: poolError,
+                        });
+                    }
+                    pool.recordFailure(member, classifyProviderError(poolError));
+                    poolLastError =
+                        poolError instanceof Error
+                            ? poolError
+                            : new Error(String(poolError));
+                    logger.warn(`[${functionTag}] ModelPool: member ${poolProviderName} failed`, { error: poolLastError.message });
+                }
+            }
+            // All pool members failed or were exhausted.
+            const poolResponseTime = Date.now() - startTime;
+            logger.error(`[${functionTag}] ModelPool: all members failed`, {
+                triedKeys: Array.from(triedKeys),
+                lastError: poolLastError?.message,
+                responseTime: poolResponseTime,
+            });
+            // Wrap in a plain Error (not a typed error class) so that
+            // runWithFallbackOrchestration does not re-activate the modelChain /
+            // providerFallback layer on top of an already-exhausted pool.
+            throw new Error(`[ModelPool] all members failed: ${poolLastError?.message ?? "no members available"}`);
+        }
+        // ─── End ModelPool path ───────────────────────────────────────────────────
         let lastError = null;
         // Try each provider in order
         for (const providerName of tryProviders) {
@@ -5659,6 +5828,11 @@ Current user's request: ${currentInput}`;
             // classify correctly. After the workflow short-circuit, so workflow
             // streams skip it.
             await context.with(streamSpanContext, () => this.setLangfuseContextFromOptions(options, () => this.applyToolRoutingExclusions(options, originalPrompt)));
+            // Pre-call request router for stream: opt-in, fails open.
+            await this.applyRequestRouter(options, originalPrompt, !options.disableTools &&
+                (!!(options.tools && Object.keys(options.tools).length > 0) ||
+                    this.getCustomTools().size > 0), !!(options.input?.images && options.input.images.length > 0), options
+                .thinking?.thinkingLevel);
             // TTS Mode 2 deferred: stream() emits text first, then synthesizes the
             // accumulated response into a single audio chunk at end-of-stream and
             // resolves `streamResult.audio` with the same TTSResult. The resolver is
@@ -5765,7 +5939,11 @@ Current user's request: ${currentInput}`;
             // Require a non-empty sessionId to avoid anonymous sessions sharing a
             // ":query" namespace (cross-session cache leak).
             const cacheKey = cacheEnabled && sessionId ? `${sessionId}:${routingQuery}` : undefined;
+            const routableServerCount = catalog.filter((s) => !(routingConfig.alwaysIncludeServerIds ?? []).includes(s.id)).length;
             // --- ITEM E: build the emitDecision callback ---
+            // Constructed BEFORE the cache-hit check so it is available for all
+            // outcome paths, including cache-hit turns (Finding 4).
+            const routingStartTime = Date.now();
             const emitDecision = (decision) => {
                 try {
                     const activeSpan = trace.getActiveSpan();
@@ -5780,100 +5958,176 @@ Current user's request: ${currentInput}`;
                     activeSpan.setAttribute("tool_routing.excluded_tool_count", decision.excludedToolCount);
                     activeSpan.setAttribute("tool_routing.cache_hit", decision.cacheHit);
                     activeSpan.setAttribute("tool_routing.duration_ms", decision.durationMs);
+                    // Optional L2 / tool-granularity fields (present only when embedding ran).
+                    if (decision.embeddingActivated !== undefined) {
+                        activeSpan.setAttribute("tool_routing.embedding_activated", decision.embeddingActivated);
+                    }
+                    if (decision.candidateToolCount !== undefined) {
+                        activeSpan.setAttribute("tool_routing.candidate_tool_count", decision.candidateToolCount);
+                    }
+                    if (decision.granularity !== undefined) {
+                        activeSpan.setAttribute("tool_routing.granularity", decision.granularity);
+                    }
                 }
                 catch {
                     // Telemetry must never affect routing behaviour.
                 }
             };
-            // Unified hit/miss block: always produces raw (pre-stickiness) exclusions
-            // so stickiness can be applied identically on both paths below.
+            // Check the cache first.
+            if (cacheEnabled && cache && cacheKey !== undefined) {
+                const cached = cache.get(cacheKey);
+                if (cached) {
+                    // Decrement stickiness turn counter even on a cache hit so the window
+                    // advances correctly regardless of whether the LLM router ran
+                    // (Finding 3). Re-apply stickiness to the cached exclusion list so
+                    // the live stickiness window, not the one at write-time, is honoured
+                    // (Finding 2 complement: we stored the pre-stickiness list, so
+                    // re-applying here gives the correct per-turn view).
+                    let cachedExcluded = cached.excludedToolNames;
+                    if (stickinessEnabled && sessionId) {
+                        try {
+                            const stickyIds = cache.getStickyServerIds(sessionId);
+                            if (stickyIds.length > 0) {
+                                cachedExcluded = cachedExcluded.filter((toolName) => !stickyIds.some((id) => toolName.startsWith(`${id}_`)));
+                            }
+                        }
+                        catch {
+                            // Stickiness failure is non-fatal.
+                        }
+                    }
+                    // Notify the emitDecision callback so custom telemetry sinks observe
+                    // every routing outcome, not just non-cached turns (Finding 4).
+                    const cachedSelectedSet = new Set(cached.selectedServerIds);
+                    const cachedExcludedServerIds = catalog
+                        .map((e) => e.id)
+                        .filter((id) => !cachedSelectedSet.has(id));
+                    emitDecision({
+                        outcome: "cache-hit",
+                        selectedServerIds: cached.selectedServerIds,
+                        excludedServerIds: cachedExcludedServerIds,
+                        hallucinatedIds: [],
+                        excludedToolCount: cachedExcluded.length,
+                        routableServerCount,
+                        cacheHit: true,
+                        durationMs: Date.now() - routingStartTime,
+                    });
+                    logger.debug("[ToolRouting] Cache hit, skipping router LLM", {
+                        hasSessionId: !!sessionId,
+                        routingQueryLength: routingQuery.length,
+                    });
+                    if (cachedExcluded.length > 0) {
+                        options.excludeTools = [
+                            ...(options.excludeTools ?? []),
+                            ...cachedExcluded,
+                        ];
+                    }
+                    return;
+                }
+            }
+            // The router call below re-enters the public generate(), whose finally
+            // block resets _disableToolCacheForCurrentRequest to false. That flag is
+            // turn-scoped (set at the top of this turn) and read by the main tool
+            // execution path that runs after routing, so save it before the router
+            // call and restore it afterward to keep the turn's cache setting intact.
+            const cacheDisabledForCurrentRequest = this._disableToolCacheForCurrentRequest;
             let routedExcludeTools;
-            let selectedServerIds = [];
             let resolvedDecision;
-            let fromCache = false;
-            const cached = cacheEnabled && cache && cacheKey !== undefined
-                ? cache.get(cacheKey)
-                : undefined;
-            if (cached) {
-                // Cache HIT — use stored raw exclusions (pre-stickiness).
-                fromCache = true;
-                routedExcludeTools = [...cached.excludedToolNames];
-                selectedServerIds = [...cached.selectedServerIds];
-                logger.debug("[ToolRouting] Cache hit, skipping router LLM", {
-                    cacheKey,
-                });
-                // Emit cache-hit telemetry with full parity to the miss path.
-                try {
-                    const activeSpan = trace.getActiveSpan();
-                    if (activeSpan) {
-                        const routableCount = catalog.filter((s) => !(routingConfig.alwaysIncludeServerIds ?? []).includes(s.id)).length;
-                        activeSpan.setAttribute("tool_routing.outcome", "cache-hit");
-                        activeSpan.setAttribute("tool_routing.routable_server_count", routableCount);
-                        activeSpan.setAttribute("tool_routing.cache_hit", true);
-                        activeSpan.setAttribute("tool_routing.selected_server_ids", spanJsonAttribute(selectedServerIds));
-                        activeSpan.setAttribute("tool_routing.excluded_tool_count", routedExcludeTools.length);
+            try {
+                // Intercept the decision so we can store it in the cache.
+                const captureDecision = (decision) => {
+                    resolvedDecision = decision;
+                    emitDecision(decision);
+                };
+                // --- ITEM B: build the embedFn for the L2 embedding fast-path ---
+                // The vector cache is persisted at the NeuroLink instance level so
+                // tool embedding vectors are computed once and reused across turns
+                // (Finding 1 fix). It is cleared by setToolRoutingServers() when the
+                // catalog changes so stale vectors are never used after an update.
+                let routingEmbedFn;
+                const embeddingCfg = routingConfig.embedding;
+                if (embeddingCfg?.enabled === true) {
+                    try {
+                        // Resolve the embedding provider: use the explicitly configured one
+                        // if present, otherwise fall back to the stream/generate call's
+                        // provider. The factory call is wrapped in try/catch so a provider
+                        // that doesn't support embedMany (it throws at call time, not
+                        // construction time) fails open when routingEmbedFn is invoked.
+                        const embProviderName = embeddingCfg.provider ??
+                            (options.provider && options.provider !== "auto"
+                                ? options.provider
+                                : undefined) ??
+                            routingConfig.routerModel?.provider;
+                        if (embProviderName) {
+                            const embProvider = await AIProviderFactory.createProvider(embProviderName, embeddingCfg.model, true, this, undefined, this.resolveCredentials(options.credentials));
+                            // Bind embedMany with the configured model (may be undefined —
+                            // the provider uses its default embedding model in that case).
+                            routingEmbedFn = (texts) => withTimeout(embProvider.embedMany(texts, embeddingCfg.model), embeddingCfg.timeoutMs ?? 10000);
+                            // Lazy-init the persistent vector cache for this instance.
+                            // Subsequent turns reuse the same Map so text→vector lookups
+                            // already populated from earlier turns are served from memory.
+                            if (!this.toolRoutingVectorCache) {
+                                this.toolRoutingVectorCache = new Map();
+                            }
+                        }
+                    }
+                    catch (embSetupError) {
+                        logger.debug("[ToolRouting] Embedding provider setup failed, L2 path disabled for this turn", {
+                            error: embSetupError instanceof Error
+                                ? embSetupError.message
+                                : String(embSetupError),
+                        });
+                        // routingEmbedFn remains undefined — fast-path is skipped.
                     }
                 }
-                catch {
-                    // Telemetry must never affect routing behaviour.
-                }
+                routedExcludeTools = await resolveToolRoutingExclusions({
+                    catalog,
+                    alwaysIncludeServerIds: routingConfig.alwaysIncludeServerIds ?? [],
+                    userQuery: routingQuery,
+                    routerPromptPrefix: routingConfig.routerPromptPrefix,
+                    routerModel: {
+                        provider: routingConfig.routerModel?.provider ??
+                            options.provider,
+                        model: routingConfig.routerModel?.model ?? options.model,
+                        region: routingConfig.routerModel?.region ?? options.region,
+                        temperature: routingConfig.routerModel?.temperature,
+                    },
+                    timeoutMs: routingConfig.timeoutMs ?? DEFAULT_TOOL_ROUTING_TIMEOUT_MS,
+                    // Forward the abort signal so a cancelled turn aborts the router
+                    // call promptly instead of waiting out the routing timeout.
+                    generateFn: (generateOptions) => this.generate({
+                        ...generateOptions,
+                        abortSignal: options.abortSignal,
+                    }),
+                    emitDecision: captureDecision,
+                    // L2 / ITEM D — only populated when embedding is configured.
+                    embedFn: routingEmbedFn,
+                    embeddingConfig: embeddingCfg,
+                    granularity: routingConfig.granularity ?? "server",
+                    // Pass the persistent vector cache so tool embeddings are reused
+                    // across turns (Finding 1).
+                    embeddingVectorCache: routingEmbedFn !== undefined
+                        ? this.toolRoutingVectorCache
+                        : undefined,
+                });
             }
-            else {
-                // Cache MISS — run the router LLM.
-                // The router call re-enters the public generate(), whose finally block
-                // resets _disableToolCacheForCurrentRequest to false. That flag is
-                // turn-scoped and read by the main tool execution path after routing, so
-                // save it before the router call and restore it afterward.
-                const cacheDisabledForCurrentRequest = this._disableToolCacheForCurrentRequest;
-                try {
-                    // Intercept the decision so we can store it in the cache.
-                    const captureDecision = (decision) => {
-                        resolvedDecision = decision;
-                        emitDecision(decision);
-                    };
-                    routedExcludeTools = await resolveToolRoutingExclusions({
-                        catalog,
-                        alwaysIncludeServerIds: routingConfig.alwaysIncludeServerIds ?? [],
-                        userQuery: routingQuery,
-                        routerPromptPrefix: routingConfig.routerPromptPrefix,
-                        routerModel: {
-                            provider: routingConfig.routerModel?.provider ??
-                                options.provider,
-                            model: routingConfig.routerModel?.model ?? options.model,
-                            region: routingConfig.routerModel?.region ?? options.region,
-                            temperature: routingConfig.routerModel?.temperature,
-                        },
-                        timeoutMs: routingConfig.timeoutMs ?? DEFAULT_TOOL_ROUTING_TIMEOUT_MS,
-                        // Forward the abort signal so a cancelled turn aborts the router
-                        // call promptly instead of waiting out the routing timeout.
-                        generateFn: (generateOptions) => this.generate({
-                            ...generateOptions,
-                            abortSignal: options.abortSignal,
-                        }),
-                        emitDecision: captureDecision,
-                    });
-                }
-                finally {
-                    this._disableToolCacheForCurrentRequest =
-                        cacheDisabledForCurrentRequest;
-                }
-                if (resolvedDecision?.outcome === "applied") {
-                    selectedServerIds = resolvedDecision.selectedServerIds;
-                }
+            finally {
+                this._disableToolCacheForCurrentRequest =
+                    cacheDisabledForCurrentRequest;
             }
             // Aborted during the router call — skip applying now-stale exclusions;
             // the main generation path enforces the abort itself.
             if (options.abortSignal?.aborted) {
                 return;
             }
-            // Capture raw (pre-stickiness) exclusions for cache storage on MISS path.
-            const rawExclusions = [...routedExcludeTools];
-            // Apply stickiness on BOTH hit and miss paths: the sticky ids were
-            // recorded on a prior turn and represent servers that should stay warm for
-            // the current turn. Consuming (decrementing) them before recordSelection
-            // ensures the window covers the correct set of future turns rather than
-            // burning one count on the same turn the selection is recorded
-            // (off-by-one fix).
+            // Snapshot the raw (pre-stickiness) exclusion list before applying
+            // stickiness overrides. The cache stores this snapshot so future cache
+            // hits can re-apply the then-current stickiness state (Finding 2).
+            const preStickinessExcludeTools = routedExcludeTools;
+            // Apply stickiness FIRST: the sticky ids were recorded on a prior turn and
+            // represent servers that should stay warm for the current turn. Consuming
+            // (decrementing) them before recordSelection ensures the window covers the
+            // correct set of future turns rather than burning one count on the same
+            // turn the selection is recorded (off-by-one fix).
             if (stickinessEnabled && cache && sessionId) {
                 try {
                     const stickyIds = cache.getStickyServerIds(sessionId);
@@ -5888,15 +6142,16 @@ Current user's request: ${currentInput}`;
                     // Stickiness failure is non-fatal.
                 }
             }
-            // --- ITEM C: store result for future turns (MISS path only) ---
-            // Cache stores RAW (pre-stickiness) exclusions so the cached value is
-            // correct regardless of which sessions hit it on subsequent turns.
-            if (!fromCache && resolvedDecision?.outcome === "applied" && cache) {
+            // --- ITEM C: store result for future turns ---
+            if (resolvedDecision?.outcome === "applied" && cache) {
+                // Store the PRE-stickiness exclusion list in the cache so future hits
+                // re-apply the live stickiness window rather than the one from this
+                // turn (Finding 2).
                 if (cacheEnabled && cacheKey !== undefined) {
                     try {
                         cache.set(cacheKey, {
-                            excludedToolNames: rawExclusions,
-                            selectedServerIds,
+                            excludedToolNames: preStickinessExcludeTools,
+                            selectedServerIds: resolvedDecision.selectedServerIds,
                         });
                     }
                     catch {
@@ -5908,7 +6163,7 @@ Current user's request: ${currentInput}`;
                 // the same turn it is set (the off-by-one fix above).
                 if (stickinessEnabled && sessionId) {
                     try {
-                        cache.recordSelection(sessionId, selectedServerIds);
+                        cache.recordSelection(sessionId, resolvedDecision.selectedServerIds);
                     }
                     catch {
                         // Stickiness failure is non-fatal.
@@ -6005,6 +6260,12 @@ Current user's request: ${currentInput}`;
             return;
         }
         this.toolRoutingConfig.servers = servers;
+        // Clear the persisted vector cache so tool vectors are recomputed against
+        // the new catalog on the next turn (stale vectors must never be reused).
+        this.toolRoutingVectorCache = undefined;
+        // Cached routing decisions are catalog-dependent too; force the next turn
+        // to recompute exclusions against the new server/tool set.
+        this.toolRoutingCacheInstance = undefined;
     }
     async validateStreamRequestOptions(options, startTime) {
         await this.validateStreamInput(options);
@@ -7119,6 +7380,114 @@ Current user's request: ${currentInput}`;
                 }
             }
         }
+        // ─── ModelPool stream path ────────────────────────────────────────────────
+        // When a pool is configured, replaces the single provider.stream() call
+        // with a retry loop over pool members. Compaction/budget-check above ran
+        // once on the initially-selected `providerName`; the pool may redirect to
+        // a different provider, but those checks are conservative enough that the
+        // redirected provider will also accept the payload (if it has a smaller
+        // window, the budget check would have already thrown before we get here).
+        // On exhaustion, throws the last member error — does NOT fall through to
+        // the static provider path below.
+        if (this.modelPool) {
+            const streamPool = this.modelPool;
+            const maxStreamAttempts = streamPool.maxAttempts;
+            const streamTriedKeys = new Set();
+            let streamLastError = null;
+            for (let attempt = 0; attempt < maxStreamAttempts; attempt++) {
+                if (options.abortSignal?.aborted) {
+                    throw new DOMException("The operation was aborted", "AbortError");
+                }
+                const streamMember = streamPool.selectNext(streamTriedKeys);
+                if (!streamMember) {
+                    break;
+                }
+                streamTriedKeys.add(streamPool.memberKey(streamMember));
+                // Override provider/model/region from the pool member.
+                // When the member omits model/region, use undefined (provider default)
+                // rather than inheriting the caller's value for a different provider.
+                const poolStreamProviderName = streamMember.provider;
+                const poolStreamModel = streamMember.model ?? undefined;
+                const poolStreamRegion = streamMember.region ?? undefined;
+                logger.debug(`[createMCPStream] ModelPool: attempting member ${poolStreamProviderName}`, { model: poolStreamModel, attempt });
+                try {
+                    const poolStreamProvider = await AIProviderFactory.createProvider(poolStreamProviderName, poolStreamModel, !options.disableTools, this, poolStreamRegion, this.resolveCredentials(options.credentials));
+                    poolStreamProvider.setTraceContext(this._metricsTraceContext);
+                    poolStreamProvider.setupToolExecutor({
+                        customTools: this.getCustomTools(),
+                        executeTool: (toolName, params) => this.executeTool(toolName, params, {
+                            disableToolCache: options.disableToolCache,
+                        }),
+                    }, "NeuroLink.createMCPStream");
+                    const poolStreamResult = await poolStreamProvider.stream({
+                        ...options,
+                        provider: poolStreamProviderName,
+                        model: poolStreamModel,
+                        region: poolStreamRegion,
+                        systemPrompt: enhancedSystemPrompt,
+                        conversationMessages,
+                    });
+                    logger.debug("[createMCPStream] ModelPool stream handle obtained", {
+                        provider: poolStreamProviderName,
+                    });
+                    // Wrap the stream so we can record success/failure based on what
+                    // actually happens during consumption, not merely on obtaining the
+                    // handle.  Recording success at handle-acquisition time (before any
+                    // tokens are delivered) would mean a mid-stream provider drop is
+                    // never reflected as a cooldown.
+                    const capturedMember = streamMember;
+                    const wrappedStream = (async function* () {
+                        try {
+                            yield* poolStreamResult.stream;
+                            streamPool.recordSuccess(capturedMember);
+                        }
+                        catch (streamConsumeErr) {
+                            streamPool.recordFailure(capturedMember, classifyProviderError(streamConsumeErr));
+                            throw streamConsumeErr;
+                        }
+                    })();
+                    return {
+                        stream: wrappedStream,
+                        provider: poolStreamProviderName,
+                        usage: poolStreamResult.usage,
+                        model: poolStreamResult.model || poolStreamModel,
+                        finishReason: poolStreamResult.finishReason,
+                        toolCalls: poolStreamResult.toolCalls ?? [],
+                        toolResults: poolStreamResult.toolResults ?? [],
+                        analytics: poolStreamResult.analytics,
+                    };
+                }
+                catch (poolStreamError) {
+                    if (isAbortError(poolStreamError)) {
+                        throw poolStreamError;
+                    }
+                    if (isNonRetryableProviderError(poolStreamError)) {
+                        const poolStreamErrMsg = poolStreamError instanceof Error
+                            ? poolStreamError.message
+                            : String(poolStreamError);
+                        streamPool.recordFailure(streamMember, classifyProviderError(poolStreamError));
+                        // Wrap in a plain Error so runWithFallbackOrchestration /
+                        // streamWithIterationFallback does not re-activate the modelChain
+                        // layer on top of an already-exhausted pool.
+                        throw new Error(`[ModelPool] non-retryable: ${poolStreamErrMsg}`, {
+                            cause: poolStreamError,
+                        });
+                    }
+                    streamPool.recordFailure(streamMember, classifyProviderError(poolStreamError));
+                    streamLastError =
+                        poolStreamError instanceof Error
+                            ? poolStreamError
+                            : new Error(String(poolStreamError));
+                    logger.warn(`[createMCPStream] ModelPool: member ${poolStreamProviderName} failed`, { error: streamLastError.message });
+                }
+            }
+            // All pool stream members failed.
+            // Wrap in a plain Error (not a typed error class) so that the
+            // surrounding fallback orchestrator does not start a second retry layer
+            // over the already-exhausted pool.
+            throw new Error(`[ModelPool] all stream members failed: ${streamLastError?.message ?? "no stream members available"}`);
+        }
+        // ─── End ModelPool stream path ────────────────────────────────────────────
         // 🔧 FIX: Pass enhanced system prompt to real streaming
         // Tools will be accessed through the streamText call in executeStream
         const streamResult = await provider.stream({