@juspay/neurolink 9.73.0 → 9.74.0

package/dist/lib/core/toolRoutingCache.js

@@ -0,0 +1,155 @@
+/**
+ * LRU+TTL cache for pre-call tool routing decisions, plus session stickiness
+ * tracking to reduce turn-to-turn server flapping.
+ *
+ * The cache is keyed by a caller-supplied string (typically a normalized hash
+ * of sessionId + routing query). On a hit the cached exclusion list is
+ * returned, skipping the router LLM entirely. On a miss the caller resolves
+ * normally, stores the result, and `recordSelection` is called with the
+ * selected server ids so stickiness can suppress their exclusion for the next
+ * N turns.
+ *
+ * All operations are synchronous and pure (no I/O). The clock is injected via
+ * `now` so tests can control time without `Date.now` side effects.
+ */
+const DEFAULT_TTL_MS = 60_000;
+const DEFAULT_MAX_ENTRIES = 256;
+const DEFAULT_STICKY_TURNS = 3;
+/**
+ * Internal cap on the number of sessions tracked in the sticky map.
+ * Not configurable via public API — avoids unbounded memory growth when many
+ * ephemeral session ids are generated. Oldest entry (Map insertion order) is
+ * evicted when the cap is reached.
+ */
+const MAX_STICKY_ENTRIES = 1024;
+export class ToolRoutingCache {
+    ttlMs;
+    maxEntries;
+    stickyTurns;
+    now;
+    /** Main LRU+TTL store keyed by routing key. */
+    store = new Map();
+    /** Per-session stickiness tracking keyed by session id. */
+    sticky = new Map();
+    /** Monotonic counter for LRU ordering — incremented on each access. */
+    accessCounter = 0;
+    constructor(opts = {}) {
+        // Fall back to defaults for invalid (<=0 or NaN) values so callers cannot
+        // accidentally disable the cache by passing bad config.
+        const validOrDefault = (v, def) => v !== undefined && Number.isFinite(v) && v > 0 ? v : def;
+        this.ttlMs = validOrDefault(opts.ttlMs, DEFAULT_TTL_MS);
+        this.maxEntries = validOrDefault(opts.maxEntries, DEFAULT_MAX_ENTRIES);
+        this.stickyTurns = validOrDefault(opts.stickyTurns, DEFAULT_STICKY_TURNS);
+        this.now = opts.now ?? Date.now;
+    }
+    /**
+     * Returns the cached routing result for the given key, or `undefined` on a
+     * miss or expiry. Expired entries are deleted on access (lazy eviction).
+     */
+    get(key) {
+        const entry = this.store.get(key);
+        if (!entry) {
+            return undefined;
+        }
+        if (this.now() >= entry.expiresAt) {
+            this.store.delete(key);
+            return undefined;
+        }
+        // Bump access order for LRU.
+        entry.accessOrder = ++this.accessCounter;
+        // Return shallow copies so callers cannot mutate cached state.
+        return {
+            excludedToolNames: [...entry.excludedToolNames],
+            selectedServerIds: [...entry.selectedServerIds],
+        };
+    }
+    /**
+     * Stores a routing result under the given key. Evicts the least-recently-
+     * used entry when the store is at capacity. Silently no-ops on any error
+     * (caller is responsible for fail-open behaviour).
+     */
+    set(key, value) {
+        try {
+            if (this.store.size >= this.maxEntries && !this.store.has(key)) {
+                this.evictLRU();
+            }
+            // Defensive copies: guard against callers mutating arrays after storing.
+            this.store.set(key, {
+                excludedToolNames: [...value.excludedToolNames],
+                selectedServerIds: [...value.selectedServerIds],
+                expiresAt: this.now() + this.ttlMs,
+                accessOrder: ++this.accessCounter,
+            });
+        }
+        catch {
+            // Silently no-ops on any error — cache writes are non-fatal.
+        }
+    }
+    /**
+     * Records the server ids selected by the router for a session so they stay
+     * warm for the next `stickyTurns` turns. Called after a successful
+     * (non-cached) routing resolution.
+     */
+    recordSelection(sessionId, serverIds) {
+        if (!sessionId || serverIds.length === 0) {
+            return;
+        }
+        // Evict the oldest session when at the internal cap (Map preserves insertion
+        // order, so the first key is always the oldest).
+        if (this.sticky.size >= MAX_STICKY_ENTRIES && !this.sticky.has(sessionId)) {
+            const oldestKey = this.sticky.keys().next().value;
+            if (oldestKey !== undefined) {
+                this.sticky.delete(oldestKey);
+            }
+        }
+        this.sticky.set(sessionId, {
+            serverIds: [...serverIds],
+            turnsRemaining: this.stickyTurns,
+        });
+    }
+    /**
+     * Returns the server ids that should be kept warm (not excluded) for the
+     * given session due to stickiness. Decrements the turn counter; when it
+     * reaches zero the entry is removed. Returns an empty array when there is no
+     * active sticky state for the session.
+     */
+    getStickyServerIds(sessionId) {
+        if (!sessionId) {
+            return [];
+        }
+        const entry = this.sticky.get(sessionId);
+        if (!entry) {
+            return [];
+        }
+        const ids = [...entry.serverIds];
+        if (entry.turnsRemaining <= 1) {
+            this.sticky.delete(sessionId);
+        }
+        else {
+            entry.turnsRemaining -= 1;
+        }
+        return ids;
+    }
+    // ---------------------------------------------------------------------------
+    // Private helpers
+    // ---------------------------------------------------------------------------
+    /**
+     * Scans the store for the entry with the lowest accessOrder and removes it.
+     * O(n) scan is acceptable at the default maxEntries (256); very large caches
+     * should consider an O(1) doubly-linked-list approach instead.
+     */
+    evictLRU() {
+        let lruKey;
+        let lruOrder = Infinity;
+        for (const [key, entry] of this.store) {
+            if (entry.accessOrder < lruOrder) {
+                lruOrder = entry.accessOrder;
+                lruKey = key;
+            }
+        }
+        if (lruKey !== undefined) {
+            this.store.delete(lruKey);
+        }
+    }
+}
+//# sourceMappingURL=toolRoutingCache.js.map

package/dist/lib/index.d.ts

@@ -216,6 +216,8 @@ export declare function createBestAIProvider(requestedProvider?: string, modelNa
 export { CircuitBreakerManager, calculateExpiresAt, createMCPServer, createOAuthProviderFromConfig, DEFAULT_HTTP_RETRY_CONFIG, DEFAULT_RATE_LIMIT_CONFIG, executeMCP, FileTokenStorage, getMCPStats, getServerInfo, globalCircuitBreakerManager, globalRateLimiterManager, HTTPRateLimiter, InMemoryTokenStorage, initializeMCPEcosystem, isRetryableHTTPError, isRetryableStatusCode, isTokenExpired, listMCPs, CircuitBreakerOpenError, MCPCircuitBreaker, mcpLogger, NeuroLinkOAuthProvider, RateLimiterManager, validateServerTools, validateTool as validateMCPTool, withHTTPRetry, MCPToolRegistry, ExternalServerManager, MCPClientFactory, ToolRouter, createToolRouter, DEFAULT_ROUTER_CONFIG, ToolCache, createToolCache, DEFAULT_CACHE_CONFIG, ToolResultCache, createToolResultCache, RequestBatcher, ToolCallBatcher, createRequestBatcher, createToolCallBatcher, DEFAULT_BATCH_CONFIG, inferAnnotations, createAnnotatedTool, validateAnnotations, filterToolsByAnnotations, mergeAnnotations, getAnnotationSummary, requiresConfirmation, isSafeToRetry, getToolSafetyLevel, ElicitationManager, globalElicitationManager, EnhancedToolDiscovery, MCPRegistryClient, globalMCPRegistryClient, getWellKnownServer, getAllWellKnownServers, MCPServerBase, MultiServerManager, globalMultiServerManager, AgentExposureManager, exposeAgentAsTool, exposeAgentsAsTools, exposeWorkflowAsTool, exposeWorkflowsAsTools, globalAgentExposureManager, ServerCapabilitiesManager, createTextResource, createJsonResource, createPrompt, neuroLinkToolToMCP, mcpToolToNeuroLink, batchConvertToMCP, batchConvertToNeuroLink, sanitizeToolName, validateToolName, createToolFromFunction, mcpProtocolToolToServerTool, serverToolToMCPProtocol, TOOL_COMPATIBILITY, ToolIntegrationManager, globalToolIntegrationManager, wrapToolWithElicitation, wrapToolsWithElicitation, createElicitationContext, confirmationMiddleware, validationMiddleware, loggingMiddleware, createRetryMiddleware, createTimeoutMiddleware, createToolMiddlewareChain, ElicitationProtocolAdapter, globalElicitationProtocol, isElicitationProtocolMessage, protocolMessageToElicitation, elicitationResponseToProtocol, createElicitationRequest, createElicitationResponse, createElicitationCancel, createTextInputRequest, createSelectRequest, createConfirmationRequest, createFormRequest, } from "./mcp/index.js";
 export { logger } from "./utils/logger.js";
 export { getPoolStats } from "./utils/redis.js";
+export { ToolRoutingCache } from "./core/toolRoutingCache.js";
+export { resolveToolRoutingExclusions, buildToolRoutingCatalog, buildRoutingQueryFromHistory, DEFAULT_ROUTER_PROMPT_PREFIX, } from "./core/toolRouting.js";
 export declare function initializeTelemetry(): Promise<boolean>;
 export declare function getTelemetryStatus(): Promise<{
     enabled: boolean;

package/dist/lib/index.js

@@ -356,6 +356,10 @@ ToolIntegrationManager, globalToolIntegrationManager, wrapToolWithElicitation, w
 ElicitationProtocolAdapter, globalElicitationProtocol, isElicitationProtocolMessage, protocolMessageToElicitation, elicitationResponseToProtocol, createElicitationRequest, createElicitationResponse, createElicitationCancel, createTextInputRequest, createSelectRequest, createConfirmationRequest, createFormRequest, } from "./mcp/index.js";
 export { logger } from "./utils/logger.js";
 export { getPoolStats } from "./utils/redis.js";
+// Pre-call tool routing cache (ITEM C: LRU+TTL cache + session stickiness)
+export { ToolRoutingCache } from "./core/toolRoutingCache.js";
+// Pre-call tool routing resolver — exported for host integrations and testing
+export { resolveToolRoutingExclusions, buildToolRoutingCatalog, buildRoutingQueryFromHistory, DEFAULT_ROUTER_PROMPT_PREFIX, } from "./core/toolRouting.js";
 // ============================================================================
 // REAL-TIME SERVICES & TELEMETRY - Enterprise Platform Features
 // ============================================================================

package/dist/lib/neurolink.d.ts

@@ -101,6 +101,7 @@ export declare class NeuroLink {
     private conversationMemoryNeedsInit;
     private conversationMemoryConfig?;
     private toolRoutingConfig?;
+    private toolRoutingCacheInstance?;
     private enableOrchestration;
     private authProvider?;
     private pendingAuthConfig?;
@@ -805,12 +806,13 @@ export declare class NeuroLink {
     private streamWithIterationFallback;
     private executeStreamRequest;
     /**
-     * Pre-call tool routing for stream(): runs the router LLM once per turn
-     * and appends the unpicked servers' registered tool names to
-     * `options.excludeTools` — the per-call denylist enforced by
-     * `baseProvider.applyToolFiltering`. No-op unless `toolRouting.enabled`
-     * is true and a non-empty server catalog has been supplied. Never throws
-     * (the resolver fails open to an empty exclusion list).
+     * Pre-call tool routing for both stream() and generate() turns: runs the
+     * router LLM once per turn and appends the unpicked servers' registered tool
+     * names to `options.excludeTools` — the per-call denylist enforced by
+     * `baseProvider.applyToolFiltering`. No-op unless `toolRouting.enabled` is
+     * true and a non-empty server catalog has been supplied. Never throws (the
+     * resolver fails open to an empty exclusion list). Accepts both StreamOptions
+     * and GenerateOptions since both share the required fields.
      */
     private applyToolRoutingExclusions;
     /**

package/dist/lib/neurolink.js

@@ -31,6 +31,7 @@ import { repairToolPairs } from "./context/toolPairRepair.js";
 import { SYSTEM_LIMITS, DEFAULT_TOOL_ROUTING_TIMEOUT_MS, } from "./core/constants.js";
 import { ConversationMemoryManager } from "./core/conversationMemoryManager.js";
 import { buildToolRoutingCatalog, buildRoutingQueryFromHistory, resolveToolRoutingExclusions, } from "./core/toolRouting.js";
+import { ToolRoutingCache } from "./core/toolRoutingCache.js";
 import { AIProviderFactory } from "./core/factory.js";
 import { createToolEventPayload } from "./core/toolEvents.js";
 import { ProviderRegistry } from "./factories/providerRegistry.js";
@@ -59,7 +60,7 @@ import { SpanSerializer } from "./observability/utils/spanSerializer.js";
 import { flushOpenTelemetry, getLangfuseContext, getLangfuseHealthStatus, initializeOpenTelemetry, isOpenTelemetryInitialized, runWithCurrentLangfuseContext, setLangfuseContext, shutdownOpenTelemetry, stampGuestRescueIdentity, } from "./services/server/ai/observability/instrumentation.js";
 import { TaskManager } from "./tasks/taskManager.js";
 import { createTaskTools } from "./tasks/tools/taskTools.js";
-import { ATTR } from "./telemetry/attributes.js";
+import { ATTR, spanJsonAttribute } from "./telemetry/attributes.js";
 import { tracers } from "./telemetry/tracers.js";
 import { getConversationMessages, storeConversationTurn, } from "./utils/conversationMemory.js";
 // Enhanced error handling imports
@@ -441,6 +442,9 @@ export class NeuroLink {
     // The server catalog inside it can be supplied/replaced later via
     // setToolRoutingServers() for hosts that register tools after construction.
     toolRoutingConfig;
+    // Lazy-initialized routing decision cache (ITEM C). Created on first use so
+    // instances that don't use routing pay no overhead.
+    toolRoutingCacheInstance;
     // Add orchestration property
     enableOrchestration;
     // Authentication provider for secure access control
@@ -3019,7 +3023,14 @@ Current user's request: ${currentInput}`;
                 generateSpan.setStatus({ code: SpanStatusCode.OK });
                 return earlyResult;
             }
-            const result = await this.setLangfuseContextFromOptions(options, () => this.runStandardGenerateRequest(options, originalPrompt, generateSpan));
+            // Pre-call tool routing for generate(): mirrors the stream() routing path.
+            // Runs inside the generate's Langfuse context (setLangfuseContextFromOptions)
+            // so the router's own generation span nests under this turn's trace.
+            // After the early-result short-circuit so workflow/media turns skip it.
+            const result = await this.setLangfuseContextFromOptions(options, async () => {
+                await this.applyToolRoutingExclusions(options, options.input?.text ?? "");
+                return this.runStandardGenerateRequest(options, originalPrompt, generateSpan);
+            });
             generateSpan.setStatus({ code: SpanStatusCode.OK });
             return result;
         }
@@ -5688,12 +5699,13 @@ Current user's request: ${currentInput}`;
         }
     }
     /**
-     * Pre-call tool routing for stream(): runs the router LLM once per turn
-     * and appends the unpicked servers' registered tool names to
-     * `options.excludeTools` — the per-call denylist enforced by
-     * `baseProvider.applyToolFiltering`. No-op unless `toolRouting.enabled`
-     * is true and a non-empty server catalog has been supplied. Never throws
-     * (the resolver fails open to an empty exclusion list).
+     * Pre-call tool routing for both stream() and generate() turns: runs the
+     * router LLM once per turn and appends the unpicked servers' registered tool
+     * names to `options.excludeTools` — the per-call denylist enforced by
+     * `baseProvider.applyToolFiltering`. No-op unless `toolRouting.enabled` is
+     * true and a non-empty server catalog has been supplied. Never throws (the
+     * resolver fails open to an empty exclusion list). Accepts both StreamOptions
+     * and GenerateOptions since both share the required fields.
      */
     async applyToolRoutingExclusions(options, userQuery) {
         const routingConfig = this.toolRoutingConfig;
@@ -5706,8 +5718,8 @@ Current user's request: ${currentInput}`;
         }
         // Whole setup is fail-open: catalog building (getCustomTools /
         // buildToolRoutingCatalog) and the router call degrade to no exclusions
-        // rather than killing the stream, honoring this method's "never throws"
-        // contract. Genuine stream cancellations still propagate.
+        // rather than killing the stream/generate turn, honoring this method's
+        // "never throws" contract. Genuine cancellations still propagate.
         try {
             const registeredToolNames = Array.from(this.getCustomTools().keys());
             const catalog = buildToolRoutingCatalog(servers, registeredToolNames);
@@ -5724,44 +5736,180 @@ Current user's request: ${currentInput}`;
             const routingQuery = recentMessages.length > 0
                 ? buildRoutingQueryFromHistory(recentMessages, userQuery)
                 : userQuery;
-            // The router call below re-enters the public generate(), whose finally
-            // block resets _disableToolCacheForCurrentRequest to false. That flag is
-            // stream-scoped (set at the top of this turn) and read by the main tool
-            // execution path that runs after routing, so save it before the router
-            // call and restore it afterward to keep the turn's cache setting intact.
-            const cacheDisabledForCurrentRequest = this._disableToolCacheForCurrentRequest;
+            // --- ITEM C: routing decision cache ---
+            const cacheConfig = routingConfig.cache;
+            const cacheEnabled = cacheConfig?.enabled === true;
+            const stickinessEnabled = routingConfig.stickiness?.enabled === true;
+            // Lazy-init the cache instance once per NeuroLink instance.
+            if ((cacheEnabled || stickinessEnabled) &&
+                !this.toolRoutingCacheInstance) {
+                this.toolRoutingCacheInstance = new ToolRoutingCache({
+                    ttlMs: cacheConfig?.ttlMs,
+                    maxEntries: cacheConfig?.maxEntries,
+                    stickyTurns: routingConfig.stickiness?.turns,
+                });
+            }
+            const cache = this.toolRoutingCacheInstance;
+            // Derive sessionId for stickiness — same extraction path as
+            // fetchRecentRoutingHistory.
+            const requestContext = options.context;
+            const sessionId = typeof requestContext?.sessionId === "string"
+                ? requestContext.sessionId
+                : "";
+            // Cache key: session + normalized routing query.
+            // Require a non-empty sessionId to avoid anonymous sessions sharing a
+            // ":query" namespace (cross-session cache leak).
+            const cacheKey = cacheEnabled && sessionId ? `${sessionId}:${routingQuery}` : undefined;
+            // --- ITEM E: build the emitDecision callback ---
+            const emitDecision = (decision) => {
+                try {
+                    const activeSpan = trace.getActiveSpan();
+                    if (!activeSpan) {
+                        return;
+                    }
+                    activeSpan.setAttribute("tool_routing.outcome", decision.outcome);
+                    activeSpan.setAttribute("tool_routing.routable_server_count", decision.routableServerCount);
+                    activeSpan.setAttribute("tool_routing.selected_server_ids", spanJsonAttribute(decision.selectedServerIds));
+                    activeSpan.setAttribute("tool_routing.excluded_server_ids", spanJsonAttribute(decision.excludedServerIds));
+                    activeSpan.setAttribute("tool_routing.hallucinated_ids", spanJsonAttribute(decision.hallucinatedIds));
+                    activeSpan.setAttribute("tool_routing.excluded_tool_count", decision.excludedToolCount);
+                    activeSpan.setAttribute("tool_routing.cache_hit", decision.cacheHit);
+                    activeSpan.setAttribute("tool_routing.duration_ms", decision.durationMs);
+                }
+                catch {
+                    // Telemetry must never affect routing behaviour.
+                }
+            };
+            // Unified hit/miss block: always produces raw (pre-stickiness) exclusions
+            // so stickiness can be applied identically on both paths below.
             let routedExcludeTools;
-            try {
-                routedExcludeTools = await resolveToolRoutingExclusions({
-                    catalog,
-                    alwaysIncludeServerIds: routingConfig.alwaysIncludeServerIds ?? [],
-                    userQuery: routingQuery,
-                    routerPromptPrefix: routingConfig.routerPromptPrefix,
-                    routerModel: {
-                        provider: routingConfig.routerModel?.provider ??
-                            options.provider,
-                        model: routingConfig.routerModel?.model ?? options.model,
-                        region: routingConfig.routerModel?.region ?? options.region,
-                        temperature: routingConfig.routerModel?.temperature,
-                    },
-                    timeoutMs: routingConfig.timeoutMs ?? DEFAULT_TOOL_ROUTING_TIMEOUT_MS,
-                    // Forward the stream's abort signal so a cancelled stream aborts the
-                    // router call promptly instead of waiting out the routing timeout.
-                    generateFn: (generateOptions) => this.generate({
-                        ...generateOptions,
-                        abortSignal: options.abortSignal,
-                    }),
+            let selectedServerIds = [];
+            let resolvedDecision;
+            let fromCache = false;
+            const cached = cacheEnabled && cache && cacheKey !== undefined
+                ? cache.get(cacheKey)
+                : undefined;
+            if (cached) {
+                // Cache HIT — use stored raw exclusions (pre-stickiness).
+                fromCache = true;
+                routedExcludeTools = [...cached.excludedToolNames];
+                selectedServerIds = [...cached.selectedServerIds];
+                logger.debug("[ToolRouting] Cache hit, skipping router LLM", {
+                    cacheKey,
                 });
+                // Emit cache-hit telemetry with full parity to the miss path.
+                try {
+                    const activeSpan = trace.getActiveSpan();
+                    if (activeSpan) {
+                        const routableCount = catalog.filter((s) => !(routingConfig.alwaysIncludeServerIds ?? []).includes(s.id)).length;
+                        activeSpan.setAttribute("tool_routing.outcome", "cache-hit");
+                        activeSpan.setAttribute("tool_routing.routable_server_count", routableCount);
+                        activeSpan.setAttribute("tool_routing.cache_hit", true);
+                        activeSpan.setAttribute("tool_routing.selected_server_ids", spanJsonAttribute(selectedServerIds));
+                        activeSpan.setAttribute("tool_routing.excluded_tool_count", routedExcludeTools.length);
+                    }
+                }
+                catch {
+                    // Telemetry must never affect routing behaviour.
+                }
             }
-            finally {
-                this._disableToolCacheForCurrentRequest =
-                    cacheDisabledForCurrentRequest;
+            else {
+                // Cache MISS — run the router LLM.
+                // The router call re-enters the public generate(), whose finally block
+                // resets _disableToolCacheForCurrentRequest to false. That flag is
+                // turn-scoped and read by the main tool execution path after routing, so
+                // save it before the router call and restore it afterward.
+                const cacheDisabledForCurrentRequest = this._disableToolCacheForCurrentRequest;
+                try {
+                    // Intercept the decision so we can store it in the cache.
+                    const captureDecision = (decision) => {
+                        resolvedDecision = decision;
+                        emitDecision(decision);
+                    };
+                    routedExcludeTools = await resolveToolRoutingExclusions({
+                        catalog,
+                        alwaysIncludeServerIds: routingConfig.alwaysIncludeServerIds ?? [],
+                        userQuery: routingQuery,
+                        routerPromptPrefix: routingConfig.routerPromptPrefix,
+                        routerModel: {
+                            provider: routingConfig.routerModel?.provider ??
+                                options.provider,
+                            model: routingConfig.routerModel?.model ?? options.model,
+                            region: routingConfig.routerModel?.region ?? options.region,
+                            temperature: routingConfig.routerModel?.temperature,
+                        },
+                        timeoutMs: routingConfig.timeoutMs ?? DEFAULT_TOOL_ROUTING_TIMEOUT_MS,
+                        // Forward the abort signal so a cancelled turn aborts the router
+                        // call promptly instead of waiting out the routing timeout.
+                        generateFn: (generateOptions) => this.generate({
+                            ...generateOptions,
+                            abortSignal: options.abortSignal,
+                        }),
+                        emitDecision: captureDecision,
+                    });
+                }
+                finally {
+                    this._disableToolCacheForCurrentRequest =
+                        cacheDisabledForCurrentRequest;
+                }
+                if (resolvedDecision?.outcome === "applied") {
+                    selectedServerIds = resolvedDecision.selectedServerIds;
+                }
             }
             // Aborted during the router call — skip applying now-stale exclusions;
             // the main generation path enforces the abort itself.
             if (options.abortSignal?.aborted) {
                 return;
             }
+            // Capture raw (pre-stickiness) exclusions for cache storage on MISS path.
+            const rawExclusions = [...routedExcludeTools];
+            // Apply stickiness on BOTH hit and miss paths: the sticky ids were
+            // recorded on a prior turn and represent servers that should stay warm for
+            // the current turn. Consuming (decrementing) them before recordSelection
+            // ensures the window covers the correct set of future turns rather than
+            // burning one count on the same turn the selection is recorded
+            // (off-by-one fix).
+            if (stickinessEnabled && cache && sessionId) {
+                try {
+                    const stickyIds = cache.getStickyServerIds(sessionId);
+                    if (stickyIds.length > 0) {
+                        // Remove from routedExcludeTools any tool belonging to a sticky server.
+                        // Precompute prefixes to avoid rebuilding the set inside the predicate.
+                        const stickyPrefixes = stickyIds.map((id) => `${id}_`);
+                        routedExcludeTools = routedExcludeTools.filter((toolName) => !stickyPrefixes.some((prefix) => toolName.startsWith(prefix)));
+                    }
+                }
+                catch {
+                    // Stickiness failure is non-fatal.
+                }
+            }
+            // --- ITEM C: store result for future turns (MISS path only) ---
+            // Cache stores RAW (pre-stickiness) exclusions so the cached value is
+            // correct regardless of which sessions hit it on subsequent turns.
+            if (!fromCache && resolvedDecision?.outcome === "applied" && cache) {
+                if (cacheEnabled && cacheKey !== undefined) {
+                    try {
+                        cache.set(cacheKey, {
+                            excludedToolNames: rawExclusions,
+                            selectedServerIds,
+                        });
+                    }
+                    catch {
+                        // Cache write failure is non-fatal.
+                    }
+                }
+                // Record selected servers for stickiness on subsequent turns. This must
+                // come after getStickyServerIds so the window count is not consumed on
+                // the same turn it is set (the off-by-one fix above).
+                if (stickinessEnabled && sessionId) {
+                    try {
+                        cache.recordSelection(sessionId, selectedServerIds);
+                    }
+                    catch {
+                        // Stickiness failure is non-fatal.
+                    }
+                }
+            }
             if (routedExcludeTools.length > 0) {
                 options.excludeTools = [
                     ...(options.excludeTools ?? []),

package/dist/lib/types/toolRouting.d.ts

@@ -64,6 +64,30 @@ export type ToolRoutingConfig = {
      * always appended by the SDK regardless of this value.
      */
     routerPromptPrefix?: string;
+    /**
+     * LRU+TTL cache for routing decisions. When enabled, identical routing
+     * queries within the TTL window skip the router LLM entirely and reuse
+     * the cached exclusion list.
+     */
+    cache?: {
+        /** Whether the cache is active. Default: false. */
+        enabled?: boolean;
+        /** Time-to-live in milliseconds for each cached entry. Default: 60000. */
+        ttlMs?: number;
+        /** Maximum number of entries in the LRU cache. Default: 256. */
+        maxEntries?: number;
+    };
+    /**
+     * Session stickiness: once the router picks a set of servers for a session,
+     * those servers are kept warm (not excluded) for the next N turns to prevent
+     * flapping.
+     */
+    stickiness?: {
+        /** Whether session stickiness is active. Default: false. */
+        enabled?: boolean;
+        /** Number of turns for which a previously-selected server stays warm. Default: 3. */
+        turns?: number;
+    };
 };
 /** Catalog entry pairing a server descriptor with its registered tool names. */
 export type ToolRoutingCatalogEntry = {
@@ -72,6 +96,63 @@ export type ToolRoutingCatalogEntry = {
     /** Registered tool names for this server, i.e. `${serverId}_${toolName}`. */
     toolNames: string[];
 };
+/** Internal cache entry for `ToolRoutingCache`. */
+export type ToolRoutingCacheEntry = {
+    excludedToolNames: string[];
+    selectedServerIds: string[];
+    /** Absolute expiry timestamp (from the injected `now()` clock). */
+    expiresAt: number;
+    /** LRU eviction order — lower = older. Bumped on each get/set. */
+    accessOrder: number;
+};
+/** Internal stickiness entry for `ToolRoutingCache`. */
+export type ToolRoutingStickiness = {
+    serverIds: string[];
+    /** Turn counter — decremented on each routing turn, removed when it hits 0. */
+    turnsRemaining: number;
+};
+/** Constructor options for `ToolRoutingCache`. */
+export type ToolRoutingCacheOptions = {
+    /** Time-to-live in milliseconds for each cached entry. Default: 60_000. */
+    ttlMs?: number;
+    /** Maximum number of entries kept in the LRU before eviction. Default: 256. */
+    maxEntries?: number;
+    /** Number of turns a selected server remains sticky per session. Default: 3. */
+    stickyTurns?: number;
+    /**
+     * Clock function for TTL calculations. Defaults to `Date.now`.
+     * Inject a deterministic function in tests to control time.
+     */
+    now?: () => number;
+};
+/**
+ * Outcome classifier for a single routing resolution. Used in
+ * `ToolRoutingDecision` and emitted as an OTel span attribute.
+ */
+export type ToolRoutingOutcome = "applied" | "skipped-no-query" | "skipped-single-server" | "empty-pick" | "failed-open-parse" | "failed-open-timeout" | "failed-open-error" | "cache-hit";
+/**
+ * Machine-readable summary of one routing resolution. Emitted via the
+ * `emitDecision` callback so the caller can attach it as OTel span attributes
+ * or record it in any other telemetry sink.
+ */
+export type ToolRoutingDecision = {
+    /** How the routing turn concluded. */
+    outcome: ToolRoutingOutcome;
+    /** Server ids the router kept (selected as relevant). */
+    selectedServerIds: string[];
+    /** Server ids whose tools were excluded (router considered them irrelevant). */
+    excludedServerIds: string[];
+    /** Server ids the router returned that did not exist in the catalog. */
+    hallucinatedIds: string[];
+    /** Total number of individual tool names added to the exclusion list. */
+    excludedToolCount: number;
+    /** Number of servers that were offered to the router (always-include excluded). */
+    routableServerCount: number;
+    /** True when the result was served from cache, skipping the router LLM. */
+    cacheHit: boolean;
+    /** Wall-clock time spent in the routing resolution in milliseconds. */
+    durationMs: number;
+};
 /** Parameters for `resolveToolRoutingExclusions()`. */
 export type ToolRoutingResolutionParams = {
     /** Full catalog; always-include servers are filtered out internally. */
@@ -88,4 +169,11 @@ export type ToolRoutingResolutionParams = {
     timeoutMs: number;
     /** Invokes the router LLM — `NeuroLink.generate` bound by the caller. */
     generateFn: (options: GenerateOptions) => Promise<GenerateResult>;
+    /**
+     * Optional callback invoked once per resolution with a structured summary of
+     * the routing decision. Called on every return path (applied, skipped,
+     * failed-open). Must never throw — any error inside is swallowed by
+     * the resolver.
+     */
+    emitDecision?: (decision: ToolRoutingDecision) => void;
 };

package/dist/neurolink.d.ts

@@ -101,6 +101,7 @@ export declare class NeuroLink {
     private conversationMemoryNeedsInit;
     private conversationMemoryConfig?;
     private toolRoutingConfig?;
+    private toolRoutingCacheInstance?;
     private enableOrchestration;
     private authProvider?;
     private pendingAuthConfig?;
@@ -805,12 +806,13 @@ export declare class NeuroLink {
     private streamWithIterationFallback;
     private executeStreamRequest;
     /**
-     * Pre-call tool routing for stream(): runs the router LLM once per turn
-     * and appends the unpicked servers' registered tool names to
-     * `options.excludeTools` — the per-call denylist enforced by
-     * `baseProvider.applyToolFiltering`. No-op unless `toolRouting.enabled`
-     * is true and a non-empty server catalog has been supplied. Never throws
-     * (the resolver fails open to an empty exclusion list).
+     * Pre-call tool routing for both stream() and generate() turns: runs the
+     * router LLM once per turn and appends the unpicked servers' registered tool
+     * names to `options.excludeTools` — the per-call denylist enforced by
+     * `baseProvider.applyToolFiltering`. No-op unless `toolRouting.enabled` is
+     * true and a non-empty server catalog has been supplied. Never throws (the
+     * resolver fails open to an empty exclusion list). Accepts both StreamOptions
+     * and GenerateOptions since both share the required fields.
      */
     private applyToolRoutingExclusions;
     /**