@juspay/neurolink 9.73.0 → 9.74.0

package/dist/neurolink.js

@@ -31,6 +31,7 @@ import { repairToolPairs } from "./context/toolPairRepair.js";
 import { SYSTEM_LIMITS, DEFAULT_TOOL_ROUTING_TIMEOUT_MS, } from "./core/constants.js";
 import { ConversationMemoryManager } from "./core/conversationMemoryManager.js";
 import { buildToolRoutingCatalog, buildRoutingQueryFromHistory, resolveToolRoutingExclusions, } from "./core/toolRouting.js";
+import { ToolRoutingCache } from "./core/toolRoutingCache.js";
 import { AIProviderFactory } from "./core/factory.js";
 import { createToolEventPayload } from "./core/toolEvents.js";
 import { ProviderRegistry } from "./factories/providerRegistry.js";
@@ -59,7 +60,7 @@ import { SpanSerializer } from "./observability/utils/spanSerializer.js";
 import { flushOpenTelemetry, getLangfuseContext, getLangfuseHealthStatus, initializeOpenTelemetry, isOpenTelemetryInitialized, runWithCurrentLangfuseContext, setLangfuseContext, shutdownOpenTelemetry, stampGuestRescueIdentity, } from "./services/server/ai/observability/instrumentation.js";
 import { TaskManager } from "./tasks/taskManager.js";
 import { createTaskTools } from "./tasks/tools/taskTools.js";
-import { ATTR } from "./telemetry/attributes.js";
+import { ATTR, spanJsonAttribute } from "./telemetry/attributes.js";
 import { tracers } from "./telemetry/tracers.js";
 import { getConversationMessages, storeConversationTurn, } from "./utils/conversationMemory.js";
 // Enhanced error handling imports
@@ -441,6 +442,9 @@ export class NeuroLink {
     // The server catalog inside it can be supplied/replaced later via
     // setToolRoutingServers() for hosts that register tools after construction.
     toolRoutingConfig;
+    // Lazy-initialized routing decision cache (ITEM C). Created on first use so
+    // instances that don't use routing pay no overhead.
+    toolRoutingCacheInstance;
     // Add orchestration property
     enableOrchestration;
     // Authentication provider for secure access control
@@ -3019,7 +3023,14 @@ Current user's request: ${currentInput}`;
                 generateSpan.setStatus({ code: SpanStatusCode.OK });
                 return earlyResult;
             }
-            const result = await this.setLangfuseContextFromOptions(options, () => this.runStandardGenerateRequest(options, originalPrompt, generateSpan));
+            // Pre-call tool routing for generate(): mirrors the stream() routing path.
+            // Runs inside the generate's Langfuse context (setLangfuseContextFromOptions)
+            // so the router's own generation span nests under this turn's trace.
+            // After the early-result short-circuit so workflow/media turns skip it.
+            const result = await this.setLangfuseContextFromOptions(options, async () => {
+                await this.applyToolRoutingExclusions(options, options.input?.text ?? "");
+                return this.runStandardGenerateRequest(options, originalPrompt, generateSpan);
+            });
             generateSpan.setStatus({ code: SpanStatusCode.OK });
             return result;
         }
@@ -5688,12 +5699,13 @@ Current user's request: ${currentInput}`;
         }
     }
     /**
-     * Pre-call tool routing for stream(): runs the router LLM once per turn
-     * and appends the unpicked servers' registered tool names to
-     * `options.excludeTools` — the per-call denylist enforced by
-     * `baseProvider.applyToolFiltering`. No-op unless `toolRouting.enabled`
-     * is true and a non-empty server catalog has been supplied. Never throws
-     * (the resolver fails open to an empty exclusion list).
+     * Pre-call tool routing for both stream() and generate() turns: runs the
+     * router LLM once per turn and appends the unpicked servers' registered tool
+     * names to `options.excludeTools` — the per-call denylist enforced by
+     * `baseProvider.applyToolFiltering`. No-op unless `toolRouting.enabled` is
+     * true and a non-empty server catalog has been supplied. Never throws (the
+     * resolver fails open to an empty exclusion list). Accepts both StreamOptions
+     * and GenerateOptions since both share the required fields.
      */
     async applyToolRoutingExclusions(options, userQuery) {
         const routingConfig = this.toolRoutingConfig;
@@ -5706,8 +5718,8 @@ Current user's request: ${currentInput}`;
         }
         // Whole setup is fail-open: catalog building (getCustomTools /
         // buildToolRoutingCatalog) and the router call degrade to no exclusions
-        // rather than killing the stream, honoring this method's "never throws"
-        // contract. Genuine stream cancellations still propagate.
+        // rather than killing the stream/generate turn, honoring this method's
+        // "never throws" contract. Genuine cancellations still propagate.
         try {
             const registeredToolNames = Array.from(this.getCustomTools().keys());
             const catalog = buildToolRoutingCatalog(servers, registeredToolNames);
@@ -5724,44 +5736,180 @@ Current user's request: ${currentInput}`;
             const routingQuery = recentMessages.length > 0
                 ? buildRoutingQueryFromHistory(recentMessages, userQuery)
                 : userQuery;
-            // The router call below re-enters the public generate(), whose finally
-            // block resets _disableToolCacheForCurrentRequest to false. That flag is
-            // stream-scoped (set at the top of this turn) and read by the main tool
-            // execution path that runs after routing, so save it before the router
-            // call and restore it afterward to keep the turn's cache setting intact.
-            const cacheDisabledForCurrentRequest = this._disableToolCacheForCurrentRequest;
+            // --- ITEM C: routing decision cache ---
+            const cacheConfig = routingConfig.cache;
+            const cacheEnabled = cacheConfig?.enabled === true;
+            const stickinessEnabled = routingConfig.stickiness?.enabled === true;
+            // Lazy-init the cache instance once per NeuroLink instance.
+            if ((cacheEnabled || stickinessEnabled) &&
+                !this.toolRoutingCacheInstance) {
+                this.toolRoutingCacheInstance = new ToolRoutingCache({
+                    ttlMs: cacheConfig?.ttlMs,
+                    maxEntries: cacheConfig?.maxEntries,
+                    stickyTurns: routingConfig.stickiness?.turns,
+                });
+            }
+            const cache = this.toolRoutingCacheInstance;
+            // Derive sessionId for stickiness — same extraction path as
+            // fetchRecentRoutingHistory.
+            const requestContext = options.context;
+            const sessionId = typeof requestContext?.sessionId === "string"
+                ? requestContext.sessionId
+                : "";
+            // Cache key: session + normalized routing query.
+            // Require a non-empty sessionId to avoid anonymous sessions sharing a
+            // ":query" namespace (cross-session cache leak).
+            const cacheKey = cacheEnabled && sessionId ? `${sessionId}:${routingQuery}` : undefined;
+            // --- ITEM E: build the emitDecision callback ---
+            const emitDecision = (decision) => {
+                try {
+                    const activeSpan = trace.getActiveSpan();
+                    if (!activeSpan) {
+                        return;
+                    }
+                    activeSpan.setAttribute("tool_routing.outcome", decision.outcome);
+                    activeSpan.setAttribute("tool_routing.routable_server_count", decision.routableServerCount);
+                    activeSpan.setAttribute("tool_routing.selected_server_ids", spanJsonAttribute(decision.selectedServerIds));
+                    activeSpan.setAttribute("tool_routing.excluded_server_ids", spanJsonAttribute(decision.excludedServerIds));
+                    activeSpan.setAttribute("tool_routing.hallucinated_ids", spanJsonAttribute(decision.hallucinatedIds));
+                    activeSpan.setAttribute("tool_routing.excluded_tool_count", decision.excludedToolCount);
+                    activeSpan.setAttribute("tool_routing.cache_hit", decision.cacheHit);
+                    activeSpan.setAttribute("tool_routing.duration_ms", decision.durationMs);
+                }
+                catch {
+                    // Telemetry must never affect routing behaviour.
+                }
+            };
+            // Unified hit/miss block: always produces raw (pre-stickiness) exclusions
+            // so stickiness can be applied identically on both paths below.
             let routedExcludeTools;
-            try {
-                routedExcludeTools = await resolveToolRoutingExclusions({
-                    catalog,
-                    alwaysIncludeServerIds: routingConfig.alwaysIncludeServerIds ?? [],
-                    userQuery: routingQuery,
-                    routerPromptPrefix: routingConfig.routerPromptPrefix,
-                    routerModel: {
-                        provider: routingConfig.routerModel?.provider ??
-                            options.provider,
-                        model: routingConfig.routerModel?.model ?? options.model,
-                        region: routingConfig.routerModel?.region ?? options.region,
-                        temperature: routingConfig.routerModel?.temperature,
-                    },
-                    timeoutMs: routingConfig.timeoutMs ?? DEFAULT_TOOL_ROUTING_TIMEOUT_MS,
-                    // Forward the stream's abort signal so a cancelled stream aborts the
-                    // router call promptly instead of waiting out the routing timeout.
-                    generateFn: (generateOptions) => this.generate({
-                        ...generateOptions,
-                        abortSignal: options.abortSignal,
-                    }),
+            let selectedServerIds = [];
+            let resolvedDecision;
+            let fromCache = false;
+            const cached = cacheEnabled && cache && cacheKey !== undefined
+                ? cache.get(cacheKey)
+                : undefined;
+            if (cached) {
+                // Cache HIT — use stored raw exclusions (pre-stickiness).
+                fromCache = true;
+                routedExcludeTools = [...cached.excludedToolNames];
+                selectedServerIds = [...cached.selectedServerIds];
+                logger.debug("[ToolRouting] Cache hit, skipping router LLM", {
+                    cacheKey,
                 });
+                // Emit cache-hit telemetry with full parity to the miss path.
+                try {
+                    const activeSpan = trace.getActiveSpan();
+                    if (activeSpan) {
+                        const routableCount = catalog.filter((s) => !(routingConfig.alwaysIncludeServerIds ?? []).includes(s.id)).length;
+                        activeSpan.setAttribute("tool_routing.outcome", "cache-hit");
+                        activeSpan.setAttribute("tool_routing.routable_server_count", routableCount);
+                        activeSpan.setAttribute("tool_routing.cache_hit", true);
+                        activeSpan.setAttribute("tool_routing.selected_server_ids", spanJsonAttribute(selectedServerIds));
+                        activeSpan.setAttribute("tool_routing.excluded_tool_count", routedExcludeTools.length);
+                    }
+                }
+                catch {
+                    // Telemetry must never affect routing behaviour.
+                }
             }
-            finally {
-                this._disableToolCacheForCurrentRequest =
-                    cacheDisabledForCurrentRequest;
+            else {
+                // Cache MISS — run the router LLM.
+                // The router call re-enters the public generate(), whose finally block
+                // resets _disableToolCacheForCurrentRequest to false. That flag is
+                // turn-scoped and read by the main tool execution path after routing, so
+                // save it before the router call and restore it afterward.
+                const cacheDisabledForCurrentRequest = this._disableToolCacheForCurrentRequest;
+                try {
+                    // Intercept the decision so we can store it in the cache.
+                    const captureDecision = (decision) => {
+                        resolvedDecision = decision;
+                        emitDecision(decision);
+                    };
+                    routedExcludeTools = await resolveToolRoutingExclusions({
+                        catalog,
+                        alwaysIncludeServerIds: routingConfig.alwaysIncludeServerIds ?? [],
+                        userQuery: routingQuery,
+                        routerPromptPrefix: routingConfig.routerPromptPrefix,
+                        routerModel: {
+                            provider: routingConfig.routerModel?.provider ??
+                                options.provider,
+                            model: routingConfig.routerModel?.model ?? options.model,
+                            region: routingConfig.routerModel?.region ?? options.region,
+                            temperature: routingConfig.routerModel?.temperature,
+                        },
+                        timeoutMs: routingConfig.timeoutMs ?? DEFAULT_TOOL_ROUTING_TIMEOUT_MS,
+                        // Forward the abort signal so a cancelled turn aborts the router
+                        // call promptly instead of waiting out the routing timeout.
+                        generateFn: (generateOptions) => this.generate({
+                            ...generateOptions,
+                            abortSignal: options.abortSignal,
+                        }),
+                        emitDecision: captureDecision,
+                    });
+                }
+                finally {
+                    this._disableToolCacheForCurrentRequest =
+                        cacheDisabledForCurrentRequest;
+                }
+                if (resolvedDecision?.outcome === "applied") {
+                    selectedServerIds = resolvedDecision.selectedServerIds;
+                }
             }
             // Aborted during the router call — skip applying now-stale exclusions;
             // the main generation path enforces the abort itself.
             if (options.abortSignal?.aborted) {
                 return;
             }
+            // Capture raw (pre-stickiness) exclusions for cache storage on MISS path.
+            const rawExclusions = [...routedExcludeTools];
+            // Apply stickiness on BOTH hit and miss paths: the sticky ids were
+            // recorded on a prior turn and represent servers that should stay warm for
+            // the current turn. Consuming (decrementing) them before recordSelection
+            // ensures the window covers the correct set of future turns rather than
+            // burning one count on the same turn the selection is recorded
+            // (off-by-one fix).
+            if (stickinessEnabled && cache && sessionId) {
+                try {
+                    const stickyIds = cache.getStickyServerIds(sessionId);
+                    if (stickyIds.length > 0) {
+                        // Remove from routedExcludeTools any tool belonging to a sticky server.
+                        // Precompute prefixes to avoid rebuilding the set inside the predicate.
+                        const stickyPrefixes = stickyIds.map((id) => `${id}_`);
+                        routedExcludeTools = routedExcludeTools.filter((toolName) => !stickyPrefixes.some((prefix) => toolName.startsWith(prefix)));
+                    }
+                }
+                catch {
+                    // Stickiness failure is non-fatal.
+                }
+            }
+            // --- ITEM C: store result for future turns (MISS path only) ---
+            // Cache stores RAW (pre-stickiness) exclusions so the cached value is
+            // correct regardless of which sessions hit it on subsequent turns.
+            if (!fromCache && resolvedDecision?.outcome === "applied" && cache) {
+                if (cacheEnabled && cacheKey !== undefined) {
+                    try {
+                        cache.set(cacheKey, {
+                            excludedToolNames: rawExclusions,
+                            selectedServerIds,
+                        });
+                    }
+                    catch {
+                        // Cache write failure is non-fatal.
+                    }
+                }
+                // Record selected servers for stickiness on subsequent turns. This must
+                // come after getStickyServerIds so the window count is not consumed on
+                // the same turn it is set (the off-by-one fix above).
+                if (stickinessEnabled && sessionId) {
+                    try {
+                        cache.recordSelection(sessionId, selectedServerIds);
+                    }
+                    catch {
+                        // Stickiness failure is non-fatal.
+                    }
+                }
+            }
             if (routedExcludeTools.length > 0) {
                 options.excludeTools = [
                     ...(options.excludeTools ?? []),

package/dist/types/toolRouting.d.ts

@@ -64,6 +64,30 @@ export type ToolRoutingConfig = {
      * always appended by the SDK regardless of this value.
      */
     routerPromptPrefix?: string;
+    /**
+     * LRU+TTL cache for routing decisions. When enabled, identical routing
+     * queries within the TTL window skip the router LLM entirely and reuse
+     * the cached exclusion list.
+     */
+    cache?: {
+        /** Whether the cache is active. Default: false. */
+        enabled?: boolean;
+        /** Time-to-live in milliseconds for each cached entry. Default: 60000. */
+        ttlMs?: number;
+        /** Maximum number of entries in the LRU cache. Default: 256. */
+        maxEntries?: number;
+    };
+    /**
+     * Session stickiness: once the router picks a set of servers for a session,
+     * those servers are kept warm (not excluded) for the next N turns to prevent
+     * flapping.
+     */
+    stickiness?: {
+        /** Whether session stickiness is active. Default: false. */
+        enabled?: boolean;
+        /** Number of turns for which a previously-selected server stays warm. Default: 3. */
+        turns?: number;
+    };
 };
 /** Catalog entry pairing a server descriptor with its registered tool names. */
 export type ToolRoutingCatalogEntry = {
@@ -72,6 +96,63 @@ export type ToolRoutingCatalogEntry = {
     /** Registered tool names for this server, i.e. `${serverId}_${toolName}`. */
     toolNames: string[];
 };
+/** Internal cache entry for `ToolRoutingCache`. */
+export type ToolRoutingCacheEntry = {
+    excludedToolNames: string[];
+    selectedServerIds: string[];
+    /** Absolute expiry timestamp (from the injected `now()` clock). */
+    expiresAt: number;
+    /** LRU eviction order — lower = older. Bumped on each get/set. */
+    accessOrder: number;
+};
+/** Internal stickiness entry for `ToolRoutingCache`. */
+export type ToolRoutingStickiness = {
+    serverIds: string[];
+    /** Turn counter — decremented on each routing turn, removed when it hits 0. */
+    turnsRemaining: number;
+};
+/** Constructor options for `ToolRoutingCache`. */
+export type ToolRoutingCacheOptions = {
+    /** Time-to-live in milliseconds for each cached entry. Default: 60_000. */
+    ttlMs?: number;
+    /** Maximum number of entries kept in the LRU before eviction. Default: 256. */
+    maxEntries?: number;
+    /** Number of turns a selected server remains sticky per session. Default: 3. */
+    stickyTurns?: number;
+    /**
+     * Clock function for TTL calculations. Defaults to `Date.now`.
+     * Inject a deterministic function in tests to control time.
+     */
+    now?: () => number;
+};
+/**
+ * Outcome classifier for a single routing resolution. Used in
+ * `ToolRoutingDecision` and emitted as an OTel span attribute.
+ */
+export type ToolRoutingOutcome = "applied" | "skipped-no-query" | "skipped-single-server" | "empty-pick" | "failed-open-parse" | "failed-open-timeout" | "failed-open-error" | "cache-hit";
+/**
+ * Machine-readable summary of one routing resolution. Emitted via the
+ * `emitDecision` callback so the caller can attach it as OTel span attributes
+ * or record it in any other telemetry sink.
+ */
+export type ToolRoutingDecision = {
+    /** How the routing turn concluded. */
+    outcome: ToolRoutingOutcome;
+    /** Server ids the router kept (selected as relevant). */
+    selectedServerIds: string[];
+    /** Server ids whose tools were excluded (router considered them irrelevant). */
+    excludedServerIds: string[];
+    /** Server ids the router returned that did not exist in the catalog. */
+    hallucinatedIds: string[];
+    /** Total number of individual tool names added to the exclusion list. */
+    excludedToolCount: number;
+    /** Number of servers that were offered to the router (always-include excluded). */
+    routableServerCount: number;
+    /** True when the result was served from cache, skipping the router LLM. */
+    cacheHit: boolean;
+    /** Wall-clock time spent in the routing resolution in milliseconds. */
+    durationMs: number;
+};
 /** Parameters for `resolveToolRoutingExclusions()`. */
 export type ToolRoutingResolutionParams = {
     /** Full catalog; always-include servers are filtered out internally. */
@@ -88,4 +169,11 @@ export type ToolRoutingResolutionParams = {
     timeoutMs: number;
     /** Invokes the router LLM — `NeuroLink.generate` bound by the caller. */
     generateFn: (options: GenerateOptions) => Promise<GenerateResult>;
+    /**
+     * Optional callback invoked once per resolution with a structured summary of
+     * the routing decision. Called on every return path (applied, skipped,
+     * failed-open). Must never throw — any error inside is swallowed by
+     * the resolver.
+     */
+    emitDecision?: (decision: ToolRoutingDecision) => void;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "9.73.0",
+  "version": "9.74.0",
   "packageManager": "pnpm@10.15.1",
   "description": "Universal AI Development Platform with working MCP integration, multi-provider support, voice (TTS/STT/realtime), and professional CLI. 58+ external MCP servers discoverable, multimodal file processing, RAG pipelines. Build, test, and deploy AI applications with 21+ providers: OpenAI, Anthropic, Google AI Studio, Google Vertex, AWS Bedrock, Azure OpenAI, Mistral, LiteLLM, SageMaker, Hugging Face, Ollama, OpenAI-compatible, OpenRouter, DeepSeek, NVIDIA NIM, LM Studio, llama.cpp, plus voice (OpenAI TTS, ElevenLabs, Deepgram, Azure Speech).",
   "author": {
@@ -120,7 +120,9 @@
     "test:envguard": "npx tsx test/helpers/envGuard.test.ts",
     "test:ci": "pnpm run test && pnpm run test:client && pnpm run test:hitl",
     "// CI tier — fast, no live AI calls, safe for every commit": "",
-    "test:unit": "pnpm run test:envguard && pnpm run test:bugfixes && pnpm run test:mcp:infra && pnpm run test:mcp:bash && pnpm run test:mcp:limits && pnpm run test:mcp:spans && pnpm run test:autoresearch:redis",
+    "test:unit:vitest": "pnpm exec vitest run test/toolRouting.test.ts",
+    "test:tool-routing": "pnpm run test:unit:vitest && npx tsx test/continuous-test-suite-tool-routing.ts",
+    "test:unit": "pnpm run test:envguard && pnpm run test:bugfixes && pnpm run test:mcp:infra && pnpm run test:mcp:bash && pnpm run test:mcp:limits && pnpm run test:mcp:spans && pnpm run test:autoresearch:redis && pnpm run test:unit:vitest",
     "// CI tier — live providers, runs only when API keys are present (test:credentials and test:dynamic make real provider calls when keys are set, so they live here, not in test:unit)": "",
     "test:live": "pnpm run test:providers && pnpm run test:mcp:http && pnpm run test:mcp:sdk && pnpm run test:mcp:cli && pnpm run test:observability && pnpm run test:context && pnpm run test:memory && pnpm run test:tool-reliability && pnpm run test:evaluation && pnpm run test:autoresearch && pnpm run test:credentials && pnpm run test:dynamic",
     "// CI tier — product output (image/video/TTS/PPT) — costs $$ per run": "",