@juspay/neurolink 9.76.0 → 9.78.0

package/dist/index.d.ts

@@ -219,6 +219,7 @@ export { logger } from "./utils/logger.js";
 export { getPoolStats } from "./utils/redis.js";
 export { ToolRoutingCache } from "./core/toolRoutingCache.js";
 export { resolveToolRoutingExclusions, buildToolRoutingCatalog, buildRoutingQueryFromHistory, DEFAULT_ROUTER_PROMPT_PREFIX, } from "./core/toolRouting.js";
+export { cosineSimilarity, ToolEmbeddingIndex, selectRelevantToolNames, } from "./core/toolRoutingEmbedding.js";
 export declare function initializeTelemetry(): Promise<boolean>;
 export declare function getTelemetryStatus(): Promise<{
     enabled: boolean;
@@ -337,6 +338,36 @@ export { BALANCED_ADAPTIVE_WORKFLOW, createAdaptiveWorkflow, QUALITY_MAX_WORKFLO
 export { CONSENSUS_3_FAST_WORKFLOW, CONSENSUS_3_WORKFLOW, createConsensus3WithPrompt, } from "./workflow/workflows/consensusWorkflow.js";
 export { AGGRESSIVE_FALLBACK_WORKFLOW, FAST_FALLBACK_WORKFLOW, } from "./workflow/workflows/fallbackWorkflow.js";
 export { createMultiJudgeWorkflow, MULTI_JUDGE_3_WORKFLOW, MULTI_JUDGE_5_WORKFLOW, } from "./workflow/workflows/multiJudgeWorkflow.js";
+/**
+ * ModelPool and RequestRouter — opt-in multi-provider failover with
+ * error-class-aware cooldown, and a pluggable pre-call provider/model router.
+ *
+ * @example ModelPool
+ * ```typescript
+ * import { ModelPool, classifyProviderError } from '@juspay/neurolink';
+ *
+ * const pool = new ModelPool({
+ *   members: [
+ *     { provider: 'anthropic', model: 'claude-sonnet-4-5' },
+ *     { provider: 'vertex',    model: 'gemini-2.5-flash' },
+ *   ],
+ *   strategy: 'priority',
+ *   cooldownMs: 30_000,
+ * });
+ * ```
+ *
+ * @example RequestRouter
+ * ```typescript
+ * import { createDefaultRequestRouter } from '@juspay/neurolink';
+ *
+ * const router = createDefaultRequestRouter({
+ *   visionTier: { provider: 'vertex', model: 'gemini-2.5-pro' },
+ *   largeTier:  { provider: 'anthropic', model: 'claude-opus-4-5' },
+ *   smallTier:  { provider: 'anthropic', model: 'claude-haiku-3-5' },
+ * });
+ * ```
+ */
+export { classifyProviderError, ModelPool, createDefaultRequestRouter, } from "./routing/index.js";
 /**
  * Server Adapters for exposing NeuroLink as HTTP APIs
  *

package/dist/index.js

@@ -362,6 +362,8 @@ export { getPoolStats } from "./utils/redis.js";
 export { ToolRoutingCache } from "./core/toolRoutingCache.js";
 // Pre-call tool routing resolver — exported for host integrations and testing
 export { resolveToolRoutingExclusions, buildToolRoutingCatalog, buildRoutingQueryFromHistory, DEFAULT_ROUTER_PROMPT_PREFIX, } from "./core/toolRouting.js";
+// Pre-call tool routing — embedding fast-path (ITEM B: L2 semantic retrieval)
+export { cosineSimilarity, ToolEmbeddingIndex, selectRelevantToolNames, } from "./core/toolRoutingEmbedding.js";
 // ============================================================================
 // REAL-TIME SERVICES & TELEMETRY - Enterprise Platform Features
 // ============================================================================
@@ -541,6 +543,39 @@ export { AGGRESSIVE_FALLBACK_WORKFLOW, FAST_FALLBACK_WORKFLOW, } from "./workflo
 // Pre-built workflows - Multi-judge
 export { createMultiJudgeWorkflow, MULTI_JUDGE_3_WORKFLOW, MULTI_JUDGE_5_WORKFLOW, } from "./workflow/workflows/multiJudgeWorkflow.js";
 // ============================================================================
+// MODEL POOL & REQUEST ROUTER - Pluggable pre-call routing + failover
+// ============================================================================
+/**
+ * ModelPool and RequestRouter — opt-in multi-provider failover with
+ * error-class-aware cooldown, and a pluggable pre-call provider/model router.
+ *
+ * @example ModelPool
+ * ```typescript
+ * import { ModelPool, classifyProviderError } from '@juspay/neurolink';
+ *
+ * const pool = new ModelPool({
+ *   members: [
+ *     { provider: 'anthropic', model: 'claude-sonnet-4-5' },
+ *     { provider: 'vertex',    model: 'gemini-2.5-flash' },
+ *   ],
+ *   strategy: 'priority',
+ *   cooldownMs: 30_000,
+ * });
+ * ```
+ *
+ * @example RequestRouter
+ * ```typescript
+ * import { createDefaultRequestRouter } from '@juspay/neurolink';
+ *
+ * const router = createDefaultRequestRouter({
+ *   visionTier: { provider: 'vertex', model: 'gemini-2.5-pro' },
+ *   largeTier:  { provider: 'anthropic', model: 'claude-opus-4-5' },
+ *   smallTier:  { provider: 'anthropic', model: 'claude-haiku-3-5' },
+ * });
+ * ```
+ */
+export { classifyProviderError, ModelPool, createDefaultRequestRouter, } from "./routing/index.js";
+// ============================================================================
 // SERVER ADAPTERS - HTTP API Framework Integration
 // ============================================================================
 // Server Types

package/dist/lib/core/toolRouting.d.ts

@@ -16,6 +16,15 @@
  * Fail-open by design: missing query, <=1 routable server, validation
  * failure, empty/invalid pick, or any thrown error returns an EMPTY list
  * (exclude nothing -> all tools, identical to routing disabled). Never throws.
+ *
+ * L2 embedding fast-path (ITEM B): when `params.embedFn` is supplied and the
+ * catalog's total tool count exceeds `embeddingConfig.minToolsToActivate`, a
+ * hybrid cosine+BM25 retriever narrows the candidate set BEFORE the LLM router.
+ * Any embedding failure falls back to the standard LLM-router path.
+ *
+ * Tool granularity (ITEM D): when `params.granularity === "tool"`, individual
+ * unpicked tools are excluded rather than whole servers. Falls back to "server"
+ * if the embedding path is disabled or fails.
  */
 import type { ToolRoutingCatalogEntry, ToolRoutingResolutionParams, ToolRoutingServerDescriptor } from "../types/index.js";
 /**

package/dist/lib/core/toolRouting.js

@@ -16,13 +16,27 @@
  * Fail-open by design: missing query, <=1 routable server, validation
  * failure, empty/invalid pick, or any thrown error returns an EMPTY list
  * (exclude nothing -> all tools, identical to routing disabled). Never throws.
+ *
+ * L2 embedding fast-path (ITEM B): when `params.embedFn` is supplied and the
+ * catalog's total tool count exceeds `embeddingConfig.minToolsToActivate`, a
+ * hybrid cosine+BM25 retriever narrows the candidate set BEFORE the LLM router.
+ * Any embedding failure falls back to the standard LLM-router path.
+ *
+ * Tool granularity (ITEM D): when `params.granularity === "tool"`, individual
+ * unpicked tools are excluded rather than whole servers. Falls back to "server"
+ * if the embedding path is disabled or fails.
  */
 import { z } from "zod";
 import { logger } from "../utils/logger.js";
 import { withTimeout } from "../utils/async/index.js";
+import { selectRelevantToolNames } from "./toolRoutingEmbedding.js";
 const routerOutputSchema = z.object({
     servers: z.array(z.string()),
 });
+/** Default minimum catalog tool count that activates the embedding fast-path. */
+const DEFAULT_EMBEDDING_MIN_TOOLS = 20;
+/** Default number of top-ranked tool candidates the embedding retriever keeps. */
+const DEFAULT_EMBEDDING_TOP_K = 20;
 /**
  * Upper bound on the user-query text interpolated into the router prompt.
  * The query is untrusted and can attempt prompt injection — the blast radius
@@ -169,12 +183,116 @@ function safeEmitDecision(emitDecision, decision) {
         // Intentionally swallowed — telemetry must never affect routing behaviour.
     }
 }
+// ---------------------------------------------------------------------------
+// L2 embedding fast-path helpers (ITEM B + D)
+// ---------------------------------------------------------------------------
+// ToolRoutingEmbeddingFastPathResult is imported from src/lib/types/toolRouting.ts
+// (Critical Rule 2 — all type aliases live in src/lib/types/).
+/**
+ * Runs the L2 embedding fast-path over `routableServers` for `userQuery`.
+ *
+ * Returns a result with `embeddingActivated: false` on any skip or error —
+ * the caller must treat this as a "fall back to LLM router" signal.
+ *
+ * This function is PURE with respect to provider access: the caller injects
+ * `embedFn` so no provider classes are imported here.
+ */
+async function runEmbeddingFastPath(userQuery, routableServers, alwaysIncludeServerIds, embedFn, opts) {
+    const { granularity, vectorCache } = opts;
+    const noop = {
+        excludedToolNames: [],
+        embeddingActivated: false,
+        candidateToolCount: 0,
+        granularity,
+    };
+    // Count routable tools (always-include servers are already excluded from
+    // routableServers at call site, but be defensive).
+    const routableToolCount = routableServers.reduce((sum, server) => sum + server.toolNames.length, 0);
+    if (routableToolCount < opts.minToolsToActivate) {
+        logger.debug("[ToolRouting] L2 embedding skipped — catalog below threshold", {
+            toolCount: routableToolCount,
+            minToolsToActivate: opts.minToolsToActivate,
+        });
+        return noop;
+    }
+    // Build retrieval items: one per individual tool, with text =
+    // "<server description> — <tool name>" so both server context and the tool
+    // identifier itself inform the embedding.
+    const items = routableServers.flatMap((server) => server.toolNames.map((toolName) => ({
+        name: toolName,
+        text: `${server.description} — ${toolName}`,
+    })));
+    try {
+        const topK = Math.min(opts.topK, items.length);
+        const topToolNames = await selectRelevantToolNames(userQuery, items, {
+            topK,
+            weights: opts.weights,
+            embedFn,
+            vectorCache,
+        });
+        const topToolSet = new Set(topToolNames);
+        if (granularity === "tool") {
+            // Tool-granularity (ITEM D): exclude every catalog tool NOT in the top-K,
+            // regardless of server. Always-include server tools are not in
+            // routableServers so they are untouched.
+            const excludedToolNames = items
+                .map((item) => item.name)
+                .filter((name) => !topToolSet.has(name));
+            logger.debug("[ToolRouting] L2 embedding applied (tool granularity)", {
+                totalTools: items.length,
+                topK,
+                keptTools: topToolNames.length,
+                excludedTools: excludedToolNames.length,
+            });
+            return {
+                excludedToolNames,
+                embeddingActivated: true,
+                candidateToolCount: topToolNames.length,
+                granularity: "tool",
+            };
+        }
+        // Server-granularity (default): map surviving tools back to their servers.
+        // A server is "kept" if at least one of its tools survived the top-K cut.
+        // The tools of every server with ZERO surviving tools are excluded.
+        const keptServerIds = new Set(routableServers
+            .filter((server) => server.toolNames.some((toolName) => topToolSet.has(toolName)))
+            .map((server) => server.id));
+        // Never exclude always-include servers (defensive — they should not be in
+        // routableServers, but guard against caller mistakes).
+        const alwaysSet = new Set(alwaysIncludeServerIds);
+        const excludedServers = routableServers.filter((server) => !keptServerIds.has(server.id) && !alwaysSet.has(server.id));
+        const excludedToolNames = excludedServers.flatMap((server) => server.toolNames);
+        logger.debug("[ToolRouting] L2 embedding applied (server granularity)", {
+            totalTools: items.length,
+            topK,
+            keptServers: keptServerIds.size,
+            excludedServers: excludedServers.length,
+            excludedTools: excludedToolNames.length,
+        });
+        return {
+            excludedToolNames,
+            embeddingActivated: true,
+            candidateToolCount: topToolNames.length,
+            granularity: "server",
+        };
+    }
+    catch (embeddingError) {
+        logger.warn("[ToolRouting] L2 embedding fast-path failed, falling back to LLM router", {
+            error: embeddingError instanceof Error
+                ? embeddingError.message
+                : String(embeddingError),
+        });
+        return noop;
+    }
+}
 /**
  * Resolves which registered tool names to EXCLUDE for a single stream() turn.
  * Returns an empty list on any skip/failure path — see module doc.
  */
 export async function resolveToolRoutingExclusions(params) {
-    const { catalog, alwaysIncludeServerIds, userQuery, routerPromptPrefix, routerModel, timeoutMs, generateFn, emitDecision, } = params;
+    const { catalog, alwaysIncludeServerIds, userQuery, routerPromptPrefix, routerModel, timeoutMs, generateFn, emitDecision, 
+    // L2 / ITEM D parameters (all optional — omitting reproduces today's behavior)
+    embedFn, embeddingConfig, granularity = "server", embeddingVectorCache, } = params;
     const routableServers = catalog.filter((server) => !alwaysIncludeServerIds.includes(server.id));
     const routingStartTime = Date.now();
     try {
@@ -198,6 +316,64 @@ export async function resolveToolRoutingExclusions(params) {
             });
             return [];
         }
+        // -------------------------------------------------------------------------
+        // L2 EMBEDDING FAST-PATH (ITEM B + D)
+        // -------------------------------------------------------------------------
+        // Attempt the embedding retriever when an embedFn is provided. On success,
+        // return the embedding-derived exclusion list immediately WITHOUT an LLM
+        // call. On any failure (embedFn throws, threshold not met, etc.), fall
+        // through to the existing LLM-router path below.
+        if (embedFn) {
+            const embResult = await runEmbeddingFastPath(userQuery, routableServers, alwaysIncludeServerIds, embedFn, {
+                topK: embeddingConfig?.topK ?? DEFAULT_EMBEDDING_TOP_K,
+                minToolsToActivate: embeddingConfig?.minToolsToActivate ?? DEFAULT_EMBEDDING_MIN_TOOLS,
+                weights: embeddingConfig?.weights,
+                granularity,
+                vectorCache: embeddingVectorCache,
+            });
+            if (embResult.embeddingActivated) {
+                logger.debug("[ToolRouting] L2 embedding fast-path succeeded", {
+                    granularity: embResult.granularity,
+                    candidateToolCount: embResult.candidateToolCount,
+                    excludedToolCount: embResult.excludedToolNames.length,
+                    routableServerCount: routableServers.length,
+                    durationMs: Date.now() - routingStartTime,
+                });
+                // For server-granularity results, reconstruct selected/excluded server
+                // ids for the decision so telemetry fields are consistent.
+                let selectedServerIds = [];
+                let excludedServerIds = [];
+                if (embResult.granularity === "server") {
+                    const excludedToolSet = new Set(embResult.excludedToolNames);
+                    // A server is "excluded" if ALL its tools are excluded.
+                    excludedServerIds = routableServers
+                        .filter((server) => server.toolNames.every((name) => excludedToolSet.has(name)))
+                        .map((server) => server.id);
+                    selectedServerIds = routableServers
+                        .filter((server) => !excludedServerIds.includes(server.id))
+                        .map((server) => server.id);
+                }
+                safeEmitDecision(emitDecision, {
+                    outcome: "applied",
+                    selectedServerIds,
+                    excludedServerIds,
+                    hallucinatedIds: [],
+                    excludedToolCount: embResult.excludedToolNames.length,
+                    routableServerCount: routableServers.length,
+                    cacheHit: false,
+                    durationMs: Date.now() - routingStartTime,
+                    embeddingActivated: true,
+                    candidateToolCount: embResult.candidateToolCount,
+                    granularity: embResult.granularity,
+                });
+                return embResult.excludedToolNames;
+            }
+            // embedFn was provided but threshold not met OR retriever failed open —
+            // fall through to the LLM-router path below.
+        }
+        // -------------------------------------------------------------------------
+        // EXISTING LLM ROUTER PATH (L3) — runs when embedding is off or fell back
+        // -------------------------------------------------------------------------
         const routerPrompt = buildRouterPrompt(userQuery, routableServers, routerPromptPrefix);
         // `timeout` lets the provider abort its own request (frees the socket);
         // withTimeout adds a hard wall-clock ceiling over the whole call so router
@@ -290,6 +466,7 @@ export async function resolveToolRoutingExclusions(params) {
             routableServerCount: routableServers.length,
             cacheHit: false,
             durationMs: Date.now() - routingStartTime,
+            granularity: "server",
         });
         return excludedToolNames;
     }

package/dist/lib/core/toolRoutingEmbedding.d.ts

@@ -0,0 +1,112 @@
+/**
+ * Embedding-retrieval fast-path for pre-call tool routing (ITEM B / L2).
+ *
+ * For large tool catalogs, ranking by semantic similarity is far cheaper than
+ * an LLM router call and benchmarks at sub-10 ms when embedding vectors are
+ * cached. This module is intentionally PURE with no provider imports: the
+ * caller injects an `embedFn` so the retriever stays testable and free of
+ * circular dependencies.
+ *
+ * Hybrid scoring formula:
+ *   score = weights.cosine * cosineSimilarity(queryVec, toolVec)
+ *         + weights.bm25   * bm25Score(query, toolText)
+ *
+ * Both components are independently normalized to [0, 1] before combination
+ * so neither scale dominates.
+ *
+ * Fail-safe contract:
+ *   `ToolEmbeddingIndex.rank()` and `selectRelevantToolNames()` propagate any
+ *   error thrown by the injected `embedFn`. The CALLER (tool-routing
+ *   orchestration) is responsible for catching that error and degrading
+ *   gracefully to the LLM-router / server-granularity path.
+ *
+ * Determinism:
+ *   All public functions are deterministic given the same inputs and embedFn.
+ *   No Math.random(), no argless Date, no global mutable state outside the
+ *   instance-level vector cache (which is keyed by text content).
+ */
+import type { ToolRetrievalItem, ToolRetrievalRankedResult, ToolRetrievalSelectOptions, ToolRetrievalWeights } from "../types/index.js";
+/**
+ * Cosine similarity between two vectors.
+ *
+ * Returns 0 when either vector is zero-length, has zero magnitude, or the
+ * lengths differ — all of which indicate the comparison is meaningless.
+ */
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+/**
+ * An in-process index that ranks tool catalog items by hybrid semantic +
+ * lexical relevance to a query.
+ *
+ * Embedding vectors for tool descriptions are computed lazily on the first
+ * `rank()` call and cached by description text, so subsequent turns that
+ * share the same catalog pay only the cost of embedding the query itself.
+ *
+ * ### Fail-safe
+ * Any error thrown by `embedFn` propagates out of `rank()`. The CALLER must
+ * catch it and degrade to the LLM-router path; `ToolEmbeddingIndex` itself
+ * never silently swallows embedFn errors.
+ *
+ * ### Thread-safety
+ * The internal cache is a plain `Map`. Node.js is single-threaded so there
+ * are no data races, but if the same index instance is used concurrently
+ * (e.g. two turns in parallel) both calls will race to populate the cache;
+ * the last writer wins (same content either way because `embedFn` is
+ * deterministic for a given text).
+ */
+export declare class ToolEmbeddingIndex {
+    private readonly items;
+    private readonly embedFn;
+    private bm25CorpusCache?;
+    /**
+     * Cached embedding vectors keyed by the item's `text` field.
+     * Populated on first `rank()` call for texts not yet seen.
+     *
+     * Callers can share a single Map across multiple index instances (e.g.
+     * across turns in the same NeuroLink session) so tool vectors are computed
+     * once and reused. Pass the shared Map via the constructor's third argument.
+     */
+    private readonly vectorCache;
+    constructor(items: ToolRetrievalItem[], embedFn: (texts: string[]) => Promise<number[][]>, 
+    /**
+     * Optional shared vector cache. When supplied, cached vectors from
+     * previous turns are reused and any newly-computed vectors are stored
+     * into this same Map, making it warm for the next call.
+     */
+    sharedVectorCache?: Map<string, number[]>);
+    /**
+     * Returns the top-K catalog items ranked by hybrid score descending.
+     *
+     * @throws If `embedFn` throws — propagated verbatim so the caller can fail
+     *   open. No wrapping, no swallowing.
+     */
+    rank(query: string, opts: {
+        topK: number;
+        weights?: ToolRetrievalWeights;
+        timeoutMs?: number;
+    }): Promise<ToolRetrievalRankedResult[]>;
+}
+/**
+ * Selects the most relevant tool names from a catalog given a query.
+ *
+ * Creates a temporary `ToolEmbeddingIndex`, runs `rank()`, and returns just
+ * the tool names. Use `ToolEmbeddingIndex` directly when you want to reuse
+ * cached tool vectors across multiple queries (e.g. multiple turns with the
+ * same catalog).
+ *
+ * @throws If `opts.embedFn` throws — propagated so the caller can degrade to
+ *   the LLM-router path.
+ *
+ * @example
+ * ```ts
+ * const tools = await selectRelevantToolNames(
+ *   "show me yesterday's sales",
+ *   catalog.flatMap(server => server.toolNames.map(name => ({
+ *     name,
+ *     text: `${server.description} — ${name}`,
+ *   }))),
+ *   { topK: 5, embedFn: myEmbedFn },
+ * );
+ * // => ["analytics_getSales", "analytics_getRevenue", ...]
+ * ```
+ */
+export declare function selectRelevantToolNames(query: string, items: ToolRetrievalItem[], opts: ToolRetrievalSelectOptions): Promise<string[]>;