@juspay/neurolink 9.76.0 → 9.78.0

package/dist/lib/types/toolRouting.d.ts

@@ -38,6 +38,71 @@ export type ToolRoutingModelConfig = {
     /** Router sampling temperature. Default: 0. */
     temperature?: number;
 };
+/**
+ * Weights for the hybrid scoring formula used by `ToolEmbeddingIndex.rank()`.
+ * Scores are computed as: `cosine * cosine + bm25 * bm25Score` then
+ * normalized before sorting.
+ * Default: `{ cosine: 0.8, bm25: 0.2 }`.
+ */
+export type ToolRetrievalWeights = {
+    /** Weight applied to the cosine-similarity (dense) component. */
+    cosine: number;
+    /** Weight applied to the BM25 (sparse/lexical) component. */
+    bm25: number;
+};
+/**
+ * Configuration for the L2 embedding fast-path (ITEM B).
+ *
+ * When enabled and the catalog's total tool count reaches `minToolsToActivate`,
+ * a hybrid cosine + BM25 retriever ranks all tools by relevance to the query
+ * and takes the top-`topK` candidates. This is far cheaper than an LLM call
+ * (sub-10 ms warm) and fires BEFORE or INSTEAD of the LLM router.
+ *
+ * Fail-open: any embedding error (missing provider, network failure, wrong
+ * model) silently falls back to the existing LLM-router / server-granularity
+ * path — the turn is never broken.
+ */
+export type ToolRoutingEmbeddingConfig = {
+    /**
+     * Activate the embedding fast-path. Default: false (backward-compatible).
+     * Setting this to true without supplying `provider`/`model` causes the SDK
+     * to try the stream call's configured provider; if that provider does not
+     * support embeddings the layer fails open.
+     */
+    enabled?: boolean;
+    /**
+     * Maximum number of top-ranked tool candidates passed to the post-embedding
+     * decision stage. Default: 20.
+     */
+    topK?: number;
+    /**
+     * Minimum total tool count in the catalog before the embedding path
+     * activates. Below this threshold the catalog is small enough that the LLM
+     * router alone is cheap and fast. Default: 20.
+     */
+    minToolsToActivate?: number;
+    /**
+     * Weights for the hybrid scoring formula:
+     *   score = cosine * cosineSim + bm25 * bm25Score  (both normalized to [0,1])
+     * Default: `{ cosine: 0.8, bm25: 0.2 }`.
+     */
+    weights?: ToolRetrievalWeights;
+    /**
+     * Provider name to use for the embedding call (e.g. "openai", "vertex").
+     * Defaults to the stream/generate call's configured provider. The provider
+     * must support `embedMany()`.
+     */
+    provider?: string;
+    /**
+     * Embedding model name (provider-specific). When omitted the provider's
+     * default embedding model is used (e.g. text-embedding-3-small for OpenAI).
+     */
+    model?: string;
+    /**
+     * Timeout for embedding calls in milliseconds. Default: 10000.
+     */
+    timeoutMs?: number;
+};
 /** Constructor-level configuration for pre-call tool routing. */
 export type ToolRoutingConfig = {
     /** Master switch. Routing runs only when true AND the server catalog is non-empty. */
@@ -88,6 +153,24 @@ export type ToolRoutingConfig = {
         /** Number of turns for which a previously-selected server stays warm. Default: 3. */
         turns?: number;
     };
+    /**
+     * L2 embedding fast-path (ITEM B). When enabled the SDK ranks tools by
+     * semantic + lexical relevance using a hybrid cosine/BM25 score and narrows
+     * the candidate set BEFORE (or instead of) the LLM router. Disabled by
+     * default for backward compatibility.
+     */
+    embedding?: ToolRoutingEmbeddingConfig;
+    /**
+     * Routing granularity (ITEM D).
+     *
+     * - `"server"` (default) — routing excludes the tools of entire unpicked
+     *   servers. This is the original behavior.
+     * - `"tool"` — routing excludes individual tools that are not in the
+     *   embedding top-K candidate set, regardless of which server they belong
+     *   to. Requires `embedding.enabled: true`; if the embedding fast-path is
+     *   off (or fails) the granularity falls back to `"server"` automatically.
+     */
+    granularity?: "server" | "tool";
 };
 /** Catalog entry pairing a server descriptor with its registered tool names. */
 export type ToolRoutingCatalogEntry = {
@@ -152,6 +235,19 @@ export type ToolRoutingDecision = {
     cacheHit: boolean;
     /** Wall-clock time spent in the routing resolution in milliseconds. */
     durationMs: number;
+    /** True when the L2 embedding fast-path ran and produced candidate results. */
+    embeddingActivated?: boolean;
+    /**
+     * Number of tool candidates produced by the embedding retriever before the
+     * post-embedding server or tool filtering step.
+     */
+    candidateToolCount?: number;
+    /**
+     * Granularity at which exclusions were applied ("server" or "tool").
+     * Matches `ToolRoutingConfig.granularity`; present only when routing was
+     * applied (outcome === "applied").
+     */
+    granularity?: "server" | "tool";
 };
 /** Parameters for `resolveToolRoutingExclusions()`. */
 export type ToolRoutingResolutionParams = {
@@ -176,4 +272,79 @@ export type ToolRoutingResolutionParams = {
      * the resolver.
      */
     emitDecision?: (decision: ToolRoutingDecision) => void;
+    /**
+     * Injected async function that converts an array of texts into embedding
+     * vectors. Built by the caller (NeuroLink) from the configured embedding
+     * provider so the resolver stays pure and free of provider imports.
+     * When undefined the embedding fast-path is skipped entirely.
+     */
+    embedFn?: (texts: string[]) => Promise<number[][]>;
+    /**
+     * Embedding fast-path configuration forwarded from `ToolRoutingConfig`.
+     * Only consulted when `embedFn` is provided.
+     */
+    embeddingConfig?: ToolRoutingEmbeddingConfig;
+    /**
+     * Routing granularity forwarded from `ToolRoutingConfig`. Default: "server".
+     */
+    granularity?: "server" | "tool";
+    /**
+     * Optional shared vector cache for the L2 embedding fast-path. When
+     * supplied, tool embedding vectors computed on prior turns are reused rather
+     * than re-fetched from the embedding provider on every call.
+     *
+     * The NeuroLink instance manages the lifecycle: it creates the Map once and
+     * passes the same reference across turns. It clears the reference when the
+     * tool catalog changes (via `setToolRoutingServers`) so stale vectors are
+     * never used after a catalog update.
+     */
+    embeddingVectorCache?: Map<string, number[]>;
+};
+/**
+ * A single item in the tool retrieval catalog, pairing a tool name with the
+ * text (tool description + server context) used to build its embedding vector.
+ */
+export type ToolRetrievalItem = {
+    /** Fully-qualified tool name (e.g. `${serverId}_${toolName}`). */
+    name: string;
+    /** Descriptive text used as the embedding document for this tool. */
+    text: string;
+};
+/**
+ * One ranked result from `ToolEmbeddingIndex.rank()` or
+ * `selectRelevantToolNames()`.
+ */
+export type ToolRetrievalRankedResult = {
+    /** Tool name (mirrors `ToolRetrievalItem.name`). */
+    name: string;
+    /** Combined hybrid score (higher = more relevant). */
+    score: number;
+};
+/**
+ * Options passed to `selectRelevantToolNames()` — the high-level convenience
+ * wrapper around `ToolEmbeddingIndex`.
+ */
+export type ToolRetrievalSelectOptions = {
+    /** Maximum number of tool names to return. */
+    topK: number;
+    /** Optional weight override (defaults to `{ cosine: 0.8, bm25: 0.2 }`). */
+    weights?: ToolRetrievalWeights;
+    /**
+     * Async function that converts an array of text strings into embedding
+     * vectors. Must return one vector per input text in the same order.
+     * Errors thrown here propagate to the caller (so it can fail open).
+     */
+    embedFn: (texts: string[]) => Promise<number[][]>;
+    /**
+     * Optional shared vector cache (keyed by text string). When supplied the
+     * underlying `ToolEmbeddingIndex` reads from and writes to this Map so that
+     * tool vectors computed on a prior call are reused on subsequent calls for
+     * the same item text. Callers that want warm-cache behavior across turns
+     * should pass the same Map instance each time.
+     */
+    vectorCache?: Map<string, number[]>;
+    /**
+     * Timeout for each embedding provider call in milliseconds. Default: 10000.
+     */
+    timeoutMs?: number;
 };

package/dist/lib/utils/providerErrorClassification.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Shared provider-error classification utilities used by both neurolink.ts and
+ * the ModelPool routing subsystem.
+ *
+ * Extracted here to avoid duplication and to ensure that typed error classes
+ * (AuthenticationError, AuthorizationError, ModelAccessDeniedError) are
+ * checked consistently in both code paths.
+ */
+/**
+ * Detects whether an error object looks like a model-access-denied condition.
+ * Matches LiteLLM "team not allowed" / "team can only access models=[...]"
+ * plus typed-error name/code markers when the full typed class is not present.
+ */
+export declare function looksLikeModelAccessDenied(error: unknown): boolean;
+/**
+ * Returns true when the error is definitively non-retryable: typed error
+ * classes (auth, access-denied, invalid-model), non-retryable HTTP status
+ * codes, or deterministic 400-class message patterns.
+ *
+ * NOTE: ContextBudgetExceededError is intentionally NOT non-retryable —
+ * each provider has its own context window, so a budget rejection on one
+ * provider does not preclude another provider accepting the same payload.
+ */
+export declare function isNonRetryableProviderError(error: unknown): boolean;

package/dist/lib/utils/providerErrorClassification.js

@@ -0,0 +1,89 @@
+/**
+ * Shared provider-error classification utilities used by both neurolink.ts and
+ * the ModelPool routing subsystem.
+ *
+ * Extracted here to avoid duplication and to ensure that typed error classes
+ * (AuthenticationError, AuthorizationError, ModelAccessDeniedError) are
+ * checked consistently in both code paths.
+ */
+import { AuthenticationError, AuthorizationError, InvalidModelError, ModelAccessDeniedError, } from "../types/index.js";
+import { NON_RETRYABLE_HTTP_STATUS_CODES, isDeterministicClientErrorMessage, } from "./retryability.js";
+/**
+ * Detects whether an error object looks like a model-access-denied condition.
+ * Matches LiteLLM "team not allowed" / "team can only access models=[...]"
+ * plus typed-error name/code markers when the full typed class is not present.
+ */
+export function looksLikeModelAccessDenied(error) {
+    if (!error) {
+        return false;
+    }
+    const e = error;
+    if (e.name === "ModelAccessDeniedError") {
+        return true;
+    }
+    if (e.code === "MODEL_ACCESS_DENIED") {
+        return true;
+    }
+    const msg = typeof e.message === "string"
+        ? e.message
+        : error instanceof Error
+            ? error.message
+            : String(error);
+    if (!msg) {
+        return false;
+    }
+    const lower = msg.toLowerCase();
+    return ((lower.includes("team") && lower.includes("not allowed")) ||
+        lower.includes("team can only access") ||
+        /not\s+allowed\s+to\s+access\s+(this\s+)?model/i.test(msg));
+}
+/**
+ * Returns true when the error is definitively non-retryable: typed error
+ * classes (auth, access-denied, invalid-model), non-retryable HTTP status
+ * codes, or deterministic 400-class message patterns.
+ *
+ * NOTE: ContextBudgetExceededError is intentionally NOT non-retryable —
+ * each provider has its own context window, so a budget rejection on one
+ * provider does not preclude another provider accepting the same payload.
+ */
+export function isNonRetryableProviderError(error) {
+    if (error instanceof InvalidModelError) {
+        return true;
+    }
+    if (error instanceof AuthenticationError) {
+        return true;
+    }
+    if (error instanceof AuthorizationError) {
+        return true;
+    }
+    if (error instanceof ModelAccessDeniedError) {
+        return true;
+    }
+    if (error && typeof error === "object") {
+        const err = error;
+        const status = typeof err.status === "number"
+            ? err.status
+            : typeof err.statusCode === "number"
+                ? err.statusCode
+                : undefined;
+        if (status !== undefined &&
+            NON_RETRYABLE_HTTP_STATUS_CODES.includes(status)) {
+            return true;
+        }
+    }
+    if (error instanceof Error) {
+        const msg = error.message;
+        if (msg.includes("NOT_FOUND") ||
+            msg.includes("Model Not Found") ||
+            msg.includes("model not found") ||
+            msg.includes("PERMISSION_DENIED") ||
+            msg.includes("UNAUTHENTICATED")) {
+            return true;
+        }
+        if (isDeterministicClientErrorMessage(msg)) {
+            return true;
+        }
+    }
+    return false;
+}
+//# sourceMappingURL=providerErrorClassification.js.map

package/dist/neurolink.d.ts

@@ -102,6 +102,7 @@ export declare class NeuroLink {
     private conversationMemoryConfig?;
     private toolRoutingConfig?;
     private toolRoutingCacheInstance?;
+    private toolRoutingVectorCache?;
     private toolDedupConfig?;
     private enableOrchestration;
     private authProvider?;
@@ -109,6 +110,8 @@ export declare class NeuroLink {
     private authInitPromise?;
     private credentials?;
     private readonly fallbackConfig;
+    private readonly modelPool;
+    private readonly requestRouter;
     /**
      * Merge instance-level credentials with per-call credentials.
      *
@@ -612,6 +615,25 @@ export declare class NeuroLink {
     private maybeHandleEarlyGenerateResult;
     private runStandardGenerateRequest;
     private maybeApplyGenerateOrchestration;
+    /**
+     * Applies the host-configured `requestRouter` to `options` in place.
+     *
+     * The router is skipped when:
+     *   - no `requestRouter` is configured on this instance, or
+     *   - the caller explicitly set both `options.provider` AND `options.model`
+     *     (we only skip the router if BOTH are set; a caller setting only one
+     *     still lets the router fill in the other field).
+     *
+     * Fails open: any router error is logged at WARN level and the call
+     * continues with the original options unmodified.
+     *
+     * @param options — the mutable options object (generate or stream).
+     * @param promptText — the text prompt used to build the RouterInputContext.
+     * @param hasTools — true when at least one tool is available for this call.
+     * @param requiresVision — true when the call includes image attachments.
+     * @param thinkingLevel — optional thinking level string from the call options.
+     */
+    private applyRequestRouter;
     private prepareGenerateAugmentations;
     private buildGenerateTextOptions;
     private finalizeGenerateRequestResult;