@juspay/neurolink 12.9.6 → 12.11.0

package/dist/memory/memoryRetrievalTools.js

@@ -6,12 +6,15 @@ import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../
 import { withSpan } from "../telemetry/withSpan.js";
 import { tracers } from "../telemetry/tracers.js";
 import { tool } from "../utils/tool.js";
+import { MAX_ARTIFACT_SEARCH_MATCHES, readArtifactWindow, searchArtifactContent, validateSearchPattern, } from "../artifacts/artifactReader.js";
 /** Maximum characters returned per retrieval request */
 const DEFAULT_RETRIEVAL_LIMIT = 50_000;
 /** Hard maximum for user/LLM-supplied limit to prevent massive tool outputs */
 const MAX_RETRIEVAL_LIMIT = 200_000;
 /** Maximum number of search matches returned */
 const MAX_SEARCH_MATCHES = 50;
+/** Bound on one artifact backend round trip, so a stalled store never hangs the tool. */
+const ARTIFACT_READ_TIMEOUT_MS = 10_000;
 /**
  * Factory function that creates memory retrieval tools bound to a memory manager.
  *
@@ -27,13 +30,16 @@ const MAX_SEARCH_MATCHES = 50;
 export function createMemoryRetrievalTools(memoryManager, artifactStore) {
     return {
         retrieve_context: tool({
-            description: "Retrieve messages from conversation memory, or fetch the full payload of " +
-                "an externalized MCP tool output by artifact ID. Use this to:\n" +
+            description: "Retrieve messages from conversation memory, or read an externalized " +
+                "tool output / banked payload by artifact ID. Use this to:\n" +
                 "• Access full tool outputs when a result was truncated or externalized\n" +
                 "• Review previous assistant responses\n" +
-                "• Search through conversation history\n" +
-                "Supports filtering by role, pagination for large content, and regex search.\n" +
-                "To fetch an externalized artifact, provide `artifactId` (omit sessionId).",
+                "• Search a session's history, or an artifact, for literal text\n" +
+                "Supports filtering by role, offset/limit pagination for large content, " +
+                "and case-insensitive literal search (not regex).\n" +
+                "To read an artifact, provide `artifactId` (omit sessionId): pass " +
+                "`offset`/`limit` to page, or `search` to get match offsets and jump " +
+                "straight to them instead of paging.",
             inputSchema: z.object({
                 sessionId: z
                     .string()
@@ -44,8 +50,9 @@ export function createMemoryRetrievalTools(memoryManager, artifactStore) {
                     .string()
                     .optional()
                     .describe("Artifact ID from an externalized MCP tool output " +
-                    "(visible in the tool output as neurolinkArtifactId=<id>). " +
-                    "When provided, returns the full stored payload directly."),
+                    "(visible in the tool output as neurolinkArtifactId=<id>) or a " +
+                    "banked payload. When provided, reads the stored payload: a " +
+                    "window at `offset`/`limit`, or with `search`, the matches."),
                 messageId: z
                     .string()
                     .optional()
@@ -75,8 +82,13 @@ export function createMemoryRetrievalTools(memoryManager, artifactStore) {
                 search: z
                     .string()
                     .optional()
-                    .describe("Regex pattern to search within message content. " +
-                    "Returns matching lines with line numbers."),
+                    .describe("Case-insensitive literal text to find (regex metacharacters are " +
+                    "matched literally). Session history: returns matching lines " +
+                    "with line numbers. Artifact: returns up to " +
+                    `${MAX_ARTIFACT_SEARCH_MATCHES} matches, each with the character ` +
+                    "`offset` of the hit and a short snippet — pass that offset back " +
+                    "as `offset` to read around it. With `offset`, the artifact " +
+                    "search starts there; use `nextSearchOffset` to continue."),
             }),
             execute: async (args) => withSpan({
                 name: "neurolink.memory.retrieve_context",
@@ -108,35 +120,12 @@ async function executeRetrieveContext(args, memoryManager, artifactStore, otelSp
                 message: "Artifact store not configured",
             });
             return {
-                error: "Artifact store not configured — " +
-                    "mcp.outputLimits.strategy must be set to 'externalize' to use artifactId retrieval",
+                error: "Artifact store not configured — this instance has never banked " +
+                    "or externalized anything, so there is no artifact to read",
                 artifactId: args.artifactId,
             };
         }
-        const content = await withTimeout(artifactStore.retrieve(args.artifactId), 10_000, new Error(`ArtifactStore.retrieve() timed out for artifact "${args.artifactId}"`));
-        if (content === null) {
-            otelSpan.setStatus({
-                code: SpanStatusCode.ERROR,
-                message: "Artifact not found or has expired",
-            });
-            return {
-                error: "Artifact not found or has expired",
-                artifactId: args.artifactId,
-            };
-        }
-        const charLimit = Math.min(args.limit ?? DEFAULT_RETRIEVAL_LIMIT, MAX_RETRIEVAL_LIMIT);
-        const start = args.offset ?? 0;
-        const slice = content.slice(start, start + charLimit);
-        otelSpan.setAttribute("memory.artifact_size", content.length);
-        otelSpan.setAttribute("memory.returned_bytes", slice.length);
-        return {
-            artifactId: args.artifactId,
-            content: slice,
-            totalSize: content.length,
-            hasMore: start + charLimit < content.length,
-            offset: start,
-            limit: charLimit,
-        };
+        return executeArtifactRetrieval(args.artifactId, args, artifactStore, otelSpan);
     }
     // ── End artifact resolution ─────────────────────────────────────────
     if (!args.sessionId) {
@@ -272,3 +261,83 @@ async function executeRetrieveContext(args, memoryManager, artifactStore, otelSp
         return { error: "Failed to retrieve context" };
     }
 }
+/**
+ * The artifact branch of `retrieve_context`.
+ *
+ * Two modes, chosen by `search`:
+ *  - Paged read: one window through `readArtifactWindow`, which lets a backend
+ *    with range reads move only the window. `hasMore` comes from the window's
+ *    `totalLength`, so it never needs the payload either.
+ *  - Search: the whole payload is read once and scanned for the literal
+ *    pattern; the model gets match offsets and bounded snippets and can jump
+ *    straight to the hit with `offset` instead of paging to it. Before this
+ *    branch existed `search` was accepted and silently ignored here, and a
+ *    model could not tell — an unfiltered window looks like "no matches".
+ *
+ * Backend failures (a Redis outage, a timeout) are reported as errors, never
+ * as "not found": those are different facts and the model acts differently on
+ * each.
+ */
+async function executeArtifactRetrieval(artifactId, args, artifactStore, otelSpan) {
+    const notFound = () => {
+        otelSpan.setStatus({
+            code: SpanStatusCode.ERROR,
+            message: "Artifact not found or has expired",
+        });
+        return { error: "Artifact not found or has expired", artifactId };
+    };
+    try {
+        if (args.search !== undefined) {
+            const invalid = validateSearchPattern(args.search);
+            if (invalid) {
+                otelSpan.setStatus({ code: SpanStatusCode.ERROR, message: invalid });
+                return { error: invalid, artifactId };
+            }
+            // A search has to see the whole payload; the window contract is for reads.
+            const content = await withTimeout(artifactStore.retrieve(artifactId), ARTIFACT_READ_TIMEOUT_MS, new Error(`ArtifactStore.retrieve() timed out for artifact "${artifactId}"`));
+            if (content === null) {
+                return notFound();
+            }
+            const result = searchArtifactContent(content, args.search, {
+                from: args.offset,
+            });
+            otelSpan.setAttribute("memory.artifact_size", content.length);
+            otelSpan.setAttribute("memory.search_matches", result.totalMatches);
+            return {
+                artifactId,
+                search: args.search,
+                totalSize: content.length,
+                ...result,
+            };
+        }
+        const charLimit = Math.min(args.limit ?? DEFAULT_RETRIEVAL_LIMIT, MAX_RETRIEVAL_LIMIT);
+        const start = Math.max(0, args.offset ?? 0);
+        const window = await withTimeout(readArtifactWindow(artifactStore, artifactId, {
+            offset: start,
+            limit: charLimit,
+        }), ARTIFACT_READ_TIMEOUT_MS, new Error(`Artifact read timed out for artifact "${artifactId}"`));
+        if (window === null) {
+            return notFound();
+        }
+        otelSpan.setAttribute("memory.artifact_size", window.totalLength);
+        otelSpan.setAttribute("memory.returned_bytes", window.content.length);
+        otelSpan.setAttribute("memory.artifact_range_read", typeof artifactStore.retrieveRange === "function");
+        return {
+            artifactId,
+            content: window.content,
+            totalSize: window.totalLength,
+            hasMore: window.offset + window.content.length < window.totalLength,
+            offset: window.offset,
+            limit: charLimit,
+        };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        logger.error("[MemoryRetrievalTools] Artifact read failed", {
+            artifactId,
+            error: message,
+        });
+        otelSpan.setStatus({ code: SpanStatusCode.ERROR, message });
+        return { error: `Artifact read failed: ${message}`, artifactId };
+    }
+}

package/dist/neurolink.d.ts

@@ -82,8 +82,19 @@ export declare class NeuroLink {
     private mcpToolBatcher?;
     private mcpEnhancedDiscovery?;
     private mcpToolMiddlewares;
-    /** Artifact store for externalized MCP tool outputs (set when strategy=externalize). */
+    /** Artifact store for externalized MCP tool outputs and banked payloads. */
     private mcpArtifactStore?;
+    /** `artifacts` constructor config — consulted whenever a store is created. */
+    private artifactsConfig?;
+    /**
+     * True when this instance built `mcpArtifactStore` itself (local or Redis
+     * from config / STORAGE_TYPE). NeuroLink closes only stores it built — on
+     * replacement and on shutdown; a store handed in through `artifacts.store`
+     * or `setArtifactStore()` is the caller's to close.
+     */
+    private ownsArtifactStore;
+    /** Normalizer settings, kept so `setArtifactStore()` can rebuild it. */
+    private mcpOutputNormalizerConfig?;
     private _disableToolCacheForCurrentRequest;
     /**
      * (toolName + args) keys already served during the CURRENT request.
@@ -2735,6 +2746,39 @@ export declare class NeuroLink {
      * @returns The artifact store backing {@link bankArtifact} / {@link readArtifact}
      */
     getArtifactStore(): ArtifactStore;
+    /**
+     * Replace this instance's artifact store.
+     *
+     * Everything that writes or reads artifacts follows the swap: banking,
+     * `retrieve_context`, host-side `readArtifact`, and the MCP output
+     * normalizer — which is rebuilt here because it captured the previous store
+     * at construction. Assigning the field alone would miss it, and
+     * externalized tool outputs would keep landing in the old backend while
+     * read-backs looked in the new one.
+     *
+     * Call it before the first bank or externalized tool output: artifacts
+     * already in the previous store are not migrated, and their ids stop
+     * resolving through this instance. `artifacts.store` in the constructor
+     * config is the same thing without the ordering concern.
+     *
+     * Ownership: a store you hand in — here or via `artifacts.store` — stays
+     * yours to close. NeuroLink closes only the stores it built itself, when
+     * they are replaced here and on `shutdown()`.
+     *
+     * @param store - Any {@link ArtifactStore}
+     */
+    setArtifactStore(store: ArtifactStore): void;
+    /**
+     * Build the store the `artifacts` config (or `STORAGE_TYPE`) asks for. The
+     * Redis connection falls back to conversation memory's, so one
+     * `STORAGE_TYPE=redis` moves sessions and artifacts together.
+     */
+    private createConfiguredArtifactStore;
+    /**
+     * (Re)build the MCP output normalizer over `artifactStore`. A no-op unless
+     * `mcp.outputLimits` was configured — without it nothing is externalized.
+     */
+    private installOutputNormalizer;
     /**
      * Bank a payload to a file and get back a pointer to it.
      *

package/dist/neurolink.js

@@ -47,7 +47,7 @@ import { ToolResultCache } from "./mcp/caching/index.js";
 import { EnhancedToolDiscovery } from "./mcp/enhancedToolDiscovery.js";
 import { ExternalServerManager } from "./mcp/externalServerManager.js";
 import { McpOutputNormalizer, DEFAULT_MAX_MCP_OUTPUT_BYTES, DEFAULT_WARN_MCP_OUTPUT_BYTES, } from "./mcp/mcpOutputNormalizer.js";
-import { LocalTempArtifactStore } from "./artifacts/artifactStore.js";
+import { createArtifactStore, resolveArtifactStorageType, } from "./artifacts/artifactStoreFactory.js";
 import { bankArtifact as bankArtifactPayload, readArtifact as readBankedArtifact, } from "./artifacts/artifactBanking.js";
 import { ToolRouter } from "./mcp/routing/index.js";
 // Import direct tools server for automatic registration
@@ -380,8 +380,19 @@ export class NeuroLink {
     mcpToolBatcher;
     mcpEnhancedDiscovery;
     mcpToolMiddlewares = [];
-    /** Artifact store for externalized MCP tool outputs (set when strategy=externalize). */
+    /** Artifact store for externalized MCP tool outputs and banked payloads. */
     mcpArtifactStore;
+    /** `artifacts` constructor config — consulted whenever a store is created. */
+    artifactsConfig;
+    /**
+     * True when this instance built `mcpArtifactStore` itself (local or Redis
+     * from config / STORAGE_TYPE). NeuroLink closes only stores it built — on
+     * replacement and on shutdown; a store handed in through `artifacts.store`
+     * or `setArtifactStore()` is the caller's to close.
+     */
+    ownsArtifactStore = false;
+    /** Normalizer settings, kept so `setArtifactStore()` can rebuild it. */
+    mcpOutputNormalizerConfig;
     _disableToolCacheForCurrentRequest = false;
     /**
      * (toolName + args) keys already served during the CURRENT request.
@@ -1198,24 +1209,22 @@ export class NeuroLink {
             });
         }
         // ToolRouter — lazy-initialized when 2+ external servers exist (see addExternalMCPServer)
+        // Artifact storage — where externalized MCP outputs and banked payloads
+        // live. Kept so both creation sites (externalize here, and
+        // getArtifactStore() on demand) resolve the same backend.
+        this.artifactsConfig = config?.artifacts;
         // McpOutputNormalizer — active when mcp.outputLimits is configured
         if (mcpConfig?.outputLimits) {
             const strategy = mcpConfig.outputLimits.strategy ?? "externalize";
             const maxBytes = mcpConfig.outputLimits.maxBytes ?? DEFAULT_MAX_MCP_OUTPUT_BYTES;
             const warnBytes = mcpConfig.outputLimits.warnBytes ?? DEFAULT_WARN_MCP_OUTPUT_BYTES;
+            this.mcpOutputNormalizerConfig = { strategy, maxBytes, warnBytes };
             let artifactStore;
             if (strategy === "externalize") {
-                artifactStore = new LocalTempArtifactStore();
+                artifactStore = this.createConfiguredArtifactStore();
                 this.mcpArtifactStore = artifactStore;
-                logger.debug("[NeuroLink] MCP artifact store initialized (local-temp)");
             }
-            const normalizer = new McpOutputNormalizer({ strategy, maxBytes, warnBytes }, artifactStore);
-            this.externalServerManager.setOutputNormalizer(normalizer);
-            logger.debug("[NeuroLink] MCP output normalizer initialized", {
-                strategy,
-                maxBytes,
-                warnBytes,
-            });
+            this.installOutputNormalizer(artifactStore);
         }
     }
     /**
@@ -2806,6 +2815,17 @@ Current user's request: ${currentInput}`;
                     this._taskManager = undefined;
                 }
             }
+            // Release the artifact store this instance built (a pooled Redis
+            // connection, for one). An injected store is the caller's to close.
+            if (this.ownsArtifactStore && this.mcpArtifactStore?.close) {
+                try {
+                    await this.mcpArtifactStore.close();
+                    logger.debug("[NeuroLink] Artifact store closed");
+                }
+                catch (error) {
+                    logger.warn("[NeuroLink] Artifact store close failed:", error);
+                }
+            }
             // Close conversation memory manager (release Redis connections, etc.)
             if (this.conversationMemory?.close) {
                 try {
@@ -13010,13 +13030,86 @@ Current user's request: ${currentInput}`;
      */
     getArtifactStore() {
         if (!this.mcpArtifactStore) {
-            this.mcpArtifactStore = new LocalTempArtifactStore();
-            logger.debug("[NeuroLink] Artifact store created on demand (local-temp) for banking");
+            this.mcpArtifactStore = this.createConfiguredArtifactStore();
         }
         // A no-op when the constructor already registered it.
         this.registerMemoryRetrievalTools();
         return this.mcpArtifactStore;
     }
+    /**
+     * Replace this instance's artifact store.
+     *
+     * Everything that writes or reads artifacts follows the swap: banking,
+     * `retrieve_context`, host-side `readArtifact`, and the MCP output
+     * normalizer — which is rebuilt here because it captured the previous store
+     * at construction. Assigning the field alone would miss it, and
+     * externalized tool outputs would keep landing in the old backend while
+     * read-backs looked in the new one.
+     *
+     * Call it before the first bank or externalized tool output: artifacts
+     * already in the previous store are not migrated, and their ids stop
+     * resolving through this instance. `artifacts.store` in the constructor
+     * config is the same thing without the ordering concern.
+     *
+     * Ownership: a store you hand in — here or via `artifacts.store` — stays
+     * yours to close. NeuroLink closes only the stores it built itself, when
+     * they are replaced here and on `shutdown()`.
+     *
+     * @param store - Any {@link ArtifactStore}
+     */
+    setArtifactStore(store) {
+        const previous = this.mcpArtifactStore;
+        if (previous === store) {
+            return;
+        }
+        if (previous) {
+            logger.warn("[NeuroLink] Artifact store replaced — artifacts already written to " +
+                "the previous store will not resolve through this instance");
+            if (this.ownsArtifactStore && previous.close) {
+                void previous.close().catch((error) => {
+                    logger.warn("[NeuroLink] Previous artifact store close failed", {
+                        error: error instanceof Error ? error.message : String(error),
+                    });
+                });
+            }
+        }
+        this.mcpArtifactStore = store;
+        this.ownsArtifactStore = false;
+        this.installOutputNormalizer(store);
+        this.registerMemoryRetrievalTools();
+    }
+    /**
+     * Build the store the `artifacts` config (or `STORAGE_TYPE`) asks for. The
+     * Redis connection falls back to conversation memory's, so one
+     * `STORAGE_TYPE=redis` moves sessions and artifacts together.
+     */
+    createConfiguredArtifactStore() {
+        const injected = this.artifactsConfig?.store !== undefined;
+        const store = createArtifactStore(this.artifactsConfig, this.conversationMemoryConfig?.conversationMemory?.redisConfig);
+        // An injected store is returned as-is by the factory and stays the
+        // caller's; anything else was built here and is this instance's to close.
+        this.ownsArtifactStore = !injected;
+        logger.debug("[NeuroLink] Artifact store initialized", {
+            backend: injected
+                ? "custom"
+                : resolveArtifactStorageType(this.artifactsConfig),
+        });
+        return store;
+    }
+    /**
+     * (Re)build the MCP output normalizer over `artifactStore`. A no-op unless
+     * `mcp.outputLimits` was configured — without it nothing is externalized.
+     */
+    installOutputNormalizer(artifactStore) {
+        if (!this.mcpOutputNormalizerConfig) {
+            return;
+        }
+        this.externalServerManager.setOutputNormalizer(new McpOutputNormalizer(this.mcpOutputNormalizerConfig, artifactStore));
+        logger.debug("[NeuroLink] MCP output normalizer initialized", {
+            ...this.mcpOutputNormalizerConfig,
+            hasArtifactStore: artifactStore !== undefined,
+        });
+    }
     /**
      * Bank a payload to a file and get back a pointer to it.
      *
@@ -13355,7 +13448,24 @@ Current user's request: ${currentInput}`;
                     logger.warn("[NeuroLink] Error shutting down external MCP servers:", error);
                 }
             }
-            // 3. Clear all event listeners to prevent memory leaks
+            // 3. Release the artifact store this instance built (a pooled Redis
+            // connection, for one). Same ownership rule as shutdown(): an injected
+            // store is the caller's to close. Without this step every
+            // construct / use / dispose cycle left the pooled reference acquired.
+            if (this.ownsArtifactStore && this.mcpArtifactStore?.close) {
+                try {
+                    await this.mcpArtifactStore.close();
+                    logger.debug("[NeuroLink] Artifact store closed");
+                }
+                catch (error) {
+                    const err = error instanceof Error
+                        ? error
+                        : new Error(`Artifact store close error: ${String(error)}`);
+                    cleanupErrors.push(err);
+                    logger.warn("[NeuroLink] Error closing artifact store:", error);
+                }
+            }
+            // 4. Clear all event listeners to prevent memory leaks
             if (this.emitter) {
                 try {
                     logger.debug("[NeuroLink] Removing all event listeners...");
@@ -13373,7 +13483,7 @@ Current user's request: ${currentInput}`;
                     logger.warn("[NeuroLink] Error removing event listeners:", error);
                 }
             }
-            // 4. Clear all circuit breakers
+            // 5. Clear all circuit breakers
             if (this.toolCircuitBreakers && this.toolCircuitBreakers.size > 0) {
                 try {
                     logger.debug(`[NeuroLink] Clearing ${this.toolCircuitBreakers.size} circuit breakers...`);
@@ -13388,7 +13498,7 @@ Current user's request: ${currentInput}`;
                     logger.warn("[NeuroLink] Error clearing circuit breakers:", error);
                 }
             }
-            // 5. Clear all Maps and caches
+            // 6. Clear all Maps and caches
             try {
                 logger.debug("[NeuroLink] Clearing maps and caches...");
                 if (this.toolExecutionMetrics) {
@@ -13439,7 +13549,7 @@ Current user's request: ${currentInput}`;
                     this._taskManager = undefined;
                 }
             }
-            // 6. Reset initialization flags
+            // 7. Reset initialization flags
             try {
                 logger.debug("[NeuroLink] Resetting initialization state...");
                 this.mcpInitialized = false;
@@ -13455,7 +13565,7 @@ Current user's request: ${currentInput}`;
                 cleanupErrors.push(err);
                 logger.warn("[NeuroLink] Error resetting state:", error);
             }
-            // 6. Log completion
+            // 7. Log completion
             if (cleanupErrors.length === 0) {
                 logger.debug("[NeuroLink] ✅ Resource disposal completed successfully");
             }