@juspay/neurolink 9.5.3 → 9.7.0

package/dist/lib/neurolink.js

@@ -14,12 +14,17 @@ catch {
     // Environment variables should be set externally in production
 }
 import { EventEmitter } from "events";
-import { isNonNullObject } from "./utils/typeUtils.js";
 import pLimit from "p-limit";
 import { CIRCUIT_BREAKER, CIRCUIT_BREAKER_RESET_MS, MEMORY_THRESHOLDS, NANOSECOND_TO_MS_DIVISOR, PERFORMANCE_THRESHOLDS, PROVIDER_TIMEOUTS, RETRY_ATTEMPTS, RETRY_DELAYS, TOOL_TIMEOUTS, } from "./constants/index.js";
+import { checkContextBudget } from "./context/budgetChecker.js";
+import { ContextCompactor, } from "./context/contextCompactor.js";
+import { isContextOverflowError } from "./context/errorDetection.js";
+import { repairToolPairs } from "./context/toolPairRepair.js";
 import { SYSTEM_LIMITS } from "./core/constants.js";
 import { AIProviderFactory } from "./core/factory.js";
 import { ProviderRegistry } from "./factories/providerRegistry.js";
+import { FileReferenceRegistry } from "./files/fileReferenceRegistry.js";
+import { createFileTools } from "./files/fileTools.js";
 import { HITLManager } from "./hitl/hitlManager.js";
 import { ExternalServerManager } from "./mcp/externalServerManager.js";
 // Import direct tools server for automatic registration
@@ -29,7 +34,7 @@ import { initializeMem0 } from "./memory/mem0Initializer.js";
 import { flushOpenTelemetry, getLangfuseHealthStatus, initializeOpenTelemetry, isOpenTelemetryInitialized, setLangfuseContext, shutdownOpenTelemetry, } from "./services/server/ai/observability/instrumentation.js";
 import { getConversationMessages, storeConversationTurn, } from "./utils/conversationMemory.js";
 // Enhanced error handling imports
-import { CircuitBreaker, ErrorFactory, isRetriableError, logStructuredError, NeuroLinkError, withRetry, withTimeout, } from "./utils/errorHandling.js";
+import { CircuitBreaker, ErrorFactory, isAbortError, isRetriableError, logStructuredError, NeuroLinkError, withRetry, withTimeout, } from "./utils/errorHandling.js";
 // Factory processing imports
 import { createCleanStreamOptions, enhanceTextGenerationOptions, processFactoryOptions, processStreamingFactoryOptions, validateFactoryConfig, } from "./utils/factoryProcessing.js";
 import { logger, mcpLogger } from "./utils/logger.js";
@@ -42,9 +47,9 @@ import { BinaryTaskClassifier } from "./utils/taskClassifier.js";
 // Tool detection and execution imports
 // Transformation utilities
 import { extractToolNames, optimizeToolForCollection, transformAvailableTools, transformParamsForLogging, transformToolExecutions, transformToolExecutionsForMCP, transformToolsForMCP, transformToolsToDescriptions, transformToolsToExpectedFormat, } from "./utils/transformationUtils.js";
-import { runWorkflow } from "./workflow/core/workflowRunner.js";
+import { isNonNullObject } from "./utils/typeUtils.js";
 import { getWorkflow } from "./workflow/core/workflowRegistry.js";
-// Core types imported from core/types.js
+import { runWorkflow } from "./workflow/core/workflowRunner.js";
 /**
  * NeuroLink - Universal AI Development Platform
  *
@@ -166,6 +171,10 @@ export class NeuroLink {
     // Mem0 memory instance and config for conversation context
     mem0Instance;
     mem0Config;
+    // File Reference Registry for lazy on-demand file processing
+    fileRegistry;
+    // Cached file tools to avoid redundant createFileTools() calls per generate/stream
+    cachedFileTools = null;
     /**
      * Extract and set Langfuse context from options with proper async scoping
      */
@@ -288,6 +297,7 @@ export class NeuroLink {
     observabilityConfig;
     constructor(config) {
         this.toolRegistry = config?.toolRegistry || new MCPToolRegistry();
+        this.fileRegistry = new FileReferenceRegistry();
         this.observabilityConfig = config?.observability;
         // Initialize orchestration setting
         this.enableOrchestration = config?.enableOrchestration ?? false;
@@ -304,6 +314,7 @@ export class NeuroLink {
         this.initializeConversationMemory(config, constructorId, constructorStartTime, constructorHrTimeStart);
         this.initializeExternalServerManager(constructorId, constructorStartTime, constructorHrTimeStart);
         this.initializeHITL(config, constructorId, constructorStartTime, constructorHrTimeStart);
+        this.registerFileTools();
         this.initializeLangfuse(constructorId, constructorStartTime, constructorHrTimeStart);
         this.logConstructorComplete(constructorId, constructorStartTime, constructorHrTimeStart);
     }
@@ -457,6 +468,56 @@ export class NeuroLink {
             });
         }
     }
+    /**
+     * Register file reference tools with the MCP tool registry.
+     *
+     * Creates file access tools (list_attached_files, read_file_section,
+     * search_in_file, get_file_preview) bound to the FileReferenceRegistry
+     * and registers them as direct tools so they're available to LLMs.
+     */
+    registerFileTools() {
+        const fileTools = createFileTools(this.fileRegistry);
+        // Use void to handle async registration without blocking constructor
+        const registrations = Object.entries(fileTools).map(async ([toolName, toolDef]) => {
+            const toolId = `direct.${toolName}`;
+            const toolInfo = {
+                name: toolName,
+                description: toolDef.description || `File tool: ${toolName}`,
+                inputSchema: {},
+                serverId: "direct",
+                category: "built-in",
+            };
+            await this.toolRegistry.registerTool(toolId, toolInfo, {
+                execute: async (params) => {
+                    try {
+                        const result = await toolDef.execute(params, {
+                            toolCallId: "file-tool",
+                            messages: [],
+                        });
+                        return {
+                            success: true,
+                            data: result,
+                            metadata: { toolName, serverId: "direct", executionTime: 0 },
+                        };
+                    }
+                    catch (error) {
+                        return {
+                            success: false,
+                            error: error instanceof Error ? error.message : String(error),
+                            metadata: { toolName, serverId: "direct", executionTime: 0 },
+                        };
+                    }
+                },
+                description: toolDef.description,
+                inputSchema: {},
+            });
+        });
+        // Fire-and-forget: registrations complete before any generate/stream call
+        // because those calls await initializeMCP() which is slower
+        void Promise.all(registrations).then(() => {
+            logger.debug(`[NeuroLink] Registered ${Object.keys(fileTools).length} file reference tools`);
+        });
+    }
     /** Format memory context for prompt inclusion */
     formatMemoryContext(memoryContext, currentInput) {
         return `Context from previous conversations:
@@ -1507,6 +1568,8 @@ Current user's request: ${currentInput}`;
                 output: options.output,
                 tools: options.tools, // Includes RAG tools if rag config was provided
                 disableTools: options.disableTools,
+                toolFilter: options.toolFilter,
+                excludeTools: options.excludeTools,
                 enableAnalytics: options.enableAnalytics,
                 enableEvaluation: options.enableEvaluation,
                 context: options.context,
@@ -1515,6 +1578,9 @@ Current user's request: ${currentInput}`;
                 input: options.input, // This includes text, images, and content arrays
                 region: options.region,
                 tts: options.tts,
+                fileRegistry: this.fileRegistry,
+                abortSignal: options.abortSignal,
+                skipToolPromptInjection: options.skipToolPromptInjection,
             };
             // Apply factory enhancement using centralized utilities
             const textOptions = enhanceTextGenerationOptions(baseOptions, factoryResult);
@@ -1552,6 +1618,7 @@ Current user's request: ${currentInput}`;
             // Convert back to GenerateResult
             const generateResult = {
                 content: textResult.content,
+                finishReason: textResult.finishReason,
                 provider: textResult.provider,
                 model: textResult.model,
                 usage: textResult.usage
@@ -1594,31 +1661,33 @@ Current user's request: ${currentInput}`;
                 video: textResult.video,
                 ppt: textResult.ppt,
             };
-            if (this.conversationMemoryConfig?.conversationMemory?.mem0Enabled &&
-                options.context?.userId &&
-                generateResult.content) {
-                // Non-blocking memory storage - run in background
-                setImmediate(async () => {
-                    try {
-                        const mem0 = await this.ensureMem0Ready();
-                        if (mem0) {
-                            await this.storeMem0ConversationTurn(mem0, originalPrompt, generateResult.content, options.context?.userId, {
-                                timestamp: new Date().toISOString(),
-                                provider: generateResult.provider,
-                                model: generateResult.model,
-                                type: "conversation_turn",
-                            });
-                        }
-                    }
-                    catch (error) {
-                        // Non-blocking: Log error but don't fail the generation
-                        logger.warn("Mem0 memory storage failed:", error);
-                    }
-                });
-            }
+            this.scheduleGenerateMem0Storage(options, originalPrompt, generateResult);
             return generateResult;
         });
     }
+    /**
+     * Schedule non-blocking Mem0 memory storage after generate completes.
+     */
+    scheduleGenerateMem0Storage(options, originalPrompt, generateResult) {
+        if (this.conversationMemoryConfig?.conversationMemory?.mem0Enabled &&
+            options.context?.userId &&
+            generateResult.content.trim()) {
+            setImmediate(async () => {
+                try {
+                    const mem0 = await this.ensureMem0Ready();
+                    if (mem0) {
+                        await this.storeMem0ConversationTurn(mem0, originalPrompt ?? "", generateResult.content.trim(), options.context?.userId, {
+                            timestamp: new Date().toISOString(),
+                            type: "conversation_turn_generate",
+                        });
+                    }
+                }
+                catch (error) {
+                    logger.warn("Mem0 memory storage failed:", error);
+                }
+            });
+        }
+    }
     /**
      * Generate with workflow engine integration
      * Returns both original and processed responses for AB testing
@@ -1894,21 +1963,108 @@ Current user's request: ${currentInput}`;
             await this.initializeConversationMemoryForGeneration(generateInternalId, generateInternalStartTime, generateInternalHrTimeStart);
             const mcpResult = await this.attemptMCPGeneration(options, generateInternalId, generateInternalStartTime, generateInternalHrTimeStart, functionTag);
             if (mcpResult) {
+                logger.info(`[NeuroLink.generateTextInternal] generate() - COMPLETE SUCCESS (MCP path)`, {
+                    provider: mcpResult.provider,
+                    model: mcpResult.model,
+                    responseTimeMs: Date.now() - generateInternalStartTime,
+                    tokensUsed: mcpResult.usage?.total || 0,
+                    toolsUsed: mcpResult.toolsUsed?.length || 0,
+                });
                 await storeConversationTurn(this.conversationMemory, options, mcpResult, new Date(generateInternalStartTime));
                 this.emitter.emit("response:end", mcpResult.content || "");
                 return mcpResult;
             }
             const directResult = await this.directProviderGeneration(options);
             logger.debug(`[${functionTag}] Direct generation successful`);
+            logger.info(`[NeuroLink.generateTextInternal] generate() - COMPLETE SUCCESS`, {
+                provider: directResult.provider,
+                model: directResult.model,
+                responseTimeMs: Date.now() - generateInternalStartTime,
+                tokensUsed: directResult.usage?.total || 0,
+                toolsUsed: directResult.toolsUsed?.length || 0,
+            });
             await storeConversationTurn(this.conversationMemory, options, directResult, new Date(generateInternalStartTime));
             this.emitter.emit("response:end", directResult.content || "");
             this.emitter.emit("message", `Text generation completed successfully`);
             return directResult;
         }
         catch (error) {
-            logger.error(`[${functionTag}] All generation methods failed`, {
-                error: error instanceof Error ? error.message : String(error),
-            });
+            // Check if this is a context overflow error - attempt recovery
+            if (isContextOverflowError(error) && this.conversationMemory) {
+                logger.warn(`[${functionTag}] Context overflow detected, attempting aggressive compaction`, {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+                try {
+                    const conversationMessages = await getConversationMessages(this.conversationMemory, options);
+                    // Calculate a meaningful compaction target from the model's budget
+                    const recoveryBudget = checkContextBudget({
+                        provider: options.provider || "openai",
+                        model: options.model,
+                        maxTokens: options.maxTokens,
+                        currentPrompt: options.prompt,
+                        systemPrompt: options.systemPrompt,
+                    });
+                    const compactionTarget = Math.floor(recoveryBudget.availableInputTokens * 0.7);
+                    const compactor = new ContextCompactor({
+                        enableSummarize: false, // Skip LLM call for recovery
+                        truncationFraction: 0.75, // Aggressive truncation
+                    });
+                    const compactionResult = await compactor.compact(conversationMessages, compactionTarget);
+                    if (compactionResult.compacted) {
+                        const repairedResult = repairToolPairs(compactionResult.messages);
+                        logger.info(`[${functionTag}] Aggressive compaction complete, retrying`, {
+                            tokensSaved: compactionResult.tokensSaved,
+                            compactionTarget,
+                        });
+                        // Retry with compacted context - pass compacted messages to avoid re-fetching
+                        return await this.directProviderGeneration({
+                            ...options,
+                            conversationMessages: repairedResult.messages,
+                        });
+                    }
+                }
+                catch (retryError) {
+                    logger.error(`[${functionTag}] Recovery attempt also failed`, {
+                        error: retryError instanceof Error
+                            ? retryError.message
+                            : String(retryError),
+                    });
+                }
+            }
+            // If the generation was aborted (e.g., coding task short-circuit via AbortController),
+            // still store the conversation turn so that:
+            // 1. The Redis conversation entry is created (if first turn)
+            // 2. setImmediate triggers generateConversationTitle() for the session
+            // 3. The caller's syncTitleFromRedis() can find the SDK-generated title
+            if (isAbortError(error)) {
+                logger.info(`[${functionTag}] Generation aborted — storing conversation turn for title generation`, {
+                    hasMemory: !!this.conversationMemory,
+                    memoryType: this.conversationMemory?.constructor?.name || "NONE",
+                    sessionId: options.context?.sessionId ||
+                        "unknown",
+                });
+                try {
+                    const abortedResult = {
+                        content: "[generation was interrupted]",
+                        provider: options.provider || "unknown",
+                        model: options.model || "unknown",
+                        responseTime: Date.now() - generateInternalStartTime,
+                    };
+                    await withTimeout(storeConversationTurn(this.conversationMemory, options, abortedResult, new Date(generateInternalStartTime)), 5000);
+                }
+                catch (storeError) {
+                    logger.warn(`[${functionTag}] Failed to store conversation turn after abort`, {
+                        error: storeError instanceof Error
+                            ? storeError.message
+                            : String(storeError),
+                    });
+                }
+            }
+            else {
+                logger.error(`[${functionTag}] All generation methods failed`, {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
             this.emitter.emit("response:end", "");
             this.emitter.emit("error", error instanceof Error ? error : new Error(String(error)));
             throw error;
@@ -2006,10 +2162,35 @@ Current user's request: ${currentInput}`;
                 }
             }
             catch (error) {
+                // Immediately propagate AbortError — never retry aborted requests
+                if (isAbortError(error)) {
+                    logger.debug(`[${functionTag}] AbortError detected on attempt ${attempt}, stopping retries`);
+                    throw error;
+                }
                 logger.debug(`[${functionTag}] MCP generation failed on attempt ${attempt}/${maxAttempts}`, {
                     error: error instanceof Error ? error.message : String(error),
                     willRetry: attempt < maxAttempts,
                 });
+                // Check for non-retryable errors — skip remaining retries immediately
+                // NoSuchToolError / InvalidToolArgumentsError from Vercel AI SDK are never
+                // retryable — the model hallucinated a tool name or gave bad params, and
+                // the same tools would be passed on every retry.
+                const isToolError = error instanceof Error &&
+                    (error.name === "AI_NoSuchToolError" ||
+                        error.name === "AI_InvalidToolArgumentsError" ||
+                        error.message.includes("NoSuchToolError") ||
+                        error.message.includes("Model tried to call unavailable tool"));
+                const isNonRetryable = isContextOverflowError(error) ||
+                    isToolError ||
+                    (error instanceof Error &&
+                        error.isRetryable ===
+                            false) ||
+                    (error instanceof Error &&
+                        error.statusCode === 400);
+                if (isNonRetryable) {
+                    logger.debug(`[${functionTag}] Non-retryable error detected, skipping remaining retries`);
+                    break;
+                }
                 if (attempt >= maxAttempts) {
                     logger.debug(`[${functionTag}] All MCP attempts exhausted, falling back to direct generation`);
                     break;
@@ -2055,7 +2236,9 @@ Current user's request: ${currentInput}`;
                 ? await getBestProvider()
                 : options.provider;
             // Get available tools
-            const availableTools = await this.getAllAvailableTools();
+            let availableTools = await this.getAllAvailableTools();
+            // Apply per-call tool filtering for system prompt tool descriptions
+            availableTools = this.applyToolInfoFiltering(availableTools, options);
             const targetTool = availableTools.find((t) => t.name.includes("SuccessRateSRByTime") ||
                 t.name.includes("juspay-analytics"));
             logger.debug("Available tools for AI prompt generation", {
@@ -2070,15 +2253,51 @@ Current user's request: ${currentInput}`;
                     }
                     : null,
             });
-            // Create tool-aware system prompt
-            const enhancedSystemPrompt = this.createToolAwareSystemPrompt(options.systemPrompt, availableTools);
+            // Create tool-aware system prompt (skip if skipToolPromptInjection is true)
+            const enhancedSystemPrompt = options.skipToolPromptInjection
+                ? options.systemPrompt || ""
+                : this.createToolAwareSystemPrompt(options.systemPrompt, availableTools);
             logger.debug("Tool-aware system prompt created", {
                 originalPromptLength: options.systemPrompt?.length || 0,
                 enhancedPromptLength: enhancedSystemPrompt.length,
+                skippedToolInjection: !!options.skipToolPromptInjection,
                 enhancedPromptPreview: enhancedSystemPrompt.substring(0, 500) + "...",
             });
             // Get conversation messages for context
-            const conversationMessages = await getConversationMessages(this.conversationMemory, options);
+            let conversationMessages = await getConversationMessages(this.conversationMemory, options);
+            // Pre-generation budget check
+            const budgetResult = checkContextBudget({
+                provider: providerName,
+                model: options.model,
+                maxTokens: options.maxTokens,
+                systemPrompt: enhancedSystemPrompt,
+                conversationMessages: conversationMessages,
+                currentPrompt: options.prompt,
+                toolDefinitions: availableTools,
+            });
+            if (budgetResult.shouldCompact && this.conversationMemory) {
+                logger.info("[NeuroLink] Context budget exceeded, triggering auto-compaction", {
+                    usageRatio: budgetResult.usageRatio,
+                    estimatedTokens: budgetResult.estimatedInputTokens,
+                    availableTokens: budgetResult.availableInputTokens,
+                });
+                const compactor = new ContextCompactor({
+                    provider: providerName,
+                    summarizationProvider: this.conversationMemoryConfig?.conversationMemory
+                        ?.summarizationProvider,
+                    summarizationModel: this.conversationMemoryConfig?.conversationMemory
+                        ?.summarizationModel,
+                });
+                const compactionResult = await compactor.compact(conversationMessages, budgetResult.availableInputTokens, this.conversationMemoryConfig?.conversationMemory);
+                if (compactionResult.compacted) {
+                    const repairedResult = repairToolPairs(compactionResult.messages);
+                    conversationMessages = repairedResult.messages;
+                    logger.info("[NeuroLink] Context compacted successfully", {
+                        stagesUsed: compactionResult.stagesUsed,
+                        tokensSaved: compactionResult.tokensSaved,
+                    });
+                }
+            }
             // Create provider and generate
             const provider = await AIProviderFactory.createProvider(providerName, options.model, !options.disableTools, // Pass disableTools as inverse of enableMCP
             this, // Pass SDK instance
@@ -2126,8 +2345,10 @@ Current user's request: ${currentInput}`;
             return {
                 content: result.content || "", // Ensure content is never undefined
                 provider: providerName,
+                model: result.model,
                 usage: result.usage,
                 responseTime,
+                finishReason: result.finishReason,
                 toolsUsed: result.toolsUsed || [],
                 toolExecutions: transformedToolExecutions,
                 enhancedWithTools: Boolean(hasToolExecutions), // Mark as enhanced if tools were actually used
@@ -2141,6 +2362,26 @@ Current user's request: ${currentInput}`;
             };
         }
         catch (error) {
+            // Immediately propagate AbortError — never swallow aborted requests
+            if (isAbortError(error)) {
+                mcpLogger.debug(`[${functionTag}] AbortError detected, rethrowing`);
+                throw error;
+            }
+            // Propagate non-retryable errors (NoSuchToolError, InvalidToolArgumentsError)
+            // so the caller's retry loop can detect them and break immediately instead
+            // of retrying the same deterministic failure.
+            const isToolError = error instanceof Error &&
+                (error.name === "AI_NoSuchToolError" ||
+                    error.name === "AI_InvalidToolArgumentsError" ||
+                    (error.message &&
+                        (error.message.includes("NoSuchToolError") ||
+                            error.message.includes("Model tried to call unavailable tool"))));
+            if (isToolError) {
+                mcpLogger.warn(`[${functionTag}] Non-retryable tool error, rethrowing`, {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+                throw error;
+            }
             mcpLogger.warn(`[${functionTag}] MCP generation failed`, {
                 error: error instanceof Error ? error.message : String(error),
             });
@@ -2192,8 +2433,32 @@ Current user's request: ${currentInput}`;
         for (const providerName of tryProviders) {
             try {
                 logger.debug(`[${functionTag}] Attempting provider: ${providerName}`);
-                // Get conversation messages for context
-                const conversationMessages = await getConversationMessages(this.conversationMemory, options);
+                // Get conversation messages for context (use pre-compacted if provided)
+                const optionsWithMessages = options;
+                let conversationMessages = optionsWithMessages.conversationMessages
+                    ?.length
+                    ? optionsWithMessages.conversationMessages
+                    : await getConversationMessages(this.conversationMemory, options);
+                // Pre-generation budget check
+                const budgetCheck = checkContextBudget({
+                    provider: providerName,
+                    model: options.model,
+                    maxTokens: options.maxTokens,
+                    systemPrompt: options.systemPrompt,
+                    conversationMessages: conversationMessages,
+                    currentPrompt: options.prompt,
+                    toolDefinitions: options.tools
+                        ? Object.values(options.tools)
+                        : undefined,
+                });
+                if (budgetCheck.shouldCompact && this.conversationMemory) {
+                    const compactor = new ContextCompactor({ provider: providerName });
+                    const compactionResult = await compactor.compact(conversationMessages, budgetCheck.availableInputTokens);
+                    if (compactionResult.compacted) {
+                        const repairedResult = repairToolPairs(compactionResult.messages);
+                        conversationMessages = repairedResult.messages;
+                    }
+                }
                 const provider = await AIProviderFactory.createProvider(providerName, options.model, !options.disableTools, // Pass disableTools as inverse of enableMCP
                 this, // Pass SDK instance
                 options.region);
@@ -2223,6 +2488,7 @@ Current user's request: ${currentInput}`;
                     model: result.model,
                     usage: result.usage,
                     responseTime,
+                    finishReason: result.finishReason,
                     toolsUsed: result.toolsUsed || [],
                     enhancedWithTools: false,
                     analytics: result.analytics,
@@ -2235,6 +2501,11 @@ Current user's request: ${currentInput}`;
                 };
             }
             catch (error) {
+                // Immediately propagate AbortError — never fall back to next provider on abort
+                if (isAbortError(error)) {
+                    logger.debug(`[${functionTag}] AbortError detected on provider ${providerName}, stopping fallback`);
+                    throw error;
+                }
                 lastError = error instanceof Error ? error : new Error(String(error));
                 logger.warn(`[${functionTag}] Provider ${providerName} failed`, {
                     error: lastError.message,
@@ -2254,6 +2525,34 @@ Current user's request: ${currentInput}`;
     /**
      * Create tool-aware system prompt that informs AI about available tools
      */
+    /**
+     * Apply per-call tool filtering (whitelist/blacklist) to a ToolInfo array.
+     * Used to filter the tool list before building the system prompt.
+     */
+    applyToolInfoFiltering(tools, options) {
+        if ((!options.toolFilter || options.toolFilter.length === 0) &&
+            (!options.excludeTools || options.excludeTools.length === 0)) {
+            return tools;
+        }
+        let filtered = tools;
+        if (options.toolFilter && options.toolFilter.length > 0) {
+            const allowSet = new Set(options.toolFilter);
+            filtered = filtered.filter((t) => allowSet.has(t.name));
+        }
+        if (options.excludeTools && options.excludeTools.length > 0) {
+            const denySet = new Set(options.excludeTools);
+            filtered = filtered.filter((t) => !denySet.has(t.name));
+        }
+        if (filtered.length !== tools.length) {
+            logger.debug(`Tool info filtering applied for system prompt`, {
+                beforeCount: tools.length,
+                afterCount: filtered.length,
+                toolFilter: options.toolFilter,
+                excludeTools: options.excludeTools,
+            });
+        }
+        return filtered;
+    }
     createToolAwareSystemPrompt(originalSystemPrompt, availableTools) {
         // AI prompt generation with tool analysis and structured logging
         const promptGenerationData = {
@@ -2390,6 +2689,8 @@ Current user's request: ${currentInput}`;
         const hrTimeStart = process.hrtime.bigint();
         const streamId = `neurolink-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
         const originalPrompt = options.input.text; // Store the original prompt for memory storage
+        // Inject file registry for lazy on-demand file processing
+        options.fileRegistry = this.fileRegistry;
         await this.validateStreamInput(options);
         this.emitStreamStartEvents(options, startTime);
         // Check if workflow is requested
@@ -2398,171 +2699,14 @@ Current user's request: ${currentInput}`;
         }
         // Set session and user IDs from context for Langfuse spans and execute with proper async scoping
         return await this.setLangfuseContextFromOptions(options, async () => {
-            let enhancedOptions;
-            let factoryResult;
             try {
-                // Initialize conversation memory if needed (for lazy loading)
-                await this.initializeConversationMemoryForGeneration(streamId, startTime, hrTimeStart);
-                // Initialize MCP
-                await this.initializeMCP();
-                const _originalPrompt = options.input.text;
-                if (this.conversationMemoryConfig?.conversationMemory?.mem0Enabled &&
-                    options.context?.userId) {
-                    try {
-                        const mem0 = await this.ensureMem0Ready();
-                        if (!mem0) {
-                            // Continue without memories if mem0 is not available
-                            logger.debug("Mem0 not available, continuing without memory retrieval");
-                        }
-                        else {
-                            const memories = await mem0.search(options.input.text, {
-                                user_id: options.context.userId,
-                                limit: 5,
-                            });
-                            if (memories && memories.length > 0) {
-                                // Enhance the input with memory context
-                                const memoryContext = this.extractMemoryContext(memories);
-                                options.input.text = this.formatMemoryContext(memoryContext, options.input.text);
-                            }
-                        }
-                    }
-                    catch (error) {
-                        // Non-blocking: Log error but continue with streaming
-                        logger.warn("Mem0 memory retrieval failed:", error);
-                    }
-                }
-                // Apply orchestration if enabled and no specific provider/model requested
-                if (this.enableOrchestration && !options.provider && !options.model) {
-                    try {
-                        const orchestratedOptions = await this.applyStreamOrchestration(options);
-                        logger.debug("Stream orchestration applied", {
-                            originalProvider: options.provider || "auto",
-                            orchestratedProvider: orchestratedOptions.provider,
-                            orchestratedModel: orchestratedOptions.model,
-                            prompt: options.input.text?.substring(0, 100),
-                        });
-                        // Use orchestrated options
-                        Object.assign(options, orchestratedOptions);
-                    }
-                    catch (error) {
-                        logger.warn("Stream orchestration failed, continuing with original options", {
-                            error: error instanceof Error ? error.message : String(error),
-                            originalProvider: options.provider || "auto",
-                        });
-                        // Continue with original options if orchestration fails
-                    }
-                }
-                // 🔧 AUTO-DISABLE TOOLS: For Ollama models that don't support tools (same logic as generate())
-                // This prevents overwhelming smaller models with massive tool descriptions in the system message
-                if ((options.provider === "ollama" ||
-                    options.provider?.toLowerCase().includes("ollama")) &&
-                    !options.disableTools) {
-                    const { ModelConfigurationManager } = await import("./core/modelConfiguration.js");
-                    const modelConfig = ModelConfigurationManager.getInstance();
-                    const ollamaConfig = modelConfig.getProviderConfiguration("ollama");
-                    const toolCapableModels = ollamaConfig?.modelBehavior?.toolCapableModels || [];
-                    // Only disable tools if we have explicit evidence the model doesn't support them
-                    // If toolCapableModels is empty or model is not specified, don't make assumptions
-                    const modelName = options.model;
-                    if (toolCapableModels.length > 0 && modelName) {
-                        const modelSupportsTools = toolCapableModels.some((capableModel) => modelName.toLowerCase().includes(capableModel.toLowerCase()));
-                        if (!modelSupportsTools) {
-                            options.disableTools = true;
-                            logger.debug("Auto-disabled tools for Ollama model that doesn't support them (stream)", {
-                                model: options.model,
-                                toolCapableModels: toolCapableModels.slice(0, 3), // Show first 3 for brevity
-                            });
-                        }
-                    }
-                }
-                // RAG Integration: If rag config is provided, prepare the RAG search tool (stream)
-                if (options.rag?.files?.length) {
-                    try {
-                        const { prepareRAGTool } = await import("./rag/ragIntegration.js");
-                        const ragResult = await prepareRAGTool(options.rag, options.provider);
-                        // Inject the RAG tool into the tools record
-                        if (!options.tools) {
-                            options.tools = {};
-                        }
-                        options.tools[ragResult.toolName] =
-                            ragResult.tool;
-                        // Inject RAG-aware system prompt so the AI uses the RAG tool first
-                        const ragStreamInstruction = [
-                            `\n\nIMPORTANT: You have a tool called "${ragResult.toolName}" that searches through`,
-                            `${ragResult.filesLoaded} loaded document(s) containing ${ragResult.chunksIndexed} indexed chunks.`,
-                            `ALWAYS use the "${ragResult.toolName}" tool FIRST to answer the user's question before using any other tools.`,
-                            `This tool searches your local knowledge base of pre-loaded documents and is the primary source of truth.`,
-                            `Do NOT use websearchGrounding or any web search tools when the answer can be found in the loaded documents.`,
-                        ].join(" ");
-                        options.systemPrompt =
-                            (options.systemPrompt || "") + ragStreamInstruction;
-                        logger.info("[RAG] Tool injected into stream()", {
-                            toolName: ragResult.toolName,
-                            filesLoaded: ragResult.filesLoaded,
-                            chunksIndexed: ragResult.chunksIndexed,
-                        });
-                    }
-                    catch (error) {
-                        logger.warn("[RAG] Failed to prepare RAG tool, continuing without RAG", {
-                            error: error instanceof Error ? error.message : String(error),
-                        });
-                    }
-                }
-                factoryResult = processStreamingFactoryOptions(options);
-                enhancedOptions = createCleanStreamOptions(options);
-                if (options.input?.text) {
-                    const { toolResults: _toolResults, enhancedPrompt } = await this.detectAndExecuteTools(options.input.text, undefined);
-                    if (enhancedPrompt !== options.input.text) {
-                        enhancedOptions.input.text = enhancedPrompt;
-                    }
-                }
+                // Prepare options: init memory, MCP, Mem0, orchestration, Ollama auto-disable, tool detection
+                const { enhancedOptions, factoryResult } = await this.prepareStreamOptions(options, streamId, startTime, hrTimeStart);
                 const { stream: mcpStream, provider: providerName } = await this.createMCPStream(enhancedOptions);
                 let accumulatedContent = "";
                 let chunkCount = 0;
-                const eventSequence = [];
-                let eventSeqCounter = 0;
-                const captureEvent = (type, data) => {
-                    eventSequence.push({
-                        type,
-                        seq: eventSeqCounter++,
-                        timestamp: Date.now(),
-                        ...(data && typeof data === "object" ? data : { data }),
-                    });
-                };
-                const onResponseChunk = (...args) => {
-                    const chunk = args[0];
-                    captureEvent("response:chunk", { content: chunk });
-                };
-                const onToolStart = (...args) => {
-                    const data = args[0];
-                    captureEvent("tool:start", data);
-                };
-                const onToolEnd = (...args) => {
-                    const data = args[0];
-                    captureEvent("tool:end", data);
-                    if (data.result && data.result.uiComponent === true) {
-                        captureEvent("ui-component", {
-                            toolName: data.toolName,
-                            componentData: data.result,
-                            timestamp: Date.now(),
-                        });
-                    }
-                };
-                const onUIComponent = (...args) => {
-                    captureEvent("ui-component", args[0]);
-                };
-                const onHITLRequest = (...args) => {
-                    captureEvent("hitl:confirmation-request", args[0]);
-                };
-                const onHITLResponse = (...args) => {
-                    captureEvent("hitl:confirmation-response", args[0]);
-                };
-                this.emitter.on("response:chunk", onResponseChunk);
-                this.emitter.on("tool:start", onToolStart);
-                this.emitter.on("tool:end", onToolEnd);
-                this.emitter.on("ui-component", onUIComponent);
-                this.emitter.on("hitl:confirmation-request", onHITLRequest);
-                this.emitter.on("hitl:confirmation-response", onHITLResponse);
+                // Set up event capture listeners
+                const { eventSequence, cleanup: cleanupListeners } = this.setupStreamEventListeners();
                 const metadata = {
                     fallbackAttempted: false,
                     guardrailsBlocked: false,
@@ -2582,127 +2726,30 @@ Current user's request: ${currentInput}`;
                             yield chunk;
                         }
                         if (chunkCount === 0 && !metadata.fallbackAttempted) {
-                            metadata.fallbackAttempted = true;
-                            const errorMsg = "Stream completed with 0 chunks (possible guardrails block)";
-                            metadata.error = errorMsg;
-                            const fallbackRoute = ModelRouter.getFallbackRoute(originalPrompt || enhancedOptions.input.text || "", {
-                                provider: providerName,
-                                model: enhancedOptions.model || "gpt-4o",
-                                reasoning: "primary failed",
-                                confidence: 0.5,
-                            }, { fallbackStrategy: "auto" });
-                            logger.warn("Retrying with fallback provider", {
-                                originalProvider: providerName,
-                                fallbackProvider: fallbackRoute.provider,
-                                reason: errorMsg,
+                            yield* self.handleStreamFallback(metadata, originalPrompt, enhancedOptions, providerName, accumulatedContent, (content) => {
+                                accumulatedContent += content;
                             });
-                            try {
-                                const fallbackProvider = await AIProviderFactory.createProvider(fallbackRoute.provider, fallbackRoute.model);
-                                // Ensure fallback provider can execute tools
-                                fallbackProvider.setupToolExecutor({
-                                    customTools: self.getCustomTools(),
-                                    executeTool: self.executeTool.bind(self),
-                                }, "NeuroLink.fallbackStream");
-                                // Get conversation messages for context (same as primary stream)
-                                const conversationMessages = await getConversationMessages(self.conversationMemory, {
-                                    prompt: enhancedOptions.input.text,
-                                    context: enhancedOptions.context,
-                                });
-                                const fallbackResult = await fallbackProvider.stream({
-                                    ...enhancedOptions,
-                                    model: fallbackRoute.model,
-                                    conversationMessages,
-                                });
-                                let fallbackChunkCount = 0;
-                                for await (const fallbackChunk of fallbackResult.stream) {
-                                    fallbackChunkCount++;
-                                    if (fallbackChunk &&
-                                        "content" in fallbackChunk &&
-                                        typeof fallbackChunk.content === "string") {
-                                        accumulatedContent += fallbackChunk.content;
-                                        self.emitter.emit("response:chunk", fallbackChunk.content);
-                                    }
-                                    yield fallbackChunk;
-                                }
-                                if (fallbackChunkCount === 0) {
-                                    throw new Error(`Fallback provider ${fallbackRoute.provider} also returned 0 chunks`);
-                                }
-                                // Fallback succeeded - likely guardrails blocked primary
-                                metadata.guardrailsBlocked = true;
-                            }
-                            catch (fallbackError) {
-                                const fallbackErrorMsg = fallbackError instanceof Error
-                                    ? fallbackError.message
-                                    : String(fallbackError);
-                                metadata.error = `${errorMsg}; Fallback failed: ${fallbackErrorMsg}`;
-                                logger.error("Fallback provider failed", {
-                                    fallbackProvider: fallbackRoute.provider,
-                                    error: fallbackErrorMsg,
-                                });
-                                throw fallbackError;
-                            }
                         }
                     }
                     finally {
-                        self.emitter.off("response:chunk", onResponseChunk);
-                        self.emitter.off("tool:start", onToolStart);
-                        self.emitter.off("tool:end", onToolEnd);
-                        self.emitter.off("ui-component", onUIComponent);
-                        self.emitter.off("hitl:confirmation-request", onHITLRequest);
-                        self.emitter.off("hitl:confirmation-response", onHITLResponse);
-                        // Store memory after stream consumption is complete
-                        if (self.conversationMemory && enhancedOptions.context?.sessionId) {
-                            const sessionId = enhancedOptions.context?.sessionId;
-                            const userId = enhancedOptions.context?.userId;
-                            let providerDetails;
-                            if (enhancedOptions.model) {
-                                providerDetails = {
-                                    provider: providerName,
-                                    model: enhancedOptions.model,
-                                };
-                            }
-                            try {
-                                await self.conversationMemory.storeConversationTurn({
-                                    sessionId,
-                                    userId,
-                                    userMessage: originalPrompt ?? "",
-                                    aiResponse: accumulatedContent,
-                                    startTimeStamp: new Date(startTime),
-                                    providerDetails,
-                                    enableSummarization: enhancedOptions.enableSummarization,
-                                    events: eventSequence.length > 0 ? eventSequence : undefined,
-                                });
-                                logger.debug("[NeuroLink.stream] Stored conversation turn with events", {
-                                    sessionId,
-                                    eventCount: eventSequence.length,
-                                    eventTypes: [...new Set(eventSequence.map((e) => e.type))],
-                                });
-                            }
-                            catch (error) {
-                                logger.warn("Failed to store stream conversation turn", {
-                                    error: error instanceof Error ? error.message : String(error),
-                                });
-                            }
-                        }
-                        if (self.conversationMemoryConfig?.conversationMemory?.mem0Enabled &&
-                            enhancedOptions.context?.userId &&
-                            accumulatedContent.trim()) {
-                            // Non-blocking memory storage - run in background
-                            setImmediate(async () => {
-                                try {
-                                    const mem0 = await self.ensureMem0Ready();
-                                    if (mem0) {
-                                        await self.storeMem0ConversationTurn(mem0, originalPrompt, accumulatedContent.trim(), enhancedOptions.context?.userId, {
-                                            timestamp: new Date().toISOString(),
-                                            type: "conversation_turn_stream",
-                                        });
-                                    }
-                                }
-                                catch (error) {
-                                    logger.warn("Mem0 memory storage failed:", error);
-                                }
+                        cleanupListeners();
+                        if (accumulatedContent.trim()) {
+                            logger.info(`[NeuroLink.stream] stream() - COMPLETE SUCCESS`, {
+                                provider: providerName,
+                                model: enhancedOptions.model,
+                                responseTimeMs: Date.now() - startTime,
+                                contentLength: accumulatedContent.length,
+                                fallback: metadata.fallbackAttempted,
                             });
                         }
+                        await self.storeStreamConversationMemory({
+                            enhancedOptions,
+                            providerName,
+                            originalPrompt,
+                            accumulatedContent,
+                            startTime,
+                            eventSequence,
+                        });
                     }
                 })();
                 const streamResult = await this.processStreamResult(processedStream, enhancedOptions, factoryResult);
@@ -2725,6 +2772,294 @@ Current user's request: ${currentInput}`;
             }
         });
     }
+    /**
+     * Prepare stream options: initialize memory, MCP, Mem0 retrieval, orchestration,
+     * Ollama tool auto-disable, factory processing, and tool detection.
+     */
+    async prepareStreamOptions(options, streamId, startTime, hrTimeStart) {
+        // Initialize conversation memory if needed (for lazy loading)
+        await this.initializeConversationMemoryForGeneration(streamId, startTime, hrTimeStart);
+        // Initialize MCP
+        await this.initializeMCP();
+        if (this.conversationMemoryConfig?.conversationMemory?.mem0Enabled &&
+            options.context?.userId) {
+            try {
+                const mem0 = await this.ensureMem0Ready();
+                if (!mem0) {
+                    // Continue without memories if mem0 is not available
+                    logger.debug("Mem0 not available, continuing without memory retrieval");
+                }
+                else {
+                    const memories = await mem0.search(options.input.text, {
+                        user_id: options.context.userId,
+                        limit: 5,
+                    });
+                    if (memories && memories.length > 0) {
+                        // Enhance the input with memory context
+                        const memoryContext = this.extractMemoryContext(memories);
+                        options.input.text = this.formatMemoryContext(memoryContext, options.input.text);
+                    }
+                }
+            }
+            catch (error) {
+                // Non-blocking: Log error but continue with streaming
+                logger.warn("Mem0 memory retrieval failed:", error);
+            }
+        }
+        // Apply orchestration if enabled and no specific provider/model requested
+        if (this.enableOrchestration && !options.provider && !options.model) {
+            try {
+                const orchestratedOptions = await this.applyStreamOrchestration(options);
+                logger.debug("Stream orchestration applied", {
+                    originalProvider: options.provider || "auto",
+                    orchestratedProvider: orchestratedOptions.provider,
+                    orchestratedModel: orchestratedOptions.model,
+                    prompt: options.input.text?.substring(0, 100),
+                });
+                // Use orchestrated options
+                Object.assign(options, orchestratedOptions);
+            }
+            catch (error) {
+                logger.warn("Stream orchestration failed, continuing with original options", {
+                    error: error instanceof Error ? error.message : String(error),
+                    originalProvider: options.provider || "auto",
+                });
+                // Continue with original options if orchestration fails
+            }
+        }
+        // Auto-disable tools for Ollama models that don't support them
+        await this.autoDisableOllamaStreamTools(options);
+        const factoryResult = processStreamingFactoryOptions(options);
+        const enhancedOptions = createCleanStreamOptions(options);
+        if (options.input?.text) {
+            const { toolResults: _toolResults, enhancedPrompt } = await this.detectAndExecuteTools(options.input.text, undefined);
+            if (enhancedPrompt !== options.input.text) {
+                enhancedOptions.input.text = enhancedPrompt;
+            }
+        }
+        return { enhancedOptions, factoryResult };
+    }
+    /**
+     * Auto-disable tools for Ollama models that don't support them (stream mode).
+     * Prevents overwhelming smaller models with massive tool descriptions in the system message.
+     */
+    async autoDisableOllamaStreamTools(options) {
+        if ((options.provider === "ollama" ||
+            options.provider?.toLowerCase().includes("ollama")) &&
+            !options.disableTools) {
+            const { ModelConfigurationManager } = await import("./core/modelConfiguration.js");
+            const modelConfig = ModelConfigurationManager.getInstance();
+            const ollamaConfig = modelConfig.getProviderConfiguration("ollama");
+            const toolCapableModels = ollamaConfig?.modelBehavior?.toolCapableModels || [];
+            // Only disable tools if we have explicit evidence the model doesn't support them
+            // If toolCapableModels is empty or model is not specified, don't make assumptions
+            const modelName = options.model;
+            if (toolCapableModels.length > 0 && modelName) {
+                const modelSupportsTools = toolCapableModels.some((capableModel) => modelName.toLowerCase().includes(capableModel.toLowerCase()));
+                if (!modelSupportsTools) {
+                    options.disableTools = true;
+                    logger.debug("Auto-disabled tools for Ollama model that doesn't support them (stream)", {
+                        model: options.model,
+                        toolCapableModels: toolCapableModels.slice(0, 3), // Show first 3 for brevity
+                    });
+                }
+            }
+        }
+    }
+    /**
+     * Set up event listeners for stream event capture (tool calls, HITL, UI components).
+     * Returns the shared event sequence array and a cleanup function to remove all listeners.
+     */
+    setupStreamEventListeners() {
+        const eventSequence = [];
+        let eventSeqCounter = 0;
+        const captureEvent = (type, data) => {
+            eventSequence.push({
+                type,
+                seq: eventSeqCounter++,
+                timestamp: Date.now(),
+                ...(data && typeof data === "object" ? data : { data }),
+            });
+        };
+        const onResponseChunk = (...args) => {
+            const chunk = args[0];
+            captureEvent("response:chunk", { content: chunk });
+        };
+        const onToolStart = (...args) => {
+            const data = args[0];
+            captureEvent("tool:start", data);
+        };
+        const onToolEnd = (...args) => {
+            const data = args[0];
+            captureEvent("tool:end", data);
+            if (data.result && data.result.uiComponent === true) {
+                captureEvent("ui-component", {
+                    toolName: data.toolName,
+                    componentData: data.result,
+                    timestamp: Date.now(),
+                });
+            }
+        };
+        const onUIComponent = (...args) => {
+            captureEvent("ui-component", args[0]);
+        };
+        const onHITLRequest = (...args) => {
+            captureEvent("hitl:confirmation-request", args[0]);
+        };
+        const onHITLResponse = (...args) => {
+            captureEvent("hitl:confirmation-response", args[0]);
+        };
+        this.emitter.on("response:chunk", onResponseChunk);
+        this.emitter.on("tool:start", onToolStart);
+        this.emitter.on("tool:end", onToolEnd);
+        this.emitter.on("ui-component", onUIComponent);
+        this.emitter.on("hitl:confirmation-request", onHITLRequest);
+        this.emitter.on("hitl:confirmation-response", onHITLResponse);
+        const cleanup = () => {
+            this.emitter.off("response:chunk", onResponseChunk);
+            this.emitter.off("tool:start", onToolStart);
+            this.emitter.off("tool:end", onToolEnd);
+            this.emitter.off("ui-component", onUIComponent);
+            this.emitter.off("hitl:confirmation-request", onHITLRequest);
+            this.emitter.off("hitl:confirmation-response", onHITLResponse);
+        };
+        return { eventSequence, cleanup };
+    }
+    /**
+     * Handle fallback when the primary stream returns 0 chunks.
+     * Yields chunks from a fallback provider and updates metadata accordingly.
+     */
+    async *handleStreamFallback(metadata, originalPrompt, enhancedOptions, providerName, _accumulatedContent, appendContent) {
+        metadata.fallbackAttempted = true;
+        const errorMsg = "Stream completed with 0 chunks (possible guardrails block)";
+        metadata.error = errorMsg;
+        const fallbackRoute = ModelRouter.getFallbackRoute(originalPrompt || enhancedOptions.input.text || "", {
+            provider: providerName,
+            model: enhancedOptions.model || "gpt-4o",
+            reasoning: "primary failed",
+            confidence: 0.5,
+        }, { fallbackStrategy: "auto" });
+        logger.warn("Retrying with fallback provider", {
+            originalProvider: providerName,
+            fallbackProvider: fallbackRoute.provider,
+            reason: errorMsg,
+        });
+        try {
+            const fallbackProvider = await AIProviderFactory.createProvider(fallbackRoute.provider, fallbackRoute.model);
+            // Ensure fallback provider can execute tools
+            fallbackProvider.setupToolExecutor({
+                customTools: this.getCustomTools(),
+                executeTool: this.executeTool.bind(this),
+            }, "NeuroLink.fallbackStream");
+            // Get conversation messages for context (same as primary stream)
+            const conversationMessages = await getConversationMessages(this.conversationMemory, {
+                prompt: enhancedOptions.input.text,
+                context: enhancedOptions.context,
+            });
+            const fallbackResult = await fallbackProvider.stream({
+                ...enhancedOptions,
+                model: fallbackRoute.model,
+                conversationMessages,
+            });
+            let fallbackChunkCount = 0;
+            for await (const fallbackChunk of fallbackResult.stream) {
+                fallbackChunkCount++;
+                if (fallbackChunk &&
+                    "content" in fallbackChunk &&
+                    typeof fallbackChunk.content === "string") {
+                    appendContent(fallbackChunk.content);
+                    this.emitter.emit("response:chunk", fallbackChunk.content);
+                }
+                yield fallbackChunk;
+            }
+            if (fallbackChunkCount === 0) {
+                throw new Error(`Fallback provider ${fallbackRoute.provider} also returned 0 chunks`);
+            }
+            // Fallback succeeded - likely guardrails blocked primary
+            metadata.guardrailsBlocked = true;
+        }
+        catch (fallbackError) {
+            const fallbackErrorMsg = fallbackError instanceof Error
+                ? fallbackError.message
+                : String(fallbackError);
+            metadata.error = `${errorMsg}; Fallback failed: ${fallbackErrorMsg}`;
+            logger.error("Fallback provider failed", {
+                fallbackProvider: fallbackRoute.provider,
+                error: fallbackErrorMsg,
+            });
+            throw fallbackError;
+        }
+    }
+    /**
+     * Store conversation memory after stream consumption is complete (called from finally block).
+     * Handles both conversation memory storage and Mem0 background storage.
+     */
+    async storeStreamConversationMemory(params) {
+        const { enhancedOptions, providerName, originalPrompt, accumulatedContent, startTime, eventSequence, } = params;
+        // Guard: skip storing if no meaningful content was produced (no text AND no tool activity)
+        const hasToolEvents = eventSequence.some((e) => e.type === "tool:start" || e.type === "tool:end");
+        if (!accumulatedContent.trim() && !hasToolEvents) {
+            logger.warn("[NeuroLink.stream] Skipping conversation turn storage — no text content or tool activity", {
+                sessionId: enhancedOptions.context
+                    ?.sessionId,
+            });
+            return;
+        }
+        // Store memory after stream consumption is complete
+        if (this.conversationMemory && enhancedOptions.context?.sessionId) {
+            const sessionId = enhancedOptions.context
+                ?.sessionId;
+            const userId = enhancedOptions.context
+                ?.userId;
+            let providerDetails;
+            if (enhancedOptions.model) {
+                providerDetails = {
+                    provider: providerName,
+                    model: enhancedOptions.model,
+                };
+            }
+            try {
+                await this.conversationMemory.storeConversationTurn({
+                    sessionId,
+                    userId,
+                    userMessage: originalPrompt ?? "",
+                    aiResponse: accumulatedContent,
+                    startTimeStamp: new Date(startTime),
+                    providerDetails,
+                    enableSummarization: enhancedOptions.enableSummarization,
+                    events: eventSequence.length > 0 ? eventSequence : undefined,
+                });
+                logger.debug("[NeuroLink.stream] Stored conversation turn with events", {
+                    sessionId,
+                    eventCount: eventSequence.length,
+                    eventTypes: [...new Set(eventSequence.map((e) => e.type))],
+                });
+            }
+            catch (error) {
+                logger.warn("Failed to store stream conversation turn", {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        if (this.conversationMemoryConfig?.conversationMemory?.mem0Enabled &&
+            enhancedOptions.context?.userId &&
+            accumulatedContent.trim()) {
+            setImmediate(async () => {
+                try {
+                    const mem0 = await this.ensureMem0Ready();
+                    if (mem0) {
+                        await this.storeMem0ConversationTurn(mem0, originalPrompt ?? "", accumulatedContent.trim(), enhancedOptions.context?.userId, {
+                            timestamp: new Date().toISOString(),
+                            type: "conversation_turn_stream",
+                        });
+                    }
+                }
+                catch (error) {
+                    logger.warn("Mem0 memory storage failed:", error);
+                }
+            });
+        }
+    }
     /**
      * Validate stream input with comprehensive error reporting
      */
@@ -2772,14 +3107,37 @@ Current user's request: ${currentInput}`;
         }, "NeuroLink.createMCPStream");
         // 🔧 FIX: Get available tools and create tool-aware system prompt
         // Use SAME pattern as tryMCPGeneration (generate mode)
-        const availableTools = await this.getAllAvailableTools();
-        const enhancedSystemPrompt = this.createToolAwareSystemPrompt(options.systemPrompt, availableTools);
+        let availableTools = await this.getAllAvailableTools();
+        // Apply per-call tool filtering for system prompt tool descriptions
+        availableTools = this.applyToolInfoFiltering(availableTools, options);
+        // Skip tool prompt injection if skipToolPromptInjection is true
+        const enhancedSystemPrompt = options.skipToolPromptInjection
+            ? options.systemPrompt || ""
+            : this.createToolAwareSystemPrompt(options.systemPrompt, availableTools);
         // Get conversation messages for context
-        const conversationMessages = await getConversationMessages(this.conversationMemory, {
+        let conversationMessages = await getConversationMessages(this.conversationMemory, {
             ...options,
             prompt: options.input.text,
             context: options.context,
         });
+        // Pre-generation budget check for streaming
+        const streamBudget = checkContextBudget({
+            provider: providerName,
+            model: options.model,
+            maxTokens: options.maxTokens,
+            systemPrompt: enhancedSystemPrompt,
+            conversationMessages: conversationMessages,
+            currentPrompt: options.input.text,
+            toolDefinitions: availableTools,
+        });
+        if (streamBudget.shouldCompact && this.conversationMemory) {
+            const compactor = new ContextCompactor({ provider: providerName });
+            const compactionResult = await compactor.compact(conversationMessages, streamBudget.availableInputTokens);
+            if (compactionResult.compacted) {
+                const repairedResult = repairToolPairs(compactionResult.messages);
+                conversationMessages = repairedResult.messages;
+            }
+        }
         // 🔧 FIX: Pass enhanced system prompt to real streaming
         // Tools will be accessed through the streamText call in executeStream
         const streamResult = await provider.stream({
@@ -2876,8 +3234,19 @@ Current user's request: ${currentInput}`;
                 }
             }
             finally {
+                if (fallbackAccumulatedContent.trim()) {
+                    logger.info(`[NeuroLink.handleStreamError] stream() - COMPLETE SUCCESS (fallback)`, {
+                        provider: providerName,
+                        model: options.model,
+                        responseTimeMs: Date.now() - startTime,
+                        contentLength: fallbackAccumulatedContent.length,
+                    });
+                }
                 // Store memory after fallback stream consumption is complete
-                if (self.conversationMemory && enhancedOptions?.context?.sessionId) {
+                // Guard: skip storing if fallback accumulated content is empty
+                if (self.conversationMemory &&
+                    enhancedOptions?.context?.sessionId &&
+                    fallbackAccumulatedContent.trim()) {
                     const sessionId = enhancedOptions?.context?.sessionId;
                     const userId = enhancedOptions?.context
                         ?.userId;
@@ -3436,6 +3805,36 @@ Current user's request: ${currentInput}`;
                 },
             });
         }
+        // Inject file reference tools so they reach the Vercel AI SDK's tools parameter.
+        // These tools are bound to this.fileRegistry and allow the LLM to read/search
+        // files on demand instead of having all file content dumped into the prompt.
+        //
+        // createFileTools() returns Vercel AI SDK tool() objects with Zod `parameters`.
+        // We pass `parameters` as `inputSchema` so processCustomTools() in ToolsManager
+        // recognises it as a Zod schema (priority 2) and serialises it correctly for
+        // every provider — including Vertex AI which rejects bare `{}` schemas.
+        // Cache to avoid redundant allocations per generate/stream call (FRT-6).
+        if (!this.cachedFileTools) {
+            this.cachedFileTools = createFileTools(this.fileRegistry);
+        }
+        const fileTools = this.cachedFileTools;
+        for (const [toolName, toolDef] of Object.entries(fileTools)) {
+            if (!toolMap.has(toolName)) {
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const toolParams = toolDef.parameters;
+                toolMap.set(toolName, {
+                    name: toolName,
+                    description: toolDef.description || `File tool: ${toolName}`,
+                    inputSchema: toolParams ?? { type: "object", properties: {} },
+                    execute: async (params) => {
+                        return await toolDef.execute(params, {
+                            toolCallId: `file-tool-${Date.now()}`,
+                            messages: [],
+                        });
+                    },
+                });
+            }
+        }
         return toolMap;
     }
     /**
@@ -5012,6 +5411,69 @@ Current user's request: ${currentInput}`;
     getToolRegistry() {
         return this.toolRegistry;
     }
+    /**
+     * Manually trigger context compaction for a session.
+     * Runs the full 4-stage compaction pipeline.
+     */
+    async compactSession(sessionId, config) {
+        if (!this.conversationMemory) {
+            return null;
+        }
+        const messages = await this.conversationMemory.buildContextMessages(sessionId);
+        if (!messages || messages.length === 0) {
+            return null;
+        }
+        const compactor = new ContextCompactor(config);
+        const targetTokens = Math.floor(messages.length * 100); // Rough target
+        const result = await compactor.compact(messages, targetTokens, this.conversationMemoryConfig?.conversationMemory);
+        if (result.compacted) {
+            repairToolPairs(result.messages);
+        }
+        return result;
+    }
+    /**
+     * Get context usage statistics for a session.
+     * Returns token counts, usage ratio, and breakdown by category.
+     */
+    async getContextStats(sessionId, provider, model) {
+        if (!this.conversationMemory) {
+            return null;
+        }
+        const messages = await this.conversationMemory.buildContextMessages(sessionId);
+        if (!messages || messages.length === 0) {
+            return null;
+        }
+        const budgetResult = checkContextBudget({
+            provider: provider || "openai",
+            model,
+            conversationMessages: messages,
+        });
+        return {
+            estimatedInputTokens: budgetResult.estimatedInputTokens,
+            availableInputTokens: budgetResult.availableInputTokens,
+            usageRatio: budgetResult.usageRatio,
+            shouldCompact: budgetResult.shouldCompact,
+            messageCount: messages.length,
+        };
+    }
+    /**
+     * Check if a session needs compaction.
+     */
+    needsCompaction(sessionId, provider, model) {
+        if (!this.conversationMemory) {
+            return false;
+        }
+        const session = this.conversationMemory.getSession?.(sessionId);
+        if (!session) {
+            return false;
+        }
+        const budgetResult = checkContextBudget({
+            provider: provider || "openai",
+            model,
+            conversationMessages: session.messages,
+        });
+        return budgetResult.shouldCompact;
+    }
     /**
      * Get the external server manager instance
      * Used internally by server adapters for external MCP server management