@juspay/neurolink 9.5.2 → 9.6.0

package/dist/files/streamingReader.js

@@ -0,0 +1,321 @@
+/**
+ * Streaming File Reader
+ *
+ * Reads files with token-budget awareness using Node.js streams.
+ * Memory usage is proportional to what's returned, not what's on disk.
+ *
+ * Inspired by Kilocode's token-budget streaming reader pattern:
+ * uses createReadStream + readline, counts tokens per line,
+ * stops when budget is exhausted.
+ *
+ * @example
+ * ```typescript
+ * // Read lines 100-200 of a large file with a 5000 token budget
+ * const result = await StreamingReader.readLines('/path/to/file.csv', {
+ *   startLine: 100,
+ *   endLine: 200,
+ *   tokenBudget: 5000,
+ *   provider: 'anthropic',
+ * });
+ * // result.content = "line 100 content\nline 101 content\n..."
+ * // result.truncated = true if budget was hit before line 200
+ * ```
+ */
+import { createReadStream } from "node:fs";
+import { stat } from "node:fs/promises";
+import { createInterface } from "node:readline";
+import { CHARS_PER_TOKEN, estimateTokens } from "../utils/tokenEstimation.js";
+/** Default token budget if none specified */
+const DEFAULT_TOKEN_BUDGET = 50_000;
+/** Maximum context lines before/after a search match */
+const DEFAULT_SEARCH_CONTEXT = 3;
+/** Maximum search matches to return */
+const DEFAULT_MAX_MATCHES = 50;
+/**
+ * Streaming file reader with token-budget-aware line reading.
+ *
+ * All methods use Node.js streams to avoid loading entire files into memory.
+ * A 2GB CSV file: only the requested lines/matches are read from disk.
+ */
+export class StreamingReader {
+    /**
+     * Read lines from a file with a token budget.
+     *
+     * Uses createReadStream + readline for memory-efficient reading.
+     * Stops reading when the token budget is exhausted or endLine is reached.
+     *
+     * @param filePath - Absolute path to the file
+     * @param options - Reading options (startLine, endLine, tokenBudget, provider)
+     * @returns FileReadResult with content, line info, and truncation status
+     */
+    static async readLines(filePath, options = {}) {
+        const { tokenBudget = DEFAULT_TOKEN_BUDGET, startLine = 1, endLine, encoding = "utf-8", provider, } = options;
+        // Validate file exists
+        await stat(filePath);
+        const lines = [];
+        let currentLine = 0;
+        let totalLines = 0;
+        let usedTokens = 0;
+        let truncated = false;
+        const actualStartLine = startLine;
+        let actualEndLine = startLine;
+        const readStream = createReadStream(filePath, { encoding });
+        const rl = createInterface({
+            input: readStream,
+            crlfDelay: Infinity,
+        });
+        try {
+            for await (const line of rl) {
+                currentLine++;
+                totalLines = currentLine;
+                // Skip lines before startLine
+                if (currentLine < startLine) {
+                    continue;
+                }
+                // Stop at endLine if specified
+                if (endLine !== undefined && currentLine > endLine) {
+                    break;
+                }
+                // Estimate tokens for this line (include newline char)
+                const lineTokens = Math.ceil((line.length + 1) / CHARS_PER_TOKEN);
+                // Check if adding this line would exceed budget
+                if (usedTokens + lineTokens > tokenBudget && lines.length > 0) {
+                    truncated = true;
+                    break;
+                }
+                lines.push(line);
+                usedTokens += lineTokens;
+                actualEndLine = currentLine;
+                // Apply provider multiplier periodically (every 100 lines) for efficiency
+                if (lines.length % 100 === 0) {
+                    const adjustedTokens = estimateTokens(lines.join("\n"), provider);
+                    if (adjustedTokens > tokenBudget) {
+                        truncated = true;
+                        break;
+                    }
+                }
+            }
+        }
+        finally {
+            rl.close();
+            readStream.destroy();
+        }
+        // If we didn't reach EOF, count remaining lines
+        if (truncated || (endLine !== undefined && currentLine <= (endLine || 0))) {
+            // We need total line count — do a quick count of remaining lines
+            if (!truncated && !endLine) {
+                // Already counted all lines
+            }
+            else {
+                totalLines = await StreamingReader.countLines(filePath, encoding);
+            }
+        }
+        const content = lines.join("\n");
+        const estimatedTokensFinal = estimateTokens(content, provider);
+        return {
+            content,
+            startLine: actualStartLine,
+            endLine: actualEndLine,
+            totalLines,
+            truncated,
+            estimatedTokens: estimatedTokensFinal,
+        };
+    }
+    /**
+     * Count total lines in a file without reading entire content into memory.
+     *
+     * @param filePath - Absolute path to the file
+     * @param encoding - File encoding (default: utf-8)
+     * @returns Total line count
+     */
+    static async countLines(filePath, encoding = "utf-8") {
+        let count = 0;
+        const readStream = createReadStream(filePath, { encoding });
+        const rl = createInterface({
+            input: readStream,
+            crlfDelay: Infinity,
+        });
+        try {
+            for await (const _line of rl) {
+                count++;
+            }
+        }
+        finally {
+            rl.close();
+            readStream.destroy();
+        }
+        return count;
+    }
+    /**
+     * Search for a pattern within a file, returning matching lines with context.
+     *
+     * Uses streaming to avoid loading the entire file into memory.
+     * Returns up to maxMatches results, each with contextBefore/contextAfter lines.
+     *
+     * @param filePath - Absolute path to the file
+     * @param pattern - Regex pattern or string to search for
+     * @param options - Search options
+     * @returns FileSearchResult with matches, total count, and truncation status
+     */
+    static async searchInFile(filePath, pattern, options = {}) {
+        const { maxMatches = DEFAULT_MAX_MATCHES, contextLines = DEFAULT_SEARCH_CONTEXT, encoding = "utf-8", } = options;
+        // Validate file exists
+        await stat(filePath);
+        const regex = pattern instanceof RegExp
+            ? pattern
+            : new RegExp(StreamingReader.escapeRegex(pattern), "i");
+        const matches = [];
+        let totalMatches = 0;
+        let truncated = false;
+        // Ring buffer for context-before lines
+        const contextBuffer = [];
+        let currentLine = 0;
+        // Track lines already captured as context-after for previous matches
+        // to avoid re-processing
+        let pendingContextAfter = null;
+        const readStream = createReadStream(filePath, { encoding });
+        const rl = createInterface({
+            input: readStream,
+            crlfDelay: Infinity,
+        });
+        try {
+            for await (const line of rl) {
+                currentLine++;
+                // Fill context-after for previous match
+                if (pendingContextAfter && pendingContextAfter.remaining > 0) {
+                    pendingContextAfter.match.contextAfter.push(line);
+                    pendingContextAfter.remaining--;
+                    if (pendingContextAfter.remaining === 0) {
+                        pendingContextAfter = null;
+                    }
+                }
+                // Test for match
+                if (regex.test(line)) {
+                    totalMatches++;
+                    if (matches.length < maxMatches) {
+                        const match = {
+                            lineNumber: currentLine,
+                            line,
+                            contextBefore: [...contextBuffer],
+                            contextAfter: [],
+                        };
+                        matches.push(match);
+                        // Set up context-after collection
+                        if (contextLines > 0) {
+                            pendingContextAfter = {
+                                match,
+                                remaining: contextLines,
+                            };
+                        }
+                    }
+                    else {
+                        truncated = true;
+                        // Continue counting total matches but don't store details
+                    }
+                }
+                // Update context buffer (ring buffer for before-context)
+                contextBuffer.push(line);
+                if (contextBuffer.length > contextLines) {
+                    contextBuffer.shift();
+                }
+            }
+        }
+        finally {
+            rl.close();
+            readStream.destroy();
+        }
+        return {
+            matches,
+            totalMatches,
+            truncated,
+        };
+    }
+    /**
+     * Read a preview of a file (first N characters / lines).
+     *
+     * Optimized for speed: reads only the first chunk of the file.
+     *
+     * @param filePath - Absolute path to the file
+     * @param maxChars - Maximum characters to return (default: 2000)
+     * @param encoding - File encoding (default: utf-8)
+     * @returns Preview string
+     */
+    static async readPreview(filePath, maxChars = 2000, encoding = "utf-8") {
+        // For preview, read a small chunk directly — no need for line-by-line
+        const readStream = createReadStream(filePath, {
+            encoding,
+            start: 0,
+            end: maxChars + 100, // Read a bit extra to avoid cutting mid-line
+        });
+        let content = "";
+        try {
+            for await (const chunk of readStream) {
+                content += chunk;
+                if (content.length >= maxChars) {
+                    break;
+                }
+            }
+        }
+        finally {
+            readStream.destroy();
+        }
+        // Truncate to maxChars, preferring to break at line boundary
+        if (content.length > maxChars) {
+            const lastNewline = content.lastIndexOf("\n", maxChars);
+            if (lastNewline > maxChars * 0.8) {
+                content = content.substring(0, lastNewline);
+            }
+            else {
+                content = content.substring(0, maxChars);
+            }
+        }
+        return content;
+    }
+    /**
+     * Read a file from a Buffer with line range and token budget.
+     *
+     * For files already in memory (e.g., from URL download or Buffer input).
+     * Does NOT use streams — the buffer is already in memory.
+     *
+     * @param buffer - File content as Buffer
+     * @param options - Reading options
+     * @returns FileReadResult
+     */
+    static readFromBuffer(buffer, options = {}) {
+        const { tokenBudget = DEFAULT_TOKEN_BUDGET, startLine = 1, endLine, encoding = "utf-8", provider, } = options;
+        const text = buffer.toString(encoding);
+        const allLines = text.split("\n");
+        const totalLines = allLines.length;
+        const start = Math.max(0, startLine - 1); // Convert to 0-indexed
+        const end = endLine !== undefined ? Math.min(endLine, totalLines) : totalLines;
+        let usedTokens = 0;
+        let truncated = false;
+        const lines = [];
+        for (let i = start; i < end; i++) {
+            const line = allLines[i];
+            const lineTokens = Math.ceil((line.length + 1) / CHARS_PER_TOKEN);
+            if (usedTokens + lineTokens > tokenBudget && lines.length > 0) {
+                truncated = true;
+                break;
+            }
+            lines.push(line);
+            usedTokens += lineTokens;
+        }
+        const content = lines.join("\n");
+        const estimatedTokensFinal = estimateTokens(content, provider);
+        return {
+            content,
+            startLine,
+            endLine: start + lines.length,
+            totalLines,
+            truncated,
+            estimatedTokens: estimatedTokensFinal,
+        };
+    }
+    /**
+     * Escape a string for use in a regular expression.
+     */
+    static escapeRegex(str) {
+        return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    }
+}

package/dist/files/types.d.ts

@@ -0,0 +1,23 @@
+/**
+ * File Reference Architecture Types
+ *
+ * Types for the lazy on-demand file processing system.
+ * Files are registered as lightweight references with metadata and previews.
+ * Full content is processed on-demand when the LLM requests it via tools.
+ */
+/**
+ * Size tier thresholds in bytes
+ */
+export declare const SIZE_TIER_THRESHOLDS: {
+    /** < 10 KB: inline in prompt */
+    readonly TINY_MAX: number;
+    /** 10 KB – 100 KB: full load with truncation */
+    readonly SMALL_MAX: number;
+    /** 100 KB – 5 MB: outline + on-demand */
+    readonly MEDIUM_MAX: number;
+    /** 5 MB – 100 MB: streaming + chunked summarization */
+    readonly LARGE_MAX: number;
+    /** 100 MB – 2 GB: reference only */
+    readonly HUGE_MAX: number;
+};
+export type { SizeTier, FileReferenceStatus, OutlineSection, FileReference, FileRegistrationOptions, FileReadResult, FileSearchResult, FileSearchMatch, StreamingReaderOptions, FileExtractionParams, FileExtractionResult, FileRegistryOptions, } from "../types/fileReferenceTypes.js";

package/dist/files/types.js

@@ -0,0 +1,23 @@
+/**
+ * File Reference Architecture Types
+ *
+ * Types for the lazy on-demand file processing system.
+ * Files are registered as lightweight references with metadata and previews.
+ * Full content is processed on-demand when the LLM requests it via tools.
+ */
+/**
+ * Size tier thresholds in bytes
+ */
+export const SIZE_TIER_THRESHOLDS = {
+    /** < 10 KB: inline in prompt */
+    TINY_MAX: 10 * 1024,
+    /** 10 KB – 100 KB: full load with truncation */
+    SMALL_MAX: 100 * 1024,
+    /** 100 KB – 5 MB: outline + on-demand */
+    MEDIUM_MAX: 5 * 1024 * 1024,
+    /** 5 MB – 100 MB: streaming + chunked summarization */
+    LARGE_MAX: 100 * 1024 * 1024,
+    /** 100 MB – 2 GB: reference only */
+    HUGE_MAX: 2 * 1024 * 1024 * 1024,
+    // > 2 GB: oversized, rejected
+};

package/dist/image-gen/imageGenTools.d.ts

@@ -25,7 +25,7 @@ import type { ImageGenConfig, ImageGenToolContext, ImageGenToolParams, ImageGenT
 /**
  * Tool definition interface compatible with AI SDK / MCP
  */
-export interface ImageGenToolDefinition {
+export type ImageGenToolDefinition = {
     name: string;
     description: string;
     inputSchema: {
@@ -38,7 +38,7 @@ export interface ImageGenToolDefinition {
         required: string[];
     };
     execute: (params: ImageGenToolParams, context?: ImageGenToolContext) => Promise<ImageGenToolResponse>;
-}
+};
 /**
  * Create an image generation tool for use with AI models
  *

package/dist/image-gen/types.d.ts

@@ -23,7 +23,7 @@ export type StylePreset = "realistic" | "photorealistic" | "artistic" | "cartoon
 /**
  * Options for image generation requests
  */
-export interface ImageGenOptions {
+export type ImageGenOptions = {
     /**
      * Text prompt describing the image to generate
      * Should be detailed and specific for best results
@@ -79,11 +79,11 @@ export interface ImageGenOptions {
      * Higher values = more creative/random
      */
     temperature?: number;
-}
+};
 /**
  * Result of an image generation request
  */
-export interface ImageGenResult {
+export type ImageGenResult = {
     /**
      * Whether generation was successful
      */
@@ -121,11 +121,11 @@ export interface ImageGenResult {
      * Additional metadata from the provider
      */
     metadata?: Record<string, unknown>;
-}
+};
 /**
  * Configuration for the ImageGenService
  */
-export interface ImageGenConfig {
+export type ImageGenConfig = {
     /**
      * Whether image generation is enabled
      */
@@ -162,7 +162,7 @@ export interface ImageGenConfig {
      * Maximum number of reference PDFs allowed
      */
     maxReferencePdfs?: number;
-}
+};
 /**
  * Default configuration for image generation
  */
@@ -170,7 +170,7 @@ export declare const DEFAULT_IMAGE_GEN_CONFIG: ImageGenConfig;
 /**
  * Tool parameters for AI model use
  */
-export interface ImageGenToolParams {
+export type ImageGenToolParams = {
     /**
      * Detailed description of the image to generate
      */
@@ -187,11 +187,11 @@ export interface ImageGenToolParams {
      * Style like "realistic", "artistic", "cartoon" (optional)
      */
     style?: StylePreset | string;
-}
+};
 /**
  * Response from the image generation tool
  */
-export interface ImageGenToolResponse {
+export type ImageGenToolResponse = {
     /**
      * Whether the tool execution was successful
      */
@@ -209,11 +209,11 @@ export interface ImageGenToolResponse {
      * Error message if execution failed
      */
     error?: string;
-}
+};
 /**
  * Context for tool execution (optional)
  */
-export interface ImageGenToolContext {
+export type ImageGenToolContext = {
     /**
      * Reference images to use for generation
      */
@@ -234,4 +234,4 @@ export interface ImageGenToolContext {
      * Additional metadata
      */
     metadata?: Record<string, unknown>;
-}
+};

package/dist/lib/agent/directTools.d.ts

@@ -221,11 +221,11 @@ export declare const directAgentTools: {
     }, "strip", z.ZodTypeAny, {
         content: string;
         path: string;
-        mode: "create" | "overwrite" | "append";
+        mode: "create" | "append" | "overwrite";
     }, {
         content: string;
         path: string;
-        mode?: "create" | "overwrite" | "append" | undefined;
+        mode?: "create" | "append" | "overwrite" | undefined;
     }>, {
         success: boolean;
         error: string;
@@ -236,7 +236,7 @@ export declare const directAgentTools: {
     } | {
         success: boolean;
         path: string;
-        mode: "create" | "overwrite" | "append";
+        mode: "create" | "append" | "overwrite";
         size: number;
         written: number;
         error?: undefined;
@@ -251,7 +251,7 @@ export declare const directAgentTools: {
         execute: (args: {
             content: string;
             path: string;
-            mode: "create" | "overwrite" | "append";
+            mode: "create" | "append" | "overwrite";
         }, options: import("ai").ToolExecutionOptions) => PromiseLike<{
             success: boolean;
             error: string;
@@ -262,7 +262,7 @@ export declare const directAgentTools: {
         } | {
             success: boolean;
             path: string;
-            mode: "create" | "overwrite" | "append";
+            mode: "create" | "append" | "overwrite";
             size: number;
             written: number;
             error?: undefined;
@@ -282,8 +282,8 @@ export declare const directAgentTools: {
         maxRows: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
     }, "strip", z.ZodTypeAny, {
         operation: "count_by_column" | "sum_by_column" | "average_by_column" | "min_max_by_column" | "describe";
-        maxRows: number;
         filePath: string;
+        maxRows: number;
         column: string;
     }, {
         operation: "count_by_column" | "sum_by_column" | "average_by_column" | "min_max_by_column" | "describe";
@@ -309,8 +309,8 @@ export declare const directAgentTools: {
     }> & {
         execute: (args: {
             operation: "count_by_column" | "sum_by_column" | "average_by_column" | "min_max_by_column" | "describe";
-            maxRows: number;
             filePath: string;
+            maxRows: number;
             column: string;
         }, options: import("ai").ToolExecutionOptions) => PromiseLike<{
             success: boolean;

package/dist/lib/config/conversationMemory.d.ts

@@ -5,6 +5,8 @@
 import type { ConversationMemoryConfig } from "../types/conversation.js";
 /**
  * Default maximum number of turns per session
+ * @deprecated Use tokenThreshold-based memory management instead.
+ * This constant is kept for backward compatibility with legacy turn-based memory.
  */
 export declare const DEFAULT_MAX_TURNS_PER_SESSION = 50;
 /**

package/dist/lib/config/conversationMemory.js

@@ -4,6 +4,8 @@
  */
 /**
  * Default maximum number of turns per session
+ * @deprecated Use tokenThreshold-based memory management instead.
+ * This constant is kept for backward compatibility with legacy turn-based memory.
  */
 export const DEFAULT_MAX_TURNS_PER_SESSION = 50;
 /**
@@ -77,11 +79,9 @@ export function getConversationMemoryDefaults() {
             : undefined,
         summarizationProvider: process.env.NEUROLINK_SUMMARIZATION_PROVIDER || "vertex",
         summarizationModel: process.env.NEUROLINK_SUMMARIZATION_MODEL || "gemini-2.5-flash",
-        // Deprecated (for backward compatibility)
-        maxTurnsPerSession: Number(process.env.NEUROLINK_MEMORY_MAX_TURNS_PER_SESSION) ||
-            DEFAULT_MAX_TURNS_PER_SESSION,
-        summarizationThresholdTurns: Number(process.env.NEUROLINK_SUMMARIZATION_THRESHOLD_TURNS) || 20,
-        summarizationTargetTurns: Number(process.env.NEUROLINK_SUMMARIZATION_TARGET_TURNS) || 10,
+        // @deprecated — Turn-based fields are superseded by tokenThreshold-based memory.
+        // Kept for backward compatibility; will be removed in a future major version.
+        maxTurnsPerSession: DEFAULT_MAX_TURNS_PER_SESSION,
     };
 }
 //# sourceMappingURL=conversationMemory.js.map

package/dist/lib/constants/contextWindows.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Context Window Registry
+ *
+ * Accurate per-provider, per-model context window sizes (INPUT token limits).
+ * These are distinct from OUTPUT token limits in tokens.ts.
+ *
+ * Sources:
+ * - Anthropic: https://docs.anthropic.com/en/docs/about-claude/models
+ * - OpenAI: https://platform.openai.com/docs/models
+ * - Google: https://ai.google.dev/gemini-api/docs/models
+ * - Others: Provider documentation as of Feb 2026
+ */
+/** Default context window when provider/model is unknown */
+export declare const DEFAULT_CONTEXT_WINDOW = 128000;
+/** Maximum output reserve when maxTokens not specified */
+export declare const MAX_DEFAULT_OUTPUT_RESERVE = 64000;
+/** Default output reserve ratio (35% of context) */
+export declare const DEFAULT_OUTPUT_RESERVE_RATIO = 0.35;
+/**
+ * Per-provider, per-model context window sizes.
+ * The "_default" key is the fallback for unknown models within a provider.
+ */
+export declare const MODEL_CONTEXT_WINDOWS: Record<string, Record<string, number>>;
+/**
+ * Resolve context window size for a provider/model combination.
+ *
+ * Priority:
+ *  1. Exact model match under provider
+ *  2. Provider's _default
+ *  3. Global DEFAULT_CONTEXT_WINDOW
+ */
+export declare function getContextWindowSize(provider: string, model?: string): number;
+/**
+ * Calculate output token reserve for a given context window.
+ *
+ * @param contextWindow - Total context window size
+ * @param maxTokens - Explicit maxTokens from user config (if set)
+ * @returns Number of tokens reserved for output
+ */
+export declare function getOutputReserve(contextWindow: number, maxTokens?: number): number;
+/**
+ * Calculate available input tokens for a given provider/model.
+ *
+ * available = contextWindow - outputReserve
+ */
+export declare function getAvailableInputTokens(provider: string, model?: string, maxTokens?: number): number;