@juspay/neurolink 9.5.2 → 9.6.0

package/dist/lib/files/fileTools.js

@@ -0,0 +1,450 @@
+/**
+ * On-Demand File Access Tools
+ *
+ * Exposes file operations as LLM tools via the Vercel AI SDK `tool()` pattern.
+ * These tools allow the LLM to access file content lazily — only reading
+ * the specific sections, lines, or search results it needs.
+ *
+ * Tools:
+ * - list_attached_files: List all registered files with metadata
+ * - read_file_section: Read specific line range with token budget
+ * - search_in_file: Search for patterns within a file
+ * - get_file_preview: Get the preview/summary of a file
+ * - extract_file_content: Targeted extraction for any file type (video frames,
+ *     PDF pages, spreadsheet ranges, archive entries, PPTX slides)
+ *
+ * @module files/fileTools
+ */
+import { tool } from "ai";
+import { z } from "zod";
+/**
+ * Create file access tools bound to a FileReferenceRegistry instance.
+ *
+ * These tools follow the same pattern as the existing directAgentTools
+ * (getCurrentTime, readFile, etc.) in src/lib/agent/directTools.ts.
+ * They use `tool()` from the Vercel AI SDK with zod parameter schemas.
+ *
+ * @param registry - The FileReferenceRegistry instance to bind to
+ * @returns Record of tool name to tool definition
+ *
+ * @example
+ * ```typescript
+ * const registry = new FileReferenceRegistry();
+ * const tools = createFileTools(registry);
+ * // tools.list_attached_files, tools.read_file_section, etc.
+ * ```
+ */
+export function createFileTools(registry) {
+    return {
+        list_attached_files: createListAttachedFilesTool(registry),
+        read_file_section: createReadFileSectionTool(registry),
+        search_in_file: createSearchInFileTool(registry),
+        get_file_preview: createGetFilePreviewTool(registry),
+        extract_file_content: createExtractFileContentTool(registry),
+    };
+}
+// ---------------------------------------------------------------------------
+// Individual tool factory helpers
+// ---------------------------------------------------------------------------
+/**
+ * List all files that have been attached/registered for this conversation.
+ * Returns a formatted table with filename, type, size, and status.
+ */
+function createListAttachedFilesTool(registry) {
+    return tool({
+        description: "List all files attached to this conversation with their metadata. " +
+            "Shows filename, type, size, estimated tokens, and processing status. " +
+            "Each file has a unique UUID 'id' field — use that ID (or the filename) " +
+            "as the file_id parameter in read_file_section, search_in_file, and get_file_preview. " +
+            "Always call this first to discover available files.",
+        parameters: z.object({}),
+        execute: async () => {
+            try {
+                const files = registry.list();
+                if (files.length === 0) {
+                    return {
+                        success: true,
+                        message: "No files are attached to this conversation.",
+                        fileCount: 0,
+                    };
+                }
+                const fileList = files.map((f, i) => ({
+                    index: i + 1,
+                    id: f.id,
+                    filename: f.filename,
+                    type: f.detectedType,
+                    size: formatSize(f.sizeBytes),
+                    sizeBytes: f.sizeBytes,
+                    sizeTier: f.sizeTier,
+                    estimatedTokens: f.estimatedTokens,
+                    status: f.status,
+                    hasPreview: !!f.preview,
+                    hasSummary: !!f.summary,
+                    totalLines: f.totalLines ?? null,
+                }));
+                return {
+                    success: true,
+                    fileCount: files.length,
+                    files: fileList,
+                    formatted: registry.listFormatted(),
+                };
+            }
+            catch (error) {
+                return {
+                    success: false,
+                    error: error instanceof Error ? error.message : String(error),
+                };
+            }
+        },
+    });
+}
+/**
+ * Read a specific section (line range) of an attached file.
+ * Supports token budget to prevent reading too much content.
+ */
+function createReadFileSectionTool(registry) {
+    return tool({
+        description: "Read specific lines from an attached file. Specify the file_id " +
+            "(UUID from list_attached_files, or the exact filename), start line, and end line. " +
+            "Uses a token budget to prevent reading too much content at once. " +
+            "For large files, read in sections rather than all at once.",
+        parameters: z.object({
+            file_id: z
+                .string()
+                .describe("The file ID (UUID from list_attached_files) or the exact filename"),
+            start_line: z
+                .number()
+                .int()
+                .min(1)
+                .default(1)
+                .describe("Starting line number (1-indexed, default: 1)"),
+            end_line: z
+                .number()
+                .int()
+                .min(1)
+                .optional()
+                .describe("Ending line number (1-indexed, default: read until token budget is exhausted)"),
+            token_budget: z
+                .number()
+                .int()
+                .min(100)
+                .max(100_000)
+                .default(10_000)
+                .describe("Maximum tokens to return (default: 10000, max: 100000)"),
+        }),
+        execute: async ({ file_id, start_line, end_line, token_budget }) => {
+            try {
+                // Resolve file_id: supports both UUID and filename
+                const ref = registry.getByIdOrFilename(file_id);
+                if (!ref) {
+                    return {
+                        success: false,
+                        error: `File not found: "${file_id}". Use list_attached_files to see available file IDs.`,
+                    };
+                }
+                const resolvedId = ref.id;
+                const result = await registry.readSection(resolvedId, start_line, end_line, token_budget);
+                return {
+                    success: true,
+                    content: result.content,
+                    startLine: result.startLine,
+                    endLine: result.endLine,
+                    totalLines: result.totalLines,
+                    truncated: result.truncated,
+                    estimatedTokens: result.estimatedTokens,
+                    guidance: result.truncated
+                        ? `Content was truncated at token budget (${token_budget}). ` +
+                            `Read from line ${result.endLine + 1} to continue.`
+                        : undefined,
+                };
+            }
+            catch (error) {
+                return {
+                    success: false,
+                    error: error instanceof Error ? error.message : String(error),
+                };
+            }
+        },
+    });
+}
+/**
+ * Search for a text pattern or regex within an attached file.
+ * Returns matching lines with surrounding context.
+ */
+function createSearchInFileTool(registry) {
+    return tool({
+        description: "Search for a text pattern within an attached file. " +
+            "Specify file_id as the UUID from list_attached_files or the exact filename. " +
+            "Supports plain text and regex patterns. Returns matching lines " +
+            "with context lines before and after each match. " +
+            "Use this to find specific content without reading the entire file.",
+        parameters: z.object({
+            file_id: z
+                .string()
+                .describe("The file ID (UUID from list_attached_files) or the exact filename"),
+            pattern: z
+                .string()
+                .describe("Text or regex pattern to search for (case-insensitive by default)"),
+            max_matches: z
+                .number()
+                .int()
+                .min(1)
+                .max(100)
+                .default(20)
+                .describe("Maximum number of matches to return (default: 20)"),
+        }),
+        execute: async ({ file_id, pattern, max_matches }) => {
+            try {
+                // Resolve file_id: supports both UUID and filename
+                const ref = registry.getByIdOrFilename(file_id);
+                if (!ref) {
+                    return {
+                        success: false,
+                        error: `File not found: "${file_id}". Use list_attached_files to see available file IDs.`,
+                    };
+                }
+                const resolvedId = ref.id;
+                const result = await registry.search(resolvedId, pattern, max_matches);
+                const formattedMatches = result.matches.map((m) => ({
+                    lineNumber: m.lineNumber,
+                    line: m.line,
+                    context: m.contextBefore.length > 0 || m.contextAfter.length > 0
+                        ? {
+                            before: m.contextBefore,
+                            after: m.contextAfter,
+                        }
+                        : undefined,
+                }));
+                return {
+                    success: true,
+                    pattern,
+                    totalMatches: result.totalMatches,
+                    matchesReturned: result.matches.length,
+                    truncated: result.truncated,
+                    matches: formattedMatches,
+                };
+            }
+            catch (error) {
+                return {
+                    success: false,
+                    error: error instanceof Error ? error.message : String(error),
+                };
+            }
+        },
+    });
+}
+/**
+ * Get the preview or summary of an attached file.
+ * For files that have been summarized, returns the summary.
+ * Otherwise returns the initial preview.
+ */
+function createGetFilePreviewTool(registry) {
+    return tool({
+        description: "Get the preview or summary of an attached file. " +
+            "Specify file_id as the UUID from list_attached_files or the exact filename. " +
+            "Returns the file's initial preview (first ~2000 chars) or its " +
+            "LLM-generated summary if available. Includes file metadata.",
+        parameters: z.object({
+            file_id: z
+                .string()
+                .describe("The file ID (UUID from list_attached_files) or the exact filename"),
+        }),
+        execute: async ({ file_id }) => {
+            try {
+                // Resolve file_id: supports both UUID and filename
+                const ref = registry.getByIdOrFilename(file_id);
+                if (!ref) {
+                    return {
+                        success: false,
+                        error: `File not found: "${file_id}". Use list_attached_files to see available file IDs.`,
+                    };
+                }
+                // Ensure binary files are processed so preview contains real content
+                // instead of placeholder metadata like "[PDF document: 4.39 MB]"
+                await registry.ensureProcessed(ref.id);
+                return {
+                    success: true,
+                    filename: ref.filename,
+                    type: ref.detectedType,
+                    size: formatSize(ref.sizeBytes),
+                    sizeBytes: ref.sizeBytes,
+                    sizeTier: ref.sizeTier,
+                    mimeType: ref.mimeType,
+                    estimatedTokens: ref.estimatedTokens,
+                    status: ref.status,
+                    totalLines: ref.totalLines ?? null,
+                    preview: ref.preview,
+                    summary: ref.summary ?? null,
+                    hasSummary: !!ref.summary,
+                };
+            }
+            catch (error) {
+                return {
+                    success: false,
+                    error: error instanceof Error ? error.message : String(error),
+                };
+            }
+        },
+    });
+}
+/**
+ * Extract targeted content from any attached file type.
+ * Supports type-specific parameters for deep content access.
+ */
+function createExtractFileContentTool(registry) {
+    return tool({
+        description: "Extract specific content from an attached file. This is the universal extraction tool " +
+            "that handles all file types with type-specific parameters:\n" +
+            "- VIDEO: Use start_time/end_time to get frames from a time range\n" +
+            "- PDF: Use pages (e.g., [1,3,5]) or page_range to get specific pages\n" +
+            "- SPREADSHEET (xlsx): Use sheet, row_range, and columns for targeted data\n" +
+            "- PPTX: Use pages to get specific slides\n" +
+            "- ARCHIVE (zip): Use entry_path to extract a specific file from the archive\n" +
+            "- TEXT/CODE: Use page_range as line range for targeted reading\n\n" +
+            "For video extraction, the result includes images (frames) that will be visible to you. " +
+            "Always call list_attached_files first to discover file IDs.",
+        parameters: z.object({
+            file_id: z
+                .string()
+                .describe("File ID (UUID) or exact filename from list_attached_files"),
+            // Video parameters
+            start_time: z
+                .number()
+                .optional()
+                .describe("Start timestamp in seconds (video only)"),
+            end_time: z
+                .number()
+                .optional()
+                .describe("End timestamp in seconds (video only)"),
+            frame_count: z
+                .number()
+                .int()
+                .min(1)
+                .max(20)
+                .optional()
+                .describe("Number of frames to extract in time range (video only, default: 5, max: 20)"),
+            // PDF / PPTX parameters
+            pages: z
+                .array(z.number().int().min(1))
+                .optional()
+                .describe("Specific page/slide numbers to extract (1-indexed)"),
+            page_range: z
+                .object({
+                start: z.number().int().min(1),
+                end: z.number().int().min(1),
+            })
+                .optional()
+                .describe("Page/slide range to extract (1-indexed, inclusive)"),
+            // Spreadsheet parameters
+            sheet: z
+                .string()
+                .optional()
+                .describe("Sheet name or 0-based index as string e.g. '0', '1' (spreadsheet only, default: first sheet)"),
+            row_range: z
+                .object({
+                start: z.number().int().min(1),
+                end: z.number().int().min(1),
+            })
+                .optional()
+                .describe("Row range (1-indexed, spreadsheet only)"),
+            columns: z
+                .array(z.string())
+                .optional()
+                .describe("Specific column letters to include (e.g., ['A', 'B', 'D'], spreadsheet only)"),
+            // Archive parameters
+            entry_path: z
+                .string()
+                .optional()
+                .describe("File path within archive to extract (archive only)"),
+            // General
+            format: z
+                .enum(["text", "detailed", "summary"])
+                .optional()
+                .describe("Output format hint (default: text)"),
+        }),
+        execute: async (params) => {
+            try {
+                // Convert sheet from string to number if it's a numeric index
+                const extractParams = {
+                    ...params,
+                    sheet: params.sheet !== undefined
+                        ? /^\d+$/.test(params.sheet)
+                            ? parseInt(params.sheet, 10)
+                            : params.sheet
+                        : undefined,
+                };
+                const result = await registry.extractContent(extractParams);
+                if (!result.success) {
+                    return {
+                        success: false,
+                        error: result.error,
+                        text: undefined,
+                        metadata: undefined,
+                        imageCount: 0,
+                        _images: undefined,
+                    };
+                }
+                return {
+                    success: true,
+                    text: result.text,
+                    metadata: result.metadata,
+                    imageCount: result.images?.length ?? 0,
+                    // Store raw buffers for experimental_toToolResultContent to convert
+                    _images: result.images,
+                    error: undefined,
+                };
+            }
+            catch (error) {
+                return {
+                    success: false,
+                    error: error instanceof Error ? error.message : String(error),
+                    text: undefined,
+                    metadata: undefined,
+                    imageCount: 0,
+                    _images: undefined,
+                };
+            }
+        },
+        experimental_toToolResultContent: (result) => {
+            const parts = [];
+            // Add text content
+            if (result.text) {
+                parts.push({ type: "text", text: result.text });
+            }
+            else if (result.error) {
+                parts.push({ type: "text", text: `Error: ${result.error}` });
+            }
+            // Add images as image content parts so the LLM can see them
+            if (result._images && result._images.length > 0) {
+                for (const img of result._images) {
+                    parts.push({
+                        type: "image",
+                        data: img.toString("base64"),
+                        mimeType: "image/jpeg",
+                    });
+                }
+            }
+            // Fallback: always have at least one text part
+            if (parts.length === 0) {
+                parts.push({ type: "text", text: "(No content extracted)" });
+            }
+            return parts;
+        },
+    });
+}
+/**
+ * Format byte size as human-readable string.
+ */
+function formatSize(bytes) {
+    if (bytes < 1024) {
+        return `${bytes} B`;
+    }
+    if (bytes < 1024 * 1024) {
+        return `${(bytes / 1024).toFixed(1)} KB`;
+    }
+    if (bytes < 1024 * 1024 * 1024) {
+        return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+    }
+    return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
+}
+//# sourceMappingURL=fileTools.js.map

package/dist/lib/files/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * File Reference Architecture
+ *
+ * Lazy on-demand file processing system. Files are registered as lightweight
+ * references with metadata and previews. Full content is processed on-demand
+ * when the LLM requests it via tools.
+ *
+ * @module files
+ */
+export { FileReferenceRegistry } from "./fileReferenceRegistry.js";
+export { createFileTools } from "./fileTools.js";
+export { StreamingReader } from "./streamingReader.js";
+export type { FileReadResult, FileReference, FileReferenceStatus, FileRegistrationOptions, FileRegistryOptions, FileSearchMatch, FileSearchResult, OutlineSection, SizeTier, StreamingReaderOptions, } from "./types.js";
+export { SIZE_TIER_THRESHOLDS } from "./types.js";

package/dist/lib/files/index.js

@@ -0,0 +1,14 @@
+/**
+ * File Reference Architecture
+ *
+ * Lazy on-demand file processing system. Files are registered as lightweight
+ * references with metadata and previews. Full content is processed on-demand
+ * when the LLM requests it via tools.
+ *
+ * @module files
+ */
+export { FileReferenceRegistry } from "./fileReferenceRegistry.js";
+export { createFileTools } from "./fileTools.js";
+export { StreamingReader } from "./streamingReader.js";
+export { SIZE_TIER_THRESHOLDS } from "./types.js";
+//# sourceMappingURL=index.js.map

package/dist/lib/files/streamingReader.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Streaming File Reader
+ *
+ * Reads files with token-budget awareness using Node.js streams.
+ * Memory usage is proportional to what's returned, not what's on disk.
+ *
+ * Inspired by Kilocode's token-budget streaming reader pattern:
+ * uses createReadStream + readline, counts tokens per line,
+ * stops when budget is exhausted.
+ *
+ * @example
+ * ```typescript
+ * // Read lines 100-200 of a large file with a 5000 token budget
+ * const result = await StreamingReader.readLines('/path/to/file.csv', {
+ *   startLine: 100,
+ *   endLine: 200,
+ *   tokenBudget: 5000,
+ *   provider: 'anthropic',
+ * });
+ * // result.content = "line 100 content\nline 101 content\n..."
+ * // result.truncated = true if budget was hit before line 200
+ * ```
+ */
+import type { FileReadResult, FileSearchResult, StreamingReaderOptions } from "./types.js";
+/**
+ * Streaming file reader with token-budget-aware line reading.
+ *
+ * All methods use Node.js streams to avoid loading entire files into memory.
+ * A 2GB CSV file: only the requested lines/matches are read from disk.
+ */
+export declare class StreamingReader {
+    /**
+     * Read lines from a file with a token budget.
+     *
+     * Uses createReadStream + readline for memory-efficient reading.
+     * Stops reading when the token budget is exhausted or endLine is reached.
+     *
+     * @param filePath - Absolute path to the file
+     * @param options - Reading options (startLine, endLine, tokenBudget, provider)
+     * @returns FileReadResult with content, line info, and truncation status
+     */
+    static readLines(filePath: string, options?: StreamingReaderOptions): Promise<FileReadResult>;
+    /**
+     * Count total lines in a file without reading entire content into memory.
+     *
+     * @param filePath - Absolute path to the file
+     * @param encoding - File encoding (default: utf-8)
+     * @returns Total line count
+     */
+    static countLines(filePath: string, encoding?: BufferEncoding): Promise<number>;
+    /**
+     * Search for a pattern within a file, returning matching lines with context.
+     *
+     * Uses streaming to avoid loading the entire file into memory.
+     * Returns up to maxMatches results, each with contextBefore/contextAfter lines.
+     *
+     * @param filePath - Absolute path to the file
+     * @param pattern - Regex pattern or string to search for
+     * @param options - Search options
+     * @returns FileSearchResult with matches, total count, and truncation status
+     */
+    static searchInFile(filePath: string, pattern: string | RegExp, options?: {
+        maxMatches?: number;
+        contextLines?: number;
+        encoding?: BufferEncoding;
+    }): Promise<FileSearchResult>;
+    /**
+     * Read a preview of a file (first N characters / lines).
+     *
+     * Optimized for speed: reads only the first chunk of the file.
+     *
+     * @param filePath - Absolute path to the file
+     * @param maxChars - Maximum characters to return (default: 2000)
+     * @param encoding - File encoding (default: utf-8)
+     * @returns Preview string
+     */
+    static readPreview(filePath: string, maxChars?: number, encoding?: BufferEncoding): Promise<string>;
+    /**
+     * Read a file from a Buffer with line range and token budget.
+     *
+     * For files already in memory (e.g., from URL download or Buffer input).
+     * Does NOT use streams — the buffer is already in memory.
+     *
+     * @param buffer - File content as Buffer
+     * @param options - Reading options
+     * @returns FileReadResult
+     */
+    static readFromBuffer(buffer: Buffer, options?: StreamingReaderOptions): FileReadResult;
+    /**
+     * Escape a string for use in a regular expression.
+     */
+    private static escapeRegex;
+}