@juspay/neurolink 9.1.1 → 9.3.0

package/dist/rag/chunking/latexChunker.js

@@ -0,0 +1,251 @@
+/**
+ * LaTeX-aware Chunker
+ *
+ * Splits LaTeX documents based on structure (sections, environments, math).
+ * Best for academic papers, scientific documents, and mathematical content.
+ */
+import { randomUUID } from "crypto";
+/**
+ * LaTeX-aware chunker implementation
+ * Splits based on LaTeX structure (sections, environments)
+ */
+export class LaTeXChunker {
+    strategy = "latex";
+    defaultSplitEnvironments = [
+        "section",
+        "subsection",
+        "subsubsection",
+        "chapter",
+        "part",
+    ];
+    mathEnvironments = [
+        "equation",
+        "equation*",
+        "align",
+        "align*",
+        "gather",
+        "gather*",
+        "multline",
+        "multline*",
+        "displaymath",
+    ];
+    async chunk(text, config) {
+        const { maxSize = 1000, overlap = 0, splitEnvironments = this.defaultSplitEnvironments, preserveMath = true, includePreamble = true, trimWhitespace = true, metadata = {}, } = config || {};
+        const documentId = randomUUID();
+        const chunks = [];
+        if (!text || text.length === 0) {
+            return chunks;
+        }
+        // Extract preamble if present
+        const preambleMatch = text.match(/^([\s\S]*?)\\begin\{document\}([\s\S]*?)\\end\{document\}/);
+        let preamble = "";
+        let documentContent = text;
+        if (preambleMatch) {
+            preamble = preambleMatch[1].trim();
+            documentContent = preambleMatch[2];
+            // Add preamble as first chunk if requested
+            if (includePreamble && preamble.length > 0) {
+                chunks.push({
+                    id: randomUUID(),
+                    text: preamble,
+                    metadata: {
+                        documentId,
+                        chunkIndex: 0,
+                        startPosition: 0,
+                        endPosition: preamble.length,
+                        documentType: "latex",
+                        latexEnvironment: "preamble",
+                        custom: metadata,
+                    },
+                });
+            }
+        }
+        // Protect math environments
+        let processedContent = documentContent;
+        const mathBlocks = [];
+        if (preserveMath) {
+            // Protect display math environments
+            for (const env of this.mathEnvironments) {
+                const envPattern = new RegExp(`\\\\begin\\{${env}\\}[\\s\\S]*?\\\\end\\{${env}\\}`, "g");
+                processedContent = processedContent.replace(envPattern, (match) => {
+                    const placeholder = `__MATH_${mathBlocks.length}__`;
+                    mathBlocks.push({ placeholder, content: match });
+                    return placeholder;
+                });
+            }
+            // Protect inline math
+            processedContent = processedContent.replace(/\$\$[\s\S]*?\$\$/g, (match) => {
+                const placeholder = `__MATH_${mathBlocks.length}__`;
+                mathBlocks.push({ placeholder, content: match });
+                return placeholder;
+            });
+            processedContent = processedContent.replace(/\$[^$]+\$/g, (match) => {
+                const placeholder = `__MATH_${mathBlocks.length}__`;
+                mathBlocks.push({ placeholder, content: match });
+                return placeholder;
+            });
+            // Protect \[ \] math
+            processedContent = processedContent.replace(/\\\[[\s\S]*?\\\]/g, (match) => {
+                const placeholder = `__MATH_${mathBlocks.length}__`;
+                mathBlocks.push({ placeholder, content: match });
+                return placeholder;
+            });
+        }
+        // Split by sectioning commands
+        const sections = this.splitBySections(processedContent, splitEnvironments);
+        let chunkIndex = chunks.length;
+        let currentPosition = includePreamble && preamble.length > 0 ? preamble.length : 0;
+        for (const section of sections) {
+            const { title, content, environment } = section;
+            // Restore math blocks
+            let restoredContent = content;
+            for (const { placeholder, content: mathContent } of mathBlocks) {
+                restoredContent = restoredContent.replace(placeholder, mathContent);
+            }
+            // Split if content is too large
+            const contentChunks = this.splitContent(restoredContent, maxSize, overlap);
+            for (let i = 0; i < contentChunks.length; i++) {
+                let chunkText = contentChunks[i];
+                // Include section command in first chunk
+                if (i === 0 && title && environment) {
+                    chunkText = `\\${environment}{${title}}\n${chunkText}`;
+                }
+                const finalText = trimWhitespace ? chunkText.trim() : chunkText;
+                if (finalText.length > 0) {
+                    chunks.push({
+                        id: randomUUID(),
+                        text: finalText,
+                        metadata: {
+                            documentId,
+                            chunkIndex,
+                            startPosition: currentPosition,
+                            endPosition: currentPosition + chunkText.length,
+                            documentType: "latex",
+                            latexEnvironment: environment ?? undefined,
+                            header: title ?? undefined,
+                            custom: metadata,
+                        },
+                    });
+                    chunkIndex++;
+                }
+                currentPosition += chunkText.length;
+            }
+        }
+        // Update total chunks count
+        chunks.forEach((chunk) => {
+            chunk.metadata.totalChunks = chunks.length;
+        });
+        return chunks;
+    }
+    /**
+     * Split LaTeX by sectioning commands
+     */
+    splitBySections(content, splitEnvironments) {
+        const sections = [];
+        // Build pattern for sectioning commands
+        const envPattern = splitEnvironments.join("|");
+        const sectionPattern = new RegExp(`\\\\(${envPattern})\\*?\\{([^}]*)\\}`, "g");
+        let lastIndex = 0;
+        let lastTitle = null;
+        let lastEnvironment = null;
+        let match;
+        // Reset regex
+        sectionPattern.lastIndex = 0;
+        while ((match = sectionPattern.exec(content)) !== null) {
+            // Content before this section
+            if (match.index > lastIndex) {
+                const sectionContent = content.slice(lastIndex, match.index);
+                if (sectionContent.trim()) {
+                    sections.push({
+                        title: lastTitle,
+                        content: sectionContent.trim(),
+                        environment: lastEnvironment,
+                    });
+                }
+            }
+            lastEnvironment = match[1];
+            lastTitle = match[2];
+            lastIndex = match.index + match[0].length;
+        }
+        // Don't forget content after the last section
+        if (lastIndex < content.length) {
+            const remaining = content.slice(lastIndex);
+            if (remaining.trim()) {
+                sections.push({
+                    title: lastTitle,
+                    content: remaining.trim(),
+                    environment: lastEnvironment,
+                });
+            }
+        }
+        // If no sections found, return entire content
+        if (sections.length === 0 && content.trim()) {
+            sections.push({
+                title: null,
+                content: content.trim(),
+                environment: null,
+            });
+        }
+        return sections;
+    }
+    /**
+     * Split content that exceeds max size
+     */
+    splitContent(content, maxSize, overlap) {
+        const effectiveMaxSize = Math.max(maxSize, 1);
+        const effectiveOverlap = Math.min(Math.max(overlap, 0), effectiveMaxSize - 1);
+        if (content.length <= effectiveMaxSize) {
+            return [content];
+        }
+        const chunks = [];
+        let start = 0;
+        while (start < content.length) {
+            let end = Math.min(start + effectiveMaxSize, content.length);
+            // Try to break at paragraph boundary
+            if (end < content.length) {
+                const searchStart = Math.max(start, end - 200);
+                const searchText = content.slice(searchStart, end);
+                // Look for paragraph break
+                const paragraphBreak = searchText.lastIndexOf("\n\n");
+                if (paragraphBreak > 0) {
+                    end = searchStart + paragraphBreak;
+                }
+                else {
+                    // Look for sentence break
+                    const sentenceBreak = searchText.search(/[.!?]\s+[A-Z\\]/);
+                    if (sentenceBreak > 0) {
+                        end = searchStart + sentenceBreak + 1;
+                    }
+                }
+            }
+            chunks.push(content.slice(start, end));
+            start = Math.max(start + 1, end - effectiveOverlap);
+        }
+        return chunks;
+    }
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        const latexConfig = config;
+        if (latexConfig.maxSize !== undefined && latexConfig.maxSize <= 0) {
+            errors.push("maxSize must be greater than 0");
+        }
+        if (latexConfig.overlap !== undefined && latexConfig.overlap < 0) {
+            errors.push("overlap must be non-negative");
+        }
+        if (latexConfig.overlap !== undefined &&
+            latexConfig.maxSize !== undefined &&
+            latexConfig.overlap >= latexConfig.maxSize) {
+            errors.push("overlap must be less than maxSize");
+        }
+        if (latexConfig.splitEnvironments !== undefined &&
+            latexConfig.splitEnvironments.length === 0) {
+            warnings.push("No split environments specified, using defaults");
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+}

package/dist/rag/chunking/markdownChunker.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Markdown-aware Chunker
+ *
+ * Splits markdown documents based on header structure while preserving formatting.
+ * Best for documentation, README files, and structured markdown content.
+ */
+import type { BaseChunkerConfig, Chunk, Chunker, ChunkerValidationResult, MarkdownChunkerConfig } from "../types.js";
+/**
+ * Markdown-aware chunker implementation
+ * Splits based on markdown structure (headers, code blocks, etc.)
+ */
+export declare class MarkdownChunker implements Chunker {
+    readonly strategy: "markdown";
+    chunk(text: string, config?: MarkdownChunkerConfig): Promise<Chunk[]>;
+    private splitByHeaders;
+    private splitContent;
+    private stripMarkdown;
+    validateConfig(config: BaseChunkerConfig): ChunkerValidationResult;
+}

package/dist/rag/chunking/markdownChunker.js

@@ -0,0 +1,201 @@
+/**
+ * Markdown-aware Chunker
+ *
+ * Splits markdown documents based on header structure while preserving formatting.
+ * Best for documentation, README files, and structured markdown content.
+ */
+import { randomUUID } from "crypto";
+/**
+ * Markdown-aware chunker implementation
+ * Splits based on markdown structure (headers, code blocks, etc.)
+ */
+export class MarkdownChunker {
+    strategy = "markdown";
+    async chunk(text, config) {
+        const { maxSize = 1000, overlap = 0, headerLevels = [1, 2, 3], preserveCodeBlocks = true, includeHeader = true, stripFormatting = false, trimWhitespace = true, metadata = {}, } = config || {};
+        const documentId = randomUUID();
+        const chunks = [];
+        if (!text || text.length === 0) {
+            return chunks;
+        }
+        // Build header regex pattern
+        const headerPattern = new RegExp(`^(#{${Math.min(...headerLevels)},${Math.max(...headerLevels)}})\\s+(.+)$`, "gm");
+        // Split by headers while preserving them
+        const sections = this.splitByHeaders(text, headerPattern, includeHeader);
+        let chunkIndex = 0;
+        let currentPosition = 0;
+        for (const section of sections) {
+            const { header, content, level } = section;
+            // Handle code blocks
+            let processedContent = content;
+            const codeBlocks = [];
+            if (preserveCodeBlocks) {
+                processedContent = content.replace(/```[\s\S]*?```|`[^`]+`/g, (match) => {
+                    const placeholder = `__CODE_BLOCK_${codeBlocks.length}__`;
+                    codeBlocks.push({ placeholder, code: match });
+                    return placeholder;
+                });
+            }
+            // Split content if too large
+            const effectiveMaxSize = Math.max(maxSize - (header?.length || 0), 100);
+            const contentChunks = this.splitContent(processedContent, effectiveMaxSize, overlap);
+            for (const contentChunk of contentChunks) {
+                let chunkText = header && includeHeader
+                    ? `${header}\n\n${contentChunk}`
+                    : contentChunk;
+                // Restore code blocks
+                for (const { placeholder, code } of codeBlocks) {
+                    chunkText = chunkText.replace(placeholder, code);
+                }
+                // Strip formatting if requested
+                if (stripFormatting) {
+                    chunkText = this.stripMarkdown(chunkText);
+                }
+                const finalText = trimWhitespace ? chunkText.trim() : chunkText;
+                if (finalText.length > 0) {
+                    chunks.push({
+                        id: randomUUID(),
+                        text: finalText,
+                        metadata: {
+                            documentId,
+                            chunkIndex,
+                            startPosition: currentPosition,
+                            endPosition: currentPosition + chunkText.length,
+                            documentType: "markdown",
+                            headerLevel: level ?? undefined,
+                            header: header?.replace(/^#+\s*/, "") ?? undefined,
+                            custom: metadata,
+                        },
+                    });
+                    chunkIndex++;
+                }
+                currentPosition += chunkText.length;
+            }
+        }
+        // Update total chunks count
+        chunks.forEach((chunk) => {
+            chunk.metadata.totalChunks = chunks.length;
+        });
+        return chunks;
+    }
+    splitByHeaders(text, headerPattern, _includeHeader) {
+        const sections = [];
+        let lastIndex = 0;
+        let match;
+        let currentHeader = null;
+        let currentLevel = null;
+        // Reset regex
+        headerPattern.lastIndex = 0;
+        while ((match = headerPattern.exec(text)) !== null) {
+            // Content before this header
+            if (match.index > lastIndex) {
+                const content = text.slice(lastIndex, match.index);
+                if (content.trim()) {
+                    sections.push({
+                        header: currentHeader,
+                        content: content.trim(),
+                        level: currentLevel,
+                    });
+                }
+            }
+            currentHeader = match[0];
+            currentLevel = match[1].length; // Number of # characters
+            lastIndex = match.index + match[0].length;
+        }
+        // Don't forget content after the last header
+        if (lastIndex < text.length) {
+            const content = text.slice(lastIndex);
+            if (content.trim()) {
+                sections.push({
+                    header: currentHeader,
+                    content: content.trim(),
+                    level: currentLevel,
+                });
+            }
+        }
+        // If no headers found, return entire text as one section
+        if (sections.length === 0 && text.trim()) {
+            sections.push({
+                header: null,
+                content: text.trim(),
+                level: null,
+            });
+        }
+        return sections;
+    }
+    splitContent(content, maxSize, overlap) {
+        const effectiveMaxSize = Math.max(maxSize, 1);
+        const effectiveOverlap = Math.min(Math.max(overlap, 0), effectiveMaxSize - 1);
+        if (content.length <= effectiveMaxSize) {
+            return [content];
+        }
+        const chunks = [];
+        let start = 0;
+        while (start < content.length) {
+            let end = Math.min(start + effectiveMaxSize, content.length);
+            // Try to break at a paragraph or sentence boundary
+            if (end < content.length) {
+                const searchStart = Math.max(start, end - 200);
+                const searchText = content.slice(searchStart, end);
+                // Look for paragraph break first
+                const paragraphBreak = searchText.lastIndexOf("\n\n");
+                if (paragraphBreak > 0) {
+                    end = searchStart + paragraphBreak;
+                }
+                else {
+                    // Look for sentence break
+                    const sentenceBreak = searchText.search(/[.!?]\s+[A-Z]/);
+                    if (sentenceBreak > 0) {
+                        end = searchStart + sentenceBreak + 1;
+                    }
+                }
+            }
+            chunks.push(content.slice(start, end));
+            start = Math.max(start + 1, end - effectiveOverlap);
+        }
+        return chunks;
+    }
+    stripMarkdown(text) {
+        return text
+            .replace(/^#+\s+/gm, "") // Headers
+            .replace(/\*\*(.+?)\*\*/g, "$1") // Bold
+            .replace(/\*(.+?)\*/g, "$1") // Italic
+            .replace(/__(.+?)__/g, "$1") // Bold (underscore)
+            .replace(/_(.+?)_/g, "$1") // Italic (underscore)
+            .replace(/`(.+?)`/g, "$1") // Inline code
+            .replace(/```[\s\S]*?```/g, "") // Code blocks
+            .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") // Links
+            .replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1"); // Images
+    }
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        const mdConfig = config;
+        if (mdConfig.maxSize !== undefined && mdConfig.maxSize <= 0) {
+            errors.push("maxSize must be greater than 0");
+        }
+        if (mdConfig.headerLevels !== undefined) {
+            if (mdConfig.headerLevels.length === 0) {
+                errors.push("headerLevels must not be empty");
+            }
+            for (const level of mdConfig.headerLevels) {
+                if (level < 1 || level > 6) {
+                    errors.push(`Invalid header level: ${level}. Must be between 1 and 6`);
+                }
+            }
+        }
+        if (mdConfig.overlap !== undefined && mdConfig.overlap < 0) {
+            errors.push("overlap must be non-negative");
+        }
+        if (mdConfig.overlap !== undefined &&
+            mdConfig.maxSize !== undefined &&
+            mdConfig.overlap >= mdConfig.maxSize) {
+            errors.push("overlap must be less than maxSize");
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+}

package/dist/rag/chunking/recursiveChunker.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Recursive Chunker
+ *
+ * Smart text splitting using hierarchical separators.
+ * Tries each separator in order, recursively splitting chunks that are too large.
+ * Best for general-purpose text that has natural boundaries.
+ */
+import type { Chunker, Chunk, ChunkerValidationResult, RecursiveChunkerConfig, BaseChunkerConfig } from "../types.js";
+/**
+ * Recursive chunker implementation
+ * Smart splitting based on content structure using hierarchical separators
+ */
+export declare class RecursiveChunker implements Chunker {
+    readonly strategy: "recursive";
+    private readonly defaultSeparators;
+    chunk(text: string, config?: RecursiveChunkerConfig): Promise<Chunk[]>;
+    private recursiveSplit;
+    validateConfig(config: BaseChunkerConfig): ChunkerValidationResult;
+}

package/dist/rag/chunking/recursiveChunker.js

@@ -0,0 +1,148 @@
+/**
+ * Recursive Chunker
+ *
+ * Smart text splitting using hierarchical separators.
+ * Tries each separator in order, recursively splitting chunks that are too large.
+ * Best for general-purpose text that has natural boundaries.
+ */
+import { randomUUID } from "crypto";
+/**
+ * Recursive chunker implementation
+ * Smart splitting based on content structure using hierarchical separators
+ */
+export class RecursiveChunker {
+    strategy = "recursive";
+    defaultSeparators = ["\n\n", "\n", ". ", " ", ""];
+    async chunk(text, config) {
+        const { maxSize = 1000, overlap = 200, separators = this.defaultSeparators, isSeparatorRegex = false, trimWhitespace = true, metadata = {}, } = config || {};
+        const documentId = randomUUID();
+        const chunks = [];
+        if (!text || text.length === 0) {
+            return chunks;
+        }
+        const splitTexts = this.recursiveSplit(text, separators, maxSize, overlap, isSeparatorRegex);
+        let chunkIndex = 0;
+        let currentPosition = 0;
+        for (const splitText of splitTexts) {
+            const chunkText = trimWhitespace ? splitText.trim() : splitText;
+            if (chunkText.length > 0) {
+                const startPosition = text.indexOf(splitText, currentPosition);
+                chunks.push({
+                    id: randomUUID(),
+                    text: chunkText,
+                    metadata: {
+                        documentId,
+                        chunkIndex,
+                        startPosition: startPosition >= 0 ? startPosition : currentPosition,
+                        endPosition: startPosition >= 0
+                            ? startPosition + splitText.length
+                            : currentPosition + splitText.length,
+                        documentType: "text",
+                        custom: metadata,
+                    },
+                });
+                chunkIndex++;
+                if (startPosition >= 0) {
+                    currentPosition = startPosition + splitText.length - overlap;
+                }
+            }
+        }
+        // Update total chunks count
+        chunks.forEach((chunk) => {
+            chunk.metadata.totalChunks = chunks.length;
+        });
+        return chunks;
+    }
+    recursiveSplit(text, separators, maxSize, overlap, isRegex) {
+        const results = [];
+        if (text.length <= maxSize) {
+            return [text];
+        }
+        // Find the best separator to use
+        let separator = separators[separators.length - 1]; // Default to last (usually "")
+        let newSeparators = separators;
+        for (let i = 0; i < separators.length; i++) {
+            const sep = separators[i];
+            const hasMatch = isRegex
+                ? new RegExp(sep).test(text)
+                : text.includes(sep);
+            if (sep === "" || hasMatch) {
+                separator = sep;
+                newSeparators = separators.slice(i + 1);
+                break;
+            }
+        }
+        // Split the text
+        const splits = isRegex
+            ? text.split(new RegExp(separator))
+            : text.split(separator);
+        // Merge splits into chunks
+        let currentChunk = "";
+        for (const split of splits) {
+            const potentialChunk = currentChunk
+                ? currentChunk + separator + split
+                : split;
+            if (potentialChunk.length <= maxSize) {
+                currentChunk = potentialChunk;
+            }
+            else {
+                // Current chunk is ready
+                if (currentChunk.length > 0) {
+                    results.push(currentChunk);
+                }
+                // Handle split that's still too large
+                if (split.length > maxSize) {
+                    const subSplits = this.recursiveSplit(split, newSeparators, maxSize, overlap, isRegex);
+                    results.push(...subSplits.slice(0, -1));
+                    currentChunk = subSplits[subSplits.length - 1] || "";
+                }
+                else {
+                    // Add overlap from previous chunk
+                    if (results.length > 0 && overlap > 0) {
+                        const lastChunk = results[results.length - 1];
+                        const overlapText = lastChunk.slice(-overlap);
+                        currentChunk = overlapText + separator + split;
+                    }
+                    else {
+                        currentChunk = split;
+                    }
+                }
+            }
+        }
+        // Don't forget the last chunk
+        if (currentChunk.length > 0) {
+            results.push(currentChunk);
+        }
+        return results;
+    }
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        const recConfig = config;
+        if (recConfig.maxSize !== undefined && recConfig.maxSize <= 0) {
+            errors.push("maxSize must be greater than 0");
+        }
+        if (recConfig.overlap !== undefined && recConfig.overlap < 0) {
+            errors.push("overlap must be non-negative");
+        }
+        if (recConfig.separators !== undefined &&
+            recConfig.separators.length === 0) {
+            errors.push("separators array must not be empty");
+        }
+        if (recConfig.isSeparatorRegex && recConfig.separators) {
+            for (const sep of recConfig.separators) {
+                try {
+                    new RegExp(sep);
+                }
+                catch {
+                    errors.push(`Invalid regex separator: ${sep}`);
+                }
+            }
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+}

package/dist/rag/chunking/semanticChunker.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Semantic Chunker
+ *
+ * LLM-powered semantic chunking that groups related content together.
+ * Uses embedding similarity to determine natural breakpoints.
+ * Best for complex documents where meaning should drive segmentation.
+ */
+import type { BaseChunkerConfig, Chunk, Chunker, ChunkerValidationResult, SemanticChunkerConfig } from "../types.js";
+/**
+ * Semantic chunker implementation
+ * Uses embedding similarity to find natural content boundaries
+ */
+export declare class SemanticChunker implements Chunker {
+    readonly strategy: "semantic";
+    chunk(text: string, config?: SemanticChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Split text into initial segments for embedding
+     */
+    private splitIntoSegments;
+    /**
+     * Get embeddings for segments
+     */
+    private getEmbeddings;
+    /**
+     * Find semantic breakpoints using cosine similarity
+     */
+    private findSemanticBreakpoints;
+    /**
+     * Group segments based on breakpoints and size limits
+     */
+    private groupSegments;
+    /**
+     * Calculate cosine similarity between two vectors
+     */
+    private cosineSimilarity;
+    /**
+     * Fallback to simple chunking when embeddings fail
+     */
+    private fallbackChunk;
+    validateConfig(config: BaseChunkerConfig): ChunkerValidationResult;
+}