@juspay/neurolink 9.30.0 → 9.31.0

package/dist/client/rag/chunking/latexChunker.js

@@ -0,0 +1,251 @@
+/**
+ * LaTeX-aware Chunker
+ *
+ * Splits LaTeX documents based on structure (sections, environments, math).
+ * Best for academic papers, scientific documents, and mathematical content.
+ */
+import { randomUUID } from "crypto";
+/**
+ * LaTeX-aware chunker implementation
+ * Splits based on LaTeX structure (sections, environments)
+ */
+export class LaTeXChunker {
+    strategy = "latex";
+    defaultSplitEnvironments = [
+        "section",
+        "subsection",
+        "subsubsection",
+        "chapter",
+        "part",
+    ];
+    mathEnvironments = [
+        "equation",
+        "equation*",
+        "align",
+        "align*",
+        "gather",
+        "gather*",
+        "multline",
+        "multline*",
+        "displaymath",
+    ];
+    async chunk(text, config) {
+        const { maxSize = 1000, overlap = 0, splitEnvironments = this.defaultSplitEnvironments, preserveMath = true, includePreamble = true, trimWhitespace = true, metadata = {}, } = config || {};
+        const documentId = randomUUID();
+        const chunks = [];
+        if (!text || text.length === 0) {
+            return chunks;
+        }
+        // Extract preamble if present
+        const preambleMatch = text.match(/^([\s\S]*?)\\begin\{document\}([\s\S]*?)\\end\{document\}/);
+        let preamble = "";
+        let documentContent = text;
+        if (preambleMatch) {
+            preamble = preambleMatch[1].trim();
+            documentContent = preambleMatch[2];
+            // Add preamble as first chunk if requested
+            if (includePreamble && preamble.length > 0) {
+                chunks.push({
+                    id: randomUUID(),
+                    text: preamble,
+                    metadata: {
+                        documentId,
+                        chunkIndex: 0,
+                        startPosition: 0,
+                        endPosition: preamble.length,
+                        documentType: "latex",
+                        latexEnvironment: "preamble",
+                        custom: metadata,
+                    },
+                });
+            }
+        }
+        // Protect math environments
+        let processedContent = documentContent;
+        const mathBlocks = [];
+        if (preserveMath) {
+            // Protect display math environments
+            for (const env of this.mathEnvironments) {
+                const envPattern = new RegExp(`\\\\begin\\{${env}\\}[\\s\\S]*?\\\\end\\{${env}\\}`, "g");
+                processedContent = processedContent.replace(envPattern, (match) => {
+                    const placeholder = `__MATH_${mathBlocks.length}__`;
+                    mathBlocks.push({ placeholder, content: match });
+                    return placeholder;
+                });
+            }
+            // Protect inline math
+            processedContent = processedContent.replace(/\$\$[\s\S]*?\$\$/g, (match) => {
+                const placeholder = `__MATH_${mathBlocks.length}__`;
+                mathBlocks.push({ placeholder, content: match });
+                return placeholder;
+            });
+            processedContent = processedContent.replace(/\$[^$]+\$/g, (match) => {
+                const placeholder = `__MATH_${mathBlocks.length}__`;
+                mathBlocks.push({ placeholder, content: match });
+                return placeholder;
+            });
+            // Protect \[ \] math
+            processedContent = processedContent.replace(/\\\[[\s\S]*?\\\]/g, (match) => {
+                const placeholder = `__MATH_${mathBlocks.length}__`;
+                mathBlocks.push({ placeholder, content: match });
+                return placeholder;
+            });
+        }
+        // Split by sectioning commands
+        const sections = this.splitBySections(processedContent, splitEnvironments);
+        let chunkIndex = chunks.length;
+        let currentPosition = includePreamble && preamble.length > 0 ? preamble.length : 0;
+        for (const section of sections) {
+            const { title, content, environment } = section;
+            // Restore math blocks
+            let restoredContent = content;
+            for (const { placeholder, content: mathContent } of mathBlocks) {
+                restoredContent = restoredContent.replace(placeholder, mathContent);
+            }
+            // Split if content is too large
+            const contentChunks = this.splitContent(restoredContent, maxSize, overlap);
+            for (let i = 0; i < contentChunks.length; i++) {
+                let chunkText = contentChunks[i];
+                // Include section command in first chunk
+                if (i === 0 && title && environment) {
+                    chunkText = `\\${environment}{${title}}\n${chunkText}`;
+                }
+                const finalText = trimWhitespace ? chunkText.trim() : chunkText;
+                if (finalText.length > 0) {
+                    chunks.push({
+                        id: randomUUID(),
+                        text: finalText,
+                        metadata: {
+                            documentId,
+                            chunkIndex,
+                            startPosition: currentPosition,
+                            endPosition: currentPosition + chunkText.length,
+                            documentType: "latex",
+                            latexEnvironment: environment ?? undefined,
+                            header: title ?? undefined,
+                            custom: metadata,
+                        },
+                    });
+                    chunkIndex++;
+                }
+                currentPosition += chunkText.length;
+            }
+        }
+        // Update total chunks count
+        chunks.forEach((chunk) => {
+            chunk.metadata.totalChunks = chunks.length;
+        });
+        return chunks;
+    }
+    /**
+     * Split LaTeX by sectioning commands
+     */
+    splitBySections(content, splitEnvironments) {
+        const sections = [];
+        // Build pattern for sectioning commands
+        const envPattern = splitEnvironments.join("|");
+        const sectionPattern = new RegExp(`\\\\(${envPattern})\\*?\\{([^}]*)\\}`, "g");
+        let lastIndex = 0;
+        let lastTitle = null;
+        let lastEnvironment = null;
+        let match;
+        // Reset regex
+        sectionPattern.lastIndex = 0;
+        while ((match = sectionPattern.exec(content)) !== null) {
+            // Content before this section
+            if (match.index > lastIndex) {
+                const sectionContent = content.slice(lastIndex, match.index);
+                if (sectionContent.trim()) {
+                    sections.push({
+                        title: lastTitle,
+                        content: sectionContent.trim(),
+                        environment: lastEnvironment,
+                    });
+                }
+            }
+            lastEnvironment = match[1];
+            lastTitle = match[2];
+            lastIndex = match.index + match[0].length;
+        }
+        // Don't forget content after the last section
+        if (lastIndex < content.length) {
+            const remaining = content.slice(lastIndex);
+            if (remaining.trim()) {
+                sections.push({
+                    title: lastTitle,
+                    content: remaining.trim(),
+                    environment: lastEnvironment,
+                });
+            }
+        }
+        // If no sections found, return entire content
+        if (sections.length === 0 && content.trim()) {
+            sections.push({
+                title: null,
+                content: content.trim(),
+                environment: null,
+            });
+        }
+        return sections;
+    }
+    /**
+     * Split content that exceeds max size
+     */
+    splitContent(content, maxSize, overlap) {
+        const effectiveMaxSize = Math.max(maxSize, 1);
+        const effectiveOverlap = Math.min(Math.max(overlap, 0), effectiveMaxSize - 1);
+        if (content.length <= effectiveMaxSize) {
+            return [content];
+        }
+        const chunks = [];
+        let start = 0;
+        while (start < content.length) {
+            let end = Math.min(start + effectiveMaxSize, content.length);
+            // Try to break at paragraph boundary
+            if (end < content.length) {
+                const searchStart = Math.max(start, end - 200);
+                const searchText = content.slice(searchStart, end);
+                // Look for paragraph break
+                const paragraphBreak = searchText.lastIndexOf("\n\n");
+                if (paragraphBreak > 0) {
+                    end = searchStart + paragraphBreak;
+                }
+                else {
+                    // Look for sentence break
+                    const sentenceBreak = searchText.search(/[.!?]\s+[A-Z\\]/);
+                    if (sentenceBreak > 0) {
+                        end = searchStart + sentenceBreak + 1;
+                    }
+                }
+            }
+            chunks.push(content.slice(start, end));
+            start = Math.max(start + 1, end - effectiveOverlap);
+        }
+        return chunks;
+    }
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        const latexConfig = config;
+        if (latexConfig.maxSize !== undefined && latexConfig.maxSize <= 0) {
+            errors.push("maxSize must be greater than 0");
+        }
+        if (latexConfig.overlap !== undefined && latexConfig.overlap < 0) {
+            errors.push("overlap must be non-negative");
+        }
+        if (latexConfig.overlap !== undefined &&
+            latexConfig.maxSize !== undefined &&
+            latexConfig.overlap >= latexConfig.maxSize) {
+            errors.push("overlap must be less than maxSize");
+        }
+        if (latexConfig.splitEnvironments !== undefined &&
+            latexConfig.splitEnvironments.length === 0) {
+            warnings.push("No split environments specified, using defaults");
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+}

package/dist/client/rag/chunking/markdownChunker.js

@@ -0,0 +1,373 @@
+/**
+ * Markdown-aware Chunker
+ *
+ * Splits markdown documents based on header structure while preserving formatting.
+ * Best for documentation, README files, and structured markdown content.
+ */
+import { randomUUID } from "crypto";
+/**
+ * Markdown-aware chunker implementation
+ * Splits based on markdown structure (headers, code blocks, etc.)
+ */
+export class MarkdownChunker {
+    strategy = "markdown";
+    async chunk(text, config) {
+        const { maxSize = 1000, overlap = 0, headerLevels = [1, 2, 3], preserveCodeBlocks = true, includeHeader = true, stripFormatting = false, trimWhitespace = true, metadata = {}, } = config || {};
+        const documentId = randomUUID();
+        const chunks = [];
+        if (!text || text.length === 0) {
+            return chunks;
+        }
+        // Build header regex pattern
+        const headerPattern = new RegExp(`^(#{${Math.min(...headerLevels)},${Math.max(...headerLevels)}})\\s+(.+)$`, "gm");
+        // Split by headers while preserving them
+        const sections = this.splitByHeaders(text, headerPattern, includeHeader);
+        let chunkIndex = 0;
+        let currentPosition = 0;
+        for (const section of sections) {
+            const { header, content, level } = section;
+            // Handle code blocks
+            let processedContent = content;
+            const codeBlocks = [];
+            if (preserveCodeBlocks) {
+                processedContent = content.replace(/```[\s\S]*?```|`[^`]+`/g, (match) => {
+                    const placeholder = `__CODE_BLOCK_${codeBlocks.length}__`;
+                    codeBlocks.push({ placeholder, code: match });
+                    return placeholder;
+                });
+            }
+            // Split content if too large
+            const effectiveMaxSize = Math.max(maxSize - (header?.length || 0), 100);
+            const contentChunks = this.splitContent(processedContent, effectiveMaxSize, overlap);
+            for (const contentChunk of contentChunks) {
+                let chunkText = header && includeHeader
+                    ? `${header}\n\n${contentChunk}`
+                    : contentChunk;
+                // Restore code blocks
+                for (const { placeholder, code } of codeBlocks) {
+                    chunkText = chunkText.replace(placeholder, code);
+                }
+                // Strip formatting if requested
+                if (stripFormatting) {
+                    chunkText = this.stripMarkdown(chunkText);
+                }
+                const finalText = trimWhitespace ? chunkText.trim() : chunkText;
+                if (finalText.length > 0) {
+                    chunks.push({
+                        id: randomUUID(),
+                        text: finalText,
+                        metadata: {
+                            documentId,
+                            chunkIndex,
+                            startPosition: currentPosition,
+                            endPosition: currentPosition + chunkText.length,
+                            documentType: "markdown",
+                            headerLevel: level ?? undefined,
+                            header: header?.replace(/^#+\s*/, "") ?? undefined,
+                            custom: metadata,
+                        },
+                    });
+                    chunkIndex++;
+                }
+                currentPosition += chunkText.length;
+            }
+        }
+        // Update total chunks count
+        chunks.forEach((chunk) => {
+            chunk.metadata.totalChunks = chunks.length;
+        });
+        return chunks;
+    }
+    splitByHeaders(text, headerPattern, _includeHeader) {
+        const sections = [];
+        let lastIndex = 0;
+        let match;
+        let currentHeader = null;
+        let currentLevel = null;
+        // Reset regex
+        headerPattern.lastIndex = 0;
+        while ((match = headerPattern.exec(text)) !== null) {
+            // Content before this header
+            if (match.index > lastIndex) {
+                const content = text.slice(lastIndex, match.index);
+                if (content.trim()) {
+                    sections.push({
+                        header: currentHeader,
+                        content: content.trim(),
+                        level: currentLevel,
+                    });
+                }
+            }
+            currentHeader = match[0];
+            currentLevel = match[1].length; // Number of # characters
+            lastIndex = match.index + match[0].length;
+        }
+        // Don't forget content after the last header
+        if (lastIndex < text.length) {
+            const content = text.slice(lastIndex);
+            if (content.trim()) {
+                sections.push({
+                    header: currentHeader,
+                    content: content.trim(),
+                    level: currentLevel,
+                });
+            }
+        }
+        // If no headers found, return entire text as one section
+        if (sections.length === 0 && text.trim()) {
+            sections.push({
+                header: null,
+                content: text.trim(),
+                level: null,
+            });
+        }
+        return sections;
+    }
+    splitContent(content, maxSize, overlap) {
+        const effectiveMaxSize = Math.max(maxSize, 1);
+        const effectiveOverlap = Math.min(Math.max(overlap, 0), effectiveMaxSize - 1);
+        if (content.length <= effectiveMaxSize) {
+            return [content];
+        }
+        // Use table-aware splitting
+        const lines = content.split("\n");
+        const tableRanges = this.detectTableRanges(lines);
+        if (tableRanges.length > 0) {
+            return this.splitContentTableAware(content, lines, tableRanges, effectiveMaxSize, effectiveOverlap);
+        }
+        return this.splitPlainContent(content, effectiveMaxSize, effectiveOverlap);
+    }
+    /**
+     * Detect contiguous table blocks in lines.
+     * Returns array of { start, end } line index ranges (inclusive).
+     */
+    detectTableRanges(lines) {
+        // Simple pipe-prefixed line check (single character class — no backtracking)
+        const TABLE_ROW_RE = /^\|[^\r\n]{1,10000}/;
+        // Per-cell separator regex applied AFTER splitting on "|" — safe because
+        // each cell is short and bounded by pipe delimiters (CodeQL: js/polynomial-redos)
+        const SEPARATOR_CELL_RE = /^[\t ]*:?-+:?[\t ]*$/;
+        const ranges = [];
+        let i = 0;
+        while (i < lines.length) {
+            if (i + 1 < lines.length &&
+                TABLE_ROW_RE.test(lines[i]) &&
+                this.isTableSeparator(lines[i + 1], SEPARATOR_CELL_RE)) {
+                const start = i;
+                i += 2;
+                while (i < lines.length && TABLE_ROW_RE.test(lines[i])) {
+                    i++;
+                }
+                ranges.push({ start, end: i - 1 });
+            }
+            else {
+                i++;
+            }
+        }
+        return ranges;
+    }
+    /** Check if a line is a markdown table separator (e.g. |---|---|). */
+    isTableSeparator(line, cellRe) {
+        const trimmed = line.trimEnd();
+        if (!trimmed.startsWith("|")) {
+            return false;
+        }
+        // Split by "|" → ["", "---", "---", ""] for "|---|---|"
+        const cells = trimmed.split("|");
+        cells.shift(); // remove leading empty element
+        if (cells.length > 0 && cells[cells.length - 1].trim() === "") {
+            cells.pop(); // remove trailing empty element
+        }
+        if (cells.length === 0) {
+            return false;
+        }
+        return cells.every((cell) => cellRe.test(cell));
+    }
+    /**
+     * Split content while preserving markdown tables.
+     */
+    splitContentTableAware(content, lines, tableRanges, maxSize, overlap) {
+        // Build segments: alternating non-table and table blocks
+        const segments = [];
+        let lineIdx = 0;
+        for (const range of tableRanges) {
+            if (lineIdx < range.start) {
+                const text = lines.slice(lineIdx, range.start).join("\n").trim();
+                if (text) {
+                    segments.push({ text, isTable: false });
+                }
+            }
+            const tableText = lines.slice(range.start, range.end + 1).join("\n");
+            segments.push({ text: tableText, isTable: true });
+            lineIdx = range.end + 1;
+        }
+        if (lineIdx < lines.length) {
+            const text = lines.slice(lineIdx).join("\n").trim();
+            if (text) {
+                segments.push({ text, isTable: false });
+            }
+        }
+        const result = [];
+        let current = "";
+        for (const seg of segments) {
+            if (!seg.isTable) {
+                const pieces = this.splitPlainContent(seg.text, maxSize, overlap);
+                for (const piece of pieces) {
+                    if (current.length === 0) {
+                        current = piece;
+                    }
+                    else if (current.length + 1 + piece.length <= maxSize) {
+                        current += "\n" + piece;
+                    }
+                    else {
+                        result.push(current);
+                        current = piece;
+                    }
+                }
+            }
+            else {
+                if (seg.text.length <= maxSize) {
+                    if (current.length === 0) {
+                        current = seg.text;
+                    }
+                    else if (current.length + 2 + seg.text.length <= maxSize) {
+                        current += "\n\n" + seg.text;
+                    }
+                    else {
+                        result.push(current);
+                        current = seg.text;
+                    }
+                }
+                else {
+                    if (current) {
+                        result.push(current);
+                        current = "";
+                    }
+                    const tableChunks = this.splitTableByRows(seg.text, maxSize);
+                    result.push(...tableChunks);
+                }
+            }
+        }
+        if (current) {
+            result.push(current);
+        }
+        return result.length > 0 ? result : [content];
+    }
+    /**
+     * Split a table on row boundaries, repeating header + separator in each chunk.
+     */
+    splitTableByRows(tableText, maxSize) {
+        const rows = tableText.split("\n");
+        if (rows.length < 3) {
+            return [tableText];
+        }
+        const headerRow = rows[0];
+        const separatorRow = rows[1];
+        const headerBlock = headerRow + "\n" + separatorRow;
+        const dataRows = rows.slice(2);
+        if (headerBlock.length > maxSize) {
+            return this.splitPlainContent(tableText, maxSize, 0);
+        }
+        const chunks = [];
+        let currentChunk = headerBlock;
+        for (const row of dataRows) {
+            // Guard: single row exceeds budget — flush and emit as standalone chunk
+            const singleRowChunk = `${headerBlock}\n${row}`;
+            if (singleRowChunk.length > maxSize) {
+                if (currentChunk.length > headerBlock.length) {
+                    chunks.push(currentChunk);
+                }
+                chunks.push(singleRowChunk);
+                currentChunk = headerBlock;
+                continue;
+            }
+            const candidate = currentChunk + "\n" + row;
+            if (candidate.length <= maxSize) {
+                currentChunk = candidate;
+            }
+            else {
+                if (currentChunk.length > headerBlock.length) {
+                    chunks.push(currentChunk);
+                }
+                currentChunk = headerBlock + "\n" + row;
+            }
+        }
+        if (currentChunk.length > headerBlock.length) {
+            chunks.push(currentChunk);
+        }
+        return chunks.length > 0 ? chunks : [tableText];
+    }
+    splitPlainContent(content, maxSize, overlap) {
+        if (content.length <= maxSize) {
+            return [content];
+        }
+        const chunks = [];
+        let start = 0;
+        while (start < content.length) {
+            let end = Math.min(start + maxSize, content.length);
+            // Try to break at a paragraph or sentence boundary
+            if (end < content.length) {
+                const searchStart = Math.max(start, end - 200);
+                const searchText = content.slice(searchStart, end);
+                // Look for paragraph break first
+                const paragraphBreak = searchText.lastIndexOf("\n\n");
+                if (paragraphBreak > 0) {
+                    end = searchStart + paragraphBreak;
+                }
+                else {
+                    // Look for sentence break
+                    const sentenceBreak = searchText.search(/[.!?]\s+[A-Z]/);
+                    if (sentenceBreak > 0) {
+                        end = searchStart + sentenceBreak + 1;
+                    }
+                }
+            }
+            chunks.push(content.slice(start, end));
+            start = Math.max(start + 1, end - overlap);
+        }
+        return chunks;
+    }
+    stripMarkdown(text) {
+        return text
+            .replace(/^#+\s+/gm, "") // Headers
+            .replace(/\*\*(.+?)\*\*/g, "$1") // Bold
+            .replace(/\*(.+?)\*/g, "$1") // Italic
+            .replace(/__(.+?)__/g, "$1") // Bold (underscore)
+            .replace(/_(.+?)_/g, "$1") // Italic (underscore)
+            .replace(/`(.+?)`/g, "$1") // Inline code
+            .replace(/```[\s\S]*?```/g, "") // Code blocks
+            .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") // Links
+            .replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1"); // Images
+    }
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        const mdConfig = config;
+        if (mdConfig.maxSize !== undefined && mdConfig.maxSize <= 0) {
+            errors.push("maxSize must be greater than 0");
+        }
+        if (mdConfig.headerLevels !== undefined) {
+            if (mdConfig.headerLevels.length === 0) {
+                errors.push("headerLevels must not be empty");
+            }
+            for (const level of mdConfig.headerLevels) {
+                if (level < 1 || level > 6) {
+                    errors.push(`Invalid header level: ${level}. Must be between 1 and 6`);
+                }
+            }
+        }
+        if (mdConfig.overlap !== undefined && mdConfig.overlap < 0) {
+            errors.push("overlap must be non-negative");
+        }
+        if (mdConfig.overlap !== undefined &&
+            mdConfig.maxSize !== undefined &&
+            mdConfig.overlap >= mdConfig.maxSize) {
+            errors.push("overlap must be less than maxSize");
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+}