@juspay/neurolink 9.32.0 → 9.32.1

package/dist/client/rag/chunking/latexChunker.js

@@ -1,251 +0,0 @@
-/**
- * LaTeX-aware Chunker
- *
- * Splits LaTeX documents based on structure (sections, environments, math).
- * Best for academic papers, scientific documents, and mathematical content.
- */
-import { randomUUID } from "crypto";
-/**
- * LaTeX-aware chunker implementation
- * Splits based on LaTeX structure (sections, environments)
- */
-export class LaTeXChunker {
-    strategy = "latex";
-    defaultSplitEnvironments = [
-        "section",
-        "subsection",
-        "subsubsection",
-        "chapter",
-        "part",
-    ];
-    mathEnvironments = [
-        "equation",
-        "equation*",
-        "align",
-        "align*",
-        "gather",
-        "gather*",
-        "multline",
-        "multline*",
-        "displaymath",
-    ];
-    async chunk(text, config) {
-        const { maxSize = 1000, overlap = 0, splitEnvironments = this.defaultSplitEnvironments, preserveMath = true, includePreamble = true, trimWhitespace = true, metadata = {}, } = config || {};
-        const documentId = randomUUID();
-        const chunks = [];
-        if (!text || text.length === 0) {
-            return chunks;
-        }
-        // Extract preamble if present
-        const preambleMatch = text.match(/^([\s\S]*?)\\begin\{document\}([\s\S]*?)\\end\{document\}/);
-        let preamble = "";
-        let documentContent = text;
-        if (preambleMatch) {
-            preamble = preambleMatch[1].trim();
-            documentContent = preambleMatch[2];
-            // Add preamble as first chunk if requested
-            if (includePreamble && preamble.length > 0) {
-                chunks.push({
-                    id: randomUUID(),
-                    text: preamble,
-                    metadata: {
-                        documentId,
-                        chunkIndex: 0,
-                        startPosition: 0,
-                        endPosition: preamble.length,
-                        documentType: "latex",
-                        latexEnvironment: "preamble",
-                        custom: metadata,
-                    },
-                });
-            }
-        }
-        // Protect math environments
-        let processedContent = documentContent;
-        const mathBlocks = [];
-        if (preserveMath) {
-            // Protect display math environments
-            for (const env of this.mathEnvironments) {
-                const envPattern = new RegExp(`\\\\begin\\{${env}\\}[\\s\\S]*?\\\\end\\{${env}\\}`, "g");
-                processedContent = processedContent.replace(envPattern, (match) => {
-                    const placeholder = `__MATH_${mathBlocks.length}__`;
-                    mathBlocks.push({ placeholder, content: match });
-                    return placeholder;
-                });
-            }
-            // Protect inline math
-            processedContent = processedContent.replace(/\$\$[\s\S]*?\$\$/g, (match) => {
-                const placeholder = `__MATH_${mathBlocks.length}__`;
-                mathBlocks.push({ placeholder, content: match });
-                return placeholder;
-            });
-            processedContent = processedContent.replace(/\$[^$]+\$/g, (match) => {
-                const placeholder = `__MATH_${mathBlocks.length}__`;
-                mathBlocks.push({ placeholder, content: match });
-                return placeholder;
-            });
-            // Protect \[ \] math
-            processedContent = processedContent.replace(/\\\[[\s\S]*?\\\]/g, (match) => {
-                const placeholder = `__MATH_${mathBlocks.length}__`;
-                mathBlocks.push({ placeholder, content: match });
-                return placeholder;
-            });
-        }
-        // Split by sectioning commands
-        const sections = this.splitBySections(processedContent, splitEnvironments);
-        let chunkIndex = chunks.length;
-        let currentPosition = includePreamble && preamble.length > 0 ? preamble.length : 0;
-        for (const section of sections) {
-            const { title, content, environment } = section;
-            // Restore math blocks
-            let restoredContent = content;
-            for (const { placeholder, content: mathContent } of mathBlocks) {
-                restoredContent = restoredContent.replace(placeholder, mathContent);
-            }
-            // Split if content is too large
-            const contentChunks = this.splitContent(restoredContent, maxSize, overlap);
-            for (let i = 0; i < contentChunks.length; i++) {
-                let chunkText = contentChunks[i];
-                // Include section command in first chunk
-                if (i === 0 && title && environment) {
-                    chunkText = `\\${environment}{${title}}\n${chunkText}`;
-                }
-                const finalText = trimWhitespace ? chunkText.trim() : chunkText;
-                if (finalText.length > 0) {
-                    chunks.push({
-                        id: randomUUID(),
-                        text: finalText,
-                        metadata: {
-                            documentId,
-                            chunkIndex,
-                            startPosition: currentPosition,
-                            endPosition: currentPosition + chunkText.length,
-                            documentType: "latex",
-                            latexEnvironment: environment ?? undefined,
-                            header: title ?? undefined,
-                            custom: metadata,
-                        },
-                    });
-                    chunkIndex++;
-                }
-                currentPosition += chunkText.length;
-            }
-        }
-        // Update total chunks count
-        chunks.forEach((chunk) => {
-            chunk.metadata.totalChunks = chunks.length;
-        });
-        return chunks;
-    }
-    /**
-     * Split LaTeX by sectioning commands
-     */
-    splitBySections(content, splitEnvironments) {
-        const sections = [];
-        // Build pattern for sectioning commands
-        const envPattern = splitEnvironments.join("|");
-        const sectionPattern = new RegExp(`\\\\(${envPattern})\\*?\\{([^}]*)\\}`, "g");
-        let lastIndex = 0;
-        let lastTitle = null;
-        let lastEnvironment = null;
-        let match;
-        // Reset regex
-        sectionPattern.lastIndex = 0;
-        while ((match = sectionPattern.exec(content)) !== null) {
-            // Content before this section
-            if (match.index > lastIndex) {
-                const sectionContent = content.slice(lastIndex, match.index);
-                if (sectionContent.trim()) {
-                    sections.push({
-                        title: lastTitle,
-                        content: sectionContent.trim(),
-                        environment: lastEnvironment,
-                    });
-                }
-            }
-            lastEnvironment = match[1];
-            lastTitle = match[2];
-            lastIndex = match.index + match[0].length;
-        }
-        // Don't forget content after the last section
-        if (lastIndex < content.length) {
-            const remaining = content.slice(lastIndex);
-            if (remaining.trim()) {
-                sections.push({
-                    title: lastTitle,
-                    content: remaining.trim(),
-                    environment: lastEnvironment,
-                });
-            }
-        }
-        // If no sections found, return entire content
-        if (sections.length === 0 && content.trim()) {
-            sections.push({
-                title: null,
-                content: content.trim(),
-                environment: null,
-            });
-        }
-        return sections;
-    }
-    /**
-     * Split content that exceeds max size
-     */
-    splitContent(content, maxSize, overlap) {
-        const effectiveMaxSize = Math.max(maxSize, 1);
-        const effectiveOverlap = Math.min(Math.max(overlap, 0), effectiveMaxSize - 1);
-        if (content.length <= effectiveMaxSize) {
-            return [content];
-        }
-        const chunks = [];
-        let start = 0;
-        while (start < content.length) {
-            let end = Math.min(start + effectiveMaxSize, content.length);
-            // Try to break at paragraph boundary
-            if (end < content.length) {
-                const searchStart = Math.max(start, end - 200);
-                const searchText = content.slice(searchStart, end);
-                // Look for paragraph break
-                const paragraphBreak = searchText.lastIndexOf("\n\n");
-                if (paragraphBreak > 0) {
-                    end = searchStart + paragraphBreak;
-                }
-                else {
-                    // Look for sentence break
-                    const sentenceBreak = searchText.search(/[.!?]\s+[A-Z\\]/);
-                    if (sentenceBreak > 0) {
-                        end = searchStart + sentenceBreak + 1;
-                    }
-                }
-            }
-            chunks.push(content.slice(start, end));
-            start = Math.max(start + 1, end - effectiveOverlap);
-        }
-        return chunks;
-    }
-    validateConfig(config) {
-        const errors = [];
-        const warnings = [];
-        const latexConfig = config;
-        if (latexConfig.maxSize !== undefined && latexConfig.maxSize <= 0) {
-            errors.push("maxSize must be greater than 0");
-        }
-        if (latexConfig.overlap !== undefined && latexConfig.overlap < 0) {
-            errors.push("overlap must be non-negative");
-        }
-        if (latexConfig.overlap !== undefined &&
-            latexConfig.maxSize !== undefined &&
-            latexConfig.overlap >= latexConfig.maxSize) {
-            errors.push("overlap must be less than maxSize");
-        }
-        if (latexConfig.splitEnvironments !== undefined &&
-            latexConfig.splitEnvironments.length === 0) {
-            warnings.push("No split environments specified, using defaults");
-        }
-        return {
-            valid: errors.length === 0,
-            errors,
-            warnings,
-        };
-    }
-}

package/dist/client/rag/chunking/markdownChunker.js

@@ -1,373 +0,0 @@
-/**
- * Markdown-aware Chunker
- *
- * Splits markdown documents based on header structure while preserving formatting.
- * Best for documentation, README files, and structured markdown content.
- */
-import { randomUUID } from "crypto";
-/**
- * Markdown-aware chunker implementation
- * Splits based on markdown structure (headers, code blocks, etc.)
- */
-export class MarkdownChunker {
-    strategy = "markdown";
-    async chunk(text, config) {
-        const { maxSize = 1000, overlap = 0, headerLevels = [1, 2, 3], preserveCodeBlocks = true, includeHeader = true, stripFormatting = false, trimWhitespace = true, metadata = {}, } = config || {};
-        const documentId = randomUUID();
-        const chunks = [];
-        if (!text || text.length === 0) {
-            return chunks;
-        }
-        // Build header regex pattern
-        const headerPattern = new RegExp(`^(#{${Math.min(...headerLevels)},${Math.max(...headerLevels)}})\\s+(.+)$`, "gm");
-        // Split by headers while preserving them
-        const sections = this.splitByHeaders(text, headerPattern, includeHeader);
-        let chunkIndex = 0;
-        let currentPosition = 0;
-        for (const section of sections) {
-            const { header, content, level } = section;
-            // Handle code blocks
-            let processedContent = content;
-            const codeBlocks = [];
-            if (preserveCodeBlocks) {
-                processedContent = content.replace(/```[\s\S]*?```|`[^`]+`/g, (match) => {
-                    const placeholder = `__CODE_BLOCK_${codeBlocks.length}__`;
-                    codeBlocks.push({ placeholder, code: match });
-                    return placeholder;
-                });
-            }
-            // Split content if too large
-            const effectiveMaxSize = Math.max(maxSize - (header?.length || 0), 100);
-            const contentChunks = this.splitContent(processedContent, effectiveMaxSize, overlap);
-            for (const contentChunk of contentChunks) {
-                let chunkText = header && includeHeader
-                    ? `${header}\n\n${contentChunk}`
-                    : contentChunk;
-                // Restore code blocks
-                for (const { placeholder, code } of codeBlocks) {
-                    chunkText = chunkText.replace(placeholder, code);
-                }
-                // Strip formatting if requested
-                if (stripFormatting) {
-                    chunkText = this.stripMarkdown(chunkText);
-                }
-                const finalText = trimWhitespace ? chunkText.trim() : chunkText;
-                if (finalText.length > 0) {
-                    chunks.push({
-                        id: randomUUID(),
-                        text: finalText,
-                        metadata: {
-                            documentId,
-                            chunkIndex,
-                            startPosition: currentPosition,
-                            endPosition: currentPosition + chunkText.length,
-                            documentType: "markdown",
-                            headerLevel: level ?? undefined,
-                            header: header?.replace(/^#+\s*/, "") ?? undefined,
-                            custom: metadata,
-                        },
-                    });
-                    chunkIndex++;
-                }
-                currentPosition += chunkText.length;
-            }
-        }
-        // Update total chunks count
-        chunks.forEach((chunk) => {
-            chunk.metadata.totalChunks = chunks.length;
-        });
-        return chunks;
-    }
-    splitByHeaders(text, headerPattern, _includeHeader) {
-        const sections = [];
-        let lastIndex = 0;
-        let match;
-        let currentHeader = null;
-        let currentLevel = null;
-        // Reset regex
-        headerPattern.lastIndex = 0;
-        while ((match = headerPattern.exec(text)) !== null) {
-            // Content before this header
-            if (match.index > lastIndex) {
-                const content = text.slice(lastIndex, match.index);
-                if (content.trim()) {
-                    sections.push({
-                        header: currentHeader,
-                        content: content.trim(),
-                        level: currentLevel,
-                    });
-                }
-            }
-            currentHeader = match[0];
-            currentLevel = match[1].length; // Number of # characters
-            lastIndex = match.index + match[0].length;
-        }
-        // Don't forget content after the last header
-        if (lastIndex < text.length) {
-            const content = text.slice(lastIndex);
-            if (content.trim()) {
-                sections.push({
-                    header: currentHeader,
-                    content: content.trim(),
-                    level: currentLevel,
-                });
-            }
-        }
-        // If no headers found, return entire text as one section
-        if (sections.length === 0 && text.trim()) {
-            sections.push({
-                header: null,
-                content: text.trim(),
-                level: null,
-            });
-        }
-        return sections;
-    }
-    splitContent(content, maxSize, overlap) {
-        const effectiveMaxSize = Math.max(maxSize, 1);
-        const effectiveOverlap = Math.min(Math.max(overlap, 0), effectiveMaxSize - 1);
-        if (content.length <= effectiveMaxSize) {
-            return [content];
-        }
-        // Use table-aware splitting
-        const lines = content.split("\n");
-        const tableRanges = this.detectTableRanges(lines);
-        if (tableRanges.length > 0) {
-            return this.splitContentTableAware(content, lines, tableRanges, effectiveMaxSize, effectiveOverlap);
-        }
-        return this.splitPlainContent(content, effectiveMaxSize, effectiveOverlap);
-    }
-    /**
-     * Detect contiguous table blocks in lines.
-     * Returns array of { start, end } line index ranges (inclusive).
-     */
-    detectTableRanges(lines) {
-        // Simple pipe-prefixed line check (single character class — no backtracking)
-        const TABLE_ROW_RE = /^\|[^\r\n]{1,10000}/;
-        // Per-cell separator regex applied AFTER splitting on "|" — safe because
-        // each cell is short and bounded by pipe delimiters (CodeQL: js/polynomial-redos)
-        const SEPARATOR_CELL_RE = /^[\t ]*:?-+:?[\t ]*$/;
-        const ranges = [];
-        let i = 0;
-        while (i < lines.length) {
-            if (i + 1 < lines.length &&
-                TABLE_ROW_RE.test(lines[i]) &&
-                this.isTableSeparator(lines[i + 1], SEPARATOR_CELL_RE)) {
-                const start = i;
-                i += 2;
-                while (i < lines.length && TABLE_ROW_RE.test(lines[i])) {
-                    i++;
-                }
-                ranges.push({ start, end: i - 1 });
-            }
-            else {
-                i++;
-            }
-        }
-        return ranges;
-    }
-    /** Check if a line is a markdown table separator (e.g. |---|---|). */
-    isTableSeparator(line, cellRe) {
-        const trimmed = line.trimEnd();
-        if (!trimmed.startsWith("|")) {
-            return false;
-        }
-        // Split by "|" → ["", "---", "---", ""] for "|---|---|"
-        const cells = trimmed.split("|");
-        cells.shift(); // remove leading empty element
-        if (cells.length > 0 && cells[cells.length - 1].trim() === "") {
-            cells.pop(); // remove trailing empty element
-        }
-        if (cells.length === 0) {
-            return false;
-        }
-        return cells.every((cell) => cellRe.test(cell));
-    }
-    /**
-     * Split content while preserving markdown tables.
-     */
-    splitContentTableAware(content, lines, tableRanges, maxSize, overlap) {
-        // Build segments: alternating non-table and table blocks
-        const segments = [];
-        let lineIdx = 0;
-        for (const range of tableRanges) {
-            if (lineIdx < range.start) {
-                const text = lines.slice(lineIdx, range.start).join("\n").trim();
-                if (text) {
-                    segments.push({ text, isTable: false });
-                }
-            }
-            const tableText = lines.slice(range.start, range.end + 1).join("\n");
-            segments.push({ text: tableText, isTable: true });
-            lineIdx = range.end + 1;
-        }
-        if (lineIdx < lines.length) {
-            const text = lines.slice(lineIdx).join("\n").trim();
-            if (text) {
-                segments.push({ text, isTable: false });
-            }
-        }
-        const result = [];
-        let current = "";
-        for (const seg of segments) {
-            if (!seg.isTable) {
-                const pieces = this.splitPlainContent(seg.text, maxSize, overlap);
-                for (const piece of pieces) {
-                    if (current.length === 0) {
-                        current = piece;
-                    }
-                    else if (current.length + 1 + piece.length <= maxSize) {
-                        current += "\n" + piece;
-                    }
-                    else {
-                        result.push(current);
-                        current = piece;
-                    }
-                }
-            }
-            else {
-                if (seg.text.length <= maxSize) {
-                    if (current.length === 0) {
-                        current = seg.text;
-                    }
-                    else if (current.length + 2 + seg.text.length <= maxSize) {
-                        current += "\n\n" + seg.text;
-                    }
-                    else {
-                        result.push(current);
-                        current = seg.text;
-                    }
-                }
-                else {
-                    if (current) {
-                        result.push(current);
-                        current = "";
-                    }
-                    const tableChunks = this.splitTableByRows(seg.text, maxSize);
-                    result.push(...tableChunks);
-                }
-            }
-        }
-        if (current) {
-            result.push(current);
-        }
-        return result.length > 0 ? result : [content];
-    }
-    /**
-     * Split a table on row boundaries, repeating header + separator in each chunk.
-     */
-    splitTableByRows(tableText, maxSize) {
-        const rows = tableText.split("\n");
-        if (rows.length < 3) {
-            return [tableText];
-        }
-        const headerRow = rows[0];
-        const separatorRow = rows[1];
-        const headerBlock = headerRow + "\n" + separatorRow;
-        const dataRows = rows.slice(2);
-        if (headerBlock.length > maxSize) {
-            return this.splitPlainContent(tableText, maxSize, 0);
-        }
-        const chunks = [];
-        let currentChunk = headerBlock;
-        for (const row of dataRows) {
-            // Guard: single row exceeds budget — flush and emit as standalone chunk
-            const singleRowChunk = `${headerBlock}\n${row}`;
-            if (singleRowChunk.length > maxSize) {
-                if (currentChunk.length > headerBlock.length) {
-                    chunks.push(currentChunk);
-                }
-                chunks.push(singleRowChunk);
-                currentChunk = headerBlock;
-                continue;
-            }
-            const candidate = currentChunk + "\n" + row;
-            if (candidate.length <= maxSize) {
-                currentChunk = candidate;
-            }
-            else {
-                if (currentChunk.length > headerBlock.length) {
-                    chunks.push(currentChunk);
-                }
-                currentChunk = headerBlock + "\n" + row;
-            }
-        }
-        if (currentChunk.length > headerBlock.length) {
-            chunks.push(currentChunk);
-        }
-        return chunks.length > 0 ? chunks : [tableText];
-    }
-    splitPlainContent(content, maxSize, overlap) {
-        if (content.length <= maxSize) {
-            return [content];
-        }
-        const chunks = [];
-        let start = 0;
-        while (start < content.length) {
-            let end = Math.min(start + maxSize, content.length);
-            // Try to break at a paragraph or sentence boundary
-            if (end < content.length) {
-                const searchStart = Math.max(start, end - 200);
-                const searchText = content.slice(searchStart, end);
-                // Look for paragraph break first
-                const paragraphBreak = searchText.lastIndexOf("\n\n");
-                if (paragraphBreak > 0) {
-                    end = searchStart + paragraphBreak;
-                }
-                else {
-                    // Look for sentence break
-                    const sentenceBreak = searchText.search(/[.!?]\s+[A-Z]/);
-                    if (sentenceBreak > 0) {
-                        end = searchStart + sentenceBreak + 1;
-                    }
-                }
-            }
-            chunks.push(content.slice(start, end));
-            start = Math.max(start + 1, end - overlap);
-        }
-        return chunks;
-    }
-    stripMarkdown(text) {
-        return text
-            .replace(/^#+\s+/gm, "") // Headers
-            .replace(/\*\*(.+?)\*\*/g, "$1") // Bold
-            .replace(/\*(.+?)\*/g, "$1") // Italic
-            .replace(/__(.+?)__/g, "$1") // Bold (underscore)
-            .replace(/_(.+?)_/g, "$1") // Italic (underscore)
-            .replace(/`(.+?)`/g, "$1") // Inline code
-            .replace(/```[\s\S]*?```/g, "") // Code blocks
-            .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") // Links
-            .replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1"); // Images
-    }
-    validateConfig(config) {
-        const errors = [];
-        const warnings = [];
-        const mdConfig = config;
-        if (mdConfig.maxSize !== undefined && mdConfig.maxSize <= 0) {
-            errors.push("maxSize must be greater than 0");
-        }
-        if (mdConfig.headerLevels !== undefined) {
-            if (mdConfig.headerLevels.length === 0) {
-                errors.push("headerLevels must not be empty");
-            }
-            for (const level of mdConfig.headerLevels) {
-                if (level < 1 || level > 6) {
-                    errors.push(`Invalid header level: ${level}. Must be between 1 and 6`);
-                }
-            }
-        }
-        if (mdConfig.overlap !== undefined && mdConfig.overlap < 0) {
-            errors.push("overlap must be non-negative");
-        }
-        if (mdConfig.overlap !== undefined &&
-            mdConfig.maxSize !== undefined &&
-            mdConfig.overlap >= mdConfig.maxSize) {
-            errors.push("overlap must be less than maxSize");
-        }
-        return {
-            valid: errors.length === 0,
-            errors,
-            warnings,
-        };
-    }
-}