@juspay/neurolink 9.30.0 → 9.31.0

package/dist/client/rag/chunkers/LaTeXChunker.js

@@ -0,0 +1,63 @@
+/**
+ * LaTeX Chunker
+ *
+ * Splits LaTeX documents by sections and environments.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/**
+ * LaTeX Chunker
+ */
+export class LaTeXChunker extends BaseChunker {
+    strategy = "latex";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 0,
+        };
+    }
+    async doChunk(content, config) {
+        const maxSize = config.maxSize ?? 1000;
+        // Split by sections
+        const sectionPattern = /\\(?:section|subsection|subsubsection|chapter|paragraph)\{[^}]+\}/g;
+        const sections = [];
+        let lastIndex = 0;
+        let match;
+        while ((match = sectionPattern.exec(content)) !== null) {
+            if (match.index > lastIndex) {
+                sections.push(content.slice(lastIndex, match.index));
+            }
+            lastIndex = match.index;
+        }
+        if (lastIndex < content.length) {
+            sections.push(content.slice(lastIndex));
+        }
+        if (sections.length === 0) {
+            sections.push(content);
+        }
+        const chunks = [];
+        let offset = 0;
+        for (const section of sections) {
+            const trimmed = section.trim();
+            if (!trimmed) {
+                continue;
+            }
+            if (trimmed.length <= maxSize) {
+                const startOffset = content.indexOf(trimmed, offset);
+                chunks.push(this.createChunk(trimmed, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
+                    ? startOffset + trimmed.length
+                    : offset + trimmed.length));
+                if (startOffset >= 0) {
+                    offset = startOffset + 1;
+                }
+            }
+            else {
+                const segments = this.splitBySizeWithOverlap(trimmed, maxSize, 0);
+                for (const segment of segments) {
+                    chunks.push(this.createChunk(segment.text, chunks.length, segment.start, segment.end));
+                }
+            }
+        }
+        return chunks;
+    }
+}

package/dist/client/rag/chunkers/MarkdownChunker.js

@@ -0,0 +1,306 @@
+/**
+ * Markdown Chunker
+ *
+ * Splits markdown content by headers and structural elements.
+ * Preserves markdown tables by detecting table boundaries and splitting
+ * on row boundaries when a table exceeds the max chunk size.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/** Matches a markdown table separator row like |---|---| or |:--:|---:| */
+const TABLE_SEPARATOR_RE = /^\|[\s:]*-+[\s:]*(\|[\s:]*-+[\s:]*)*\|?\s*$/;
+/** Matches a line that looks like a table row (starts with |) */
+const TABLE_ROW_RE = /^\|.+\|?\s*$/;
+/**
+ * Detect contiguous table blocks in text.
+ * Returns an array of { start, end } line index ranges (inclusive).
+ * A table is a sequence of lines where the second line is a separator.
+ */
+function detectTableRanges(lines) {
+    const ranges = [];
+    let i = 0;
+    while (i < lines.length) {
+        // A table needs at least a header row + separator
+        if (i + 1 < lines.length &&
+            TABLE_ROW_RE.test(lines[i]) &&
+            TABLE_SEPARATOR_RE.test(lines[i + 1])) {
+            const start = i;
+            // Advance past header + separator
+            i += 2;
+            // Consume remaining data rows
+            while (i < lines.length && TABLE_ROW_RE.test(lines[i])) {
+                i++;
+            }
+            ranges.push({ start, end: i - 1 });
+        }
+        else {
+            i++;
+        }
+    }
+    return ranges;
+}
+/**
+ * Markdown Chunker
+ */
+export class MarkdownChunker extends BaseChunker {
+    strategy = "markdown";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 50,
+        };
+    }
+    async doChunk(content, config) {
+        const maxSize = config.maxSize ?? 1000;
+        // Split by headers
+        const headerPattern = /^(#{1,6})\s+(.+)$/gm;
+        const sections = [];
+        let lastIndex = 0;
+        let match = headerPattern.exec(content);
+        while (match !== null) {
+            // Add content before this header
+            if (match.index > lastIndex) {
+                const prevContent = content.slice(lastIndex, match.index).trim();
+                if (prevContent && sections.length > 0) {
+                    const lastSection = sections[sections.length - 1];
+                    if (lastSection) {
+                        lastSection.content += "\n\n" + prevContent;
+                    }
+                }
+                else if (prevContent) {
+                    sections.push({ header: "", content: prevContent, level: 0 });
+                }
+            }
+            sections.push({
+                header: match[0],
+                content: "",
+                level: match[1]?.length ?? 1,
+            });
+            lastIndex = match.index + match[0].length;
+            match = headerPattern.exec(content);
+        }
+        // Add remaining content
+        if (lastIndex < content.length) {
+            const remaining = content.slice(lastIndex).trim();
+            if (remaining) {
+                if (sections.length > 0) {
+                    const lastSection = sections[sections.length - 1];
+                    if (lastSection) {
+                        lastSection.content += remaining;
+                    }
+                }
+                else {
+                    sections.push({ header: "", content: remaining, level: 0 });
+                }
+            }
+        }
+        // Convert sections to chunks
+        const chunks = [];
+        let offset = 0;
+        for (let i = 0; i < sections.length; i++) {
+            const section = sections[i];
+            if (!section) {
+                continue;
+            }
+            const fullContent = section.header
+                ? section.header + "\n\n" + section.content.trim()
+                : section.content.trim();
+            if (!fullContent) {
+                continue;
+            }
+            // Split if too large — use table-aware splitting
+            if (fullContent.length > maxSize) {
+                const subChunks = this.splitContentTableAware(fullContent, maxSize);
+                for (const sub of subChunks) {
+                    const startOffset = content.indexOf(sub, offset);
+                    chunks.push(this.createChunk(sub, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0 ? startOffset + sub.length : offset + sub.length, "unknown", { sectionContext: section.header }));
+                    if (startOffset >= 0) {
+                        offset = startOffset + sub.length;
+                    }
+                }
+            }
+            else {
+                const startOffset = content.indexOf(fullContent, offset);
+                chunks.push(this.createChunk(fullContent, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
+                    ? startOffset + fullContent.length
+                    : offset + fullContent.length, "unknown", { sectionContext: section.header }));
+                if (startOffset >= 0) {
+                    offset = startOffset + fullContent.length;
+                }
+            }
+        }
+        return chunks;
+    }
+    /**
+     * Split content while preserving markdown tables.
+     *
+     * Strategy:
+     * 1. Identify table blocks in the content.
+     * 2. Split content into segments: non-table text and table blocks.
+     * 3. Non-table text is split using paragraph/sentence boundaries (existing logic).
+     * 4. Tables that fit in a chunk are kept intact.
+     * 5. Oversized tables are split on row boundaries, repeating the header row.
+     */
+    splitContentTableAware(content, maxSize) {
+        const lines = content.split("\n");
+        const tableRanges = detectTableRanges(lines);
+        // If no tables, fall back to existing splitting logic
+        if (tableRanges.length === 0) {
+            return this.splitPlainContent(content, maxSize, this.config.overlap ?? 0);
+        }
+        // Build segments: alternating non-table and table blocks
+        const segments = [];
+        let lineIdx = 0;
+        for (const range of tableRanges) {
+            // Non-table text before this table
+            if (lineIdx < range.start) {
+                const text = lines.slice(lineIdx, range.start).join("\n").trim();
+                if (text) {
+                    segments.push({ text, isTable: false });
+                }
+            }
+            // The table itself
+            const tableText = lines.slice(range.start, range.end + 1).join("\n");
+            segments.push({ text: tableText, isTable: true });
+            lineIdx = range.end + 1;
+        }
+        // Trailing non-table text
+        if (lineIdx < lines.length) {
+            const text = lines.slice(lineIdx).join("\n").trim();
+            if (text) {
+                segments.push({ text, isTable: false });
+            }
+        }
+        // Now produce chunks, trying to pack segments together up to maxSize
+        const result = [];
+        let current = "";
+        for (const seg of segments) {
+            if (!seg.isTable) {
+                // Non-table text: try to append, split if needed
+                const pieces = this.splitPlainContent(seg.text, maxSize, this.config.overlap ?? 0);
+                for (const piece of pieces) {
+                    if (current.length === 0) {
+                        current = piece;
+                    }
+                    else if (current.length + 1 + piece.length <= maxSize) {
+                        current += "\n" + piece;
+                    }
+                    else {
+                        result.push(current);
+                        current = piece;
+                    }
+                }
+            }
+            else {
+                // Table block
+                if (seg.text.length <= maxSize) {
+                    // Table fits — try to append to current chunk
+                    if (current.length === 0) {
+                        current = seg.text;
+                    }
+                    else if (current.length + 2 + seg.text.length <= maxSize) {
+                        current += "\n\n" + seg.text;
+                    }
+                    else {
+                        result.push(current);
+                        current = seg.text;
+                    }
+                }
+                else {
+                    // Oversized table — flush current, then split table on row boundaries
+                    if (current) {
+                        result.push(current);
+                        current = "";
+                    }
+                    const tableChunks = this.splitTableByRows(seg.text, maxSize);
+                    result.push(...tableChunks);
+                }
+            }
+        }
+        if (current) {
+            result.push(current);
+        }
+        return result.length > 0 ? result : [content];
+    }
+    /**
+     * Split a table on row boundaries, repeating header + separator in each chunk.
+     */
+    splitTableByRows(tableText, maxSize) {
+        const rows = tableText.split("\n");
+        if (rows.length < 3) {
+            // Not a proper table (need header + separator + at least 1 data row)
+            return [tableText];
+        }
+        const headerRow = rows[0];
+        const separatorRow = rows[1];
+        const headerBlock = headerRow + "\n" + separatorRow;
+        const dataRows = rows.slice(2);
+        // If even the header doesn't fit, fall back to size-based split
+        if (headerBlock.length > maxSize) {
+            return this.splitPlainContent(tableText, maxSize, this.config.overlap ?? 0);
+        }
+        const chunks = [];
+        let currentChunk = headerBlock;
+        for (const row of dataRows) {
+            // Guard: single row exceeds budget — flush and emit as standalone chunk
+            const singleRowChunk = `${headerBlock}\n${row}`;
+            if (singleRowChunk.length > maxSize) {
+                if (currentChunk.length > headerBlock.length) {
+                    chunks.push(currentChunk);
+                }
+                chunks.push(singleRowChunk);
+                currentChunk = headerBlock;
+                continue;
+            }
+            const candidate = currentChunk + "\n" + row;
+            if (candidate.length <= maxSize) {
+                currentChunk = candidate;
+            }
+            else {
+                // Flush current chunk (skip if it only contains the header)
+                if (currentChunk.length > headerBlock.length) {
+                    chunks.push(currentChunk);
+                }
+                // Start new chunk with header repeated
+                currentChunk = headerBlock + "\n" + row;
+            }
+        }
+        if (currentChunk.length > headerBlock.length) {
+            chunks.push(currentChunk);
+        }
+        return chunks.length > 0 ? chunks : [tableText];
+    }
+    /**
+     * Split non-table text using paragraph and sentence boundaries.
+     * This is the original splitContent logic extracted for reuse.
+     */
+    splitPlainContent(content, maxSize, overlap = 0) {
+        if (content.length <= maxSize) {
+            return [content];
+        }
+        const chunks = [];
+        let start = 0;
+        while (start < content.length) {
+            let end = Math.min(start + maxSize, content.length);
+            if (end < content.length) {
+                const searchStart = Math.max(start, end - 200);
+                const searchText = content.slice(searchStart, end);
+                // Look for paragraph break first
+                const paragraphBreak = searchText.lastIndexOf("\n\n");
+                if (paragraphBreak > 0) {
+                    end = searchStart + paragraphBreak;
+                }
+                else {
+                    // Look for sentence break
+                    const sentenceBreak = searchText.search(/[.!?]\s+[A-Z]/);
+                    if (sentenceBreak > 0) {
+                        end = searchStart + sentenceBreak + 1;
+                    }
+                }
+            }
+            chunks.push(content.slice(start, end));
+            start = Math.max(start + 1, end - overlap);
+        }
+        return chunks;
+    }
+}

package/dist/client/rag/chunkers/RecursiveChunker.js

@@ -0,0 +1,139 @@
+/**
+ * Recursive Chunker
+ *
+ * Recursively splits text using an ordered list of separators.
+ * Tries each separator in order until chunks are small enough.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/**
+ * Default separators for recursive splitting
+ */
+const DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", " ", ""];
+/**
+ * Recursive Chunker
+ *
+ * Splits content using ordered separators, recursively breaking
+ * down text until chunks meet size requirements.
+ */
+export class RecursiveChunker extends BaseChunker {
+    strategy = "recursive";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 100,
+            separators: DEFAULT_SEPARATORS,
+        };
+    }
+    async doChunk(content, config) {
+        const recursiveConfig = config;
+        const maxSize = config.maxSize ?? 1000;
+        const overlap = config.overlap ?? 100;
+        const separators = recursiveConfig.separators ?? DEFAULT_SEPARATORS;
+        const keepSeparators = recursiveConfig.keepSeparators ?? true;
+        const chunks = [];
+        let offset = 0;
+        const textChunks = this.recursiveSplit(content, separators, maxSize, overlap, keepSeparators);
+        for (let i = 0; i < textChunks.length; i++) {
+            const text = textChunks[i];
+            if (!text) {
+                continue;
+            }
+            const startOffset = content.indexOf(text, offset);
+            const endOffset = startOffset + text.length;
+            chunks.push(this.createChunk(text, i, startOffset, endOffset));
+            offset = Math.max(offset, startOffset + 1);
+        }
+        return chunks;
+    }
+    /**
+     * Recursively split text using separators
+     */
+    recursiveSplit(text, separators, maxSize, overlap, keepSeparators) {
+        if (text.length <= maxSize) {
+            return [text];
+        }
+        // Find the first separator that exists in the text
+        let separator = "";
+        for (const sep of separators) {
+            if (sep === "" || text.includes(sep)) {
+                separator = sep;
+                break;
+            }
+        }
+        // If no separator found or empty separator, split by size
+        if (separator === "") {
+            const result = [];
+            let start = 0;
+            while (start < text.length) {
+                const end = Math.min(start + maxSize, text.length);
+                result.push(text.slice(start, end));
+                const previousStart = start;
+                start = end - overlap;
+                if (start <= previousStart) {
+                    start = previousStart + 1;
+                }
+                if (start >= text.length) {
+                    break;
+                }
+            }
+            return result;
+        }
+        // Split by separator
+        const parts = text.split(separator);
+        const result = [];
+        let currentChunk = "";
+        for (let i = 0; i < parts.length; i++) {
+            const part = parts[i];
+            const addSeparator = keepSeparators && i < parts.length - 1;
+            const toAdd = part + (addSeparator ? separator : "");
+            if (currentChunk.length + toAdd.length <= maxSize) {
+                currentChunk += toAdd;
+            }
+            else {
+                // Current chunk is full
+                if (currentChunk.length > 0) {
+                    result.push(currentChunk);
+                }
+                // If the part itself is too large, recursively split it
+                if (toAdd.length > maxSize) {
+                    const remainingSeparators = separators.slice(separators.indexOf(separator) + 1);
+                    const subChunks = this.recursiveSplit(toAdd, remainingSeparators, maxSize, overlap, keepSeparators);
+                    result.push(...subChunks);
+                    currentChunk = "";
+                }
+                else {
+                    currentChunk = toAdd;
+                }
+            }
+        }
+        if (currentChunk.length > 0) {
+            result.push(currentChunk);
+        }
+        // Apply overlap between chunks
+        if (overlap > 0 && result.length > 1) {
+            return this.applyOverlap(result, overlap);
+        }
+        return result;
+    }
+    /**
+     * Apply overlap between chunks
+     */
+    applyOverlap(chunks, overlap) {
+        if (chunks.length <= 1) {
+            return chunks;
+        }
+        const result = [];
+        for (let i = 0; i < chunks.length; i++) {
+            let chunk = chunks[i] ?? "";
+            // Add overlap from previous chunk
+            const prevChunk = i > 0 ? chunks[i - 1] : undefined;
+            if (prevChunk) {
+                const overlapText = prevChunk.slice(-Math.min(overlap, prevChunk.length));
+                chunk = overlapText + chunk;
+            }
+            result.push(chunk);
+        }
+        return result;
+    }
+}

package/dist/client/rag/chunkers/SemanticMarkdownChunker.js

@@ -0,0 +1,138 @@
+/**
+ * Semantic Markdown Chunker
+ *
+ * Combines markdown splitting with semantic similarity for intelligent merging.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/**
+ * Semantic Markdown Chunker
+ *
+ * Extends markdown chunking with semantic awareness.
+ * Can be enhanced with embedding-based similarity.
+ */
+export class SemanticMarkdownChunker extends BaseChunker {
+    strategy = "semantic-markdown";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 100,
+        };
+    }
+    async doChunk(content, config) {
+        const maxSize = config.maxSize ?? 1000;
+        const overlap = config.overlap ?? 100;
+        // First, split by markdown headers
+        const headerPattern = /^(#{1,6})\s+(.+)$/gm;
+        const sections = [];
+        let lastIndex = 0;
+        let match = headerPattern.exec(content);
+        while (match !== null) {
+            if (match.index > lastIndex) {
+                const prevContent = content.slice(lastIndex, match.index).trim();
+                if (prevContent && sections.length > 0) {
+                    const lastSection = sections[sections.length - 1];
+                    if (lastSection) {
+                        lastSection.content += "\n\n" + prevContent;
+                    }
+                }
+                else if (prevContent) {
+                    sections.push({ header: "", content: prevContent });
+                }
+            }
+            sections.push({ header: match[0], content: "" });
+            lastIndex = match.index + match[0].length;
+            match = headerPattern.exec(content);
+        }
+        if (lastIndex < content.length) {
+            const remaining = content.slice(lastIndex).trim();
+            if (remaining) {
+                if (sections.length > 0) {
+                    const lastSection = sections[sections.length - 1];
+                    if (lastSection) {
+                        lastSection.content += remaining;
+                    }
+                }
+                else {
+                    sections.push({ header: "", content: remaining });
+                }
+            }
+        }
+        // Merge small sections that are semantically related
+        const mergedSections = this.mergeSmallSections(sections, maxSize);
+        // Convert to chunks
+        const chunks = [];
+        let offset = 0;
+        for (let i = 0; i < mergedSections.length; i++) {
+            const section = mergedSections[i];
+            if (!section) {
+                continue;
+            }
+            const fullContent = section.header
+                ? section.header + "\n\n" + section.content.trim()
+                : section.content.trim();
+            if (!fullContent) {
+                continue;
+            }
+            if (fullContent.length > maxSize) {
+                const segments = this.splitBySizeWithOverlap(fullContent, maxSize, overlap);
+                for (const segment of segments) {
+                    const startOffset = content.indexOf(segment.text.slice(0, 50), offset);
+                    chunks.push(this.createChunk(segment.text, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
+                        ? startOffset + segment.text.length
+                        : offset + segment.text.length, "unknown", { sectionContext: section.header }));
+                    if (startOffset >= 0) {
+                        offset = startOffset + 1;
+                    }
+                }
+            }
+            else {
+                const startOffset = content.indexOf(fullContent.slice(0, 50), offset);
+                chunks.push(this.createChunk(fullContent, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
+                    ? startOffset + fullContent.length
+                    : offset + fullContent.length, "unknown", { sectionContext: section.header }));
+                if (startOffset >= 0) {
+                    offset = startOffset + 1;
+                }
+            }
+        }
+        return chunks;
+    }
+    /**
+     * Merge small sections to optimize chunk sizes
+     */
+    mergeSmallSections(sections, maxSize) {
+        const result = [];
+        let current = null;
+        for (const section of sections) {
+            const fullContent = section.header
+                ? section.header + "\n\n" + section.content.trim()
+                : section.content.trim();
+            const sectionLength = fullContent.length;
+            if (!current) {
+                current = { ...section };
+                continue;
+            }
+            const currentLength = current.header
+                ? current.header.length + current.content.length + 2
+                : current.content.length;
+            // Merge if combined size is within limit
+            if (currentLength + sectionLength <= maxSize) {
+                if (section.header) {
+                    current.content += "\n\n" + section.header + "\n" + section.content;
+                }
+                else {
+                    current.content += "\n\n" + section.content;
+                }
+            }
+            else {
+                result.push(current);
+                current = { ...section };
+            }
+        }
+        if (current) {
+            result.push(current);
+        }
+        return result;
+    }
+}