@juspay/neurolink 9.32.0 → 9.32.1

package/dist/client/rag/chunkers/LaTeXChunker.js

@@ -1,63 +0,0 @@
-/**
- * LaTeX Chunker
- *
- * Splits LaTeX documents by sections and environments.
- */
-import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
-/**
- * LaTeX Chunker
- */
-export class LaTeXChunker extends BaseChunker {
-    strategy = "latex";
-    getDefaultConfig() {
-        return {
-            ...DEFAULT_CHUNKER_CONFIG,
-            maxSize: 1000,
-            overlap: 0,
-        };
-    }
-    async doChunk(content, config) {
-        const maxSize = config.maxSize ?? 1000;
-        // Split by sections
-        const sectionPattern = /\\(?:section|subsection|subsubsection|chapter|paragraph)\{[^}]+\}/g;
-        const sections = [];
-        let lastIndex = 0;
-        let match;
-        while ((match = sectionPattern.exec(content)) !== null) {
-            if (match.index > lastIndex) {
-                sections.push(content.slice(lastIndex, match.index));
-            }
-            lastIndex = match.index;
-        }
-        if (lastIndex < content.length) {
-            sections.push(content.slice(lastIndex));
-        }
-        if (sections.length === 0) {
-            sections.push(content);
-        }
-        const chunks = [];
-        let offset = 0;
-        for (const section of sections) {
-            const trimmed = section.trim();
-            if (!trimmed) {
-                continue;
-            }
-            if (trimmed.length <= maxSize) {
-                const startOffset = content.indexOf(trimmed, offset);
-                chunks.push(this.createChunk(trimmed, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
-                    ? startOffset + trimmed.length
-                    : offset + trimmed.length));
-                if (startOffset >= 0) {
-                    offset = startOffset + 1;
-                }
-            }
-            else {
-                const segments = this.splitBySizeWithOverlap(trimmed, maxSize, 0);
-                for (const segment of segments) {
-                    chunks.push(this.createChunk(segment.text, chunks.length, segment.start, segment.end));
-                }
-            }
-        }
-        return chunks;
-    }
-}

package/dist/client/rag/chunkers/MarkdownChunker.js

@@ -1,306 +0,0 @@
-/**
- * Markdown Chunker
- *
- * Splits markdown content by headers and structural elements.
- * Preserves markdown tables by detecting table boundaries and splitting
- * on row boundaries when a table exceeds the max chunk size.
- */
-import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
-/** Matches a markdown table separator row like |---|---| or |:--:|---:| */
-const TABLE_SEPARATOR_RE = /^\|[\s:]*-+[\s:]*(\|[\s:]*-+[\s:]*)*\|?\s*$/;
-/** Matches a line that looks like a table row (starts with |) */
-const TABLE_ROW_RE = /^\|.+\|?\s*$/;
-/**
- * Detect contiguous table blocks in text.
- * Returns an array of { start, end } line index ranges (inclusive).
- * A table is a sequence of lines where the second line is a separator.
- */
-function detectTableRanges(lines) {
-    const ranges = [];
-    let i = 0;
-    while (i < lines.length) {
-        // A table needs at least a header row + separator
-        if (i + 1 < lines.length &&
-            TABLE_ROW_RE.test(lines[i]) &&
-            TABLE_SEPARATOR_RE.test(lines[i + 1])) {
-            const start = i;
-            // Advance past header + separator
-            i += 2;
-            // Consume remaining data rows
-            while (i < lines.length && TABLE_ROW_RE.test(lines[i])) {
-                i++;
-            }
-            ranges.push({ start, end: i - 1 });
-        }
-        else {
-            i++;
-        }
-    }
-    return ranges;
-}
-/**
- * Markdown Chunker
- */
-export class MarkdownChunker extends BaseChunker {
-    strategy = "markdown";
-    getDefaultConfig() {
-        return {
-            ...DEFAULT_CHUNKER_CONFIG,
-            maxSize: 1000,
-            overlap: 50,
-        };
-    }
-    async doChunk(content, config) {
-        const maxSize = config.maxSize ?? 1000;
-        // Split by headers
-        const headerPattern = /^(#{1,6})\s+(.+)$/gm;
-        const sections = [];
-        let lastIndex = 0;
-        let match = headerPattern.exec(content);
-        while (match !== null) {
-            // Add content before this header
-            if (match.index > lastIndex) {
-                const prevContent = content.slice(lastIndex, match.index).trim();
-                if (prevContent && sections.length > 0) {
-                    const lastSection = sections[sections.length - 1];
-                    if (lastSection) {
-                        lastSection.content += "\n\n" + prevContent;
-                    }
-                }
-                else if (prevContent) {
-                    sections.push({ header: "", content: prevContent, level: 0 });
-                }
-            }
-            sections.push({
-                header: match[0],
-                content: "",
-                level: match[1]?.length ?? 1,
-            });
-            lastIndex = match.index + match[0].length;
-            match = headerPattern.exec(content);
-        }
-        // Add remaining content
-        if (lastIndex < content.length) {
-            const remaining = content.slice(lastIndex).trim();
-            if (remaining) {
-                if (sections.length > 0) {
-                    const lastSection = sections[sections.length - 1];
-                    if (lastSection) {
-                        lastSection.content += remaining;
-                    }
-                }
-                else {
-                    sections.push({ header: "", content: remaining, level: 0 });
-                }
-            }
-        }
-        // Convert sections to chunks
-        const chunks = [];
-        let offset = 0;
-        for (let i = 0; i < sections.length; i++) {
-            const section = sections[i];
-            if (!section) {
-                continue;
-            }
-            const fullContent = section.header
-                ? section.header + "\n\n" + section.content.trim()
-                : section.content.trim();
-            if (!fullContent) {
-                continue;
-            }
-            // Split if too large — use table-aware splitting
-            if (fullContent.length > maxSize) {
-                const subChunks = this.splitContentTableAware(fullContent, maxSize);
-                for (const sub of subChunks) {
-                    const startOffset = content.indexOf(sub, offset);
-                    chunks.push(this.createChunk(sub, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0 ? startOffset + sub.length : offset + sub.length, "unknown", { sectionContext: section.header }));
-                    if (startOffset >= 0) {
-                        offset = startOffset + sub.length;
-                    }
-                }
-            }
-            else {
-                const startOffset = content.indexOf(fullContent, offset);
-                chunks.push(this.createChunk(fullContent, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
-                    ? startOffset + fullContent.length
-                    : offset + fullContent.length, "unknown", { sectionContext: section.header }));
-                if (startOffset >= 0) {
-                    offset = startOffset + fullContent.length;
-                }
-            }
-        }
-        return chunks;
-    }
-    /**
-     * Split content while preserving markdown tables.
-     *
-     * Strategy:
-     * 1. Identify table blocks in the content.
-     * 2. Split content into segments: non-table text and table blocks.
-     * 3. Non-table text is split using paragraph/sentence boundaries (existing logic).
-     * 4. Tables that fit in a chunk are kept intact.
-     * 5. Oversized tables are split on row boundaries, repeating the header row.
-     */
-    splitContentTableAware(content, maxSize) {
-        const lines = content.split("\n");
-        const tableRanges = detectTableRanges(lines);
-        // If no tables, fall back to existing splitting logic
-        if (tableRanges.length === 0) {
-            return this.splitPlainContent(content, maxSize, this.config.overlap ?? 0);
-        }
-        // Build segments: alternating non-table and table blocks
-        const segments = [];
-        let lineIdx = 0;
-        for (const range of tableRanges) {
-            // Non-table text before this table
-            if (lineIdx < range.start) {
-                const text = lines.slice(lineIdx, range.start).join("\n").trim();
-                if (text) {
-                    segments.push({ text, isTable: false });
-                }
-            }
-            // The table itself
-            const tableText = lines.slice(range.start, range.end + 1).join("\n");
-            segments.push({ text: tableText, isTable: true });
-            lineIdx = range.end + 1;
-        }
-        // Trailing non-table text
-        if (lineIdx < lines.length) {
-            const text = lines.slice(lineIdx).join("\n").trim();
-            if (text) {
-                segments.push({ text, isTable: false });
-            }
-        }
-        // Now produce chunks, trying to pack segments together up to maxSize
-        const result = [];
-        let current = "";
-        for (const seg of segments) {
-            if (!seg.isTable) {
-                // Non-table text: try to append, split if needed
-                const pieces = this.splitPlainContent(seg.text, maxSize, this.config.overlap ?? 0);
-                for (const piece of pieces) {
-                    if (current.length === 0) {
-                        current = piece;
-                    }
-                    else if (current.length + 1 + piece.length <= maxSize) {
-                        current += "\n" + piece;
-                    }
-                    else {
-                        result.push(current);
-                        current = piece;
-                    }
-                }
-            }
-            else {
-                // Table block
-                if (seg.text.length <= maxSize) {
-                    // Table fits — try to append to current chunk
-                    if (current.length === 0) {
-                        current = seg.text;
-                    }
-                    else if (current.length + 2 + seg.text.length <= maxSize) {
-                        current += "\n\n" + seg.text;
-                    }
-                    else {
-                        result.push(current);
-                        current = seg.text;
-                    }
-                }
-                else {
-                    // Oversized table — flush current, then split table on row boundaries
-                    if (current) {
-                        result.push(current);
-                        current = "";
-                    }
-                    const tableChunks = this.splitTableByRows(seg.text, maxSize);
-                    result.push(...tableChunks);
-                }
-            }
-        }
-        if (current) {
-            result.push(current);
-        }
-        return result.length > 0 ? result : [content];
-    }
-    /**
-     * Split a table on row boundaries, repeating header + separator in each chunk.
-     */
-    splitTableByRows(tableText, maxSize) {
-        const rows = tableText.split("\n");
-        if (rows.length < 3) {
-            // Not a proper table (need header + separator + at least 1 data row)
-            return [tableText];
-        }
-        const headerRow = rows[0];
-        const separatorRow = rows[1];
-        const headerBlock = headerRow + "\n" + separatorRow;
-        const dataRows = rows.slice(2);
-        // If even the header doesn't fit, fall back to size-based split
-        if (headerBlock.length > maxSize) {
-            return this.splitPlainContent(tableText, maxSize, this.config.overlap ?? 0);
-        }
-        const chunks = [];
-        let currentChunk = headerBlock;
-        for (const row of dataRows) {
-            // Guard: single row exceeds budget — flush and emit as standalone chunk
-            const singleRowChunk = `${headerBlock}\n${row}`;
-            if (singleRowChunk.length > maxSize) {
-                if (currentChunk.length > headerBlock.length) {
-                    chunks.push(currentChunk);
-                }
-                chunks.push(singleRowChunk);
-                currentChunk = headerBlock;
-                continue;
-            }
-            const candidate = currentChunk + "\n" + row;
-            if (candidate.length <= maxSize) {
-                currentChunk = candidate;
-            }
-            else {
-                // Flush current chunk (skip if it only contains the header)
-                if (currentChunk.length > headerBlock.length) {
-                    chunks.push(currentChunk);
-                }
-                // Start new chunk with header repeated
-                currentChunk = headerBlock + "\n" + row;
-            }
-        }
-        if (currentChunk.length > headerBlock.length) {
-            chunks.push(currentChunk);
-        }
-        return chunks.length > 0 ? chunks : [tableText];
-    }
-    /**
-     * Split non-table text using paragraph and sentence boundaries.
-     * This is the original splitContent logic extracted for reuse.
-     */
-    splitPlainContent(content, maxSize, overlap = 0) {
-        if (content.length <= maxSize) {
-            return [content];
-        }
-        const chunks = [];
-        let start = 0;
-        while (start < content.length) {
-            let end = Math.min(start + maxSize, content.length);
-            if (end < content.length) {
-                const searchStart = Math.max(start, end - 200);
-                const searchText = content.slice(searchStart, end);
-                // Look for paragraph break first
-                const paragraphBreak = searchText.lastIndexOf("\n\n");
-                if (paragraphBreak > 0) {
-                    end = searchStart + paragraphBreak;
-                }
-                else {
-                    // Look for sentence break
-                    const sentenceBreak = searchText.search(/[.!?]\s+[A-Z]/);
-                    if (sentenceBreak > 0) {
-                        end = searchStart + sentenceBreak + 1;
-                    }
-                }
-            }
-            chunks.push(content.slice(start, end));
-            start = Math.max(start + 1, end - overlap);
-        }
-        return chunks;
-    }
-}

package/dist/client/rag/chunkers/RecursiveChunker.js

@@ -1,139 +0,0 @@
-/**
- * Recursive Chunker
- *
- * Recursively splits text using an ordered list of separators.
- * Tries each separator in order until chunks are small enough.
- */
-import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
-/**
- * Default separators for recursive splitting
- */
-const DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", " ", ""];
-/**
- * Recursive Chunker
- *
- * Splits content using ordered separators, recursively breaking
- * down text until chunks meet size requirements.
- */
-export class RecursiveChunker extends BaseChunker {
-    strategy = "recursive";
-    getDefaultConfig() {
-        return {
-            ...DEFAULT_CHUNKER_CONFIG,
-            maxSize: 1000,
-            overlap: 100,
-            separators: DEFAULT_SEPARATORS,
-        };
-    }
-    async doChunk(content, config) {
-        const recursiveConfig = config;
-        const maxSize = config.maxSize ?? 1000;
-        const overlap = config.overlap ?? 100;
-        const separators = recursiveConfig.separators ?? DEFAULT_SEPARATORS;
-        const keepSeparators = recursiveConfig.keepSeparators ?? true;
-        const chunks = [];
-        let offset = 0;
-        const textChunks = this.recursiveSplit(content, separators, maxSize, overlap, keepSeparators);
-        for (let i = 0; i < textChunks.length; i++) {
-            const text = textChunks[i];
-            if (!text) {
-                continue;
-            }
-            const startOffset = content.indexOf(text, offset);
-            const endOffset = startOffset + text.length;
-            chunks.push(this.createChunk(text, i, startOffset, endOffset));
-            offset = Math.max(offset, startOffset + 1);
-        }
-        return chunks;
-    }
-    /**
-     * Recursively split text using separators
-     */
-    recursiveSplit(text, separators, maxSize, overlap, keepSeparators) {
-        if (text.length <= maxSize) {
-            return [text];
-        }
-        // Find the first separator that exists in the text
-        let separator = "";
-        for (const sep of separators) {
-            if (sep === "" || text.includes(sep)) {
-                separator = sep;
-                break;
-            }
-        }
-        // If no separator found or empty separator, split by size
-        if (separator === "") {
-            const result = [];
-            let start = 0;
-            while (start < text.length) {
-                const end = Math.min(start + maxSize, text.length);
-                result.push(text.slice(start, end));
-                const previousStart = start;
-                start = end - overlap;
-                if (start <= previousStart) {
-                    start = previousStart + 1;
-                }
-                if (start >= text.length) {
-                    break;
-                }
-            }
-            return result;
-        }
-        // Split by separator
-        const parts = text.split(separator);
-        const result = [];
-        let currentChunk = "";
-        for (let i = 0; i < parts.length; i++) {
-            const part = parts[i];
-            const addSeparator = keepSeparators && i < parts.length - 1;
-            const toAdd = part + (addSeparator ? separator : "");
-            if (currentChunk.length + toAdd.length <= maxSize) {
-                currentChunk += toAdd;
-            }
-            else {
-                // Current chunk is full
-                if (currentChunk.length > 0) {
-                    result.push(currentChunk);
-                }
-                // If the part itself is too large, recursively split it
-                if (toAdd.length > maxSize) {
-                    const remainingSeparators = separators.slice(separators.indexOf(separator) + 1);
-                    const subChunks = this.recursiveSplit(toAdd, remainingSeparators, maxSize, overlap, keepSeparators);
-                    result.push(...subChunks);
-                    currentChunk = "";
-                }
-                else {
-                    currentChunk = toAdd;
-                }
-            }
-        }
-        if (currentChunk.length > 0) {
-            result.push(currentChunk);
-        }
-        // Apply overlap between chunks
-        if (overlap > 0 && result.length > 1) {
-            return this.applyOverlap(result, overlap);
-        }
-        return result;
-    }
-    /**
-     * Apply overlap between chunks
-     */
-    applyOverlap(chunks, overlap) {
-        if (chunks.length <= 1) {
-            return chunks;
-        }
-        const result = [];
-        for (let i = 0; i < chunks.length; i++) {
-            let chunk = chunks[i] ?? "";
-            // Add overlap from previous chunk
-            const prevChunk = i > 0 ? chunks[i - 1] : undefined;
-            if (prevChunk) {
-                const overlapText = prevChunk.slice(-Math.min(overlap, prevChunk.length));
-                chunk = overlapText + chunk;
-            }
-            result.push(chunk);
-        }
-        return result;
-    }
-}

package/dist/client/rag/chunkers/SemanticMarkdownChunker.js

@@ -1,138 +0,0 @@
-/**
- * Semantic Markdown Chunker
- *
- * Combines markdown splitting with semantic similarity for intelligent merging.
- */
-import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
-/**
- * Semantic Markdown Chunker
- *
- * Extends markdown chunking with semantic awareness.
- * Can be enhanced with embedding-based similarity.
- */
-export class SemanticMarkdownChunker extends BaseChunker {
-    strategy = "semantic-markdown";
-    getDefaultConfig() {
-        return {
-            ...DEFAULT_CHUNKER_CONFIG,
-            maxSize: 1000,
-            overlap: 100,
-        };
-    }
-    async doChunk(content, config) {
-        const maxSize = config.maxSize ?? 1000;
-        const overlap = config.overlap ?? 100;
-        // First, split by markdown headers
-        const headerPattern = /^(#{1,6})\s+(.+)$/gm;
-        const sections = [];
-        let lastIndex = 0;
-        let match = headerPattern.exec(content);
-        while (match !== null) {
-            if (match.index > lastIndex) {
-                const prevContent = content.slice(lastIndex, match.index).trim();
-                if (prevContent && sections.length > 0) {
-                    const lastSection = sections[sections.length - 1];
-                    if (lastSection) {
-                        lastSection.content += "\n\n" + prevContent;
-                    }
-                }
-                else if (prevContent) {
-                    sections.push({ header: "", content: prevContent });
-                }
-            }
-            sections.push({ header: match[0], content: "" });
-            lastIndex = match.index + match[0].length;
-            match = headerPattern.exec(content);
-        }
-        if (lastIndex < content.length) {
-            const remaining = content.slice(lastIndex).trim();
-            if (remaining) {
-                if (sections.length > 0) {
-                    const lastSection = sections[sections.length - 1];
-                    if (lastSection) {
-                        lastSection.content += remaining;
-                    }
-                }
-                else {
-                    sections.push({ header: "", content: remaining });
-                }
-            }
-        }
-        // Merge small sections that are semantically related
-        const mergedSections = this.mergeSmallSections(sections, maxSize);
-        // Convert to chunks
-        const chunks = [];
-        let offset = 0;
-        for (let i = 0; i < mergedSections.length; i++) {
-            const section = mergedSections[i];
-            if (!section) {
-                continue;
-            }
-            const fullContent = section.header
-                ? section.header + "\n\n" + section.content.trim()
-                : section.content.trim();
-            if (!fullContent) {
-                continue;
-            }
-            if (fullContent.length > maxSize) {
-                const segments = this.splitBySizeWithOverlap(fullContent, maxSize, overlap);
-                for (const segment of segments) {
-                    const startOffset = content.indexOf(segment.text.slice(0, 50), offset);
-                    chunks.push(this.createChunk(segment.text, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
-                        ? startOffset + segment.text.length
-                        : offset + segment.text.length, "unknown", { sectionContext: section.header }));
-                    if (startOffset >= 0) {
-                        offset = startOffset + 1;
-                    }
-                }
-            }
-            else {
-                const startOffset = content.indexOf(fullContent.slice(0, 50), offset);
-                chunks.push(this.createChunk(fullContent, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
-                    ? startOffset + fullContent.length
-                    : offset + fullContent.length, "unknown", { sectionContext: section.header }));
-                if (startOffset >= 0) {
-                    offset = startOffset + 1;
-                }
-            }
-        }
-        return chunks;
-    }
-    /**
-     * Merge small sections to optimize chunk sizes
-     */
-    mergeSmallSections(sections, maxSize) {
-        const result = [];
-        let current = null;
-        for (const section of sections) {
-            const fullContent = section.header
-                ? section.header + "\n\n" + section.content.trim()
-                : section.content.trim();
-            const sectionLength = fullContent.length;
-            if (!current) {
-                current = { ...section };
-                continue;
-            }
-            const currentLength = current.header
-                ? current.header.length + current.content.length + 2
-                : current.content.length;
-            // Merge if combined size is within limit
-            if (currentLength + sectionLength <= maxSize) {
-                if (section.header) {
-                    current.content += "\n\n" + section.header + "\n" + section.content;
-                }
-                else {
-                    current.content += "\n\n" + section.content;
-                }
-            }
-            else {
-                result.push(current);
-                current = { ...section };
-            }
-        }
-        if (current) {
-            result.push(current);
-        }
-        return result;
-    }
-}