@juspay/neurolink 9.1.1 → 9.3.0

package/dist/lib/rag/chunkers/HTMLChunker.js

@@ -0,0 +1,39 @@
+/**
+ * HTML Chunker
+ *
+ * Splits HTML content by semantic tags.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/**
+ * HTML Chunker
+ */
+export class HTMLChunker extends BaseChunker {
+    strategy = "html";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 0,
+        };
+    }
+    async doChunk(content, config) {
+        const maxSize = config.maxSize ?? 1000;
+        // Strip HTML tags for text content
+        const textContent = this.stripHtml(content);
+        // Use simple character-based splitting for now
+        const segments = this.splitBySizeWithOverlap(textContent, maxSize, 0);
+        return segments.map((segment, index) => this.createChunk(segment.text, index, segment.start, segment.end));
+    }
+    /**
+     * Strip HTML tags from content
+     */
+    stripHtml(html) {
+        return html
+            .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
+            .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
+            .replace(/<[^>]+>/g, " ")
+            .replace(/\s+/g, " ")
+            .trim();
+    }
+}
+//# sourceMappingURL=HTMLChunker.js.map

package/dist/lib/rag/chunkers/JSONChunker.d.ts

@@ -0,0 +1,19 @@
+/**
+ * JSON Chunker
+ *
+ * Splits JSON documents by object boundaries.
+ */
+import type { Chunk, ChunkerConfig, ChunkingStrategy } from "../types.js";
+import { BaseChunker } from "./BaseChunker.js";
+/**
+ * JSON Chunker
+ */
+export declare class JSONChunker extends BaseChunker {
+    readonly strategy: ChunkingStrategy;
+    getDefaultConfig(): ChunkerConfig;
+    protected doChunk(content: string, config: ChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Flatten JSON into array of objects
+     */
+    private flattenJson;
+}

package/dist/lib/rag/chunkers/JSONChunker.js

@@ -0,0 +1,69 @@
+/**
+ * JSON Chunker
+ *
+ * Splits JSON documents by object boundaries.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+import { ChunkingError, RAGErrorCodes } from "../errors/RAGError.js";
+/**
+ * JSON Chunker
+ */
+export class JSONChunker extends BaseChunker {
+    strategy = "json";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 0,
+        };
+    }
+    async doChunk(content, config) {
+        const maxSize = config.maxSize ?? 1000;
+        let parsed;
+        try {
+            parsed = JSON.parse(content);
+        }
+        catch {
+            throw new ChunkingError("Invalid JSON content", {
+                code: RAGErrorCodes.CHUNKING_INVALID_CONFIG,
+                strategy: this.strategy,
+            });
+        }
+        const chunks = [];
+        const items = this.flattenJson(parsed);
+        for (let i = 0; i < items.length; i++) {
+            const item = items[i];
+            if (!item) {
+                continue;
+            }
+            const jsonString = JSON.stringify(item, null, 2);
+            if (jsonString.length <= maxSize) {
+                const startOffset = content.indexOf(jsonString.slice(0, 20));
+                chunks.push(this.createChunk(jsonString, i, startOffset >= 0 ? startOffset : i * maxSize, startOffset >= 0
+                    ? startOffset + jsonString.length
+                    : (i + 1) * maxSize));
+            }
+            else {
+                // Split large objects
+                const segments = this.splitBySizeWithOverlap(jsonString, maxSize, 0);
+                for (const segment of segments) {
+                    chunks.push(this.createChunk(segment.text, chunks.length, segment.start, segment.end));
+                }
+            }
+        }
+        return chunks;
+    }
+    /**
+     * Flatten JSON into array of objects
+     */
+    flattenJson(data) {
+        if (Array.isArray(data)) {
+            return data;
+        }
+        if (typeof data === "object" && data !== null) {
+            return [data];
+        }
+        return [{ value: data }];
+    }
+}
+//# sourceMappingURL=JSONChunker.js.map

package/dist/lib/rag/chunkers/LaTeXChunker.d.ts

@@ -0,0 +1,15 @@
+/**
+ * LaTeX Chunker
+ *
+ * Splits LaTeX documents by sections and environments.
+ */
+import type { Chunk, ChunkerConfig, ChunkingStrategy } from "../types.js";
+import { BaseChunker } from "./BaseChunker.js";
+/**
+ * LaTeX Chunker
+ */
+export declare class LaTeXChunker extends BaseChunker {
+    readonly strategy: ChunkingStrategy;
+    getDefaultConfig(): ChunkerConfig;
+    protected doChunk(content: string, config: ChunkerConfig): Promise<Chunk[]>;
+}

package/dist/lib/rag/chunkers/LaTeXChunker.js

@@ -0,0 +1,64 @@
+/**
+ * LaTeX Chunker
+ *
+ * Splits LaTeX documents by sections and environments.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/**
+ * LaTeX Chunker
+ */
+export class LaTeXChunker extends BaseChunker {
+    strategy = "latex";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 0,
+        };
+    }
+    async doChunk(content, config) {
+        const maxSize = config.maxSize ?? 1000;
+        // Split by sections
+        const sectionPattern = /\\(?:section|subsection|subsubsection|chapter|paragraph)\{[^}]+\}/g;
+        const sections = [];
+        let lastIndex = 0;
+        let match;
+        while ((match = sectionPattern.exec(content)) !== null) {
+            if (match.index > lastIndex) {
+                sections.push(content.slice(lastIndex, match.index));
+            }
+            lastIndex = match.index;
+        }
+        if (lastIndex < content.length) {
+            sections.push(content.slice(lastIndex));
+        }
+        if (sections.length === 0) {
+            sections.push(content);
+        }
+        const chunks = [];
+        let offset = 0;
+        for (const section of sections) {
+            const trimmed = section.trim();
+            if (!trimmed) {
+                continue;
+            }
+            if (trimmed.length <= maxSize) {
+                const startOffset = content.indexOf(trimmed, offset);
+                chunks.push(this.createChunk(trimmed, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
+                    ? startOffset + trimmed.length
+                    : offset + trimmed.length));
+                if (startOffset >= 0) {
+                    offset = startOffset + 1;
+                }
+            }
+            else {
+                const segments = this.splitBySizeWithOverlap(trimmed, maxSize, 0);
+                for (const segment of segments) {
+                    chunks.push(this.createChunk(segment.text, chunks.length, segment.start, segment.end));
+                }
+            }
+        }
+        return chunks;
+    }
+}
+//# sourceMappingURL=LaTeXChunker.js.map

package/dist/lib/rag/chunkers/MarkdownChunker.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Markdown Chunker
+ *
+ * Splits markdown content by headers and structural elements.
+ */
+import type { Chunk, ChunkerConfig, ChunkingStrategy } from "../types.js";
+import { BaseChunker } from "./BaseChunker.js";
+/**
+ * Markdown Chunker
+ */
+export declare class MarkdownChunker extends BaseChunker {
+    readonly strategy: ChunkingStrategy;
+    getDefaultConfig(): ChunkerConfig;
+    protected doChunk(content: string, config: ChunkerConfig): Promise<Chunk[]>;
+}

package/dist/lib/rag/chunkers/MarkdownChunker.js

@@ -0,0 +1,103 @@
+/**
+ * Markdown Chunker
+ *
+ * Splits markdown content by headers and structural elements.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/**
+ * Markdown Chunker
+ */
+export class MarkdownChunker extends BaseChunker {
+    strategy = "markdown";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 0,
+        };
+    }
+    async doChunk(content, config) {
+        const maxSize = config.maxSize ?? 1000;
+        // Split by headers
+        const headerPattern = /^(#{1,6})\s+(.+)$/gm;
+        const sections = [];
+        let lastIndex = 0;
+        let match = headerPattern.exec(content);
+        while (match !== null) {
+            // Add content before this header
+            if (match.index > lastIndex) {
+                const prevContent = content.slice(lastIndex, match.index).trim();
+                if (prevContent && sections.length > 0) {
+                    const lastSection = sections[sections.length - 1];
+                    if (lastSection) {
+                        lastSection.content += "\n\n" + prevContent;
+                    }
+                }
+                else if (prevContent) {
+                    sections.push({ header: "", content: prevContent, level: 0 });
+                }
+            }
+            sections.push({
+                header: match[0],
+                content: "",
+                level: match[1]?.length ?? 1,
+            });
+            lastIndex = match.index + match[0].length;
+            match = headerPattern.exec(content);
+        }
+        // Add remaining content
+        if (lastIndex < content.length) {
+            const remaining = content.slice(lastIndex).trim();
+            if (remaining) {
+                if (sections.length > 0) {
+                    const lastSection = sections[sections.length - 1];
+                    if (lastSection) {
+                        lastSection.content += remaining;
+                    }
+                }
+                else {
+                    sections.push({ header: "", content: remaining, level: 0 });
+                }
+            }
+        }
+        // Convert sections to chunks
+        const chunks = [];
+        let offset = 0;
+        for (let i = 0; i < sections.length; i++) {
+            const section = sections[i];
+            if (!section) {
+                continue;
+            }
+            const fullContent = section.header
+                ? section.header + "\n\n" + section.content.trim()
+                : section.content.trim();
+            if (!fullContent) {
+                continue;
+            }
+            // Split if too large
+            if (fullContent.length > maxSize) {
+                const subChunks = this.splitBySizeWithOverlap(fullContent, maxSize, 0);
+                for (const sub of subChunks) {
+                    const startOffset = content.indexOf(sub.text, offset);
+                    chunks.push(this.createChunk(sub.text, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
+                        ? startOffset + sub.text.length
+                        : offset + sub.text.length, "unknown", { sectionContext: section.header }));
+                    if (startOffset >= 0) {
+                        offset = startOffset + 1;
+                    }
+                }
+            }
+            else {
+                const startOffset = content.indexOf(fullContent, offset);
+                chunks.push(this.createChunk(fullContent, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
+                    ? startOffset + fullContent.length
+                    : offset + fullContent.length, "unknown", { sectionContext: section.header }));
+                if (startOffset >= 0) {
+                    offset = startOffset + 1;
+                }
+            }
+        }
+        return chunks;
+    }
+}
+//# sourceMappingURL=MarkdownChunker.js.map

package/dist/lib/rag/chunkers/RecursiveChunker.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Recursive Chunker
+ *
+ * Recursively splits text using an ordered list of separators.
+ * Tries each separator in order until chunks are small enough.
+ */
+import type { Chunk, ChunkerConfig, ChunkingStrategy } from "../types.js";
+import { BaseChunker } from "./BaseChunker.js";
+/**
+ * Recursive Chunker
+ *
+ * Splits content using ordered separators, recursively breaking
+ * down text until chunks meet size requirements.
+ */
+export declare class RecursiveChunker extends BaseChunker {
+    readonly strategy: ChunkingStrategy;
+    getDefaultConfig(): ChunkerConfig;
+    protected doChunk(content: string, config: ChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Recursively split text using separators
+     */
+    private recursiveSplit;
+    /**
+     * Apply overlap between chunks
+     */
+    private applyOverlap;
+}

package/dist/lib/rag/chunkers/RecursiveChunker.js

@@ -0,0 +1,140 @@
+/**
+ * Recursive Chunker
+ *
+ * Recursively splits text using an ordered list of separators.
+ * Tries each separator in order until chunks are small enough.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/**
+ * Default separators for recursive splitting
+ */
+const DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", " ", ""];
+/**
+ * Recursive Chunker
+ *
+ * Splits content using ordered separators, recursively breaking
+ * down text until chunks meet size requirements.
+ */
+export class RecursiveChunker extends BaseChunker {
+    strategy = "recursive";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 100,
+            separators: DEFAULT_SEPARATORS,
+        };
+    }
+    async doChunk(content, config) {
+        const recursiveConfig = config;
+        const maxSize = config.maxSize ?? 1000;
+        const overlap = config.overlap ?? 100;
+        const separators = recursiveConfig.separators ?? DEFAULT_SEPARATORS;
+        const keepSeparators = recursiveConfig.keepSeparators ?? true;
+        const chunks = [];
+        let offset = 0;
+        const textChunks = this.recursiveSplit(content, separators, maxSize, overlap, keepSeparators);
+        for (let i = 0; i < textChunks.length; i++) {
+            const text = textChunks[i];
+            if (!text) {
+                continue;
+            }
+            const startOffset = content.indexOf(text, offset);
+            const endOffset = startOffset + text.length;
+            chunks.push(this.createChunk(text, i, startOffset, endOffset));
+            offset = Math.max(offset, startOffset + 1);
+        }
+        return chunks;
+    }
+    /**
+     * Recursively split text using separators
+     */
+    recursiveSplit(text, separators, maxSize, overlap, keepSeparators) {
+        if (text.length <= maxSize) {
+            return [text];
+        }
+        // Find the first separator that exists in the text
+        let separator = "";
+        for (const sep of separators) {
+            if (sep === "" || text.includes(sep)) {
+                separator = sep;
+                break;
+            }
+        }
+        // If no separator found or empty separator, split by size
+        if (separator === "") {
+            const result = [];
+            let start = 0;
+            while (start < text.length) {
+                const end = Math.min(start + maxSize, text.length);
+                result.push(text.slice(start, end));
+                const previousStart = start;
+                start = end - overlap;
+                if (start <= previousStart) {
+                    start = previousStart + 1;
+                }
+                if (start >= text.length) {
+                    break;
+                }
+            }
+            return result;
+        }
+        // Split by separator
+        const parts = text.split(separator);
+        const result = [];
+        let currentChunk = "";
+        for (let i = 0; i < parts.length; i++) {
+            const part = parts[i];
+            const addSeparator = keepSeparators && i < parts.length - 1;
+            const toAdd = part + (addSeparator ? separator : "");
+            if (currentChunk.length + toAdd.length <= maxSize) {
+                currentChunk += toAdd;
+            }
+            else {
+                // Current chunk is full
+                if (currentChunk.length > 0) {
+                    result.push(currentChunk);
+                }
+                // If the part itself is too large, recursively split it
+                if (toAdd.length > maxSize) {
+                    const remainingSeparators = separators.slice(separators.indexOf(separator) + 1);
+                    const subChunks = this.recursiveSplit(toAdd, remainingSeparators, maxSize, overlap, keepSeparators);
+                    result.push(...subChunks);
+                    currentChunk = "";
+                }
+                else {
+                    currentChunk = toAdd;
+                }
+            }
+        }
+        if (currentChunk.length > 0) {
+            result.push(currentChunk);
+        }
+        // Apply overlap between chunks
+        if (overlap > 0 && result.length > 1) {
+            return this.applyOverlap(result, overlap);
+        }
+        return result;
+    }
+    /**
+     * Apply overlap between chunks
+     */
+    applyOverlap(chunks, overlap) {
+        if (chunks.length <= 1) {
+            return chunks;
+        }
+        const result = [];
+        for (let i = 0; i < chunks.length; i++) {
+            let chunk = chunks[i] ?? "";
+            // Add overlap from previous chunk
+            if (i > 0 && chunks[i - 1]) {
+                const prevChunk = chunks[i - 1];
+                const overlapText = prevChunk.slice(-Math.min(overlap, prevChunk.length));
+                chunk = overlapText + chunk;
+            }
+            result.push(chunk);
+        }
+        return result;
+    }
+}
+//# sourceMappingURL=RecursiveChunker.js.map

package/dist/lib/rag/chunkers/SemanticMarkdownChunker.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Semantic Markdown Chunker
+ *
+ * Combines markdown splitting with semantic similarity for intelligent merging.
+ */
+import type { Chunk, ChunkerConfig, ChunkingStrategy } from "../types.js";
+import { BaseChunker } from "./BaseChunker.js";
+/**
+ * Semantic Markdown Chunker
+ *
+ * Extends markdown chunking with semantic awareness.
+ * Can be enhanced with embedding-based similarity.
+ */
+export declare class SemanticMarkdownChunker extends BaseChunker {
+    readonly strategy: ChunkingStrategy;
+    getDefaultConfig(): ChunkerConfig;
+    protected doChunk(content: string, config: ChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Merge small sections to optimize chunk sizes
+     */
+    private mergeSmallSections;
+}

package/dist/lib/rag/chunkers/SemanticMarkdownChunker.js

@@ -0,0 +1,139 @@
+/**
+ * Semantic Markdown Chunker
+ *
+ * Combines markdown splitting with semantic similarity for intelligent merging.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/**
+ * Semantic Markdown Chunker
+ *
+ * Extends markdown chunking with semantic awareness.
+ * Can be enhanced with embedding-based similarity.
+ */
+export class SemanticMarkdownChunker extends BaseChunker {
+    strategy = "semantic-markdown";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 100,
+        };
+    }
+    async doChunk(content, config) {
+        const maxSize = config.maxSize ?? 1000;
+        const overlap = config.overlap ?? 100;
+        // First, split by markdown headers
+        const headerPattern = /^(#{1,6})\s+(.+)$/gm;
+        const sections = [];
+        let lastIndex = 0;
+        let match = headerPattern.exec(content);
+        while (match !== null) {
+            if (match.index > lastIndex) {
+                const prevContent = content.slice(lastIndex, match.index).trim();
+                if (prevContent && sections.length > 0) {
+                    const lastSection = sections[sections.length - 1];
+                    if (lastSection) {
+                        lastSection.content += "\n\n" + prevContent;
+                    }
+                }
+                else if (prevContent) {
+                    sections.push({ header: "", content: prevContent });
+                }
+            }
+            sections.push({ header: match[0], content: "" });
+            lastIndex = match.index + match[0].length;
+            match = headerPattern.exec(content);
+        }
+        if (lastIndex < content.length) {
+            const remaining = content.slice(lastIndex).trim();
+            if (remaining) {
+                if (sections.length > 0) {
+                    const lastSection = sections[sections.length - 1];
+                    if (lastSection) {
+                        lastSection.content += remaining;
+                    }
+                }
+                else {
+                    sections.push({ header: "", content: remaining });
+                }
+            }
+        }
+        // Merge small sections that are semantically related
+        const mergedSections = this.mergeSmallSections(sections, maxSize);
+        // Convert to chunks
+        const chunks = [];
+        let offset = 0;
+        for (let i = 0; i < mergedSections.length; i++) {
+            const section = mergedSections[i];
+            if (!section) {
+                continue;
+            }
+            const fullContent = section.header
+                ? section.header + "\n\n" + section.content.trim()
+                : section.content.trim();
+            if (!fullContent) {
+                continue;
+            }
+            if (fullContent.length > maxSize) {
+                const segments = this.splitBySizeWithOverlap(fullContent, maxSize, overlap);
+                for (const segment of segments) {
+                    const startOffset = content.indexOf(segment.text.slice(0, 50), offset);
+                    chunks.push(this.createChunk(segment.text, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
+                        ? startOffset + segment.text.length
+                        : offset + segment.text.length, "unknown", { sectionContext: section.header }));
+                    if (startOffset >= 0) {
+                        offset = startOffset + 1;
+                    }
+                }
+            }
+            else {
+                const startOffset = content.indexOf(fullContent.slice(0, 50), offset);
+                chunks.push(this.createChunk(fullContent, chunks.length, startOffset >= 0 ? startOffset : offset, startOffset >= 0
+                    ? startOffset + fullContent.length
+                    : offset + fullContent.length, "unknown", { sectionContext: section.header }));
+                if (startOffset >= 0) {
+                    offset = startOffset + 1;
+                }
+            }
+        }
+        return chunks;
+    }
+    /**
+     * Merge small sections to optimize chunk sizes
+     */
+    mergeSmallSections(sections, maxSize) {
+        const result = [];
+        let current = null;
+        for (const section of sections) {
+            const fullContent = section.header
+                ? section.header + "\n\n" + section.content.trim()
+                : section.content.trim();
+            const sectionLength = fullContent.length;
+            if (!current) {
+                current = { ...section };
+                continue;
+            }
+            const currentLength = current.header
+                ? current.header.length + current.content.length + 2
+                : current.content.length;
+            // Merge if combined size is within limit
+            if (currentLength + sectionLength <= maxSize) {
+                if (section.header) {
+                    current.content += "\n\n" + section.header + "\n" + section.content;
+                }
+                else {
+                    current.content += "\n\n" + section.content;
+                }
+            }
+            else {
+                result.push(current);
+                current = { ...section };
+            }
+        }
+        if (current) {
+            result.push(current);
+        }
+        return result;
+    }
+}
+//# sourceMappingURL=SemanticMarkdownChunker.js.map

package/dist/lib/rag/chunkers/SentenceChunker.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Sentence Chunker
+ *
+ * Splits text by sentence boundaries for semantically meaningful chunks.
+ */
+import type { Chunk, ChunkerConfig, ChunkingStrategy } from "../types.js";
+import { BaseChunker } from "./BaseChunker.js";
+/**
+ * Sentence Chunker
+ */
+export declare class SentenceChunker extends BaseChunker {
+    readonly strategy: ChunkingStrategy;
+    getDefaultConfig(): ChunkerConfig;
+    protected doChunk(content: string, config: ChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Split content into sentences
+     */
+    private splitIntoSentences;
+}