@juspay/neurolink 9.2.0 → 9.3.0

package/dist/lib/rag/chunking/htmlChunker.js

@@ -0,0 +1,248 @@
+/**
+ * HTML-aware Chunker
+ *
+ * Splits HTML documents based on tag structure while preserving semantics.
+ * Best for web pages, email templates, and structured HTML content.
+ */
+import { randomUUID } from "crypto";
+/**
+ * HTML-aware chunker implementation
+ * Splits based on HTML structure (tags, elements)
+ */
+export class HTMLChunker {
+    strategy = "html";
+    defaultSplitTags = [
+        "div",
+        "p",
+        "section",
+        "article",
+        "main",
+        "aside",
+        "header",
+        "footer",
+        "nav",
+        "li",
+        "tr",
+        "td",
+        "th",
+    ];
+    defaultPreserveTags = [
+        "pre",
+        "code",
+        "table",
+        "ul",
+        "ol",
+        "blockquote",
+    ];
+    async chunk(text, config) {
+        const { maxSize = 1000, overlap = 0, splitTags = this.defaultSplitTags, preserveTags = this.defaultPreserveTags, extractTextOnly = false, includeTagMetadata = true, trimWhitespace = true, metadata = {}, } = config || {};
+        const documentId = randomUUID();
+        const chunks = [];
+        if (!text || text.length === 0) {
+            return chunks;
+        }
+        // Extract and split by structural tags
+        const sections = this.splitByTags(text, splitTags, preserveTags);
+        let chunkIndex = 0;
+        let currentPosition = 0;
+        for (const section of sections) {
+            const { content, tagName, attributes } = section;
+            // Process content
+            let processedContent = content;
+            if (extractTextOnly) {
+                processedContent = this.extractText(content);
+            }
+            // Split if content is too large
+            const contentChunks = this.splitContent(processedContent, maxSize, overlap);
+            for (const contentChunk of contentChunks) {
+                const finalText = trimWhitespace ? contentChunk.trim() : contentChunk;
+                if (finalText.length > 0) {
+                    const chunkMetadata = {
+                        ...metadata,
+                    };
+                    if (includeTagMetadata && tagName) {
+                        chunkMetadata.tagName = tagName;
+                        if (attributes && Object.keys(attributes).length > 0) {
+                            chunkMetadata.attributes = attributes;
+                        }
+                    }
+                    chunks.push({
+                        id: randomUUID(),
+                        text: finalText,
+                        metadata: {
+                            documentId,
+                            chunkIndex,
+                            startPosition: currentPosition,
+                            endPosition: currentPosition + contentChunk.length,
+                            documentType: "html",
+                            custom: chunkMetadata,
+                        },
+                    });
+                    chunkIndex++;
+                }
+                currentPosition += contentChunk.length;
+            }
+        }
+        // Update total chunks count
+        chunks.forEach((chunk) => {
+            chunk.metadata.totalChunks = chunks.length;
+        });
+        return chunks;
+    }
+    /**
+     * Split HTML by structural tags
+     */
+    splitByTags(html, splitTags, preserveTags) {
+        const sections = [];
+        // Create regex pattern for split tags
+        const tagPattern = new RegExp(`<(${splitTags.join("|")})([^>]*)>([\\s\\S]*?)</\\1>`, "gi");
+        let lastIndex = 0;
+        let match;
+        // Reset regex
+        tagPattern.lastIndex = 0;
+        while ((match = tagPattern.exec(html)) !== null) {
+            // Content before this tag
+            if (match.index > lastIndex) {
+                const beforeContent = html.slice(lastIndex, match.index).trim();
+                if (beforeContent.length > 0) {
+                    sections.push({
+                        content: beforeContent,
+                    });
+                }
+            }
+            const tagName = match[1].toLowerCase();
+            const attributeString = match[2];
+            const innerContent = match[3];
+            // Parse attributes
+            const attributes = this.parseAttributes(attributeString);
+            // Check if this tag should be preserved as a unit
+            const shouldPreserve = preserveTags.some((pt) => innerContent.toLowerCase().includes(`<${pt}`));
+            if (shouldPreserve) {
+                // Keep the full tag content
+                sections.push({
+                    content: match[0],
+                    tagName,
+                    attributes,
+                });
+            }
+            else {
+                // Just the inner content
+                sections.push({
+                    content: innerContent,
+                    tagName,
+                    attributes,
+                });
+            }
+            lastIndex = match.index + match[0].length;
+        }
+        // Don't forget content after the last tag
+        if (lastIndex < html.length) {
+            const remaining = html.slice(lastIndex).trim();
+            if (remaining.length > 0) {
+                sections.push({
+                    content: remaining,
+                });
+            }
+        }
+        // If no tags found, return entire text as one section
+        if (sections.length === 0 && html.trim()) {
+            sections.push({
+                content: html.trim(),
+            });
+        }
+        return sections;
+    }
+    /**
+     * Parse HTML attributes from string
+     */
+    parseAttributes(attributeString) {
+        const attributes = {};
+        const attrPattern = /(\w+)(?:=["']([^"']*?)["'])?/g;
+        let match;
+        while ((match = attrPattern.exec(attributeString)) !== null) {
+            const name = match[1];
+            const value = match[2] || "";
+            attributes[name] = value;
+        }
+        return attributes;
+    }
+    /**
+     * Extract plain text from HTML
+     */
+    extractText(html) {
+        return (html
+            // Remove script and style elements
+            .replace(/<script[\s\S]*?<\/script>/gi, "")
+            .replace(/<style[\s\S]*?<\/style>/gi, "")
+            // Remove HTML comments
+            .replace(/<!--[\s\S]*?-->/g, "")
+            // Replace block elements with newlines
+            .replace(/<\/(p|div|br|h[1-6]|li|tr)>/gi, "\n")
+            // Remove remaining tags
+            .replace(/<[^>]+>/g, "")
+            // Decode common HTML entities
+            .replace(/&nbsp;/gi, " ")
+            .replace(/&amp;/gi, "&")
+            .replace(/&lt;/gi, "<")
+            .replace(/&gt;/gi, ">")
+            .replace(/&quot;/gi, '"')
+            .replace(/&#039;/gi, "'")
+            // Normalize whitespace
+            .replace(/\s+/g, " ")
+            .trim());
+    }
+    /**
+     * Split content that exceeds max size
+     */
+    splitContent(content, maxSize, overlap) {
+        const effectiveMaxSize = Math.max(maxSize, 1);
+        const effectiveOverlap = Math.min(Math.max(overlap, 0), effectiveMaxSize - 1);
+        if (content.length <= effectiveMaxSize) {
+            return [content];
+        }
+        const chunks = [];
+        let start = 0;
+        while (start < content.length) {
+            let end = Math.min(start + effectiveMaxSize, content.length);
+            // Try to break at a natural boundary
+            if (end < content.length) {
+                const searchStart = Math.max(start, end - 100);
+                const searchText = content.slice(searchStart, end);
+                // Look for paragraph/sentence break
+                const breakMatch = searchText.match(/[.!?\n]\s+/);
+                if (breakMatch && breakMatch.index !== undefined) {
+                    end = searchStart + breakMatch.index + 1;
+                }
+            }
+            chunks.push(content.slice(start, end));
+            start = Math.max(start + 1, end - effectiveOverlap);
+        }
+        return chunks;
+    }
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        const htmlConfig = config;
+        if (htmlConfig.maxSize !== undefined && htmlConfig.maxSize <= 0) {
+            errors.push("maxSize must be greater than 0");
+        }
+        if (htmlConfig.overlap !== undefined && htmlConfig.overlap < 0) {
+            errors.push("overlap must be non-negative");
+        }
+        if (htmlConfig.overlap !== undefined &&
+            htmlConfig.maxSize !== undefined &&
+            htmlConfig.overlap >= htmlConfig.maxSize) {
+            errors.push("overlap must be less than maxSize");
+        }
+        if (htmlConfig.splitTags !== undefined &&
+            htmlConfig.splitTags.length === 0) {
+            warnings.push("No split tags specified, using defaults");
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+}
+//# sourceMappingURL=htmlChunker.js.map

package/dist/lib/rag/chunking/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Chunking Module Exports
+ *
+ * Provides all chunking strategies and the chunker registry.
+ */
+export { ChunkerRegistry, chunkText } from "./chunkerRegistry.js";
+export { CharacterChunker } from "./characterChunker.js";
+export { RecursiveChunker } from "./recursiveChunker.js";
+export { SentenceChunker } from "./sentenceChunker.js";
+export { TokenChunker } from "./tokenChunker.js";
+export { MarkdownChunker } from "./markdownChunker.js";
+export { HTMLChunker } from "./htmlChunker.js";
+export { JSONChunker } from "./jsonChunker.js";
+export { LaTeXChunker } from "./latexChunker.js";
+export { SemanticChunker } from "./semanticChunker.js";

package/dist/lib/rag/chunking/index.js

@@ -0,0 +1,18 @@
+/**
+ * Chunking Module Exports
+ *
+ * Provides all chunking strategies and the chunker registry.
+ */
+// Registry
+export { ChunkerRegistry, chunkText } from "./chunkerRegistry.js";
+// Individual chunkers
+export { CharacterChunker } from "./characterChunker.js";
+export { RecursiveChunker } from "./recursiveChunker.js";
+export { SentenceChunker } from "./sentenceChunker.js";
+export { TokenChunker } from "./tokenChunker.js";
+export { MarkdownChunker } from "./markdownChunker.js";
+export { HTMLChunker } from "./htmlChunker.js";
+export { JSONChunker } from "./jsonChunker.js";
+export { LaTeXChunker } from "./latexChunker.js";
+export { SemanticChunker } from "./semanticChunker.js";
+//# sourceMappingURL=index.js.map

package/dist/lib/rag/chunking/jsonChunker.d.ts

@@ -0,0 +1,20 @@
+/**
+ * JSON-aware Chunker
+ *
+ * Splits JSON documents based on structure (arrays, objects, keys).
+ * Best for API responses, configuration files, and structured data.
+ */
+import type { Chunker, Chunk, ChunkerValidationResult, JSONChunkerConfig, BaseChunkerConfig } from "../types.js";
+/**
+ * JSON-aware chunker implementation
+ * Splits based on JSON structure
+ */
+export declare class JSONChunker implements Chunker {
+    readonly strategy: "json";
+    chunk(text: string, config?: JSONChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Recursively extract chunks from JSON structure
+     */
+    private extractChunks;
+    validateConfig(config: BaseChunkerConfig): ChunkerValidationResult;
+}

package/dist/lib/rag/chunking/jsonChunker.js

@@ -0,0 +1,282 @@
+/**
+ * JSON-aware Chunker
+ *
+ * Splits JSON documents based on structure (arrays, objects, keys).
+ * Best for API responses, configuration files, and structured data.
+ */
+import { randomUUID } from "crypto";
+/**
+ * JSON-aware chunker implementation
+ * Splits based on JSON structure
+ */
+export class JSONChunker {
+    strategy = "json";
+    async chunk(text, config) {
+        const { maxSize = 1000, maxDepth = 10, splitKeys = [], preserveKeys = [], includeJsonPath = true, trimWhitespace = true, metadata = {}, } = config || {};
+        const documentId = randomUUID();
+        const chunks = [];
+        if (!text || text.length === 0) {
+            return chunks;
+        }
+        // Parse JSON
+        let jsonData;
+        try {
+            jsonData = JSON.parse(text);
+        }
+        catch {
+            // If not valid JSON, treat as plain text
+            chunks.push({
+                id: randomUUID(),
+                text: trimWhitespace ? text.trim() : text,
+                metadata: {
+                    documentId,
+                    chunkIndex: 0,
+                    totalChunks: 1,
+                    startPosition: 0,
+                    endPosition: text.length,
+                    documentType: "json",
+                    custom: {
+                        ...metadata,
+                        parseError: "Invalid JSON",
+                    },
+                },
+            });
+            return chunks;
+        }
+        // Extract chunks from JSON structure
+        const extractedChunks = this.extractChunks({
+            data: jsonData,
+            path: "",
+            depth: 0,
+            maxDepth,
+            maxSize,
+            splitKeys,
+            preserveKeys,
+            includeJsonPath,
+        });
+        // Convert to Chunk objects
+        let chunkIndex = 0;
+        let currentPosition = 0;
+        for (const extracted of extractedChunks) {
+            const chunkText = JSON.stringify(extracted.value, null, 2);
+            const finalText = trimWhitespace ? chunkText.trim() : chunkText;
+            if (finalText.length > 0) {
+                const chunkMetadata = {
+                    ...metadata,
+                };
+                if (includeJsonPath && extracted.path) {
+                    chunkMetadata.jsonPath = extracted.path;
+                }
+                chunks.push({
+                    id: randomUUID(),
+                    text: finalText,
+                    metadata: {
+                        documentId,
+                        chunkIndex,
+                        startPosition: currentPosition,
+                        endPosition: currentPosition + finalText.length,
+                        documentType: "json",
+                        jsonPath: extracted.path,
+                        custom: chunkMetadata,
+                    },
+                });
+                chunkIndex++;
+                currentPosition += finalText.length;
+            }
+        }
+        // Update total chunks count
+        chunks.forEach((chunk) => {
+            chunk.metadata.totalChunks = chunks.length;
+        });
+        return chunks;
+    }
+    /**
+     * Recursively extract chunks from JSON structure
+     */
+    extractChunks(options) {
+        const { data, path, depth, maxDepth, maxSize, splitKeys, preserveKeys, includeJsonPath, } = options;
+        const results = [];
+        // Check depth limit
+        if (depth > maxDepth) {
+            results.push({ value: data, path });
+            return results;
+        }
+        // Check if this should be preserved as a unit
+        const currentKey = path.split(".").pop() || "";
+        if (preserveKeys.includes(currentKey)) {
+            results.push({ value: data, path });
+            return results;
+        }
+        // Check size - if small enough, keep as one chunk
+        const serialized = JSON.stringify(data, null, 2);
+        if (serialized.length <= maxSize) {
+            results.push({ value: data, path });
+            return results;
+        }
+        // Handle arrays
+        if (Array.isArray(data)) {
+            // Check if array should be split by index
+            if (splitKeys.length === 0 || splitKeys.some((k) => path.endsWith(k))) {
+                // Split array into individual elements or groups
+                let currentGroup = [];
+                let currentGroupSize = 0;
+                for (let i = 0; i < data.length; i++) {
+                    const item = data[i];
+                    const itemSize = JSON.stringify(item, null, 2).length;
+                    if (currentGroupSize + itemSize > maxSize &&
+                        currentGroup.length > 0) {
+                        // Save current group
+                        results.push({
+                            value: currentGroup.length === 1 ? currentGroup[0] : currentGroup,
+                            path: `${path}[${i - currentGroup.length}:${i}]`,
+                        });
+                        currentGroup = [];
+                        currentGroupSize = 0;
+                    }
+                    // If single item is too large, recursively split it
+                    if (itemSize > maxSize) {
+                        const subChunks = this.extractChunks({
+                            data: item,
+                            path: `${path}[${i}]`,
+                            depth: depth + 1,
+                            maxDepth,
+                            maxSize,
+                            splitKeys,
+                            preserveKeys,
+                            includeJsonPath,
+                        });
+                        results.push(...subChunks);
+                    }
+                    else {
+                        currentGroup.push(item);
+                        currentGroupSize += itemSize;
+                    }
+                }
+                // Don't forget the last group
+                if (currentGroup.length > 0) {
+                    results.push({
+                        value: currentGroup.length === 1 ? currentGroup[0] : currentGroup,
+                        path: `${path}[${data.length - currentGroup.length}:${data.length}]`,
+                    });
+                }
+            }
+            else {
+                // Keep array as one unit but may need to truncate
+                results.push({ value: data, path });
+            }
+        }
+        // Handle objects
+        else if (data !== null && typeof data === "object") {
+            const obj = data;
+            const keys = Object.keys(obj);
+            // Check if any keys should be split
+            const keysToSplit = keys.filter((k) => splitKeys.length === 0 || splitKeys.includes(k));
+            if (keysToSplit.length > 0) {
+                let currentObj = {};
+                let currentObjSize = 0;
+                for (const key of keys) {
+                    const value = obj[key];
+                    const valueSize = JSON.stringify({ [key]: value }, null, 2).length;
+                    // Check if this key should be split out
+                    if (splitKeys.includes(key)) {
+                        // Save current object first if it has content
+                        if (Object.keys(currentObj).length > 0) {
+                            results.push({
+                                value: currentObj,
+                                path: path,
+                            });
+                            currentObj = {};
+                            currentObjSize = 0;
+                        }
+                        // Recursively process this value
+                        const subChunks = this.extractChunks({
+                            data: value,
+                            path: path ? `${path}.${key}` : key,
+                            depth: depth + 1,
+                            maxDepth,
+                            maxSize,
+                            splitKeys,
+                            preserveKeys,
+                            includeJsonPath,
+                        });
+                        results.push(...subChunks);
+                    }
+                    else if (currentObjSize + valueSize > maxSize &&
+                        Object.keys(currentObj).length > 0) {
+                        // Save current object
+                        results.push({
+                            value: currentObj,
+                            path: path,
+                        });
+                        currentObj = { [key]: value };
+                        currentObjSize = valueSize;
+                    }
+                    else {
+                        currentObj[key] = value;
+                        currentObjSize += valueSize;
+                    }
+                }
+                // Don't forget the last object
+                if (Object.keys(currentObj).length > 0) {
+                    results.push({
+                        value: currentObj,
+                        path: path,
+                    });
+                }
+            }
+            else {
+                // Process each key individually
+                for (const key of keys) {
+                    const value = obj[key];
+                    const keyPath = path ? `${path}.${key}` : key;
+                    const valueSize = JSON.stringify(value, null, 2).length;
+                    if (valueSize > maxSize) {
+                        // Recursively split
+                        const subChunks = this.extractChunks({
+                            data: value,
+                            path: keyPath,
+                            depth: depth + 1,
+                            maxDepth,
+                            maxSize,
+                            splitKeys,
+                            preserveKeys,
+                            includeJsonPath,
+                        });
+                        results.push(...subChunks);
+                    }
+                    else {
+                        results.push({
+                            value: { [key]: value },
+                            path: keyPath,
+                        });
+                    }
+                }
+            }
+        }
+        // Primitive values
+        else {
+            results.push({ value: data, path });
+        }
+        return results;
+    }
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        const jsonConfig = config;
+        if (jsonConfig.maxSize !== undefined && jsonConfig.maxSize <= 0) {
+            errors.push("maxSize must be greater than 0");
+        }
+        if (jsonConfig.maxDepth !== undefined && jsonConfig.maxDepth < 1) {
+            errors.push("maxDepth must be at least 1");
+        }
+        if (jsonConfig.maxDepth !== undefined && jsonConfig.maxDepth > 100) {
+            warnings.push("Very high maxDepth may cause performance issues");
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+}
+//# sourceMappingURL=jsonChunker.js.map

package/dist/lib/rag/chunking/latexChunker.d.ts

@@ -0,0 +1,26 @@
+/**
+ * LaTeX-aware Chunker
+ *
+ * Splits LaTeX documents based on structure (sections, environments, math).
+ * Best for academic papers, scientific documents, and mathematical content.
+ */
+import type { BaseChunkerConfig, Chunk, Chunker, ChunkerValidationResult, LaTeXChunkerConfig } from "../types.js";
+/**
+ * LaTeX-aware chunker implementation
+ * Splits based on LaTeX structure (sections, environments)
+ */
+export declare class LaTeXChunker implements Chunker {
+    readonly strategy: "latex";
+    private readonly defaultSplitEnvironments;
+    private readonly mathEnvironments;
+    chunk(text: string, config?: LaTeXChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Split LaTeX by sectioning commands
+     */
+    private splitBySections;
+    /**
+     * Split content that exceeds max size
+     */
+    private splitContent;
+    validateConfig(config: BaseChunkerConfig): ChunkerValidationResult;
+}