@juspay/neurolink 9.30.0 → 9.31.0

package/dist/client/rag/chunkers/SentenceChunker.js

@@ -0,0 +1,66 @@
+/**
+ * Sentence Chunker
+ *
+ * Splits text by sentence boundaries for semantically meaningful chunks.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/**
+ * Sentence Chunker
+ */
+export class SentenceChunker extends BaseChunker {
+    strategy = "sentence";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 1, // Overlap in sentences
+        };
+    }
+    async doChunk(content, config) {
+        const maxSize = config.maxSize ?? 1000;
+        // Simple sentence splitting (can be enhanced with NLP)
+        const sentences = this.splitIntoSentences(content);
+        const chunks = [];
+        let currentChunk = "";
+        let currentStart = 0;
+        let chunkIndex = 0;
+        for (const sentence of sentences) {
+            if (currentChunk.length + sentence.length <= maxSize) {
+                currentChunk += sentence;
+            }
+            else {
+                if (currentChunk.length > 0) {
+                    const startOffset = content.indexOf(currentChunk, currentStart);
+                    chunks.push(this.createChunk(currentChunk, chunkIndex++, startOffset, startOffset + currentChunk.length));
+                    currentStart = startOffset + 1;
+                }
+                currentChunk = sentence;
+            }
+        }
+        // Add remaining chunk
+        if (currentChunk.length > 0) {
+            const startOffset = content.indexOf(currentChunk, currentStart);
+            chunks.push(this.createChunk(currentChunk, chunkIndex, startOffset, startOffset + currentChunk.length));
+        }
+        return chunks;
+    }
+    /**
+     * Split content into sentences
+     */
+    splitIntoSentences(content) {
+        // Simple regex-based sentence splitting
+        // Handles common abbreviations and sentence endings
+        const sentencePattern = /[^.!?]*[.!?]+(?:\s|$)/g;
+        const sentences = [];
+        let match;
+        while ((match = sentencePattern.exec(content)) !== null) {
+            sentences.push(match[0]);
+        }
+        // Handle remaining content without sentence ending
+        const lastIndex = sentences.reduce((acc, s) => acc + s.length, 0);
+        if (lastIndex < content.length) {
+            sentences.push(content.slice(lastIndex));
+        }
+        return sentences;
+    }
+}

package/dist/client/rag/chunkers/TokenChunker.js

@@ -0,0 +1,61 @@
+/**
+ * Token Chunker
+ *
+ * Splits text by token count using a tokenizer.
+ * Useful for precise token budget management.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/**
+ * Token Chunker
+ *
+ * Approximates token-based splitting using word count.
+ * For production, integrate with a proper tokenizer (tiktoken, etc.)
+ */
+export class TokenChunker extends BaseChunker {
+    strategy = "token";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 512, // Tokens
+            overlap: 50, // Tokens
+        };
+    }
+    async doChunk(content, config) {
+        const maxTokens = config.maxSize ?? 512;
+        const overlapTokens = config.overlap ?? 50;
+        // Approximate tokenization using words
+        // In production, use a proper tokenizer like tiktoken
+        const words = content.split(/\s+/);
+        const chunks = [];
+        let currentWords = [];
+        let currentStart = 0;
+        let chunkIndex = 0;
+        for (let i = 0; i < words.length; i++) {
+            const word = words[i];
+            if (!word) {
+                continue;
+            }
+            // Estimate tokens (roughly 1.3 tokens per word on average)
+            const estimatedTokens = Math.ceil(currentWords.length * 1.3);
+            if (estimatedTokens >= maxTokens) {
+                const chunkText = currentWords.join(" ");
+                const startOffset = content.indexOf(currentWords[0] ?? "", currentStart);
+                const endOffset = startOffset + chunkText.length;
+                chunks.push(this.createChunk(chunkText, chunkIndex++, startOffset, endOffset));
+                // Keep overlap words
+                const overlapCount = Math.ceil(overlapTokens / 1.3);
+                currentWords = currentWords.slice(-overlapCount);
+                currentStart = endOffset - currentWords.join(" ").length;
+            }
+            currentWords.push(word);
+        }
+        // Add remaining chunk
+        if (currentWords.length > 0) {
+            const chunkText = currentWords.join(" ");
+            const startOffset = content.indexOf(currentWords[0] ?? "", currentStart);
+            const endOffset = startOffset + chunkText.length;
+            chunks.push(this.createChunk(chunkText, chunkIndex, startOffset, endOffset));
+        }
+        return chunks;
+    }
+}

package/dist/client/rag/chunkers/index.js

@@ -0,0 +1,15 @@
+/**
+ * Chunkers Index
+ *
+ * Exports all chunker implementations.
+ */
+export { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+export { CharacterChunker } from "./CharacterChunker.js";
+export { RecursiveChunker } from "./RecursiveChunker.js";
+export { SentenceChunker } from "./SentenceChunker.js";
+export { TokenChunker } from "./TokenChunker.js";
+export { MarkdownChunker } from "./MarkdownChunker.js";
+export { HTMLChunker } from "./HTMLChunker.js";
+export { JSONChunker } from "./JSONChunker.js";
+export { LaTeXChunker } from "./LaTeXChunker.js";
+export { SemanticMarkdownChunker } from "./SemanticMarkdownChunker.js";

package/dist/client/rag/chunking/characterChunker.js

@@ -0,0 +1,142 @@
+/**
+ * Character-based Chunker
+ *
+ * Simple character-based text splitting with configurable separator and overlap.
+ * Best for unstructured text where character count is the primary concern.
+ */
+import { randomUUID } from "crypto";
+/**
+ * Character-based chunker implementation
+ * Splits text by character count with optional separator
+ */
+export class CharacterChunker {
+    strategy = "character";
+    async chunk(text, config) {
+        const { maxSize = 1000, overlap = 0, separator = "", keepSeparator = false, trimWhitespace = true, metadata = {}, } = config || {};
+        const chunks = [];
+        const documentId = randomUUID();
+        if (!text || text.length === 0) {
+            return chunks;
+        }
+        // Split by separator if provided
+        let segments;
+        if (separator) {
+            segments = text.split(separator);
+            if (keepSeparator && separator) {
+                segments = segments.map((s, i) => i < segments.length - 1 ? s + separator : s);
+            }
+        }
+        else {
+            segments = [text];
+        }
+        let currentChunk = "";
+        let chunkIndex = 0;
+        let startPosition = 0;
+        for (const segment of segments) {
+            if (currentChunk.length + segment.length <= maxSize) {
+                currentChunk += segment;
+            }
+            else {
+                // Save current chunk if it has content
+                if (currentChunk.length > 0) {
+                    const chunkText = trimWhitespace ? currentChunk.trim() : currentChunk;
+                    if (chunkText.length > 0) {
+                        chunks.push({
+                            id: randomUUID(),
+                            text: chunkText,
+                            metadata: {
+                                documentId,
+                                chunkIndex,
+                                startPosition,
+                                endPosition: startPosition + currentChunk.length,
+                                documentType: "text",
+                                custom: metadata,
+                            },
+                        });
+                        chunkIndex++;
+                    }
+                }
+                // Handle overlap
+                if (overlap > 0 && currentChunk.length > overlap) {
+                    currentChunk = currentChunk.slice(-overlap) + segment;
+                    startPosition = startPosition + currentChunk.length - overlap;
+                }
+                else {
+                    startPosition += currentChunk.length;
+                    currentChunk = segment;
+                }
+                // If segment is larger than maxSize, split it further
+                while (currentChunk.length > maxSize) {
+                    const chunkText = trimWhitespace
+                        ? currentChunk.slice(0, maxSize).trim()
+                        : currentChunk.slice(0, maxSize);
+                    chunks.push({
+                        id: randomUUID(),
+                        text: chunkText,
+                        metadata: {
+                            documentId,
+                            chunkIndex,
+                            startPosition,
+                            endPosition: startPosition + maxSize,
+                            documentType: "text",
+                            custom: metadata,
+                        },
+                    });
+                    chunkIndex++;
+                    const overlapStart = Math.max(0, maxSize - overlap);
+                    currentChunk = currentChunk.slice(overlapStart);
+                    startPosition += overlapStart;
+                }
+            }
+        }
+        // Don't forget the last chunk
+        if (currentChunk.length > 0) {
+            const chunkText = trimWhitespace ? currentChunk.trim() : currentChunk;
+            if (chunkText.length > 0) {
+                chunks.push({
+                    id: randomUUID(),
+                    text: chunkText,
+                    metadata: {
+                        documentId,
+                        chunkIndex,
+                        startPosition,
+                        endPosition: startPosition + currentChunk.length,
+                        documentType: "text",
+                        custom: metadata,
+                    },
+                });
+            }
+        }
+        // Update total chunks count
+        chunks.forEach((chunk) => {
+            chunk.metadata.totalChunks = chunks.length;
+        });
+        return chunks;
+    }
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        const charConfig = config;
+        if (charConfig.maxSize !== undefined && charConfig.maxSize <= 0) {
+            errors.push("maxSize must be greater than 0");
+        }
+        if (charConfig.overlap !== undefined && charConfig.overlap < 0) {
+            errors.push("overlap must be non-negative");
+        }
+        if (charConfig.overlap !== undefined && charConfig.maxSize !== undefined) {
+            if (charConfig.overlap >= charConfig.maxSize) {
+                errors.push("overlap must be less than maxSize");
+            }
+        }
+        if (charConfig.minSize !== undefined && charConfig.maxSize !== undefined) {
+            if (charConfig.minSize > charConfig.maxSize) {
+                warnings.push("minSize is greater than maxSize, some chunks may be smaller than minSize");
+            }
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+}

package/dist/client/rag/chunking/chunkerRegistry.js

@@ -0,0 +1,194 @@
+/**
+ * Chunker Registry
+ *
+ * Central registry for all chunking strategies following NeuroLink's registry pattern.
+ * Provides factory methods for creating chunker instances.
+ */
+import { SemanticMarkdownChunker } from "../chunkers/SemanticMarkdownChunker.js";
+import { CharacterChunker } from "./characterChunker.js";
+import { HTMLChunker } from "./htmlChunker.js";
+import { JSONChunker } from "./jsonChunker.js";
+import { LaTeXChunker } from "./latexChunker.js";
+import { MarkdownChunker } from "./markdownChunker.js";
+import { RecursiveChunker } from "./recursiveChunker.js";
+import { SemanticChunker } from "./semanticChunker.js";
+import { SentenceChunker } from "./sentenceChunker.js";
+import { TokenChunker } from "./tokenChunker.js";
+/**
+ * Registry for chunking strategies
+ * Follows NeuroLink's factory pattern with lazy initialization
+ */
+export class ChunkerRegistry {
+    static chunkers = new Map();
+    static initialized = false;
+    /**
+     * Initialize all built-in chunkers
+     */
+    static initialize() {
+        if (ChunkerRegistry.initialized) {
+            return;
+        }
+        ChunkerRegistry.register("character", () => new CharacterChunker());
+        ChunkerRegistry.register("recursive", () => new RecursiveChunker());
+        ChunkerRegistry.register("sentence", () => new SentenceChunker());
+        ChunkerRegistry.register("token", () => new TokenChunker());
+        ChunkerRegistry.register("markdown", () => new MarkdownChunker());
+        ChunkerRegistry.register("html", () => new HTMLChunker());
+        ChunkerRegistry.register("json", () => new JSONChunker());
+        ChunkerRegistry.register("latex", () => new LaTeXChunker());
+        ChunkerRegistry.register("semantic", () => new SemanticChunker());
+        ChunkerRegistry.register("semantic-markdown", () => new SemanticMarkdownChunker());
+        ChunkerRegistry.initialized = true;
+    }
+    /**
+     * Register a custom chunker
+     * @param strategy - Strategy name
+     * @param factory - Factory function that creates chunker instance
+     */
+    static register(strategy, factory) {
+        ChunkerRegistry.chunkers.set(strategy, factory);
+    }
+    /**
+     * Get a chunker by strategy name
+     * @param strategy - Chunking strategy name
+     * @returns Chunker instance
+     * @throws Error if strategy is not registered
+     */
+    static get(strategy) {
+        ChunkerRegistry.initialize();
+        const factory = ChunkerRegistry.chunkers.get(strategy);
+        if (!factory) {
+            throw new Error(`Unknown chunking strategy: ${strategy}. Available strategies: ${ChunkerRegistry.getAvailableStrategies().join(", ")}`);
+        }
+        return factory();
+    }
+    /**
+     * Get all available chunking strategies
+     * @returns Array of strategy names
+     */
+    static getAvailableStrategies() {
+        ChunkerRegistry.initialize();
+        return Array.from(ChunkerRegistry.chunkers.keys());
+    }
+    /**
+     * Check if a strategy is registered
+     * @param strategy - Strategy name to check
+     * @returns True if strategy is registered
+     */
+    static has(strategy) {
+        ChunkerRegistry.initialize();
+        return ChunkerRegistry.chunkers.has(strategy);
+    }
+    /**
+     * Get strategy recommendation based on content type
+     * @param contentType - Document type or MIME type
+     * @returns Recommended chunking strategy
+     */
+    static getRecommendedStrategy(contentType) {
+        const normalized = contentType.toLowerCase();
+        if (normalized.includes("markdown") || normalized === "md") {
+            return "markdown";
+        }
+        if (normalized.includes("html") || normalized.includes("htm")) {
+            return "html";
+        }
+        if (normalized.includes("json")) {
+            return "json";
+        }
+        // Check for latex specifically - don't match "text" which contains "tex"
+        if (normalized.includes("latex") ||
+            normalized === "tex" ||
+            normalized.endsWith("/tex")) {
+            return "latex";
+        }
+        if (normalized.includes("code") || normalized.includes("programming")) {
+            return "recursive";
+        }
+        if (normalized.includes("document") || normalized.includes("text")) {
+            return "sentence";
+        }
+        // Default to recursive for general text
+        return "recursive";
+    }
+    /**
+     * Get default configuration for a strategy
+     * @param strategy - Chunking strategy
+     * @returns Default configuration object
+     */
+    static getDefaultConfig(strategy) {
+        const defaults = {
+            character: {
+                maxSize: 1000,
+                overlap: 0,
+                separator: "",
+                keepSeparator: false,
+            },
+            recursive: {
+                maxSize: 1000,
+                overlap: 200,
+                separators: ["\n\n", "\n", ". ", " ", ""],
+            },
+            sentence: {
+                maxSize: 1000,
+                overlap: 0,
+                minSentences: 1,
+                sentenceEnders: [".", "!", "?"],
+            },
+            token: {
+                maxTokens: 512,
+                tokenOverlap: 50,
+                tokenizer: "cl100k_base",
+            },
+            markdown: {
+                maxSize: 1000,
+                headerLevels: [1, 2, 3],
+                preserveCodeBlocks: true,
+                includeHeader: true,
+            },
+            html: {
+                maxSize: 1000,
+                splitTags: ["div", "p", "section", "article"],
+                extractTextOnly: false,
+            },
+            json: {
+                maxSize: 1000,
+                maxDepth: 10,
+                includeJsonPath: true,
+            },
+            latex: {
+                maxSize: 1000,
+                splitEnvironments: ["section", "subsection", "chapter"],
+                preserveMath: true,
+            },
+            semantic: {
+                maxSize: 1000,
+                similarityThreshold: 0.7,
+                joinThreshold: 100,
+            },
+            "semantic-markdown": {
+                maxSize: 1000,
+                overlap: 100,
+                similarityThreshold: 0.7,
+            },
+        };
+        return defaults[strategy] || { maxSize: 1000 };
+    }
+    /**
+     * Reset the registry (useful for testing)
+     */
+    static reset() {
+        ChunkerRegistry.chunkers.clear();
+        ChunkerRegistry.initialized = false;
+    }
+}
+/**
+ * Convenience function to chunk text with a given strategy
+ * @param text - Text to chunk
+ * @param strategy - Chunking strategy (default: "recursive")
+ * @param config - Strategy-specific configuration
+ * @returns Array of chunks
+ */
+export async function chunkText(text, strategy = "recursive", config) {
+    const chunker = ChunkerRegistry.get(strategy);
+    return chunker.chunk(text, config);
+}