@juspay/neurolink 9.32.0 → 9.32.1

package/dist/client/rag/chunkers/SentenceChunker.js

@@ -1,66 +0,0 @@
-/**
- * Sentence Chunker
- *
- * Splits text by sentence boundaries for semantically meaningful chunks.
- */
-import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
-/**
- * Sentence Chunker
- */
-export class SentenceChunker extends BaseChunker {
-    strategy = "sentence";
-    getDefaultConfig() {
-        return {
-            ...DEFAULT_CHUNKER_CONFIG,
-            maxSize: 1000,
-            overlap: 1, // Overlap in sentences
-        };
-    }
-    async doChunk(content, config) {
-        const maxSize = config.maxSize ?? 1000;
-        // Simple sentence splitting (can be enhanced with NLP)
-        const sentences = this.splitIntoSentences(content);
-        const chunks = [];
-        let currentChunk = "";
-        let currentStart = 0;
-        let chunkIndex = 0;
-        for (const sentence of sentences) {
-            if (currentChunk.length + sentence.length <= maxSize) {
-                currentChunk += sentence;
-            }
-            else {
-                if (currentChunk.length > 0) {
-                    const startOffset = content.indexOf(currentChunk, currentStart);
-                    chunks.push(this.createChunk(currentChunk, chunkIndex++, startOffset, startOffset + currentChunk.length));
-                    currentStart = startOffset + 1;
-                }
-                currentChunk = sentence;
-            }
-        }
-        // Add remaining chunk
-        if (currentChunk.length > 0) {
-            const startOffset = content.indexOf(currentChunk, currentStart);
-            chunks.push(this.createChunk(currentChunk, chunkIndex, startOffset, startOffset + currentChunk.length));
-        }
-        return chunks;
-    }
-    /**
-     * Split content into sentences
-     */
-    splitIntoSentences(content) {
-        // Simple regex-based sentence splitting
-        // Handles common abbreviations and sentence endings
-        const sentencePattern = /[^.!?]*[.!?]+(?:\s|$)/g;
-        const sentences = [];
-        let match;
-        while ((match = sentencePattern.exec(content)) !== null) {
-            sentences.push(match[0]);
-        }
-        // Handle remaining content without sentence ending
-        const lastIndex = sentences.reduce((acc, s) => acc + s.length, 0);
-        if (lastIndex < content.length) {
-            sentences.push(content.slice(lastIndex));
-        }
-        return sentences;
-    }
-}

package/dist/client/rag/chunkers/TokenChunker.js

@@ -1,61 +0,0 @@
-/**
- * Token Chunker
- *
- * Splits text by token count using a tokenizer.
- * Useful for precise token budget management.
- */
-import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
-/**
- * Token Chunker
- *
- * Approximates token-based splitting using word count.
- * For production, integrate with a proper tokenizer (tiktoken, etc.)
- */
-export class TokenChunker extends BaseChunker {
-    strategy = "token";
-    getDefaultConfig() {
-        return {
-            ...DEFAULT_CHUNKER_CONFIG,
-            maxSize: 512, // Tokens
-            overlap: 50, // Tokens
-        };
-    }
-    async doChunk(content, config) {
-        const maxTokens = config.maxSize ?? 512;
-        const overlapTokens = config.overlap ?? 50;
-        // Approximate tokenization using words
-        // In production, use a proper tokenizer like tiktoken
-        const words = content.split(/\s+/);
-        const chunks = [];
-        let currentWords = [];
-        let currentStart = 0;
-        let chunkIndex = 0;
-        for (let i = 0; i < words.length; i++) {
-            const word = words[i];
-            if (!word) {
-                continue;
-            }
-            // Estimate tokens (roughly 1.3 tokens per word on average)
-            const estimatedTokens = Math.ceil(currentWords.length * 1.3);
-            if (estimatedTokens >= maxTokens) {
-                const chunkText = currentWords.join(" ");
-                const startOffset = content.indexOf(currentWords[0] ?? "", currentStart);
-                const endOffset = startOffset + chunkText.length;
-                chunks.push(this.createChunk(chunkText, chunkIndex++, startOffset, endOffset));
-                // Keep overlap words
-                const overlapCount = Math.ceil(overlapTokens / 1.3);
-                currentWords = currentWords.slice(-overlapCount);
-                currentStart = endOffset - currentWords.join(" ").length;
-            }
-            currentWords.push(word);
-        }
-        // Add remaining chunk
-        if (currentWords.length > 0) {
-            const chunkText = currentWords.join(" ");
-            const startOffset = content.indexOf(currentWords[0] ?? "", currentStart);
-            const endOffset = startOffset + chunkText.length;
-            chunks.push(this.createChunk(chunkText, chunkIndex, startOffset, endOffset));
-        }
-        return chunks;
-    }
-}

package/dist/client/rag/chunkers/index.js

@@ -1,15 +0,0 @@
-/**
- * Chunkers Index
- *
- * Exports all chunker implementations.
- */
-export { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
-export { CharacterChunker } from "./CharacterChunker.js";
-export { RecursiveChunker } from "./RecursiveChunker.js";
-export { SentenceChunker } from "./SentenceChunker.js";
-export { TokenChunker } from "./TokenChunker.js";
-export { MarkdownChunker } from "./MarkdownChunker.js";
-export { HTMLChunker } from "./HTMLChunker.js";
-export { JSONChunker } from "./JSONChunker.js";
-export { LaTeXChunker } from "./LaTeXChunker.js";
-export { SemanticMarkdownChunker } from "./SemanticMarkdownChunker.js";

package/dist/client/rag/chunking/characterChunker.js

@@ -1,142 +0,0 @@
-/**
- * Character-based Chunker
- *
- * Simple character-based text splitting with configurable separator and overlap.
- * Best for unstructured text where character count is the primary concern.
- */
-import { randomUUID } from "crypto";
-/**
- * Character-based chunker implementation
- * Splits text by character count with optional separator
- */
-export class CharacterChunker {
-    strategy = "character";
-    async chunk(text, config) {
-        const { maxSize = 1000, overlap = 0, separator = "", keepSeparator = false, trimWhitespace = true, metadata = {}, } = config || {};
-        const chunks = [];
-        const documentId = randomUUID();
-        if (!text || text.length === 0) {
-            return chunks;
-        }
-        // Split by separator if provided
-        let segments;
-        if (separator) {
-            segments = text.split(separator);
-            if (keepSeparator && separator) {
-                segments = segments.map((s, i) => i < segments.length - 1 ? s + separator : s);
-            }
-        }
-        else {
-            segments = [text];
-        }
-        let currentChunk = "";
-        let chunkIndex = 0;
-        let startPosition = 0;
-        for (const segment of segments) {
-            if (currentChunk.length + segment.length <= maxSize) {
-                currentChunk += segment;
-            }
-            else {
-                // Save current chunk if it has content
-                if (currentChunk.length > 0) {
-                    const chunkText = trimWhitespace ? currentChunk.trim() : currentChunk;
-                    if (chunkText.length > 0) {
-                        chunks.push({
-                            id: randomUUID(),
-                            text: chunkText,
-                            metadata: {
-                                documentId,
-                                chunkIndex,
-                                startPosition,
-                                endPosition: startPosition + currentChunk.length,
-                                documentType: "text",
-                                custom: metadata,
-                            },
-                        });
-                        chunkIndex++;
-                    }
-                }
-                // Handle overlap
-                if (overlap > 0 && currentChunk.length > overlap) {
-                    currentChunk = currentChunk.slice(-overlap) + segment;
-                    startPosition = startPosition + currentChunk.length - overlap;
-                }
-                else {
-                    startPosition += currentChunk.length;
-                    currentChunk = segment;
-                }
-                // If segment is larger than maxSize, split it further
-                while (currentChunk.length > maxSize) {
-                    const chunkText = trimWhitespace
-                        ? currentChunk.slice(0, maxSize).trim()
-                        : currentChunk.slice(0, maxSize);
-                    chunks.push({
-                        id: randomUUID(),
-                        text: chunkText,
-                        metadata: {
-                            documentId,
-                            chunkIndex,
-                            startPosition,
-                            endPosition: startPosition + maxSize,
-                            documentType: "text",
-                            custom: metadata,
-                        },
-                    });
-                    chunkIndex++;
-                    const overlapStart = Math.max(0, maxSize - overlap);
-                    currentChunk = currentChunk.slice(overlapStart);
-                    startPosition += overlapStart;
-                }
-            }
-        }
-        // Don't forget the last chunk
-        if (currentChunk.length > 0) {
-            const chunkText = trimWhitespace ? currentChunk.trim() : currentChunk;
-            if (chunkText.length > 0) {
-                chunks.push({
-                    id: randomUUID(),
-                    text: chunkText,
-                    metadata: {
-                        documentId,
-                        chunkIndex,
-                        startPosition,
-                        endPosition: startPosition + currentChunk.length,
-                        documentType: "text",
-                        custom: metadata,
-                    },
-                });
-            }
-        }
-        // Update total chunks count
-        chunks.forEach((chunk) => {
-            chunk.metadata.totalChunks = chunks.length;
-        });
-        return chunks;
-    }
-    validateConfig(config) {
-        const errors = [];
-        const warnings = [];
-        const charConfig = config;
-        if (charConfig.maxSize !== undefined && charConfig.maxSize <= 0) {
-            errors.push("maxSize must be greater than 0");
-        }
-        if (charConfig.overlap !== undefined && charConfig.overlap < 0) {
-            errors.push("overlap must be non-negative");
-        }
-        if (charConfig.overlap !== undefined && charConfig.maxSize !== undefined) {
-            if (charConfig.overlap >= charConfig.maxSize) {
-                errors.push("overlap must be less than maxSize");
-            }
-        }
-        if (charConfig.minSize !== undefined && charConfig.maxSize !== undefined) {
-            if (charConfig.minSize > charConfig.maxSize) {
-                warnings.push("minSize is greater than maxSize, some chunks may be smaller than minSize");
-            }
-        }
-        return {
-            valid: errors.length === 0,
-            errors,
-            warnings,
-        };
-    }
-}

package/dist/client/rag/chunking/chunkerRegistry.js

@@ -1,194 +0,0 @@
-/**
- * Chunker Registry
- *
- * Central registry for all chunking strategies following NeuroLink's registry pattern.
- * Provides factory methods for creating chunker instances.
- */
-import { SemanticMarkdownChunker } from "../chunkers/SemanticMarkdownChunker.js";
-import { CharacterChunker } from "./characterChunker.js";
-import { HTMLChunker } from "./htmlChunker.js";
-import { JSONChunker } from "./jsonChunker.js";
-import { LaTeXChunker } from "./latexChunker.js";
-import { MarkdownChunker } from "./markdownChunker.js";
-import { RecursiveChunker } from "./recursiveChunker.js";
-import { SemanticChunker } from "./semanticChunker.js";
-import { SentenceChunker } from "./sentenceChunker.js";
-import { TokenChunker } from "./tokenChunker.js";
-/**
- * Registry for chunking strategies
- * Follows NeuroLink's factory pattern with lazy initialization
- */
-export class ChunkerRegistry {
-    static chunkers = new Map();
-    static initialized = false;
-    /**
-     * Initialize all built-in chunkers
-     */
-    static initialize() {
-        if (ChunkerRegistry.initialized) {
-            return;
-        }
-        ChunkerRegistry.register("character", () => new CharacterChunker());
-        ChunkerRegistry.register("recursive", () => new RecursiveChunker());
-        ChunkerRegistry.register("sentence", () => new SentenceChunker());
-        ChunkerRegistry.register("token", () => new TokenChunker());
-        ChunkerRegistry.register("markdown", () => new MarkdownChunker());
-        ChunkerRegistry.register("html", () => new HTMLChunker());
-        ChunkerRegistry.register("json", () => new JSONChunker());
-        ChunkerRegistry.register("latex", () => new LaTeXChunker());
-        ChunkerRegistry.register("semantic", () => new SemanticChunker());
-        ChunkerRegistry.register("semantic-markdown", () => new SemanticMarkdownChunker());
-        ChunkerRegistry.initialized = true;
-    }
-    /**
-     * Register a custom chunker
-     * @param strategy - Strategy name
-     * @param factory - Factory function that creates chunker instance
-     */
-    static register(strategy, factory) {
-        ChunkerRegistry.chunkers.set(strategy, factory);
-    }
-    /**
-     * Get a chunker by strategy name
-     * @param strategy - Chunking strategy name
-     * @returns Chunker instance
-     * @throws Error if strategy is not registered
-     */
-    static get(strategy) {
-        ChunkerRegistry.initialize();
-        const factory = ChunkerRegistry.chunkers.get(strategy);
-        if (!factory) {
-            throw new Error(`Unknown chunking strategy: ${strategy}. Available strategies: ${ChunkerRegistry.getAvailableStrategies().join(", ")}`);
-        }
-        return factory();
-    }
-    /**
-     * Get all available chunking strategies
-     * @returns Array of strategy names
-     */
-    static getAvailableStrategies() {
-        ChunkerRegistry.initialize();
-        return Array.from(ChunkerRegistry.chunkers.keys());
-    }
-    /**
-     * Check if a strategy is registered
-     * @param strategy - Strategy name to check
-     * @returns True if strategy is registered
-     */
-    static has(strategy) {
-        ChunkerRegistry.initialize();
-        return ChunkerRegistry.chunkers.has(strategy);
-    }
-    /**
-     * Get strategy recommendation based on content type
-     * @param contentType - Document type or MIME type
-     * @returns Recommended chunking strategy
-     */
-    static getRecommendedStrategy(contentType) {
-        const normalized = contentType.toLowerCase();
-        if (normalized.includes("markdown") || normalized === "md") {
-            return "markdown";
-        }
-        if (normalized.includes("html") || normalized.includes("htm")) {
-            return "html";
-        }
-        if (normalized.includes("json")) {
-            return "json";
-        }
-        // Check for latex specifically - don't match "text" which contains "tex"
-        if (normalized.includes("latex") ||
-            normalized === "tex" ||
-            normalized.endsWith("/tex")) {
-            return "latex";
-        }
-        if (normalized.includes("code") || normalized.includes("programming")) {
-            return "recursive";
-        }
-        if (normalized.includes("document") || normalized.includes("text")) {
-            return "sentence";
-        }
-        // Default to recursive for general text
-        return "recursive";
-    }
-    /**
-     * Get default configuration for a strategy
-     * @param strategy - Chunking strategy
-     * @returns Default configuration object
-     */
-    static getDefaultConfig(strategy) {
-        const defaults = {
-            character: {
-                maxSize: 1000,
-                overlap: 0,
-                separator: "",
-                keepSeparator: false,
-            },
-            recursive: {
-                maxSize: 1000,
-                overlap: 200,
-                separators: ["\n\n", "\n", ". ", " ", ""],
-            },
-            sentence: {
-                maxSize: 1000,
-                overlap: 0,
-                minSentences: 1,
-                sentenceEnders: [".", "!", "?"],
-            },
-            token: {
-                maxTokens: 512,
-                tokenOverlap: 50,
-                tokenizer: "cl100k_base",
-            },
-            markdown: {
-                maxSize: 1000,
-                headerLevels: [1, 2, 3],
-                preserveCodeBlocks: true,
-                includeHeader: true,
-            },
-            html: {
-                maxSize: 1000,
-                splitTags: ["div", "p", "section", "article"],
-                extractTextOnly: false,
-            },
-            json: {
-                maxSize: 1000,
-                maxDepth: 10,
-                includeJsonPath: true,
-            },
-            latex: {
-                maxSize: 1000,
-                splitEnvironments: ["section", "subsection", "chapter"],
-                preserveMath: true,
-            },
-            semantic: {
-                maxSize: 1000,
-                similarityThreshold: 0.7,
-                joinThreshold: 100,
-            },
-            "semantic-markdown": {
-                maxSize: 1000,
-                overlap: 100,
-                similarityThreshold: 0.7,
-            },
-        };
-        return defaults[strategy] || { maxSize: 1000 };
-    }
-    /**
-     * Reset the registry (useful for testing)
-     */
-    static reset() {
-        ChunkerRegistry.chunkers.clear();
-        ChunkerRegistry.initialized = false;
-    }
-}
-/**
- * Convenience function to chunk text with a given strategy
- * @param text - Text to chunk
- * @param strategy - Chunking strategy (default: "recursive")
- * @param config - Strategy-specific configuration
- * @returns Array of chunks
- */
-export async function chunkText(text, strategy = "recursive", config) {
-    const chunker = ChunkerRegistry.get(strategy);
-    return chunker.chunk(text, config);
-}