@juspay/neurolink 9.32.0 → 9.33.0

package/dist/client/rag/chunking/recursiveChunker.js

@@ -1,148 +0,0 @@
-/**
- * Recursive Chunker
- *
- * Smart text splitting using hierarchical separators.
- * Tries each separator in order, recursively splitting chunks that are too large.
- * Best for general-purpose text that has natural boundaries.
- */
-import { randomUUID } from "crypto";
-/**
- * Recursive chunker implementation
- * Smart splitting based on content structure using hierarchical separators
- */
-export class RecursiveChunker {
-    strategy = "recursive";
-    defaultSeparators = ["\n\n", "\n", ". ", " ", ""];
-    async chunk(text, config) {
-        const { maxSize = 1000, overlap = 200, separators = this.defaultSeparators, isSeparatorRegex = false, trimWhitespace = true, metadata = {}, } = config || {};
-        const documentId = randomUUID();
-        const chunks = [];
-        if (!text || text.length === 0) {
-            return chunks;
-        }
-        const splitTexts = this.recursiveSplit(text, separators, maxSize, overlap, isSeparatorRegex);
-        let chunkIndex = 0;
-        let currentPosition = 0;
-        for (const splitText of splitTexts) {
-            const chunkText = trimWhitespace ? splitText.trim() : splitText;
-            if (chunkText.length > 0) {
-                const startPosition = text.indexOf(splitText, currentPosition);
-                chunks.push({
-                    id: randomUUID(),
-                    text: chunkText,
-                    metadata: {
-                        documentId,
-                        chunkIndex,
-                        startPosition: startPosition >= 0 ? startPosition : currentPosition,
-                        endPosition: startPosition >= 0
-                            ? startPosition + splitText.length
-                            : currentPosition + splitText.length,
-                        documentType: "text",
-                        custom: metadata,
-                    },
-                });
-                chunkIndex++;
-                if (startPosition >= 0) {
-                    currentPosition = startPosition + splitText.length - overlap;
-                }
-            }
-        }
-        // Update total chunks count
-        chunks.forEach((chunk) => {
-            chunk.metadata.totalChunks = chunks.length;
-        });
-        return chunks;
-    }
-    recursiveSplit(text, separators, maxSize, overlap, isRegex) {
-        const results = [];
-        if (text.length <= maxSize) {
-            return [text];
-        }
-        // Find the best separator to use
-        let separator = separators[separators.length - 1]; // Default to last (usually "")
-        let newSeparators = separators;
-        for (let i = 0; i < separators.length; i++) {
-            const sep = separators[i];
-            const hasMatch = isRegex
-                ? new RegExp(sep).test(text)
-                : text.includes(sep);
-            if (sep === "" || hasMatch) {
-                separator = sep;
-                newSeparators = separators.slice(i + 1);
-                break;
-            }
-        }
-        // Split the text
-        const splits = isRegex
-            ? text.split(new RegExp(separator))
-            : text.split(separator);
-        // Merge splits into chunks
-        let currentChunk = "";
-        for (const split of splits) {
-            const potentialChunk = currentChunk
-                ? currentChunk + separator + split
-                : split;
-            if (potentialChunk.length <= maxSize) {
-                currentChunk = potentialChunk;
-            }
-            else {
-                // Current chunk is ready
-                if (currentChunk.length > 0) {
-                    results.push(currentChunk);
-                }
-                // Handle split that's still too large
-                if (split.length > maxSize) {
-                    const subSplits = this.recursiveSplit(split, newSeparators, maxSize, overlap, isRegex);
-                    results.push(...subSplits.slice(0, -1));
-                    currentChunk = subSplits[subSplits.length - 1] || "";
-                }
-                else {
-                    // Add overlap from previous chunk
-                    if (results.length > 0 && overlap > 0) {
-                        const lastChunk = results[results.length - 1];
-                        const overlapText = lastChunk.slice(-overlap);
-                        currentChunk = overlapText + separator + split;
-                    }
-                    else {
-                        currentChunk = split;
-                    }
-                }
-            }
-        }
-        // Don't forget the last chunk
-        if (currentChunk.length > 0) {
-            results.push(currentChunk);
-        }
-        return results;
-    }
-    validateConfig(config) {
-        const errors = [];
-        const warnings = [];
-        const recConfig = config;
-        if (recConfig.maxSize !== undefined && recConfig.maxSize <= 0) {
-            errors.push("maxSize must be greater than 0");
-        }
-        if (recConfig.overlap !== undefined && recConfig.overlap < 0) {
-            errors.push("overlap must be non-negative");
-        }
-        if (recConfig.separators !== undefined &&
-            recConfig.separators.length === 0) {
-            errors.push("separators array must not be empty");
-        }
-        if (recConfig.isSeparatorRegex && recConfig.separators) {
-            for (const sep of recConfig.separators) {
-                try {
-                    new RegExp(sep);
-                }
-                catch {
-                    errors.push(`Invalid regex separator: ${sep}`);
-                }
-            }
-        }
-        return {
-            valid: errors.length === 0,
-            errors,
-            warnings,
-        };
-    }
-}

package/dist/client/rag/chunking/semanticChunker.js

@@ -1,306 +0,0 @@
-/**
- * Semantic Chunker
- *
- * LLM-powered semantic chunking that groups related content together.
- * Uses embedding similarity to determine natural breakpoints.
- * Best for complex documents where meaning should drive segmentation.
- */
-import { randomUUID } from "crypto";
-import { ProviderFactory } from "../../factories/providerFactory.js";
-import { logger } from "../../utils/logger.js";
-/**
- * Semantic chunker implementation
- * Uses embedding similarity to find natural content boundaries
- */
-export class SemanticChunker {
-    strategy = "semantic";
-    async chunk(text, config) {
-        const { maxSize = 1000, overlap = 0, joinThreshold = 100, modelName = "text-embedding-3-small", provider = "openai", similarityThreshold = 0.7, trimWhitespace = true, metadata = {}, } = config || {};
-        const documentId = randomUUID();
-        const chunks = [];
-        if (!text || text.length === 0) {
-            return chunks;
-        }
-        // First, split into initial segments (paragraphs or sentences)
-        const segments = this.splitIntoSegments(text, joinThreshold);
-        if (segments.length <= 1) {
-            // Single segment, no need for semantic analysis
-            chunks.push({
-                id: randomUUID(),
-                text: trimWhitespace ? text.trim() : text,
-                metadata: {
-                    documentId,
-                    chunkIndex: 0,
-                    totalChunks: 1,
-                    startPosition: 0,
-                    endPosition: text.length,
-                    documentType: "text",
-                    custom: metadata,
-                },
-            });
-            return chunks;
-        }
-        try {
-            // Get embeddings for each segment
-            const embeddings = await this.getEmbeddings(segments, provider, modelName);
-            // Find semantic breakpoints
-            const breakpoints = this.findSemanticBreakpoints(embeddings, similarityThreshold);
-            // Group segments by semantic similarity
-            const groups = this.groupSegments(segments, breakpoints, maxSize);
-            // Create chunks from groups
-            let chunkIndex = 0;
-            let currentPosition = 0;
-            for (const group of groups) {
-                const chunkText = group.join("\n\n");
-                const finalText = trimWhitespace ? chunkText.trim() : chunkText;
-                if (finalText.length > 0) {
-                    chunks.push({
-                        id: randomUUID(),
-                        text: finalText,
-                        metadata: {
-                            documentId,
-                            chunkIndex,
-                            startPosition: currentPosition,
-                            endPosition: currentPosition + chunkText.length,
-                            documentType: "text",
-                            custom: {
-                                ...metadata,
-                                segmentCount: group.length,
-                            },
-                        },
-                    });
-                    chunkIndex++;
-                }
-                currentPosition += chunkText.length + 2; // +2 for separator
-            }
-            // Handle overlap if configured
-            if (overlap > 0) {
-                chunks.forEach((chunk, i) => {
-                    if (i > 0) {
-                        // Add overlap from previous chunk
-                        const prevText = chunks[i - 1].text;
-                        const overlapText = prevText.slice(-overlap);
-                        chunk.text = overlapText + "\n" + chunk.text;
-                    }
-                });
-            }
-        }
-        catch (error) {
-            // Fallback to simple chunking if embeddings fail
-            logger.warn("[SemanticChunker] Embedding failed, falling back to simple chunking", {
-                error: error instanceof Error ? error.message : String(error),
-            });
-            return this.fallbackChunk(text, maxSize, overlap, documentId, metadata, trimWhitespace);
-        }
-        // Update total chunks count
-        chunks.forEach((chunk) => {
-            chunk.metadata.totalChunks = chunks.length;
-        });
-        return chunks;
-    }
-    /**
-     * Split text into initial segments for embedding
-     */
-    splitIntoSegments(text, minSize) {
-        const segments = [];
-        // Split by double newlines (paragraphs)
-        const paragraphs = text.split(/\n\n+/);
-        let currentSegment = "";
-        for (const paragraph of paragraphs) {
-            const trimmed = paragraph.trim();
-            if (trimmed.length === 0) {
-                continue;
-            }
-            if (currentSegment.length === 0) {
-                currentSegment = trimmed;
-            }
-            else if (currentSegment.length + trimmed.length < minSize) {
-                // Join small paragraphs
-                currentSegment += "\n\n" + trimmed;
-            }
-            else {
-                // Save current and start new
-                if (currentSegment.length > 0) {
-                    segments.push(currentSegment);
-                }
-                currentSegment = trimmed;
-            }
-        }
-        // Don't forget the last segment
-        if (currentSegment.length > 0) {
-            segments.push(currentSegment);
-        }
-        return segments;
-    }
-    /**
-     * Get embeddings for segments
-     */
-    async getEmbeddings(segments, provider, modelName) {
-        const embeddingProvider = await ProviderFactory.createProvider(provider, modelName);
-        // Check if provider has embed method
-        if (typeof embeddingProvider.embed !==
-            "function") {
-            throw new Error(`Provider ${provider} does not support embeddings`);
-        }
-        const embeddings = [];
-        // Process in batches to avoid rate limits
-        const batchSize = 10;
-        for (let i = 0; i < segments.length; i += batchSize) {
-            const batch = segments.slice(i, i + batchSize);
-            for (const segment of batch) {
-                try {
-                    const embedding = await embeddingProvider.embed(segment);
-                    embeddings.push(embedding);
-                }
-                catch (error) {
-                    logger.warn("[SemanticChunker] Failed to embed segment", {
-                        error: error instanceof Error ? error.message : String(error),
-                    });
-                    // Use zero vector as fallback
-                    embeddings.push(new Array(1536).fill(0));
-                }
-            }
-        }
-        return embeddings;
-    }
-    /**
-     * Find semantic breakpoints using cosine similarity
-     */
-    findSemanticBreakpoints(embeddings, threshold) {
-        const breakpoints = [];
-        for (let i = 1; i < embeddings.length; i++) {
-            const similarity = this.cosineSimilarity(embeddings[i - 1], embeddings[i]);
-            // If similarity is below threshold, it's a breakpoint
-            if (similarity < threshold) {
-                breakpoints.push(i);
-            }
-        }
-        return breakpoints;
-    }
-    /**
-     * Group segments based on breakpoints and size limits
-     */
-    groupSegments(segments, breakpoints, maxSize) {
-        const groups = [];
-        let currentGroup = [];
-        let currentSize = 0;
-        let breakpointIndex = 0;
-        for (let i = 0; i < segments.length; i++) {
-            const segment = segments[i];
-            const segmentSize = segment.length;
-            // Check if we're at a breakpoint or exceeding size
-            const isBreakpoint = breakpointIndex < breakpoints.length &&
-                breakpoints[breakpointIndex] === i;
-            if ((currentSize + segmentSize > maxSize && currentGroup.length > 0) ||
-                (isBreakpoint && currentGroup.length > 0)) {
-                // Save current group
-                groups.push(currentGroup);
-                currentGroup = [];
-                currentSize = 0;
-            }
-            if (isBreakpoint) {
-                breakpointIndex++;
-            }
-            currentGroup.push(segment);
-            currentSize += segmentSize;
-        }
-        // Don't forget the last group
-        if (currentGroup.length > 0) {
-            groups.push(currentGroup);
-        }
-        return groups;
-    }
-    /**
-     * Calculate cosine similarity between two vectors
-     */
-    cosineSimilarity(a, b) {
-        if (a.length !== b.length) {
-            return 0;
-        }
-        let dotProduct = 0;
-        let normA = 0;
-        let normB = 0;
-        for (let i = 0; i < a.length; i++) {
-            dotProduct += a[i] * b[i];
-            normA += a[i] * a[i];
-            normB += b[i] * b[i];
-        }
-        const denominator = Math.sqrt(normA) * Math.sqrt(normB);
-        return denominator === 0 ? 0 : dotProduct / denominator;
-    }
-    /**
-     * Fallback to simple chunking when embeddings fail
-     */
-    fallbackChunk(text, maxSize, overlap, documentId, metadata, trimWhitespace) {
-        const effectiveMaxSize = Math.max(maxSize, 1);
-        const effectiveOverlap = Math.min(Math.max(overlap, 0), effectiveMaxSize - 1);
-        const chunks = [];
-        let start = 0;
-        let chunkIndex = 0;
-        while (start < text.length) {
-            let end = Math.min(start + effectiveMaxSize, text.length);
-            // Try to break at paragraph boundary
-            if (end < text.length) {
-                const searchStart = Math.max(start, end - 200);
-                const searchText = text.slice(searchStart, end);
-                const paragraphBreak = searchText.lastIndexOf("\n\n");
-                if (paragraphBreak > 0) {
-                    end = searchStart + paragraphBreak;
-                }
-            }
-            const chunkText = text.slice(start, end);
-            const finalText = trimWhitespace ? chunkText.trim() : chunkText;
-            if (finalText.length > 0) {
-                chunks.push({
-                    id: randomUUID(),
-                    text: finalText,
-                    metadata: {
-                        documentId,
-                        chunkIndex,
-                        startPosition: start,
-                        endPosition: end,
-                        documentType: "text",
-                        custom: {
-                            ...metadata,
-                            fallbackChunking: true,
-                        },
-                    },
-                });
-                chunkIndex++;
-            }
-            start = Math.max(start + 1, end - effectiveOverlap);
-        }
-        return chunks;
-    }
-    validateConfig(config) {
-        const errors = [];
-        const warnings = [];
-        const semConfig = config;
-        if (semConfig.maxSize !== undefined && semConfig.maxSize <= 0) {
-            errors.push("maxSize must be greater than 0");
-        }
-        if (semConfig.overlap !== undefined && semConfig.overlap < 0) {
-            errors.push("overlap must be non-negative");
-        }
-        if (semConfig.overlap !== undefined &&
-            semConfig.maxSize !== undefined &&
-            semConfig.overlap >= semConfig.maxSize) {
-            errors.push("overlap must be less than maxSize");
-        }
-        if (semConfig.similarityThreshold !== undefined) {
-            if (semConfig.similarityThreshold < 0 ||
-                semConfig.similarityThreshold > 1) {
-                errors.push("similarityThreshold must be between 0 and 1");
-            }
-        }
-        if (semConfig.joinThreshold !== undefined && semConfig.joinThreshold < 0) {
-            errors.push("joinThreshold must be non-negative");
-        }
-        warnings.push("Semantic chunking requires an embedding provider. Ensure API credentials are configured.");
-        return {
-            valid: errors.length === 0,
-            errors,
-            warnings,
-        };
-    }
-}

package/dist/client/rag/chunking/sentenceChunker.js

@@ -1,230 +0,0 @@
-/**
- * Sentence-based Chunker
- *
- * Splits text based on sentence boundaries while respecting size limits.
- * Best for prose and natural language content where sentence integrity matters.
- */
-import { randomUUID } from "crypto";
-/**
- * Sentence-aware chunker implementation
- * Splits text by sentences while respecting size constraints
- */
-export class SentenceChunker {
-    strategy = "sentence";
-    defaultSentenceEnders = [".", "!", "?"];
-    async chunk(text, config) {
-        const { maxSize = 1000, overlap = 0, sentenceEnders = this.defaultSentenceEnders, minSentences = 1, maxSentences, trimWhitespace = true, metadata = {}, } = config || {};
-        const chunks = [];
-        const documentId = randomUUID();
-        if (!text || text.length === 0) {
-            return chunks;
-        }
-        // Split text into sentences
-        const sentences = this.splitIntoSentences(text, sentenceEnders);
-        if (sentences.length === 0) {
-            return chunks;
-        }
-        let currentChunkSentences = [];
-        let currentChunkLength = 0;
-        let chunkIndex = 0;
-        let startPosition = 0;
-        let currentPosition = 0;
-        for (let i = 0; i < sentences.length; i++) {
-            const sentence = sentences[i];
-            const sentenceLength = sentence.length;
-            // Check if adding this sentence would exceed limits
-            const wouldExceedSize = currentChunkLength + sentenceLength + 1 > maxSize;
-            const wouldExceedSentences = maxSentences !== undefined &&
-                currentChunkSentences.length >= maxSentences;
-            if (currentChunkSentences.length > 0 &&
-                (wouldExceedSize || wouldExceedSentences)) {
-                // Save current chunk if it meets minimum requirements
-                if (currentChunkSentences.length >= minSentences) {
-                    const chunkText = currentChunkSentences.join(" ");
-                    const finalText = trimWhitespace ? chunkText.trim() : chunkText;
-                    if (finalText.length > 0) {
-                        chunks.push({
-                            id: randomUUID(),
-                            text: finalText,
-                            metadata: {
-                                documentId,
-                                chunkIndex,
-                                startPosition,
-                                endPosition: startPosition + chunkText.length,
-                                documentType: "text",
-                                custom: metadata,
-                            },
-                        });
-                        chunkIndex++;
-                    }
-                }
-                // Handle overlap by keeping some sentences
-                if (overlap > 0 && currentChunkSentences.length > 0) {
-                    // Calculate how many sentences to keep for overlap
-                    let overlapLength = 0;
-                    const overlapSentences = [];
-                    for (let j = currentChunkSentences.length - 1; j >= 0; j--) {
-                        const s = currentChunkSentences[j];
-                        if (overlapLength + s.length + 1 <= overlap) {
-                            overlapSentences.unshift(s);
-                            overlapLength += s.length + 1;
-                        }
-                        else {
-                            break;
-                        }
-                    }
-                    currentChunkSentences = overlapSentences;
-                    currentChunkLength = overlapLength;
-                    startPosition = currentPosition - overlapLength;
-                }
-                else {
-                    currentChunkSentences = [];
-                    currentChunkLength = 0;
-                    startPosition = currentPosition;
-                }
-            }
-            // Handle sentences larger than maxSize
-            if (sentenceLength > maxSize) {
-                // Split the sentence itself if necessary
-                const subChunks = this.splitLargeSentence(sentence, maxSize);
-                for (const subChunk of subChunks) {
-                    chunks.push({
-                        id: randomUUID(),
-                        text: trimWhitespace ? subChunk.trim() : subChunk,
-                        metadata: {
-                            documentId,
-                            chunkIndex,
-                            startPosition: currentPosition,
-                            endPosition: currentPosition + subChunk.length,
-                            documentType: "text",
-                            custom: metadata,
-                        },
-                    });
-                    chunkIndex++;
-                    currentPosition += subChunk.length;
-                }
-                startPosition = currentPosition;
-            }
-            else {
-                currentChunkSentences.push(sentence);
-                currentChunkLength += sentenceLength + 1; // +1 for space
-                currentPosition += sentenceLength + 1;
-            }
-        }
-        // Don't forget the last chunk
-        if (currentChunkSentences.length >= minSentences) {
-            const chunkText = currentChunkSentences.join(" ");
-            const finalText = trimWhitespace ? chunkText.trim() : chunkText;
-            if (finalText.length > 0) {
-                chunks.push({
-                    id: randomUUID(),
-                    text: finalText,
-                    metadata: {
-                        documentId,
-                        chunkIndex,
-                        startPosition,
-                        endPosition: startPosition + chunkText.length,
-                        documentType: "text",
-                        custom: metadata,
-                    },
-                });
-            }
-        }
-        // Update total chunks count
-        chunks.forEach((chunk) => {
-            chunk.metadata.totalChunks = chunks.length;
-        });
-        return chunks;
-    }
-    /**
-     * Split text into sentences based on sentence enders
-     */
-    splitIntoSentences(text, sentenceEnders) {
-        const sentences = [];
-        // Build regex pattern for sentence splitting
-        // Look for sentence enders followed by whitespace or end of string
-        const pattern = new RegExp(`([${sentenceEnders.map((e) => "\\" + e).join("")}]+)(?=\\s|$)`, "g");
-        let lastIndex = 0;
-        let match;
-        // Reset regex state
-        pattern.lastIndex = 0;
-        while ((match = pattern.exec(text)) !== null) {
-            const endIndex = match.index + match[0].length;
-            const sentence = text.slice(lastIndex, endIndex).trim();
-            if (sentence.length > 0) {
-                sentences.push(sentence);
-            }
-            lastIndex = endIndex;
-            // Skip whitespace
-            while (lastIndex < text.length && /\s/.test(text[lastIndex])) {
-                lastIndex++;
-            }
-        }
-        // Don't forget the last part
-        if (lastIndex < text.length) {
-            const remaining = text.slice(lastIndex).trim();
-            if (remaining.length > 0) {
-                sentences.push(remaining);
-            }
-        }
-        return sentences;
-    }
-    /**
-     * Split a large sentence into smaller chunks
-     */
-    splitLargeSentence(sentence, maxSize) {
-        const chunks = [];
-        const words = sentence.split(/\s+/);
-        let currentChunk = "";
-        for (const word of words) {
-            if (currentChunk.length + word.length + 1 <= maxSize) {
-                currentChunk = currentChunk ? currentChunk + " " + word : word;
-            }
-            else {
-                if (currentChunk.length > 0) {
-                    chunks.push(currentChunk);
-                }
-                // If a single word is larger than maxSize, we have to include it anyway
-                currentChunk = word;
-            }
-        }
-        if (currentChunk.length > 0) {
-            chunks.push(currentChunk);
-        }
-        return chunks;
-    }
-    validateConfig(config) {
-        const errors = [];
-        const warnings = [];
-        const sentConfig = config;
-        if (sentConfig.maxSize !== undefined && sentConfig.maxSize <= 0) {
-            errors.push("maxSize must be greater than 0");
-        }
-        if (sentConfig.overlap !== undefined && sentConfig.overlap < 0) {
-            errors.push("overlap must be non-negative");
-        }
-        if (sentConfig.overlap !== undefined &&
-            sentConfig.maxSize !== undefined &&
-            sentConfig.overlap >= sentConfig.maxSize) {
-            errors.push("overlap must be less than maxSize");
-        }
-        if (sentConfig.minSentences !== undefined && sentConfig.minSentences < 1) {
-            errors.push("minSentences must be at least 1");
-        }
-        if (sentConfig.maxSentences !== undefined &&
-            sentConfig.minSentences !== undefined) {
-            if (sentConfig.maxSentences < sentConfig.minSentences) {
-                errors.push("maxSentences must be >= minSentences");
-            }
-        }
-        if (sentConfig.sentenceEnders !== undefined &&
-            sentConfig.sentenceEnders.length === 0) {
-            warnings.push("No sentence enders specified, using defaults");
-        }
-        return {
-            valid: errors.length === 0,
-            errors,
-            warnings,
-        };
-    }
-}