@juspay/neurolink 9.1.1 → 9.3.0

package/dist/lib/rag/chunking/semanticChunker.js

@@ -0,0 +1,307 @@
+/**
+ * Semantic Chunker
+ *
+ * LLM-powered semantic chunking that groups related content together.
+ * Uses embedding similarity to determine natural breakpoints.
+ * Best for complex documents where meaning should drive segmentation.
+ */
+import { randomUUID } from "crypto";
+import { ProviderFactory } from "../../factories/providerFactory.js";
+import { logger } from "../../utils/logger.js";
+/**
+ * Semantic chunker implementation
+ * Uses embedding similarity to find natural content boundaries
+ */
+export class SemanticChunker {
+    strategy = "semantic";
+    async chunk(text, config) {
+        const { maxSize = 1000, overlap = 0, joinThreshold = 100, modelName = "text-embedding-3-small", provider = "openai", similarityThreshold = 0.7, trimWhitespace = true, metadata = {}, } = config || {};
+        const documentId = randomUUID();
+        const chunks = [];
+        if (!text || text.length === 0) {
+            return chunks;
+        }
+        // First, split into initial segments (paragraphs or sentences)
+        const segments = this.splitIntoSegments(text, joinThreshold);
+        if (segments.length <= 1) {
+            // Single segment, no need for semantic analysis
+            chunks.push({
+                id: randomUUID(),
+                text: trimWhitespace ? text.trim() : text,
+                metadata: {
+                    documentId,
+                    chunkIndex: 0,
+                    totalChunks: 1,
+                    startPosition: 0,
+                    endPosition: text.length,
+                    documentType: "text",
+                    custom: metadata,
+                },
+            });
+            return chunks;
+        }
+        try {
+            // Get embeddings for each segment
+            const embeddings = await this.getEmbeddings(segments, provider, modelName);
+            // Find semantic breakpoints
+            const breakpoints = this.findSemanticBreakpoints(embeddings, similarityThreshold);
+            // Group segments by semantic similarity
+            const groups = this.groupSegments(segments, breakpoints, maxSize);
+            // Create chunks from groups
+            let chunkIndex = 0;
+            let currentPosition = 0;
+            for (const group of groups) {
+                const chunkText = group.join("\n\n");
+                const finalText = trimWhitespace ? chunkText.trim() : chunkText;
+                if (finalText.length > 0) {
+                    chunks.push({
+                        id: randomUUID(),
+                        text: finalText,
+                        metadata: {
+                            documentId,
+                            chunkIndex,
+                            startPosition: currentPosition,
+                            endPosition: currentPosition + chunkText.length,
+                            documentType: "text",
+                            custom: {
+                                ...metadata,
+                                segmentCount: group.length,
+                            },
+                        },
+                    });
+                    chunkIndex++;
+                }
+                currentPosition += chunkText.length + 2; // +2 for separator
+            }
+            // Handle overlap if configured
+            if (overlap > 0) {
+                chunks.forEach((chunk, i) => {
+                    if (i > 0) {
+                        // Add overlap from previous chunk
+                        const prevText = chunks[i - 1].text;
+                        const overlapText = prevText.slice(-overlap);
+                        chunk.text = overlapText + "\n" + chunk.text;
+                    }
+                });
+            }
+        }
+        catch (error) {
+            // Fallback to simple chunking if embeddings fail
+            logger.warn("[SemanticChunker] Embedding failed, falling back to simple chunking", {
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return this.fallbackChunk(text, maxSize, overlap, documentId, metadata, trimWhitespace);
+        }
+        // Update total chunks count
+        chunks.forEach((chunk) => {
+            chunk.metadata.totalChunks = chunks.length;
+        });
+        return chunks;
+    }
+    /**
+     * Split text into initial segments for embedding
+     */
+    splitIntoSegments(text, minSize) {
+        const segments = [];
+        // Split by double newlines (paragraphs)
+        const paragraphs = text.split(/\n\n+/);
+        let currentSegment = "";
+        for (const paragraph of paragraphs) {
+            const trimmed = paragraph.trim();
+            if (trimmed.length === 0) {
+                continue;
+            }
+            if (currentSegment.length === 0) {
+                currentSegment = trimmed;
+            }
+            else if (currentSegment.length + trimmed.length < minSize) {
+                // Join small paragraphs
+                currentSegment += "\n\n" + trimmed;
+            }
+            else {
+                // Save current and start new
+                if (currentSegment.length > 0) {
+                    segments.push(currentSegment);
+                }
+                currentSegment = trimmed;
+            }
+        }
+        // Don't forget the last segment
+        if (currentSegment.length > 0) {
+            segments.push(currentSegment);
+        }
+        return segments;
+    }
+    /**
+     * Get embeddings for segments
+     */
+    async getEmbeddings(segments, provider, modelName) {
+        const embeddingProvider = await ProviderFactory.createProvider(provider, modelName);
+        // Check if provider has embed method
+        if (typeof embeddingProvider.embed !==
+            "function") {
+            throw new Error(`Provider ${provider} does not support embeddings`);
+        }
+        const embeddings = [];
+        // Process in batches to avoid rate limits
+        const batchSize = 10;
+        for (let i = 0; i < segments.length; i += batchSize) {
+            const batch = segments.slice(i, i + batchSize);
+            for (const segment of batch) {
+                try {
+                    const embedding = await embeddingProvider.embed(segment);
+                    embeddings.push(embedding);
+                }
+                catch (error) {
+                    logger.warn("[SemanticChunker] Failed to embed segment", {
+                        error: error instanceof Error ? error.message : String(error),
+                    });
+                    // Use zero vector as fallback
+                    embeddings.push(new Array(1536).fill(0));
+                }
+            }
+        }
+        return embeddings;
+    }
+    /**
+     * Find semantic breakpoints using cosine similarity
+     */
+    findSemanticBreakpoints(embeddings, threshold) {
+        const breakpoints = [];
+        for (let i = 1; i < embeddings.length; i++) {
+            const similarity = this.cosineSimilarity(embeddings[i - 1], embeddings[i]);
+            // If similarity is below threshold, it's a breakpoint
+            if (similarity < threshold) {
+                breakpoints.push(i);
+            }
+        }
+        return breakpoints;
+    }
+    /**
+     * Group segments based on breakpoints and size limits
+     */
+    groupSegments(segments, breakpoints, maxSize) {
+        const groups = [];
+        let currentGroup = [];
+        let currentSize = 0;
+        let breakpointIndex = 0;
+        for (let i = 0; i < segments.length; i++) {
+            const segment = segments[i];
+            const segmentSize = segment.length;
+            // Check if we're at a breakpoint or exceeding size
+            const isBreakpoint = breakpointIndex < breakpoints.length &&
+                breakpoints[breakpointIndex] === i;
+            if ((currentSize + segmentSize > maxSize && currentGroup.length > 0) ||
+                (isBreakpoint && currentGroup.length > 0)) {
+                // Save current group
+                groups.push(currentGroup);
+                currentGroup = [];
+                currentSize = 0;
+            }
+            if (isBreakpoint) {
+                breakpointIndex++;
+            }
+            currentGroup.push(segment);
+            currentSize += segmentSize;
+        }
+        // Don't forget the last group
+        if (currentGroup.length > 0) {
+            groups.push(currentGroup);
+        }
+        return groups;
+    }
+    /**
+     * Calculate cosine similarity between two vectors
+     */
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length) {
+            return 0;
+        }
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+        return denominator === 0 ? 0 : dotProduct / denominator;
+    }
+    /**
+     * Fallback to simple chunking when embeddings fail
+     */
+    fallbackChunk(text, maxSize, overlap, documentId, metadata, trimWhitespace) {
+        const effectiveMaxSize = Math.max(maxSize, 1);
+        const effectiveOverlap = Math.min(Math.max(overlap, 0), effectiveMaxSize - 1);
+        const chunks = [];
+        let start = 0;
+        let chunkIndex = 0;
+        while (start < text.length) {
+            let end = Math.min(start + effectiveMaxSize, text.length);
+            // Try to break at paragraph boundary
+            if (end < text.length) {
+                const searchStart = Math.max(start, end - 200);
+                const searchText = text.slice(searchStart, end);
+                const paragraphBreak = searchText.lastIndexOf("\n\n");
+                if (paragraphBreak > 0) {
+                    end = searchStart + paragraphBreak;
+                }
+            }
+            const chunkText = text.slice(start, end);
+            const finalText = trimWhitespace ? chunkText.trim() : chunkText;
+            if (finalText.length > 0) {
+                chunks.push({
+                    id: randomUUID(),
+                    text: finalText,
+                    metadata: {
+                        documentId,
+                        chunkIndex,
+                        startPosition: start,
+                        endPosition: end,
+                        documentType: "text",
+                        custom: {
+                            ...metadata,
+                            fallbackChunking: true,
+                        },
+                    },
+                });
+                chunkIndex++;
+            }
+            start = Math.max(start + 1, end - effectiveOverlap);
+        }
+        return chunks;
+    }
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        const semConfig = config;
+        if (semConfig.maxSize !== undefined && semConfig.maxSize <= 0) {
+            errors.push("maxSize must be greater than 0");
+        }
+        if (semConfig.overlap !== undefined && semConfig.overlap < 0) {
+            errors.push("overlap must be non-negative");
+        }
+        if (semConfig.overlap !== undefined &&
+            semConfig.maxSize !== undefined &&
+            semConfig.overlap >= semConfig.maxSize) {
+            errors.push("overlap must be less than maxSize");
+        }
+        if (semConfig.similarityThreshold !== undefined) {
+            if (semConfig.similarityThreshold < 0 ||
+                semConfig.similarityThreshold > 1) {
+                errors.push("similarityThreshold must be between 0 and 1");
+            }
+        }
+        if (semConfig.joinThreshold !== undefined && semConfig.joinThreshold < 0) {
+            errors.push("joinThreshold must be non-negative");
+        }
+        warnings.push("Semantic chunking requires an embedding provider. Ensure API credentials are configured.");
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+}
+//# sourceMappingURL=semanticChunker.js.map

package/dist/lib/rag/chunking/sentenceChunker.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Sentence-based Chunker
+ *
+ * Splits text based on sentence boundaries while respecting size limits.
+ * Best for prose and natural language content where sentence integrity matters.
+ */
+import type { BaseChunkerConfig, Chunk, Chunker, ChunkerValidationResult, SentenceChunkerConfig } from "../types.js";
+/**
+ * Sentence-aware chunker implementation
+ * Splits text by sentences while respecting size constraints
+ */
+export declare class SentenceChunker implements Chunker {
+    readonly strategy: "sentence";
+    private readonly defaultSentenceEnders;
+    chunk(text: string, config?: SentenceChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Split text into sentences based on sentence enders
+     */
+    private splitIntoSentences;
+    /**
+     * Split a large sentence into smaller chunks
+     */
+    private splitLargeSentence;
+    validateConfig(config: BaseChunkerConfig): ChunkerValidationResult;
+}

package/dist/lib/rag/chunking/sentenceChunker.js

@@ -0,0 +1,231 @@
+/**
+ * Sentence-based Chunker
+ *
+ * Splits text based on sentence boundaries while respecting size limits.
+ * Best for prose and natural language content where sentence integrity matters.
+ */
+import { randomUUID } from "crypto";
+/**
+ * Sentence-aware chunker implementation
+ * Splits text by sentences while respecting size constraints
+ */
+export class SentenceChunker {
+    strategy = "sentence";
+    defaultSentenceEnders = [".", "!", "?"];
+    async chunk(text, config) {
+        const { maxSize = 1000, overlap = 0, sentenceEnders = this.defaultSentenceEnders, minSentences = 1, maxSentences, trimWhitespace = true, metadata = {}, } = config || {};
+        const chunks = [];
+        const documentId = randomUUID();
+        if (!text || text.length === 0) {
+            return chunks;
+        }
+        // Split text into sentences
+        const sentences = this.splitIntoSentences(text, sentenceEnders);
+        if (sentences.length === 0) {
+            return chunks;
+        }
+        let currentChunkSentences = [];
+        let currentChunkLength = 0;
+        let chunkIndex = 0;
+        let startPosition = 0;
+        let currentPosition = 0;
+        for (let i = 0; i < sentences.length; i++) {
+            const sentence = sentences[i];
+            const sentenceLength = sentence.length;
+            // Check if adding this sentence would exceed limits
+            const wouldExceedSize = currentChunkLength + sentenceLength + 1 > maxSize;
+            const wouldExceedSentences = maxSentences !== undefined &&
+                currentChunkSentences.length >= maxSentences;
+            if (currentChunkSentences.length > 0 &&
+                (wouldExceedSize || wouldExceedSentences)) {
+                // Save current chunk if it meets minimum requirements
+                if (currentChunkSentences.length >= minSentences) {
+                    const chunkText = currentChunkSentences.join(" ");
+                    const finalText = trimWhitespace ? chunkText.trim() : chunkText;
+                    if (finalText.length > 0) {
+                        chunks.push({
+                            id: randomUUID(),
+                            text: finalText,
+                            metadata: {
+                                documentId,
+                                chunkIndex,
+                                startPosition,
+                                endPosition: startPosition + chunkText.length,
+                                documentType: "text",
+                                custom: metadata,
+                            },
+                        });
+                        chunkIndex++;
+                    }
+                }
+                // Handle overlap by keeping some sentences
+                if (overlap > 0 && currentChunkSentences.length > 0) {
+                    // Calculate how many sentences to keep for overlap
+                    let overlapLength = 0;
+                    const overlapSentences = [];
+                    for (let j = currentChunkSentences.length - 1; j >= 0; j--) {
+                        const s = currentChunkSentences[j];
+                        if (overlapLength + s.length + 1 <= overlap) {
+                            overlapSentences.unshift(s);
+                            overlapLength += s.length + 1;
+                        }
+                        else {
+                            break;
+                        }
+                    }
+                    currentChunkSentences = overlapSentences;
+                    currentChunkLength = overlapLength;
+                    startPosition = currentPosition - overlapLength;
+                }
+                else {
+                    currentChunkSentences = [];
+                    currentChunkLength = 0;
+                    startPosition = currentPosition;
+                }
+            }
+            // Handle sentences larger than maxSize
+            if (sentenceLength > maxSize) {
+                // Split the sentence itself if necessary
+                const subChunks = this.splitLargeSentence(sentence, maxSize);
+                for (const subChunk of subChunks) {
+                    chunks.push({
+                        id: randomUUID(),
+                        text: trimWhitespace ? subChunk.trim() : subChunk,
+                        metadata: {
+                            documentId,
+                            chunkIndex,
+                            startPosition: currentPosition,
+                            endPosition: currentPosition + subChunk.length,
+                            documentType: "text",
+                            custom: metadata,
+                        },
+                    });
+                    chunkIndex++;
+                    currentPosition += subChunk.length;
+                }
+                startPosition = currentPosition;
+            }
+            else {
+                currentChunkSentences.push(sentence);
+                currentChunkLength += sentenceLength + 1; // +1 for space
+                currentPosition += sentenceLength + 1;
+            }
+        }
+        // Don't forget the last chunk
+        if (currentChunkSentences.length >= minSentences) {
+            const chunkText = currentChunkSentences.join(" ");
+            const finalText = trimWhitespace ? chunkText.trim() : chunkText;
+            if (finalText.length > 0) {
+                chunks.push({
+                    id: randomUUID(),
+                    text: finalText,
+                    metadata: {
+                        documentId,
+                        chunkIndex,
+                        startPosition,
+                        endPosition: startPosition + chunkText.length,
+                        documentType: "text",
+                        custom: metadata,
+                    },
+                });
+            }
+        }
+        // Update total chunks count
+        chunks.forEach((chunk) => {
+            chunk.metadata.totalChunks = chunks.length;
+        });
+        return chunks;
+    }
+    /**
+     * Split text into sentences based on sentence enders
+     */
+    splitIntoSentences(text, sentenceEnders) {
+        const sentences = [];
+        // Build regex pattern for sentence splitting
+        // Look for sentence enders followed by whitespace or end of string
+        const pattern = new RegExp(`([${sentenceEnders.map((e) => "\\" + e).join("")}]+)(?=\\s|$)`, "g");
+        let lastIndex = 0;
+        let match;
+        // Reset regex state
+        pattern.lastIndex = 0;
+        while ((match = pattern.exec(text)) !== null) {
+            const endIndex = match.index + match[0].length;
+            const sentence = text.slice(lastIndex, endIndex).trim();
+            if (sentence.length > 0) {
+                sentences.push(sentence);
+            }
+            lastIndex = endIndex;
+            // Skip whitespace
+            while (lastIndex < text.length && /\s/.test(text[lastIndex])) {
+                lastIndex++;
+            }
+        }
+        // Don't forget the last part
+        if (lastIndex < text.length) {
+            const remaining = text.slice(lastIndex).trim();
+            if (remaining.length > 0) {
+                sentences.push(remaining);
+            }
+        }
+        return sentences;
+    }
+    /**
+     * Split a large sentence into smaller chunks
+     */
+    splitLargeSentence(sentence, maxSize) {
+        const chunks = [];
+        const words = sentence.split(/\s+/);
+        let currentChunk = "";
+        for (const word of words) {
+            if (currentChunk.length + word.length + 1 <= maxSize) {
+                currentChunk = currentChunk ? currentChunk + " " + word : word;
+            }
+            else {
+                if (currentChunk.length > 0) {
+                    chunks.push(currentChunk);
+                }
+                // If a single word is larger than maxSize, we have to include it anyway
+                currentChunk = word;
+            }
+        }
+        if (currentChunk.length > 0) {
+            chunks.push(currentChunk);
+        }
+        return chunks;
+    }
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        const sentConfig = config;
+        if (sentConfig.maxSize !== undefined && sentConfig.maxSize <= 0) {
+            errors.push("maxSize must be greater than 0");
+        }
+        if (sentConfig.overlap !== undefined && sentConfig.overlap < 0) {
+            errors.push("overlap must be non-negative");
+        }
+        if (sentConfig.overlap !== undefined &&
+            sentConfig.maxSize !== undefined &&
+            sentConfig.overlap >= sentConfig.maxSize) {
+            errors.push("overlap must be less than maxSize");
+        }
+        if (sentConfig.minSentences !== undefined && sentConfig.minSentences < 1) {
+            errors.push("minSentences must be at least 1");
+        }
+        if (sentConfig.maxSentences !== undefined &&
+            sentConfig.minSentences !== undefined) {
+            if (sentConfig.maxSentences < sentConfig.minSentences) {
+                errors.push("maxSentences must be >= minSentences");
+            }
+        }
+        if (sentConfig.sentenceEnders !== undefined &&
+            sentConfig.sentenceEnders.length === 0) {
+            warnings.push("No sentence enders specified, using defaults");
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+}
+//# sourceMappingURL=sentenceChunker.js.map

package/dist/lib/rag/chunking/tokenChunker.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Token-based Chunker
+ *
+ * Splits text based on token counts using simple tokenization.
+ * Best for controlling context window usage with LLMs.
+ */
+import type { Chunker, Chunk, ChunkerValidationResult, TokenChunkerConfig, BaseChunkerConfig } from "../types.js";
+/**
+ * Token-aware chunker implementation
+ * Splits text based on approximate token counts
+ *
+ * Note: Uses simple word-based tokenization as approximation.
+ * For exact token counts, integrate with tiktoken or model-specific tokenizers.
+ */
+export declare class TokenChunker implements Chunker {
+    readonly strategy: "token";
+    private readonly CHARS_PER_TOKEN;
+    chunk(text: string, config?: TokenChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Simple word-based tokenization
+     */
+    private tokenize;
+    /**
+     * Get characters per token for a tokenizer
+     */
+    private getCharsPerToken;
+    /**
+     * Estimate average tokens per word
+     */
+    private estimateTokensPerWord;
+    /**
+     * Estimate token count for text
+     */
+    estimateTokenCount(text: string, tokenizer?: string): number;
+    validateConfig(config: BaseChunkerConfig): ChunkerValidationResult;
+}