@juspay/neurolink 9.1.1 → 9.3.0

package/dist/lib/rag/chunking/tokenChunker.js

@@ -0,0 +1,184 @@
+/**
+ * Token-based Chunker
+ *
+ * Splits text based on token counts using simple tokenization.
+ * Best for controlling context window usage with LLMs.
+ */
+import { randomUUID } from "crypto";
+/**
+ * Token-aware chunker implementation
+ * Splits text based on approximate token counts
+ *
+ * Note: Uses simple word-based tokenization as approximation.
+ * For exact token counts, integrate with tiktoken or model-specific tokenizers.
+ */
+export class TokenChunker {
+    strategy = "token";
+    // Approximate characters per token for different tokenizers
+    CHARS_PER_TOKEN = {
+        cl100k_base: 4, // GPT-4, GPT-3.5-turbo
+        p50k_base: 4, // Codex
+        r50k_base: 4, // GPT-3
+        default: 4,
+    };
+    async chunk(text, config) {
+        const { maxSize, overlap = 0, tokenizer = "cl100k_base", maxTokens = 512, tokenOverlap, trimWhitespace = true, metadata = {}, } = config || {};
+        const chunks = [];
+        const documentId = randomUUID();
+        if (!text || text.length === 0) {
+            return chunks;
+        }
+        // Determine effective overlap
+        const effectiveOverlap = tokenOverlap ?? Math.floor(overlap / this.getCharsPerToken(tokenizer));
+        // Use maxSize if provided, otherwise calculate from maxTokens
+        const _effectiveMaxChars = maxSize ?? maxTokens * this.getCharsPerToken(tokenizer);
+        // Tokenize text (simple word-based approximation)
+        const words = this.tokenize(text);
+        const _tokensPerWord = this.estimateTokensPerWord(tokenizer);
+        let currentWords = [];
+        let currentTokenCount = 0;
+        let chunkIndex = 0;
+        let startPosition = 0;
+        let charPosition = 0;
+        for (const word of words) {
+            const wordTokens = Math.ceil(word.length / this.getCharsPerToken(tokenizer));
+            // Check if adding this word would exceed the limit
+            if (currentTokenCount + wordTokens > maxTokens &&
+                currentWords.length > 0) {
+                // Save current chunk
+                const chunkText = currentWords.join(" ");
+                const finalText = trimWhitespace ? chunkText.trim() : chunkText;
+                if (finalText.length > 0) {
+                    chunks.push({
+                        id: randomUUID(),
+                        text: finalText,
+                        metadata: {
+                            documentId,
+                            chunkIndex,
+                            startPosition,
+                            endPosition: charPosition,
+                            documentType: "text",
+                            custom: {
+                                ...metadata,
+                                estimatedTokens: currentTokenCount,
+                            },
+                        },
+                    });
+                    chunkIndex++;
+                }
+                // Handle token overlap
+                if (effectiveOverlap > 0 && currentWords.length > 0) {
+                    // Keep some words for overlap
+                    let overlapTokens = 0;
+                    const overlapWords = [];
+                    for (let i = currentWords.length - 1; i >= 0; i--) {
+                        const w = currentWords[i];
+                        const wTokens = Math.ceil(w.length / this.getCharsPerToken(tokenizer));
+                        if (overlapTokens + wTokens <= effectiveOverlap) {
+                            overlapWords.unshift(w);
+                            overlapTokens += wTokens;
+                        }
+                        else {
+                            break;
+                        }
+                    }
+                    currentWords = overlapWords;
+                    currentTokenCount = overlapTokens;
+                    // Adjust start position for overlap
+                    const overlapChars = overlapWords.join(" ").length + 1;
+                    startPosition = charPosition - overlapChars;
+                }
+                else {
+                    currentWords = [];
+                    currentTokenCount = 0;
+                    startPosition = charPosition;
+                }
+            }
+            currentWords.push(word);
+            currentTokenCount += wordTokens;
+            charPosition += word.length + 1; // +1 for space
+        }
+        // Don't forget the last chunk
+        if (currentWords.length > 0) {
+            const chunkText = currentWords.join(" ");
+            const finalText = trimWhitespace ? chunkText.trim() : chunkText;
+            if (finalText.length > 0) {
+                chunks.push({
+                    id: randomUUID(),
+                    text: finalText,
+                    metadata: {
+                        documentId,
+                        chunkIndex,
+                        startPosition,
+                        endPosition: charPosition,
+                        documentType: "text",
+                        custom: {
+                            ...metadata,
+                            estimatedTokens: currentTokenCount,
+                        },
+                    },
+                });
+            }
+        }
+        // Update total chunks count
+        chunks.forEach((chunk) => {
+            chunk.metadata.totalChunks = chunks.length;
+        });
+        return chunks;
+    }
+    /**
+     * Simple word-based tokenization
+     */
+    tokenize(text) {
+        // Split on whitespace and filter empty strings
+        return text.split(/\s+/).filter((w) => w.length > 0);
+    }
+    /**
+     * Get characters per token for a tokenizer
+     */
+    getCharsPerToken(tokenizer) {
+        return this.CHARS_PER_TOKEN[tokenizer] ?? this.CHARS_PER_TOKEN.default;
+    }
+    /**
+     * Estimate average tokens per word
+     */
+    estimateTokensPerWord(_tokenizer) {
+        // Average English word is ~5 characters, so roughly 1.25 tokens
+        return 1.25;
+    }
+    /**
+     * Estimate token count for text
+     */
+    estimateTokenCount(text, tokenizer = "cl100k_base") {
+        return Math.ceil(text.length / this.getCharsPerToken(tokenizer));
+    }
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        const tokenConfig = config;
+        if (tokenConfig.maxTokens !== undefined && tokenConfig.maxTokens <= 0) {
+            errors.push("maxTokens must be greater than 0");
+        }
+        if (tokenConfig.tokenOverlap !== undefined &&
+            tokenConfig.tokenOverlap < 0) {
+            errors.push("tokenOverlap must be non-negative");
+        }
+        if (tokenConfig.tokenOverlap !== undefined &&
+            tokenConfig.maxTokens !== undefined) {
+            if (tokenConfig.tokenOverlap >= tokenConfig.maxTokens) {
+                errors.push("tokenOverlap must be less than maxTokens");
+            }
+        }
+        if (tokenConfig.maxSize !== undefined && tokenConfig.maxSize <= 0) {
+            errors.push("maxSize must be greater than 0");
+        }
+        // Warn about tokenizer approximation
+        warnings.push("Token counts are approximated. For exact counts, integrate with tiktoken.");
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings,
+        };
+    }
+}
+//# sourceMappingURL=tokenChunker.js.map

package/dist/lib/rag/document/MDocument.d.ts

@@ -0,0 +1,198 @@
+/**
+ * MDocument - Main Document Processing Class
+ *
+ * Provides a fluent interface for document processing using the Factory + Registry pattern.
+ * Supports various document types, chunking strategies, and metadata extraction.
+ *
+ * @example
+ * ```typescript
+ * const doc = await MDocument.fromText(content);
+ * const chunks = await doc.chunk({
+ *   strategy: 'recursive',
+ *   config: { maxSize: 1000, overlap: 200 }
+ * });
+ * const enriched = await doc.extractMetadata({
+ *   title: true,
+ *   summary: true,
+ *   keywords: true
+ * });
+ * ```
+ */
+import type { Chunk, ChunkParams, DocumentType, ExtractParams, MDocumentConfig } from "../types.js";
+/**
+ * MDocument class for comprehensive document processing
+ *
+ * Provides a chainable API for:
+ * - Loading documents from various sources
+ * - Chunking with multiple strategies
+ * - Metadata extraction using LLMs
+ * - Embedding generation
+ */
+export declare class MDocument {
+    private state;
+    private documentId;
+    /**
+     * Create a new MDocument instance
+     * @param content - Document content
+     * @param config - Document configuration
+     */
+    constructor(content: string, config?: MDocumentConfig);
+    /**
+     * Create MDocument from plain text
+     * @param text - Plain text content
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromText(text: string, metadata?: Record<string, unknown>): MDocument;
+    /**
+     * Create MDocument from markdown content
+     * @param markdown - Markdown content
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromMarkdown(markdown: string, metadata?: Record<string, unknown>): MDocument;
+    /**
+     * Create MDocument from HTML content
+     * @param html - HTML content
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromHTML(html: string, metadata?: Record<string, unknown>): MDocument;
+    /**
+     * Create MDocument from JSON content
+     * @param json - JSON string or object
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromJSONContent(json: string | object, metadata?: Record<string, unknown>): MDocument;
+    /**
+     * Create MDocument from LaTeX content
+     * @param latex - LaTeX content
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromLaTeX(latex: string, metadata?: Record<string, unknown>): MDocument;
+    /**
+     * Create MDocument from CSV content
+     * @param csv - CSV content
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromCSV(csv: string, metadata?: Record<string, unknown>): MDocument;
+    /**
+     * Chunk the document using specified strategy
+     * @param params - Chunking parameters
+     * @returns This MDocument instance (for chaining)
+     */
+    chunk(params?: ChunkParams): Promise<MDocument>;
+    /**
+     * Extract metadata from chunks using LLM
+     * @param params - Extraction parameters
+     * @param options - Extractor options
+     * @returns This MDocument instance (for chaining)
+     */
+    extractMetadata(params: ExtractParams, options?: {
+        provider?: string;
+        modelName?: string;
+    }): Promise<MDocument>;
+    /**
+     * Generate embeddings for all chunks
+     * @param provider - Embedding provider name
+     * @param modelName - Embedding model name
+     * @returns This MDocument instance (for chaining)
+     */
+    embed(provider?: string, modelName?: string): Promise<MDocument>;
+    /**
+     * Get document ID
+     */
+    getId(): string;
+    /**
+     * Get raw document content
+     */
+    getContent(): string;
+    /**
+     * Get document type
+     */
+    getType(): DocumentType;
+    /**
+     * Get document metadata
+     */
+    getMetadata(): Record<string, unknown>;
+    /**
+     * Get processed chunks
+     */
+    getChunks(): Chunk[];
+    /**
+     * Get chunk embeddings
+     */
+    getEmbeddings(): number[][];
+    /**
+     * Get processing history
+     */
+    getHistory(): string[];
+    /**
+     * Check if document has been chunked
+     */
+    isChunked(): boolean;
+    /**
+     * Check if document has embeddings
+     */
+    hasEmbeddings(): boolean;
+    /**
+     * Get chunk count
+     */
+    getChunkCount(): number;
+    /**
+     * Set document metadata
+     * @param key - Metadata key
+     * @param value - Metadata value
+     * @returns This MDocument instance (for chaining)
+     */
+    setMetadata(key: string, value: unknown): MDocument;
+    /**
+     * Merge metadata into document
+     * @param metadata - Metadata to merge
+     * @returns This MDocument instance (for chaining)
+     */
+    mergeMetadata(metadata: Record<string, unknown>): MDocument;
+    /**
+     * Filter chunks based on predicate
+     * @param predicate - Filter function
+     * @returns New MDocument with filtered chunks
+     */
+    filterChunks(predicate: (chunk: Chunk) => boolean): MDocument;
+    /**
+     * Map transformation over chunks
+     * @param transform - Transform function
+     * @returns New MDocument with transformed chunks
+     */
+    mapChunks(transform: (chunk: Chunk) => Chunk): MDocument;
+    /**
+     * Convert to plain object for serialization
+     */
+    toJSON(): {
+        id: string;
+        content: string;
+        type: DocumentType;
+        metadata: Record<string, unknown>;
+        chunks: Chunk[];
+        history: string[];
+    };
+    /**
+     * Create MDocument from serialized JSON
+     * @param json - Serialized document data
+     * @returns MDocument instance
+     */
+    static fromJSON(json: {
+        id?: string;
+        content: string;
+        type: DocumentType;
+        metadata?: Record<string, unknown>;
+        chunks?: Chunk[];
+        history?: string[];
+    }): MDocument;
+    /**
+     * Get default chunking strategy based on document type
+     */
+    private getDefaultStrategy;
+}