@juspay/neurolink 9.32.0 → 9.33.0

package/dist/client/rag/chunking/tokenChunker.js

@@ -1,183 +0,0 @@
-/**
- * Token-based Chunker
- *
- * Splits text based on token counts using simple tokenization.
- * Best for controlling context window usage with LLMs.
- */
-import { randomUUID } from "crypto";
-/**
- * Token-aware chunker implementation
- * Splits text based on approximate token counts
- *
- * Note: Uses simple word-based tokenization as approximation.
- * For exact token counts, integrate with tiktoken or model-specific tokenizers.
- */
-export class TokenChunker {
-    strategy = "token";
-    // Approximate characters per token for different tokenizers
-    CHARS_PER_TOKEN = {
-        cl100k_base: 4, // GPT-4, GPT-3.5-turbo
-        p50k_base: 4, // Codex
-        r50k_base: 4, // GPT-3
-        default: 4,
-    };
-    async chunk(text, config) {
-        const { maxSize, overlap = 0, tokenizer = "cl100k_base", maxTokens = 512, tokenOverlap, trimWhitespace = true, metadata = {}, } = config || {};
-        const chunks = [];
-        const documentId = randomUUID();
-        if (!text || text.length === 0) {
-            return chunks;
-        }
-        // Determine effective overlap
-        const effectiveOverlap = tokenOverlap ?? Math.floor(overlap / this.getCharsPerToken(tokenizer));
-        // Use maxSize if provided, otherwise calculate from maxTokens
-        const _effectiveMaxChars = maxSize ?? maxTokens * this.getCharsPerToken(tokenizer);
-        // Tokenize text (simple word-based approximation)
-        const words = this.tokenize(text);
-        const _tokensPerWord = this.estimateTokensPerWord(tokenizer);
-        let currentWords = [];
-        let currentTokenCount = 0;
-        let chunkIndex = 0;
-        let startPosition = 0;
-        let charPosition = 0;
-        for (const word of words) {
-            const wordTokens = Math.ceil(word.length / this.getCharsPerToken(tokenizer));
-            // Check if adding this word would exceed the limit
-            if (currentTokenCount + wordTokens > maxTokens &&
-                currentWords.length > 0) {
-                // Save current chunk
-                const chunkText = currentWords.join(" ");
-                const finalText = trimWhitespace ? chunkText.trim() : chunkText;
-                if (finalText.length > 0) {
-                    chunks.push({
-                        id: randomUUID(),
-                        text: finalText,
-                        metadata: {
-                            documentId,
-                            chunkIndex,
-                            startPosition,
-                            endPosition: charPosition,
-                            documentType: "text",
-                            custom: {
-                                ...metadata,
-                                estimatedTokens: currentTokenCount,
-                            },
-                        },
-                    });
-                    chunkIndex++;
-                }
-                // Handle token overlap
-                if (effectiveOverlap > 0 && currentWords.length > 0) {
-                    // Keep some words for overlap
-                    let overlapTokens = 0;
-                    const overlapWords = [];
-                    for (let i = currentWords.length - 1; i >= 0; i--) {
-                        const w = currentWords[i];
-                        const wTokens = Math.ceil(w.length / this.getCharsPerToken(tokenizer));
-                        if (overlapTokens + wTokens <= effectiveOverlap) {
-                            overlapWords.unshift(w);
-                            overlapTokens += wTokens;
-                        }
-                        else {
-                            break;
-                        }
-                    }
-                    currentWords = overlapWords;
-                    currentTokenCount = overlapTokens;
-                    // Adjust start position for overlap
-                    const overlapChars = overlapWords.join(" ").length + 1;
-                    startPosition = charPosition - overlapChars;
-                }
-                else {
-                    currentWords = [];
-                    currentTokenCount = 0;
-                    startPosition = charPosition;
-                }
-            }
-            currentWords.push(word);
-            currentTokenCount += wordTokens;
-            charPosition += word.length + 1; // +1 for space
-        }
-        // Don't forget the last chunk
-        if (currentWords.length > 0) {
-            const chunkText = currentWords.join(" ");
-            const finalText = trimWhitespace ? chunkText.trim() : chunkText;
-            if (finalText.length > 0) {
-                chunks.push({
-                    id: randomUUID(),
-                    text: finalText,
-                    metadata: {
-                        documentId,
-                        chunkIndex,
-                        startPosition,
-                        endPosition: charPosition,
-                        documentType: "text",
-                        custom: {
-                            ...metadata,
-                            estimatedTokens: currentTokenCount,
-                        },
-                    },
-                });
-            }
-        }
-        // Update total chunks count
-        chunks.forEach((chunk) => {
-            chunk.metadata.totalChunks = chunks.length;
-        });
-        return chunks;
-    }
-    /**
-     * Simple word-based tokenization
-     */
-    tokenize(text) {
-        // Split on whitespace and filter empty strings
-        return text.split(/\s+/).filter((w) => w.length > 0);
-    }
-    /**
-     * Get characters per token for a tokenizer
-     */
-    getCharsPerToken(tokenizer) {
-        return this.CHARS_PER_TOKEN[tokenizer] ?? this.CHARS_PER_TOKEN.default;
-    }
-    /**
-     * Estimate average tokens per word
-     */
-    estimateTokensPerWord(_tokenizer) {
-        // Average English word is ~5 characters, so roughly 1.25 tokens
-        return 1.25;
-    }
-    /**
-     * Estimate token count for text
-     */
-    estimateTokenCount(text, tokenizer = "cl100k_base") {
-        return Math.ceil(text.length / this.getCharsPerToken(tokenizer));
-    }
-    validateConfig(config) {
-        const errors = [];
-        const warnings = [];
-        const tokenConfig = config;
-        if (tokenConfig.maxTokens !== undefined && tokenConfig.maxTokens <= 0) {
-            errors.push("maxTokens must be greater than 0");
-        }
-        if (tokenConfig.tokenOverlap !== undefined &&
-            tokenConfig.tokenOverlap < 0) {
-            errors.push("tokenOverlap must be non-negative");
-        }
-        if (tokenConfig.tokenOverlap !== undefined &&
-            tokenConfig.maxTokens !== undefined) {
-            if (tokenConfig.tokenOverlap >= tokenConfig.maxTokens) {
-                errors.push("tokenOverlap must be less than maxTokens");
-            }
-        }
-        if (tokenConfig.maxSize !== undefined && tokenConfig.maxSize <= 0) {
-            errors.push("maxSize must be greater than 0");
-        }
-        // Warn about tokenizer approximation
-        warnings.push("Token counts are approximated. For exact counts, integrate with tiktoken.");
-        return {
-            valid: errors.length === 0,
-            errors,
-            warnings,
-        };
-    }
-}

package/dist/client/rag/document/MDocument.js

@@ -1,392 +0,0 @@
-/**
- * MDocument - Main Document Processing Class
- *
- * Provides a fluent interface for document processing using the Factory + Registry pattern.
- * Supports various document types, chunking strategies, and metadata extraction.
- *
- * @example
- * ```typescript
- * const doc = await MDocument.fromText(content);
- * const chunks = await doc.chunk({
- *   strategy: 'recursive',
- *   config: { maxSize: 1000, overlap: 200 }
- * });
- * const enriched = await doc.extractMetadata({
- *   title: true,
- *   summary: true,
- *   keywords: true
- * });
- * ```
- */
-import { randomUUID } from "crypto";
-import { logger } from "../../utils/logger.js";
-import { ChunkerRegistry } from "../chunking/chunkerRegistry.js";
-import { LLMMetadataExtractor } from "../metadata/metadataExtractor.js";
-/**
- * MDocument class for comprehensive document processing
- *
- * Provides a chainable API for:
- * - Loading documents from various sources
- * - Chunking with multiple strategies
- * - Metadata extraction using LLMs
- * - Embedding generation
- */
-export class MDocument {
-    state;
-    documentId;
-    /**
-     * Create a new MDocument instance
-     * @param content - Document content
-     * @param config - Document configuration
-     */
-    constructor(content, config) {
-        this.documentId = randomUUID();
-        this.state = {
-            content,
-            type: config?.type ?? "text",
-            metadata: {
-                ...config?.metadata,
-                documentId: this.documentId,
-                createdAt: new Date().toISOString(),
-            },
-            chunks: [],
-            embeddings: [],
-            history: ["created"],
-        };
-    }
-    // ============================================================================
-    // Static Factory Methods
-    // ============================================================================
-    /**
-     * Create MDocument from plain text
-     * @param text - Plain text content
-     * @param metadata - Optional metadata
-     * @returns MDocument instance
-     */
-    static fromText(text, metadata) {
-        return new MDocument(text, { type: "text", metadata });
-    }
-    /**
-     * Create MDocument from markdown content
-     * @param markdown - Markdown content
-     * @param metadata - Optional metadata
-     * @returns MDocument instance
-     */
-    static fromMarkdown(markdown, metadata) {
-        return new MDocument(markdown, { type: "markdown", metadata });
-    }
-    /**
-     * Create MDocument from HTML content
-     * @param html - HTML content
-     * @param metadata - Optional metadata
-     * @returns MDocument instance
-     */
-    static fromHTML(html, metadata) {
-        return new MDocument(html, { type: "html", metadata });
-    }
-    /**
-     * Create MDocument from JSON content
-     * @param json - JSON string or object
-     * @param metadata - Optional metadata
-     * @returns MDocument instance
-     */
-    static fromJSONContent(json, metadata) {
-        const content = typeof json === "string" ? json : JSON.stringify(json, null, 2);
-        return new MDocument(content, { type: "json", metadata });
-    }
-    /**
-     * Create MDocument from LaTeX content
-     * @param latex - LaTeX content
-     * @param metadata - Optional metadata
-     * @returns MDocument instance
-     */
-    static fromLaTeX(latex, metadata) {
-        return new MDocument(latex, { type: "latex", metadata });
-    }
-    /**
-     * Create MDocument from CSV content
-     * @param csv - CSV content
-     * @param metadata - Optional metadata
-     * @returns MDocument instance
-     */
-    static fromCSV(csv, metadata) {
-        return new MDocument(csv, { type: "csv", metadata });
-    }
-    // ============================================================================
-    // Core Processing Methods
-    // ============================================================================
-    /**
-     * Chunk the document using specified strategy
-     * @param params - Chunking parameters
-     * @returns This MDocument instance (for chaining)
-     */
-    async chunk(params) {
-        const { strategy = this.getDefaultStrategy(), config = {} } = params || {};
-        logger.debug("[MDocument] Chunking document", {
-            documentId: this.documentId,
-            strategy,
-            contentLength: this.state.content.length,
-        });
-        const chunker = ChunkerRegistry.get(strategy);
-        // Merge document metadata into chunk config
-        const chunkConfig = {
-            ...config,
-            metadata: {
-                ...config.metadata,
-                source: this.state.metadata.source,
-                documentType: this.state.type,
-            },
-        };
-        this.state.chunks = await chunker.chunk(this.state.content, chunkConfig);
-        this.state.history.push(`chunked:${strategy}`);
-        logger.info("[MDocument] Document chunked", {
-            documentId: this.documentId,
-            strategy,
-            chunkCount: this.state.chunks.length,
-        });
-        return this;
-    }
-    /**
-     * Extract metadata from chunks using LLM
-     * @param params - Extraction parameters
-     * @param options - Extractor options
-     * @returns This MDocument instance (for chaining)
-     */
-    async extractMetadata(params, options) {
-        if (this.state.chunks.length === 0) {
-            logger.warn("[MDocument] No chunks to extract metadata from. Call chunk() first.");
-            return this;
-        }
-        logger.debug("[MDocument] Extracting metadata", {
-            documentId: this.documentId,
-            chunkCount: this.state.chunks.length,
-            params: Object.keys(params),
-        });
-        const extractor = new LLMMetadataExtractor(options);
-        const results = await extractor.extract(this.state.chunks, params);
-        // Merge extraction results into chunk metadata
-        for (let i = 0; i < this.state.chunks.length && i < results.length; i++) {
-            const result = results[i];
-            if (result.title) {
-                this.state.chunks[i].metadata.title = result.title;
-            }
-            if (result.summary) {
-                this.state.chunks[i].metadata.summary = result.summary;
-            }
-            if (result.keywords) {
-                this.state.chunks[i].metadata.keywords = result.keywords;
-            }
-            if (result.custom) {
-                this.state.chunks[i].metadata.custom = {
-                    ...(this.state.chunks[i].metadata.custom || {}),
-                    ...result.custom,
-                };
-            }
-        }
-        this.state.history.push(`metadata:${Object.keys(params).join(",")}`);
-        logger.info("[MDocument] Metadata extracted", {
-            documentId: this.documentId,
-            extractedFields: Object.keys(params),
-        });
-        return this;
-    }
-    /**
-     * Generate embeddings for all chunks
-     * @param provider - Embedding provider name
-     * @param modelName - Embedding model name
-     * @returns This MDocument instance (for chaining)
-     */
-    async embed(provider = "openai", modelName = "text-embedding-3-small") {
-        if (this.state.chunks.length === 0) {
-            logger.warn("[MDocument] No chunks to embed. Call chunk() first.");
-            return this;
-        }
-        // Lazy import to avoid circular dependencies
-        const { ProviderFactory } = await import("../../factories/providerFactory.js");
-        logger.debug("[MDocument] Generating embeddings", {
-            documentId: this.documentId,
-            chunkCount: this.state.chunks.length,
-            provider,
-            model: modelName,
-        });
-        const embeddingProvider = await ProviderFactory.createProvider(provider, modelName);
-        if (typeof embeddingProvider.embed !==
-            "function") {
-            throw new Error(`Provider ${provider} does not support embeddings`);
-        }
-        this.state.embeddings = [];
-        for (const chunk of this.state.chunks) {
-            const embedding = await embeddingProvider.embed(chunk.text);
-            this.state.embeddings.push(embedding);
-            chunk.embedding = embedding;
-        }
-        this.state.history.push(`embedded:${provider}:${modelName}`);
-        logger.info("[MDocument] Embeddings generated", {
-            documentId: this.documentId,
-            embeddingCount: this.state.embeddings.length,
-            dimension: this.state.embeddings[0]?.length,
-        });
-        return this;
-    }
-    // ============================================================================
-    // Accessor Methods
-    // ============================================================================
-    /**
-     * Get document ID
-     */
-    getId() {
-        return this.documentId;
-    }
-    /**
-     * Get raw document content
-     */
-    getContent() {
-        return this.state.content;
-    }
-    /**
-     * Get document type
-     */
-    getType() {
-        return this.state.type;
-    }
-    /**
-     * Get document metadata
-     */
-    getMetadata() {
-        return { ...this.state.metadata };
-    }
-    /**
-     * Get processed chunks
-     */
-    getChunks() {
-        return [...this.state.chunks];
-    }
-    /**
-     * Get chunk embeddings
-     */
-    getEmbeddings() {
-        return [...this.state.embeddings];
-    }
-    /**
-     * Get processing history
-     */
-    getHistory() {
-        return [...this.state.history];
-    }
-    /**
-     * Check if document has been chunked
-     */
-    isChunked() {
-        return this.state.chunks.length > 0;
-    }
-    /**
-     * Check if document has embeddings
-     */
-    hasEmbeddings() {
-        return this.state.embeddings.length > 0;
-    }
-    /**
-     * Get chunk count
-     */
-    getChunkCount() {
-        return this.state.chunks.length;
-    }
-    // ============================================================================
-    // Transformation Methods
-    // ============================================================================
-    /**
-     * Set document metadata
-     * @param key - Metadata key
-     * @param value - Metadata value
-     * @returns This MDocument instance (for chaining)
-     */
-    setMetadata(key, value) {
-        this.state.metadata[key] = value;
-        return this;
-    }
-    /**
-     * Merge metadata into document
-     * @param metadata - Metadata to merge
-     * @returns This MDocument instance (for chaining)
-     */
-    mergeMetadata(metadata) {
-        this.state.metadata = { ...this.state.metadata, ...metadata };
-        return this;
-    }
-    /**
-     * Filter chunks based on predicate
-     * @param predicate - Filter function
-     * @returns New MDocument with filtered chunks
-     */
-    filterChunks(predicate) {
-        const doc = new MDocument(this.state.content, {
-            type: this.state.type,
-            metadata: this.state.metadata,
-        });
-        doc.state.chunks = this.state.chunks.filter(predicate);
-        doc.state.embeddings = this.state.embeddings.filter((_, i) => predicate(this.state.chunks[i]));
-        doc.state.history = [...this.state.history, "filtered"];
-        return doc;
-    }
-    /**
-     * Map transformation over chunks
-     * @param transform - Transform function
-     * @returns New MDocument with transformed chunks
-     */
-    mapChunks(transform) {
-        const doc = new MDocument(this.state.content, {
-            type: this.state.type,
-            metadata: this.state.metadata,
-        });
-        doc.state.chunks = this.state.chunks.map(transform);
-        doc.state.embeddings = [...this.state.embeddings];
-        doc.state.history = [...this.state.history, "mapped"];
-        return doc;
-    }
-    // ============================================================================
-    // Serialization Methods
-    // ============================================================================
-    /**
-     * Convert to plain object for serialization
-     */
-    toJSON() {
-        return {
-            id: this.documentId,
-            content: this.state.content,
-            type: this.state.type,
-            metadata: this.state.metadata,
-            chunks: this.state.chunks,
-            history: this.state.history,
-        };
-    }
-    /**
-     * Create MDocument from serialized JSON
-     * @param json - Serialized document data
-     * @returns MDocument instance
-     */
-    static fromJSON(json) {
-        const doc = new MDocument(json.content, {
-            type: json.type,
-            metadata: json.metadata,
-        });
-        if (json.id) {
-            doc.documentId = json.id;
-        }
-        if (json.chunks) {
-            doc.state.chunks = json.chunks;
-        }
-        if (json.history) {
-            doc.state.history = json.history;
-        }
-        return doc;
-    }
-    // ============================================================================
-    // Private Helper Methods
-    // ============================================================================
-    /**
-     * Get default chunking strategy based on document type
-     */
-    getDefaultStrategy() {
-        return ChunkerRegistry.getRecommendedStrategy(this.state.type);
-    }
-}

package/dist/client/rag/document/index.js

@@ -1,5 +0,0 @@
-/**
- * Document Module Exports
- */
-export { MDocument } from "./MDocument.js";
-export { TextLoader, MarkdownLoader, HTMLLoader, JSONLoader, CSVLoader, PDFLoader, WebLoader, loadDocument, loadDocuments, } from "./loaders.js";