@juspay/neurolink 9.1.1 → 9.3.0

package/dist/rag/document/MDocument.js

@@ -0,0 +1,392 @@
+/**
+ * MDocument - Main Document Processing Class
+ *
+ * Provides a fluent interface for document processing using the Factory + Registry pattern.
+ * Supports various document types, chunking strategies, and metadata extraction.
+ *
+ * @example
+ * ```typescript
+ * const doc = await MDocument.fromText(content);
+ * const chunks = await doc.chunk({
+ *   strategy: 'recursive',
+ *   config: { maxSize: 1000, overlap: 200 }
+ * });
+ * const enriched = await doc.extractMetadata({
+ *   title: true,
+ *   summary: true,
+ *   keywords: true
+ * });
+ * ```
+ */
+import { randomUUID } from "crypto";
+import { logger } from "../../utils/logger.js";
+import { ChunkerRegistry } from "../chunking/chunkerRegistry.js";
+import { LLMMetadataExtractor } from "../metadata/metadataExtractor.js";
+/**
+ * MDocument class for comprehensive document processing
+ *
+ * Provides a chainable API for:
+ * - Loading documents from various sources
+ * - Chunking with multiple strategies
+ * - Metadata extraction using LLMs
+ * - Embedding generation
+ */
+export class MDocument {
+    state;
+    documentId;
+    /**
+     * Create a new MDocument instance
+     * @param content - Document content
+     * @param config - Document configuration
+     */
+    constructor(content, config) {
+        this.documentId = randomUUID();
+        this.state = {
+            content,
+            type: config?.type ?? "text",
+            metadata: {
+                ...config?.metadata,
+                documentId: this.documentId,
+                createdAt: new Date().toISOString(),
+            },
+            chunks: [],
+            embeddings: [],
+            history: ["created"],
+        };
+    }
+    // ============================================================================
+    // Static Factory Methods
+    // ============================================================================
+    /**
+     * Create MDocument from plain text
+     * @param text - Plain text content
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromText(text, metadata) {
+        return new MDocument(text, { type: "text", metadata });
+    }
+    /**
+     * Create MDocument from markdown content
+     * @param markdown - Markdown content
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromMarkdown(markdown, metadata) {
+        return new MDocument(markdown, { type: "markdown", metadata });
+    }
+    /**
+     * Create MDocument from HTML content
+     * @param html - HTML content
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromHTML(html, metadata) {
+        return new MDocument(html, { type: "html", metadata });
+    }
+    /**
+     * Create MDocument from JSON content
+     * @param json - JSON string or object
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromJSONContent(json, metadata) {
+        const content = typeof json === "string" ? json : JSON.stringify(json, null, 2);
+        return new MDocument(content, { type: "json", metadata });
+    }
+    /**
+     * Create MDocument from LaTeX content
+     * @param latex - LaTeX content
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromLaTeX(latex, metadata) {
+        return new MDocument(latex, { type: "latex", metadata });
+    }
+    /**
+     * Create MDocument from CSV content
+     * @param csv - CSV content
+     * @param metadata - Optional metadata
+     * @returns MDocument instance
+     */
+    static fromCSV(csv, metadata) {
+        return new MDocument(csv, { type: "csv", metadata });
+    }
+    // ============================================================================
+    // Core Processing Methods
+    // ============================================================================
+    /**
+     * Chunk the document using specified strategy
+     * @param params - Chunking parameters
+     * @returns This MDocument instance (for chaining)
+     */
+    async chunk(params) {
+        const { strategy = this.getDefaultStrategy(), config = {} } = params || {};
+        logger.debug("[MDocument] Chunking document", {
+            documentId: this.documentId,
+            strategy,
+            contentLength: this.state.content.length,
+        });
+        const chunker = ChunkerRegistry.get(strategy);
+        // Merge document metadata into chunk config
+        const chunkConfig = {
+            ...config,
+            metadata: {
+                ...config.metadata,
+                source: this.state.metadata.source,
+                documentType: this.state.type,
+            },
+        };
+        this.state.chunks = await chunker.chunk(this.state.content, chunkConfig);
+        this.state.history.push(`chunked:${strategy}`);
+        logger.info("[MDocument] Document chunked", {
+            documentId: this.documentId,
+            strategy,
+            chunkCount: this.state.chunks.length,
+        });
+        return this;
+    }
+    /**
+     * Extract metadata from chunks using LLM
+     * @param params - Extraction parameters
+     * @param options - Extractor options
+     * @returns This MDocument instance (for chaining)
+     */
+    async extractMetadata(params, options) {
+        if (this.state.chunks.length === 0) {
+            logger.warn("[MDocument] No chunks to extract metadata from. Call chunk() first.");
+            return this;
+        }
+        logger.debug("[MDocument] Extracting metadata", {
+            documentId: this.documentId,
+            chunkCount: this.state.chunks.length,
+            params: Object.keys(params),
+        });
+        const extractor = new LLMMetadataExtractor(options);
+        const results = await extractor.extract(this.state.chunks, params);
+        // Merge extraction results into chunk metadata
+        for (let i = 0; i < this.state.chunks.length && i < results.length; i++) {
+            const result = results[i];
+            if (result.title) {
+                this.state.chunks[i].metadata.title = result.title;
+            }
+            if (result.summary) {
+                this.state.chunks[i].metadata.summary = result.summary;
+            }
+            if (result.keywords) {
+                this.state.chunks[i].metadata.keywords = result.keywords;
+            }
+            if (result.custom) {
+                this.state.chunks[i].metadata.custom = {
+                    ...(this.state.chunks[i].metadata.custom || {}),
+                    ...result.custom,
+                };
+            }
+        }
+        this.state.history.push(`metadata:${Object.keys(params).join(",")}`);
+        logger.info("[MDocument] Metadata extracted", {
+            documentId: this.documentId,
+            extractedFields: Object.keys(params),
+        });
+        return this;
+    }
+    /**
+     * Generate embeddings for all chunks
+     * @param provider - Embedding provider name
+     * @param modelName - Embedding model name
+     * @returns This MDocument instance (for chaining)
+     */
+    async embed(provider = "openai", modelName = "text-embedding-3-small") {
+        if (this.state.chunks.length === 0) {
+            logger.warn("[MDocument] No chunks to embed. Call chunk() first.");
+            return this;
+        }
+        // Lazy import to avoid circular dependencies
+        const { ProviderFactory } = await import("../../factories/providerFactory.js");
+        logger.debug("[MDocument] Generating embeddings", {
+            documentId: this.documentId,
+            chunkCount: this.state.chunks.length,
+            provider,
+            model: modelName,
+        });
+        const embeddingProvider = await ProviderFactory.createProvider(provider, modelName);
+        if (typeof embeddingProvider.embed !==
+            "function") {
+            throw new Error(`Provider ${provider} does not support embeddings`);
+        }
+        this.state.embeddings = [];
+        for (const chunk of this.state.chunks) {
+            const embedding = await embeddingProvider.embed(chunk.text);
+            this.state.embeddings.push(embedding);
+            chunk.embedding = embedding;
+        }
+        this.state.history.push(`embedded:${provider}:${modelName}`);
+        logger.info("[MDocument] Embeddings generated", {
+            documentId: this.documentId,
+            embeddingCount: this.state.embeddings.length,
+            dimension: this.state.embeddings[0]?.length,
+        });
+        return this;
+    }
+    // ============================================================================
+    // Accessor Methods
+    // ============================================================================
+    /**
+     * Get document ID
+     */
+    getId() {
+        return this.documentId;
+    }
+    /**
+     * Get raw document content
+     */
+    getContent() {
+        return this.state.content;
+    }
+    /**
+     * Get document type
+     */
+    getType() {
+        return this.state.type;
+    }
+    /**
+     * Get document metadata
+     */
+    getMetadata() {
+        return { ...this.state.metadata };
+    }
+    /**
+     * Get processed chunks
+     */
+    getChunks() {
+        return [...this.state.chunks];
+    }
+    /**
+     * Get chunk embeddings
+     */
+    getEmbeddings() {
+        return [...this.state.embeddings];
+    }
+    /**
+     * Get processing history
+     */
+    getHistory() {
+        return [...this.state.history];
+    }
+    /**
+     * Check if document has been chunked
+     */
+    isChunked() {
+        return this.state.chunks.length > 0;
+    }
+    /**
+     * Check if document has embeddings
+     */
+    hasEmbeddings() {
+        return this.state.embeddings.length > 0;
+    }
+    /**
+     * Get chunk count
+     */
+    getChunkCount() {
+        return this.state.chunks.length;
+    }
+    // ============================================================================
+    // Transformation Methods
+    // ============================================================================
+    /**
+     * Set document metadata
+     * @param key - Metadata key
+     * @param value - Metadata value
+     * @returns This MDocument instance (for chaining)
+     */
+    setMetadata(key, value) {
+        this.state.metadata[key] = value;
+        return this;
+    }
+    /**
+     * Merge metadata into document
+     * @param metadata - Metadata to merge
+     * @returns This MDocument instance (for chaining)
+     */
+    mergeMetadata(metadata) {
+        this.state.metadata = { ...this.state.metadata, ...metadata };
+        return this;
+    }
+    /**
+     * Filter chunks based on predicate
+     * @param predicate - Filter function
+     * @returns New MDocument with filtered chunks
+     */
+    filterChunks(predicate) {
+        const doc = new MDocument(this.state.content, {
+            type: this.state.type,
+            metadata: this.state.metadata,
+        });
+        doc.state.chunks = this.state.chunks.filter(predicate);
+        doc.state.embeddings = this.state.embeddings.filter((_, i) => predicate(this.state.chunks[i]));
+        doc.state.history = [...this.state.history, "filtered"];
+        return doc;
+    }
+    /**
+     * Map transformation over chunks
+     * @param transform - Transform function
+     * @returns New MDocument with transformed chunks
+     */
+    mapChunks(transform) {
+        const doc = new MDocument(this.state.content, {
+            type: this.state.type,
+            metadata: this.state.metadata,
+        });
+        doc.state.chunks = this.state.chunks.map(transform);
+        doc.state.embeddings = [...this.state.embeddings];
+        doc.state.history = [...this.state.history, "mapped"];
+        return doc;
+    }
+    // ============================================================================
+    // Serialization Methods
+    // ============================================================================
+    /**
+     * Convert to plain object for serialization
+     */
+    toJSON() {
+        return {
+            id: this.documentId,
+            content: this.state.content,
+            type: this.state.type,
+            metadata: this.state.metadata,
+            chunks: this.state.chunks,
+            history: this.state.history,
+        };
+    }
+    /**
+     * Create MDocument from serialized JSON
+     * @param json - Serialized document data
+     * @returns MDocument instance
+     */
+    static fromJSON(json) {
+        const doc = new MDocument(json.content, {
+            type: json.type,
+            metadata: json.metadata,
+        });
+        if (json.id) {
+            doc.documentId = json.id;
+        }
+        if (json.chunks) {
+            doc.state.chunks = json.chunks;
+        }
+        if (json.history) {
+            doc.state.history = json.history;
+        }
+        return doc;
+    }
+    // ============================================================================
+    // Private Helper Methods
+    // ============================================================================
+    /**
+     * Get default chunking strategy based on document type
+     */
+    getDefaultStrategy() {
+        return ChunkerRegistry.getRecommendedStrategy(this.state.type);
+    }
+}

package/dist/rag/document/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Document Module Exports
+ */
+export { MDocument } from "./MDocument.js";
+export { type DocumentLoader, TextLoader, MarkdownLoader, HTMLLoader, JSONLoader, CSVLoader, PDFLoader, WebLoader, loadDocument, loadDocuments, type LoaderOptions, type WebLoaderOptions, type PDFLoaderOptions, type CSVLoaderOptions, } from "./loaders.js";

package/dist/rag/document/index.js

@@ -0,0 +1,5 @@
+/**
+ * Document Module Exports
+ */
+export { MDocument } from "./MDocument.js";
+export { TextLoader, MarkdownLoader, HTMLLoader, JSONLoader, CSVLoader, PDFLoader, WebLoader, loadDocument, loadDocuments, } from "./loaders.js";

package/dist/rag/document/loaders.d.ts

@@ -0,0 +1,201 @@
+/**
+ * Document Loaders
+ *
+ * Provides loaders for various document formats including:
+ * - Text files
+ * - Markdown files
+ * - HTML files and web pages
+ * - JSON files
+ * - CSV files
+ * - PDF files
+ *
+ * @example
+ * ```typescript
+ * import { loadDocument, WebLoader, PDFLoader } from 'neurolink/rag';
+ *
+ * // Load from file path
+ * const doc = await loadDocument('/path/to/document.md');
+ *
+ * // Load from URL
+ * const webDoc = await WebLoader.load('https://example.com/article');
+ *
+ * // Load PDF
+ * const pdfDoc = await PDFLoader.load('/path/to/document.pdf');
+ * ```
+ */
+import { MDocument } from "./MDocument.js";
+import type { DocumentType } from "../types.js";
+/**
+ * Document loader options
+ */
+export interface LoaderOptions {
+    /** Custom metadata to add to document */
+    metadata?: Record<string, unknown>;
+    /** Encoding for text files */
+    encoding?: BufferEncoding;
+    /** Document type override */
+    type?: DocumentType;
+}
+/**
+ * Web loader options
+ */
+export interface WebLoaderOptions extends LoaderOptions {
+    /** Request timeout in milliseconds */
+    timeout?: number;
+    /** Custom headers for request */
+    headers?: Record<string, string>;
+    /** Extract only main content (remove navigation, ads, etc.) */
+    extractMainContent?: boolean;
+    /** Selector for main content (CSS selector) */
+    contentSelector?: string;
+    /** User agent string */
+    userAgent?: string;
+}
+/**
+ * PDF loader options
+ */
+export interface PDFLoaderOptions extends LoaderOptions {
+    /** Page range to extract (e.g., "1-5" or "1,3,5") */
+    pageRange?: string;
+    /** Extract images as base64 */
+    extractImages?: boolean;
+    /** OCR for scanned documents */
+    enableOCR?: boolean;
+    /** Preserve layout formatting */
+    preserveLayout?: boolean;
+}
+/**
+ * CSV loader options
+ */
+export interface CSVLoaderOptions extends LoaderOptions {
+    /** Delimiter character */
+    delimiter?: string;
+    /** Whether first row is header */
+    hasHeader?: boolean;
+    /** Column names (if no header) */
+    columns?: string[];
+    /** Output format */
+    outputFormat?: "text" | "json" | "markdown";
+}
+/**
+ * Abstract document loader interface
+ */
+export interface DocumentLoader {
+    /**
+     * Load document from source
+     * @param source - File path, URL, or content
+     * @param options - Loader options
+     * @returns Promise resolving to MDocument
+     */
+    load(source: string, options?: LoaderOptions): Promise<MDocument>;
+    /**
+     * Check if loader can handle the source
+     * @param source - File path, URL, or content
+     * @returns True if loader can handle the source
+     */
+    canHandle(source: string): boolean;
+}
+/**
+ * Text file loader
+ */
+export declare class TextLoader implements DocumentLoader {
+    load(source: string, options?: LoaderOptions): Promise<MDocument>;
+    canHandle(source: string): boolean;
+    protected loadContent(source: string, encoding?: BufferEncoding): Promise<string>;
+    protected getSourceName(source: string): string;
+}
+/**
+ * Markdown file loader
+ */
+export declare class MarkdownLoader extends TextLoader {
+    load(source: string, options?: LoaderOptions): Promise<MDocument>;
+    canHandle(source: string): boolean;
+}
+/**
+ * HTML file loader
+ */
+export declare class HTMLLoader extends TextLoader {
+    load(source: string, options?: LoaderOptions): Promise<MDocument>;
+    canHandle(source: string): boolean;
+}
+/**
+ * JSON file loader
+ */
+export declare class JSONLoader extends TextLoader {
+    load(source: string, options?: LoaderOptions): Promise<MDocument>;
+    canHandle(source: string): boolean;
+}
+/**
+ * CSV file loader
+ */
+export declare class CSVLoader extends TextLoader {
+    load(source: string, options?: CSVLoaderOptions): Promise<MDocument>;
+    canHandle(source: string): boolean;
+    private parseCSVLine;
+    private toMarkdownTable;
+    private toTextTable;
+}
+/**
+ * PDF file loader
+ *
+ * Note: Requires external PDF processing library for full functionality.
+ * Falls back to placeholder implementation if pdf-parse is not available.
+ */
+export declare class PDFLoader implements DocumentLoader {
+    load(source: string, options?: PDFLoaderOptions): Promise<MDocument>;
+    canHandle(source: string): boolean;
+    private loadPdfParser;
+    private parsePageRange;
+}
+/**
+ * Web page loader
+ *
+ * Fetches and extracts content from web pages.
+ * Supports basic HTML parsing without external dependencies.
+ */
+export declare class WebLoader implements DocumentLoader {
+    private defaultUserAgent;
+    load(source: string, options?: WebLoaderOptions): Promise<MDocument>;
+    canHandle(source: string): boolean;
+    /**
+     * Extract main content from HTML
+     */
+    private extractMainContent;
+    /**
+     * Convert HTML to plain text
+     */
+    private htmlToText;
+}
+/**
+ * Load document from file path, URL, or content
+ *
+ * Automatically detects the document type and uses the appropriate loader.
+ *
+ * @param source - File path, URL, or raw content
+ * @param options - Loader options
+ * @returns Promise resolving to MDocument
+ *
+ * @example
+ * ```typescript
+ * // Load from file
+ * const doc = await loadDocument('/path/to/document.md');
+ *
+ * // Load from URL
+ * const webDoc = await loadDocument('https://example.com/article');
+ *
+ * // Load with options
+ * const pdfDoc = await loadDocument('/path/to/doc.pdf', {
+ *   pageRange: '1-5',
+ *   metadata: { project: 'research' }
+ * });
+ * ```
+ */
+export declare function loadDocument(source: string, options?: LoaderOptions): Promise<MDocument>;
+/**
+ * Load multiple documents
+ *
+ * @param sources - Array of file paths, URLs, or content
+ * @param options - Loader options (applied to all)
+ * @returns Promise resolving to array of MDocuments
+ */
+export declare function loadDocuments(sources: string[], options?: LoaderOptions): Promise<MDocument[]>;