@juspay/neurolink 9.30.0 → 9.31.1

package/dist/client/rag/index.js

@@ -0,0 +1,141 @@
+/**
+ * RAG Document Processing Module
+ *
+ * Provides comprehensive RAG (Retrieval-Augmented Generation) capabilities:
+ * - Document loading (text, markdown, HTML, JSON, CSV, PDF, web)
+ * - MDocument class for fluent document processing
+ * - 10 chunking strategies (character, recursive, sentence, token, markdown, html, json, latex, semantic, semantic-markdown)
+ * - LLM-powered metadata extraction (title, summary, keywords, Q&A)
+ * - Vector query tools with metadata filtering and reranking
+ * - Hybrid search (BM25 + vector fusion)
+ * - Graph RAG for knowledge graph-based retrieval
+ * - RAG pipeline orchestration
+ * - Context assembly and formatting
+ * - ChunkerFactory and ChunkerRegistry patterns for extensibility
+ * - Error handling and resilience (CircuitBreaker, RetryHandler)
+ *
+ * @example
+ * ```typescript
+ * import {
+ *   MDocument,
+ *   loadDocument,
+ *   RAGPipeline,
+ *   ChunkerRegistry,
+ *   ChunkerFactory,
+ *   CircuitBreaker
+ * } from '@juspay/neurolink';
+ *
+ * // Load and process a document
+ * const doc = await loadDocument('/path/to/document.md');
+ * await doc.chunk({ strategy: 'markdown', config: { maxSize: 1000 } });
+ * await doc.embed('openai', 'text-embedding-3-small');
+ *
+ * // Or use the full RAG pipeline
+ * const pipeline = new RAGPipeline({
+ *   embeddingModel: { provider: 'openai', modelName: 'text-embedding-3-small' },
+ *   generationModel: { provider: 'openai', modelName: 'gpt-4o-mini' }
+ * });
+ * await pipeline.ingest(['/path/to/docs/*.md']);
+ * const response = await pipeline.query('What are the key features?');
+ *
+ * // Use factory pattern for chunker creation
+ * const chunker = await ChunkerFactory.createChunker('semantic', { maxSize: 500 });
+ * const chunks = await chunker.chunk(text);
+ * ```
+ */
+export { ChunkerFactory, chunkerFactory, createChunker, getAvailableStrategies as getFactoryStrategies, getDefaultConfig as getFactoryDefaultConfig, } from "./ChunkerFactory.js";
+// ChunkerFactory and ChunkerRegistry patterns (from main worktree)
+export { ChunkerRegistry as ChunkerRegistryV2, chunkerRegistry, getAvailableChunkers, getChunker, getChunkerMetadata, } from "./ChunkerRegistry.js";
+// Base chunker and chunker implementations (from main worktree)
+export * from "./chunkers/index.js";
+// Chunking
+export { CharacterChunker, ChunkerRegistry, chunkText, HTMLChunker, JSONChunker, LaTeXChunker, MarkdownChunker, RecursiveChunker, SemanticChunker, SentenceChunker, TokenChunker, } from "./chunking/index.js";
+// Document Processing
+export { CSVLoader, HTMLLoader, JSONLoader, loadDocument, loadDocuments, MarkdownLoader, MDocument, PDFLoader, TextLoader, WebLoader, } from "./document/index.js";
+// Error handling
+export * from "./errors/index.js";
+// Graph RAG
+export { GraphRAG } from "./graphRag/index.js";
+// Metadata Extraction
+export { createMetadataExtractor, extractMetadata, getAvailableExtractors, getAvailableExtractorTypes, getExtractor, getExtractorDefaultConfig, getExtractorMetadata, getRegisteredExtractorMetadata, LLMMetadataExtractor, 
+// Factory pattern
+MetadataExtractorFactory, 
+// Registry pattern
+MetadataExtractorRegistry, metadataExtractorFactory, metadataExtractorRegistry, } from "./metadata/index.js";
+// Pipeline
+export { assembleContext, createContextWindow, createRAGPipeline, extractKeySentences, formatContextWithCitations, orderByDocumentStructure, RAGPipeline, summarizeContext, } from "./pipeline/index.js";
+// RAG Integration (for generate/stream)
+export { prepareRAGTool } from "./ragIntegration.js";
+// Reranker
+export { batchRerank, CohereRelevanceScorer, CrossEncoderReranker, createReranker, getAvailableRerankers, getAvailableRerankerTypes, getRegisteredRerankerMetadata, getReranker, getRerankerDefaultConfig, getRerankerMetadata, 
+// Factory pattern
+RerankerFactory, 
+// Registry pattern
+RerankerRegistry, rerank, rerankerFactory, rerankerRegistry, simpleRerank, } from "./reranker/index.js";
+// Resilience patterns
+export * from "./resilience/index.js";
+// Retrieval
+export { createHybridSearch, createVectorQueryTool, InMemoryBM25Index, InMemoryVectorStore, linearCombination, reciprocalRankFusion, } from "./retrieval/index.js";
+// Types
+export * from "./types.js";
+// Convenience functions
+import { ChunkerRegistry } from "./chunking/index.js";
+import { LLMMetadataExtractor } from "./metadata/index.js";
+/**
+ * Process a document through the full RAG pipeline
+ *
+ * @param text - Document text to process
+ * @param options - Processing options
+ * @returns Processed chunks with optional metadata
+ */
+export async function processDocument(text, options) {
+    const { strategy = "recursive", maxSize = 1000, overlap = 200, extract, provider, model, metadata = {}, } = options || {};
+    // Chunk the document
+    const chunker = ChunkerRegistry.get(strategy);
+    const chunks = await chunker.chunk(text, { maxSize, overlap, metadata });
+    // Extract metadata if requested
+    if (extract) {
+        const extractor = new LLMMetadataExtractor({ provider, modelName: model });
+        const results = await extractor.extract(chunks, extract);
+        // Merge metadata into chunks
+        for (let i = 0; i < chunks.length && i < results.length; i++) {
+            const result = results[i];
+            if (result.title) {
+                chunks[i].metadata.title = result.title;
+            }
+            if (result.summary) {
+                chunks[i].metadata.summary = result.summary;
+            }
+            if (result.keywords) {
+                chunks[i].metadata.keywords = result.keywords;
+            }
+        }
+    }
+    return chunks;
+}
+/**
+ * Get recommended chunking strategy based on content type
+ *
+ * @param contentType - MIME type or file extension
+ * @returns Recommended chunking strategy
+ */
+export function getRecommendedStrategy(contentType) {
+    return ChunkerRegistry.getRecommendedStrategy(contentType);
+}
+/**
+ * Get available chunking strategies
+ *
+ * @returns Array of available strategy names
+ */
+export function getAvailableStrategies() {
+    return ChunkerRegistry.getAvailableStrategies();
+}
+/**
+ * Get default configuration for a chunking strategy
+ *
+ * @param strategy - Chunking strategy name
+ * @returns Default configuration object
+ */
+export function getDefaultChunkerConfig(strategy) {
+    return ChunkerRegistry.getDefaultConfig(strategy);
+}

package/dist/client/rag/metadata/MetadataExtractorFactory.js

@@ -0,0 +1,418 @@
+/**
+ * Metadata Extractor Factory
+ *
+ * Factory for creating metadata extractor instances with configuration.
+ * Follows the BaseFactory pattern for consistent lifecycle management.
+ */
+import { BaseFactory } from "../../core/infrastructure/index.js";
+import { logger } from "../../utils/logger.js";
+import { MetadataExtractionError, RAGErrorCodes } from "../errors/RAGError.js";
+/**
+ * Default metadata extractor metadata entries
+ */
+const DEFAULT_EXTRACTOR_METADATA = {
+    llm: {
+        description: "Full LLM-powered metadata extraction supporting all extraction types",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            temperature: 0.3,
+        },
+        supportedOptions: [
+            "provider",
+            "modelName",
+            "promptTemplate",
+            "maxTokens",
+            "temperature",
+        ],
+        useCases: [
+            "Comprehensive metadata extraction",
+            "Multi-type extraction in single pass",
+            "Custom schema extraction",
+        ],
+        aliases: ["full", "comprehensive", "all"],
+        requiresModel: true,
+        extractionTypes: ["title", "summary", "keywords", "questions", "custom"],
+    },
+    title: {
+        description: "Extracts concise, descriptive titles from document content",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            maxTokens: 100,
+        },
+        supportedOptions: ["provider", "modelName", "promptTemplate", "maxTokens"],
+        useCases: [
+            "Document indexing",
+            "Content organization",
+            "Navigation systems",
+        ],
+        aliases: ["header", "heading"],
+        requiresModel: true,
+        extractionTypes: ["title"],
+    },
+    summary: {
+        description: "Generates concise summaries of document chunks",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            maxTokens: 200,
+        },
+        supportedOptions: [
+            "provider",
+            "modelName",
+            "promptTemplate",
+            "maxTokens",
+            "maxWords",
+        ],
+        useCases: [
+            "Document previews",
+            "Search result snippets",
+            "Content condensation",
+        ],
+        aliases: ["summarize", "abstract"],
+        requiresModel: true,
+        extractionTypes: ["summary"],
+    },
+    keywords: {
+        description: "Extracts key terms and phrases from content",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            maxTokens: 100,
+        },
+        supportedOptions: [
+            "provider",
+            "modelName",
+            "promptTemplate",
+            "maxKeywords",
+        ],
+        useCases: ["Tag generation", "Topic modeling", "Search optimization"],
+        aliases: ["tags", "terms", "keyphrase"],
+        requiresModel: true,
+        extractionTypes: ["keywords"],
+    },
+    questions: {
+        description: "Generates Q&A pairs from content for training or FAQs",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            maxTokens: 500,
+        },
+        supportedOptions: [
+            "provider",
+            "modelName",
+            "promptTemplate",
+            "numQuestions",
+            "includeAnswers",
+        ],
+        useCases: [
+            "FAQ generation",
+            "Training data creation",
+            "Knowledge base building",
+        ],
+        aliases: ["qa", "faq", "questions-answers"],
+        requiresModel: true,
+        extractionTypes: ["questions"],
+    },
+    custom: {
+        description: "Extracts structured data according to custom schema",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            maxTokens: 500,
+        },
+        supportedOptions: [
+            "provider",
+            "modelName",
+            "promptTemplate",
+            "schema",
+            "description",
+        ],
+        useCases: [
+            "Structured data extraction",
+            "Entity extraction",
+            "Custom field extraction",
+        ],
+        aliases: ["schema", "structured", "entity"],
+        requiresModel: true,
+        extractionTypes: ["custom"],
+    },
+    composite: {
+        description: "Combines multiple extraction types in a single pass",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+        },
+        supportedOptions: ["provider", "modelName", "extractors"],
+        useCases: [
+            "Multi-field extraction",
+            "Complete document processing",
+            "Pipeline integration",
+        ],
+        aliases: ["multi", "combined", "batch"],
+        requiresModel: true,
+        extractionTypes: ["title", "summary", "keywords", "questions", "custom"],
+    },
+};
+/**
+ * Metadata Extractor Factory
+ *
+ * Creates metadata extractor instances based on type with configuration support.
+ * Uses lazy loading via dynamic imports to avoid circular dependencies.
+ */
+export class MetadataExtractorFactory extends BaseFactory {
+    static instance = null;
+    metadataMap = new Map();
+    constructor() {
+        super();
+    }
+    /**
+     * Get singleton instance
+     */
+    static getInstance() {
+        if (!MetadataExtractorFactory.instance) {
+            MetadataExtractorFactory.instance = new MetadataExtractorFactory();
+        }
+        return MetadataExtractorFactory.instance;
+    }
+    /**
+     * Reset singleton (for testing)
+     */
+    static resetInstance() {
+        if (MetadataExtractorFactory.instance) {
+            MetadataExtractorFactory.instance.clear();
+            MetadataExtractorFactory.instance = null;
+        }
+    }
+    /**
+     * Register all default extractors
+     */
+    async registerAll() {
+        // Register full LLM extractor
+        this.registerExtractor("llm", async (config) => {
+            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
+            return this.wrapExtractor(new LLMMetadataExtractor({
+                provider: config?.provider,
+                modelName: config?.modelName,
+            }), "llm");
+        }, DEFAULT_EXTRACTOR_METADATA.llm);
+        // Register title extractor
+        this.registerExtractor("title", async (config) => {
+            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
+            return this.createSpecializedExtractor(new LLMMetadataExtractor({
+                provider: config?.provider,
+                modelName: config?.modelName,
+            }), "title", { title: true });
+        }, DEFAULT_EXTRACTOR_METADATA.title);
+        // Register summary extractor
+        this.registerExtractor("summary", async (config) => {
+            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
+            return this.createSpecializedExtractor(new LLMMetadataExtractor({
+                provider: config?.provider,
+                modelName: config?.modelName,
+            }), "summary", { summary: true });
+        }, DEFAULT_EXTRACTOR_METADATA.summary);
+        // Register keywords extractor
+        this.registerExtractor("keywords", async (config) => {
+            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
+            return this.createSpecializedExtractor(new LLMMetadataExtractor({
+                provider: config?.provider,
+                modelName: config?.modelName,
+            }), "keywords", { keywords: true });
+        }, DEFAULT_EXTRACTOR_METADATA.keywords);
+        // Register questions extractor
+        this.registerExtractor("questions", async (config) => {
+            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
+            return this.createSpecializedExtractor(new LLMMetadataExtractor({
+                provider: config?.provider,
+                modelName: config?.modelName,
+            }), "questions", { questions: true });
+        }, DEFAULT_EXTRACTOR_METADATA.questions);
+        // Register custom extractor
+        this.registerExtractor("custom", async (config) => {
+            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
+            return this.wrapExtractor(new LLMMetadataExtractor({
+                provider: config?.provider,
+                modelName: config?.modelName,
+            }), "custom");
+        }, DEFAULT_EXTRACTOR_METADATA.custom);
+        // Register composite extractor
+        this.registerExtractor("composite", async (config) => {
+            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
+            return this.wrapExtractor(new LLMMetadataExtractor({
+                provider: config?.provider,
+                modelName: config?.modelName,
+            }), "composite");
+        }, DEFAULT_EXTRACTOR_METADATA.composite);
+        logger.debug(`[MetadataExtractorFactory] Registered ${this.items.size} extractor types`);
+    }
+    /**
+     * Wrap LLMMetadataExtractor to conform to MetadataExtractor interface
+     */
+    wrapExtractor(extractor, type) {
+        return {
+            type,
+            async extract(chunks, params) {
+                return extractor.extract(chunks, params ?? {});
+            },
+        };
+    }
+    /**
+     * Create specialized extractor that only extracts specific types
+     */
+    createSpecializedExtractor(extractor, type, defaultParams) {
+        return {
+            type,
+            async extract(chunks, params) {
+                // Merge default params with any provided params
+                const mergedParams = { ...defaultParams, ...params };
+                return extractor.extract(chunks, mergedParams);
+            },
+        };
+    }
+    /**
+     * Register an extractor with metadata and aliases
+     */
+    registerExtractor(type, factory, metadata) {
+        // Store metadata
+        this.metadataMap.set(type, metadata);
+        // Register with aliases
+        this.register(type, factory, metadata.aliases, { metadata });
+        logger.debug(`[MetadataExtractorFactory] Registered extractor '${type}' with aliases: ${metadata.aliases.join(", ")}`);
+    }
+    /**
+     * Create an extractor by type or alias
+     */
+    async createExtractor(typeOrAlias, config) {
+        await this.ensureInitialized();
+        const resolvedName = this.resolveName(typeOrAlias);
+        if (!this.has(resolvedName)) {
+            const available = this.getAvailable();
+            throw new MetadataExtractionError(`Unknown metadata extractor type: '${typeOrAlias}'. Available types: ${available.join(", ")}`, {
+                code: RAGErrorCodes.METADATA_EXTRACTOR_NOT_FOUND,
+                extractorType: typeOrAlias,
+                details: {
+                    requestedType: typeOrAlias,
+                    availableTypes: available,
+                },
+            });
+        }
+        try {
+            const extractor = await this.create(resolvedName, config);
+            logger.debug(`[MetadataExtractorFactory] Created extractor '${resolvedName}' with config:`, config);
+            return extractor;
+        }
+        catch (error) {
+            // Re-throw if already a MetadataExtractionError
+            if (error instanceof MetadataExtractionError) {
+                throw error;
+            }
+            throw new MetadataExtractionError(`Failed to create extractor '${resolvedName}': ${error instanceof Error ? error.message : String(error)}`, {
+                extractorType: resolvedName,
+                cause: error instanceof Error ? error : undefined,
+                details: { type: resolvedName, config },
+            });
+        }
+    }
+    /**
+     * Get metadata for an extractor
+     */
+    getExtractorMetadata(typeOrAlias) {
+        const resolvedName = this.resolveName(typeOrAlias);
+        return this.metadataMap.get(resolvedName);
+    }
+    /**
+     * Get default configuration for an extractor
+     */
+    getDefaultConfig(typeOrAlias) {
+        const metadata = this.getExtractorMetadata(typeOrAlias);
+        return metadata?.defaultConfig;
+    }
+    /**
+     * Get available extractor types (not including aliases)
+     */
+    getAvailableTypes() {
+        return this.getAvailable();
+    }
+    /**
+     * Get all aliases mapped to their types
+     */
+    getTypeAliases() {
+        return this.getAliases();
+    }
+    /**
+     * Check if a type exists
+     */
+    hasType(typeOrAlias) {
+        const resolved = this.resolveName(typeOrAlias);
+        return this.has(resolved);
+    }
+    /**
+     * Get extractors suitable for a use case
+     */
+    getExtractorsForUseCase(useCase) {
+        const matches = [];
+        const useCaseLower = useCase.toLowerCase();
+        for (const [type, metadata] of this.metadataMap) {
+            const hasMatch = metadata.useCases.some((uc) => uc.toLowerCase().includes(useCaseLower));
+            if (hasMatch) {
+                matches.push(type);
+            }
+        }
+        return matches;
+    }
+    /**
+     * Get extractors that can produce a specific extraction type
+     */
+    getExtractorsForExtractionType(extractionType) {
+        const matches = [];
+        for (const [type, metadata] of this.metadataMap) {
+            if (metadata.extractionTypes.includes(extractionType)) {
+                matches.push(type);
+            }
+        }
+        return matches;
+    }
+    /**
+     * Get all extractor metadata
+     */
+    getAllMetadata() {
+        return new Map(this.metadataMap);
+    }
+    /**
+     * Clear factory and metadata
+     */
+    clear() {
+        super.clear();
+        this.metadataMap.clear();
+    }
+}
+/**
+ * Global metadata extractor factory singleton
+ */
+export const metadataExtractorFactory = MetadataExtractorFactory.getInstance();
+/**
+ * Convenience function to create a metadata extractor
+ */
+export async function createMetadataExtractor(typeOrAlias, config) {
+    return metadataExtractorFactory.createExtractor(typeOrAlias, config);
+}
+/**
+ * Convenience function to get available extractor types
+ */
+export function getAvailableExtractorTypes() {
+    return metadataExtractorFactory.getAvailableTypes();
+}
+/**
+ * Convenience function to get extractor metadata
+ */
+export function getExtractorMetadata(typeOrAlias) {
+    return metadataExtractorFactory.getExtractorMetadata(typeOrAlias);
+}
+/**
+ * Convenience function to get default config
+ */
+export function getExtractorDefaultConfig(typeOrAlias) {
+    return metadataExtractorFactory.getDefaultConfig(typeOrAlias);
+}