@juspay/neurolink 9.32.0 → 9.33.0

package/dist/client/rag/index.js

@@ -1,141 +0,0 @@
-/**
- * RAG Document Processing Module
- *
- * Provides comprehensive RAG (Retrieval-Augmented Generation) capabilities:
- * - Document loading (text, markdown, HTML, JSON, CSV, PDF, web)
- * - MDocument class for fluent document processing
- * - 10 chunking strategies (character, recursive, sentence, token, markdown, html, json, latex, semantic, semantic-markdown)
- * - LLM-powered metadata extraction (title, summary, keywords, Q&A)
- * - Vector query tools with metadata filtering and reranking
- * - Hybrid search (BM25 + vector fusion)
- * - Graph RAG for knowledge graph-based retrieval
- * - RAG pipeline orchestration
- * - Context assembly and formatting
- * - ChunkerFactory and ChunkerRegistry patterns for extensibility
- * - Error handling and resilience (CircuitBreaker, RetryHandler)
- *
- * @example
- * ```typescript
- * import {
- *   MDocument,
- *   loadDocument,
- *   RAGPipeline,
- *   ChunkerRegistry,
- *   ChunkerFactory,
- *   CircuitBreaker
- * } from '@juspay/neurolink';
- *
- * // Load and process a document
- * const doc = await loadDocument('/path/to/document.md');
- * await doc.chunk({ strategy: 'markdown', config: { maxSize: 1000 } });
- * await doc.embed('openai', 'text-embedding-3-small');
- *
- * // Or use the full RAG pipeline
- * const pipeline = new RAGPipeline({
- *   embeddingModel: { provider: 'openai', modelName: 'text-embedding-3-small' },
- *   generationModel: { provider: 'openai', modelName: 'gpt-4o-mini' }
- * });
- * await pipeline.ingest(['/path/to/docs/*.md']);
- * const response = await pipeline.query('What are the key features?');
- *
- * // Use factory pattern for chunker creation
- * const chunker = await ChunkerFactory.createChunker('semantic', { maxSize: 500 });
- * const chunks = await chunker.chunk(text);
- * ```
- */
-export { ChunkerFactory, chunkerFactory, createChunker, getAvailableStrategies as getFactoryStrategies, getDefaultConfig as getFactoryDefaultConfig, } from "./ChunkerFactory.js";
-// ChunkerFactory and ChunkerRegistry patterns (from main worktree)
-export { ChunkerRegistry as ChunkerRegistryV2, chunkerRegistry, getAvailableChunkers, getChunker, getChunkerMetadata, } from "./ChunkerRegistry.js";
-// Base chunker and chunker implementations (from main worktree)
-export * from "./chunkers/index.js";
-// Chunking
-export { CharacterChunker, ChunkerRegistry, chunkText, HTMLChunker, JSONChunker, LaTeXChunker, MarkdownChunker, RecursiveChunker, SemanticChunker, SentenceChunker, TokenChunker, } from "./chunking/index.js";
-// Document Processing
-export { CSVLoader, HTMLLoader, JSONLoader, loadDocument, loadDocuments, MarkdownLoader, MDocument, PDFLoader, TextLoader, WebLoader, } from "./document/index.js";
-// Error handling
-export * from "./errors/index.js";
-// Graph RAG
-export { GraphRAG } from "./graphRag/index.js";
-// Metadata Extraction
-export { createMetadataExtractor, extractMetadata, getAvailableExtractors, getAvailableExtractorTypes, getExtractor, getExtractorDefaultConfig, getExtractorMetadata, getRegisteredExtractorMetadata, LLMMetadataExtractor, 
-// Factory pattern
-MetadataExtractorFactory, 
-// Registry pattern
-MetadataExtractorRegistry, metadataExtractorFactory, metadataExtractorRegistry, } from "./metadata/index.js";
-// Pipeline
-export { assembleContext, createContextWindow, createRAGPipeline, extractKeySentences, formatContextWithCitations, orderByDocumentStructure, RAGPipeline, summarizeContext, } from "./pipeline/index.js";
-// RAG Integration (for generate/stream)
-export { prepareRAGTool } from "./ragIntegration.js";
-// Reranker
-export { batchRerank, CohereRelevanceScorer, CrossEncoderReranker, createReranker, getAvailableRerankers, getAvailableRerankerTypes, getRegisteredRerankerMetadata, getReranker, getRerankerDefaultConfig, getRerankerMetadata, 
-// Factory pattern
-RerankerFactory, 
-// Registry pattern
-RerankerRegistry, rerank, rerankerFactory, rerankerRegistry, simpleRerank, } from "./reranker/index.js";
-// Resilience patterns
-export * from "./resilience/index.js";
-// Retrieval
-export { createHybridSearch, createVectorQueryTool, InMemoryBM25Index, InMemoryVectorStore, linearCombination, reciprocalRankFusion, } from "./retrieval/index.js";
-// Types
-export * from "./types.js";
-// Convenience functions
-import { ChunkerRegistry } from "./chunking/index.js";
-import { LLMMetadataExtractor } from "./metadata/index.js";
-/**
- * Process a document through the full RAG pipeline
- *
- * @param text - Document text to process
- * @param options - Processing options
- * @returns Processed chunks with optional metadata
- */
-export async function processDocument(text, options) {
-    const { strategy = "recursive", maxSize = 1000, overlap = 200, extract, provider, model, metadata = {}, } = options || {};
-    // Chunk the document
-    const chunker = ChunkerRegistry.get(strategy);
-    const chunks = await chunker.chunk(text, { maxSize, overlap, metadata });
-    // Extract metadata if requested
-    if (extract) {
-        const extractor = new LLMMetadataExtractor({ provider, modelName: model });
-        const results = await extractor.extract(chunks, extract);
-        // Merge metadata into chunks
-        for (let i = 0; i < chunks.length && i < results.length; i++) {
-            const result = results[i];
-            if (result.title) {
-                chunks[i].metadata.title = result.title;
-            }
-            if (result.summary) {
-                chunks[i].metadata.summary = result.summary;
-            }
-            if (result.keywords) {
-                chunks[i].metadata.keywords = result.keywords;
-            }
-        }
-    }
-    return chunks;
-}
-/**
- * Get recommended chunking strategy based on content type
- *
- * @param contentType - MIME type or file extension
- * @returns Recommended chunking strategy
- */
-export function getRecommendedStrategy(contentType) {
-    return ChunkerRegistry.getRecommendedStrategy(contentType);
-}
-/**
- * Get available chunking strategies
- *
- * @returns Array of available strategy names
- */
-export function getAvailableStrategies() {
-    return ChunkerRegistry.getAvailableStrategies();
-}
-/**
- * Get default configuration for a chunking strategy
- *
- * @param strategy - Chunking strategy name
- * @returns Default configuration object
- */
-export function getDefaultChunkerConfig(strategy) {
-    return ChunkerRegistry.getDefaultConfig(strategy);
-}

package/dist/client/rag/metadata/MetadataExtractorFactory.js

@@ -1,418 +0,0 @@
-/**
- * Metadata Extractor Factory
- *
- * Factory for creating metadata extractor instances with configuration.
- * Follows the BaseFactory pattern for consistent lifecycle management.
- */
-import { BaseFactory } from "../../core/infrastructure/index.js";
-import { logger } from "../../utils/logger.js";
-import { MetadataExtractionError, RAGErrorCodes } from "../errors/RAGError.js";
-/**
- * Default metadata extractor metadata entries
- */
-const DEFAULT_EXTRACTOR_METADATA = {
-    llm: {
-        description: "Full LLM-powered metadata extraction supporting all extraction types",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            temperature: 0.3,
-        },
-        supportedOptions: [
-            "provider",
-            "modelName",
-            "promptTemplate",
-            "maxTokens",
-            "temperature",
-        ],
-        useCases: [
-            "Comprehensive metadata extraction",
-            "Multi-type extraction in single pass",
-            "Custom schema extraction",
-        ],
-        aliases: ["full", "comprehensive", "all"],
-        requiresModel: true,
-        extractionTypes: ["title", "summary", "keywords", "questions", "custom"],
-    },
-    title: {
-        description: "Extracts concise, descriptive titles from document content",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            maxTokens: 100,
-        },
-        supportedOptions: ["provider", "modelName", "promptTemplate", "maxTokens"],
-        useCases: [
-            "Document indexing",
-            "Content organization",
-            "Navigation systems",
-        ],
-        aliases: ["header", "heading"],
-        requiresModel: true,
-        extractionTypes: ["title"],
-    },
-    summary: {
-        description: "Generates concise summaries of document chunks",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            maxTokens: 200,
-        },
-        supportedOptions: [
-            "provider",
-            "modelName",
-            "promptTemplate",
-            "maxTokens",
-            "maxWords",
-        ],
-        useCases: [
-            "Document previews",
-            "Search result snippets",
-            "Content condensation",
-        ],
-        aliases: ["summarize", "abstract"],
-        requiresModel: true,
-        extractionTypes: ["summary"],
-    },
-    keywords: {
-        description: "Extracts key terms and phrases from content",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            maxTokens: 100,
-        },
-        supportedOptions: [
-            "provider",
-            "modelName",
-            "promptTemplate",
-            "maxKeywords",
-        ],
-        useCases: ["Tag generation", "Topic modeling", "Search optimization"],
-        aliases: ["tags", "terms", "keyphrase"],
-        requiresModel: true,
-        extractionTypes: ["keywords"],
-    },
-    questions: {
-        description: "Generates Q&A pairs from content for training or FAQs",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            maxTokens: 500,
-        },
-        supportedOptions: [
-            "provider",
-            "modelName",
-            "promptTemplate",
-            "numQuestions",
-            "includeAnswers",
-        ],
-        useCases: [
-            "FAQ generation",
-            "Training data creation",
-            "Knowledge base building",
-        ],
-        aliases: ["qa", "faq", "questions-answers"],
-        requiresModel: true,
-        extractionTypes: ["questions"],
-    },
-    custom: {
-        description: "Extracts structured data according to custom schema",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            maxTokens: 500,
-        },
-        supportedOptions: [
-            "provider",
-            "modelName",
-            "promptTemplate",
-            "schema",
-            "description",
-        ],
-        useCases: [
-            "Structured data extraction",
-            "Entity extraction",
-            "Custom field extraction",
-        ],
-        aliases: ["schema", "structured", "entity"],
-        requiresModel: true,
-        extractionTypes: ["custom"],
-    },
-    composite: {
-        description: "Combines multiple extraction types in a single pass",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-        },
-        supportedOptions: ["provider", "modelName", "extractors"],
-        useCases: [
-            "Multi-field extraction",
-            "Complete document processing",
-            "Pipeline integration",
-        ],
-        aliases: ["multi", "combined", "batch"],
-        requiresModel: true,
-        extractionTypes: ["title", "summary", "keywords", "questions", "custom"],
-    },
-};
-/**
- * Metadata Extractor Factory
- *
- * Creates metadata extractor instances based on type with configuration support.
- * Uses lazy loading via dynamic imports to avoid circular dependencies.
- */
-export class MetadataExtractorFactory extends BaseFactory {
-    static instance = null;
-    metadataMap = new Map();
-    constructor() {
-        super();
-    }
-    /**
-     * Get singleton instance
-     */
-    static getInstance() {
-        if (!MetadataExtractorFactory.instance) {
-            MetadataExtractorFactory.instance = new MetadataExtractorFactory();
-        }
-        return MetadataExtractorFactory.instance;
-    }
-    /**
-     * Reset singleton (for testing)
-     */
-    static resetInstance() {
-        if (MetadataExtractorFactory.instance) {
-            MetadataExtractorFactory.instance.clear();
-            MetadataExtractorFactory.instance = null;
-        }
-    }
-    /**
-     * Register all default extractors
-     */
-    async registerAll() {
-        // Register full LLM extractor
-        this.registerExtractor("llm", async (config) => {
-            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
-            return this.wrapExtractor(new LLMMetadataExtractor({
-                provider: config?.provider,
-                modelName: config?.modelName,
-            }), "llm");
-        }, DEFAULT_EXTRACTOR_METADATA.llm);
-        // Register title extractor
-        this.registerExtractor("title", async (config) => {
-            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
-            return this.createSpecializedExtractor(new LLMMetadataExtractor({
-                provider: config?.provider,
-                modelName: config?.modelName,
-            }), "title", { title: true });
-        }, DEFAULT_EXTRACTOR_METADATA.title);
-        // Register summary extractor
-        this.registerExtractor("summary", async (config) => {
-            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
-            return this.createSpecializedExtractor(new LLMMetadataExtractor({
-                provider: config?.provider,
-                modelName: config?.modelName,
-            }), "summary", { summary: true });
-        }, DEFAULT_EXTRACTOR_METADATA.summary);
-        // Register keywords extractor
-        this.registerExtractor("keywords", async (config) => {
-            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
-            return this.createSpecializedExtractor(new LLMMetadataExtractor({
-                provider: config?.provider,
-                modelName: config?.modelName,
-            }), "keywords", { keywords: true });
-        }, DEFAULT_EXTRACTOR_METADATA.keywords);
-        // Register questions extractor
-        this.registerExtractor("questions", async (config) => {
-            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
-            return this.createSpecializedExtractor(new LLMMetadataExtractor({
-                provider: config?.provider,
-                modelName: config?.modelName,
-            }), "questions", { questions: true });
-        }, DEFAULT_EXTRACTOR_METADATA.questions);
-        // Register custom extractor
-        this.registerExtractor("custom", async (config) => {
-            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
-            return this.wrapExtractor(new LLMMetadataExtractor({
-                provider: config?.provider,
-                modelName: config?.modelName,
-            }), "custom");
-        }, DEFAULT_EXTRACTOR_METADATA.custom);
-        // Register composite extractor
-        this.registerExtractor("composite", async (config) => {
-            const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
-            return this.wrapExtractor(new LLMMetadataExtractor({
-                provider: config?.provider,
-                modelName: config?.modelName,
-            }), "composite");
-        }, DEFAULT_EXTRACTOR_METADATA.composite);
-        logger.debug(`[MetadataExtractorFactory] Registered ${this.items.size} extractor types`);
-    }
-    /**
-     * Wrap LLMMetadataExtractor to conform to MetadataExtractor interface
-     */
-    wrapExtractor(extractor, type) {
-        return {
-            type,
-            async extract(chunks, params) {
-                return extractor.extract(chunks, params ?? {});
-            },
-        };
-    }
-    /**
-     * Create specialized extractor that only extracts specific types
-     */
-    createSpecializedExtractor(extractor, type, defaultParams) {
-        return {
-            type,
-            async extract(chunks, params) {
-                // Merge default params with any provided params
-                const mergedParams = { ...defaultParams, ...params };
-                return extractor.extract(chunks, mergedParams);
-            },
-        };
-    }
-    /**
-     * Register an extractor with metadata and aliases
-     */
-    registerExtractor(type, factory, metadata) {
-        // Store metadata
-        this.metadataMap.set(type, metadata);
-        // Register with aliases
-        this.register(type, factory, metadata.aliases, { metadata });
-        logger.debug(`[MetadataExtractorFactory] Registered extractor '${type}' with aliases: ${metadata.aliases.join(", ")}`);
-    }
-    /**
-     * Create an extractor by type or alias
-     */
-    async createExtractor(typeOrAlias, config) {
-        await this.ensureInitialized();
-        const resolvedName = this.resolveName(typeOrAlias);
-        if (!this.has(resolvedName)) {
-            const available = this.getAvailable();
-            throw new MetadataExtractionError(`Unknown metadata extractor type: '${typeOrAlias}'. Available types: ${available.join(", ")}`, {
-                code: RAGErrorCodes.METADATA_EXTRACTOR_NOT_FOUND,
-                extractorType: typeOrAlias,
-                details: {
-                    requestedType: typeOrAlias,
-                    availableTypes: available,
-                },
-            });
-        }
-        try {
-            const extractor = await this.create(resolvedName, config);
-            logger.debug(`[MetadataExtractorFactory] Created extractor '${resolvedName}' with config:`, config);
-            return extractor;
-        }
-        catch (error) {
-            // Re-throw if already a MetadataExtractionError
-            if (error instanceof MetadataExtractionError) {
-                throw error;
-            }
-            throw new MetadataExtractionError(`Failed to create extractor '${resolvedName}': ${error instanceof Error ? error.message : String(error)}`, {
-                extractorType: resolvedName,
-                cause: error instanceof Error ? error : undefined,
-                details: { type: resolvedName, config },
-            });
-        }
-    }
-    /**
-     * Get metadata for an extractor
-     */
-    getExtractorMetadata(typeOrAlias) {
-        const resolvedName = this.resolveName(typeOrAlias);
-        return this.metadataMap.get(resolvedName);
-    }
-    /**
-     * Get default configuration for an extractor
-     */
-    getDefaultConfig(typeOrAlias) {
-        const metadata = this.getExtractorMetadata(typeOrAlias);
-        return metadata?.defaultConfig;
-    }
-    /**
-     * Get available extractor types (not including aliases)
-     */
-    getAvailableTypes() {
-        return this.getAvailable();
-    }
-    /**
-     * Get all aliases mapped to their types
-     */
-    getTypeAliases() {
-        return this.getAliases();
-    }
-    /**
-     * Check if a type exists
-     */
-    hasType(typeOrAlias) {
-        const resolved = this.resolveName(typeOrAlias);
-        return this.has(resolved);
-    }
-    /**
-     * Get extractors suitable for a use case
-     */
-    getExtractorsForUseCase(useCase) {
-        const matches = [];
-        const useCaseLower = useCase.toLowerCase();
-        for (const [type, metadata] of this.metadataMap) {
-            const hasMatch = metadata.useCases.some((uc) => uc.toLowerCase().includes(useCaseLower));
-            if (hasMatch) {
-                matches.push(type);
-            }
-        }
-        return matches;
-    }
-    /**
-     * Get extractors that can produce a specific extraction type
-     */
-    getExtractorsForExtractionType(extractionType) {
-        const matches = [];
-        for (const [type, metadata] of this.metadataMap) {
-            if (metadata.extractionTypes.includes(extractionType)) {
-                matches.push(type);
-            }
-        }
-        return matches;
-    }
-    /**
-     * Get all extractor metadata
-     */
-    getAllMetadata() {
-        return new Map(this.metadataMap);
-    }
-    /**
-     * Clear factory and metadata
-     */
-    clear() {
-        super.clear();
-        this.metadataMap.clear();
-    }
-}
-/**
- * Global metadata extractor factory singleton
- */
-export const metadataExtractorFactory = MetadataExtractorFactory.getInstance();
-/**
- * Convenience function to create a metadata extractor
- */
-export async function createMetadataExtractor(typeOrAlias, config) {
-    return metadataExtractorFactory.createExtractor(typeOrAlias, config);
-}
-/**
- * Convenience function to get available extractor types
- */
-export function getAvailableExtractorTypes() {
-    return metadataExtractorFactory.getAvailableTypes();
-}
-/**
- * Convenience function to get extractor metadata
- */
-export function getExtractorMetadata(typeOrAlias) {
-    return metadataExtractorFactory.getExtractorMetadata(typeOrAlias);
-}
-/**
- * Convenience function to get default config
- */
-export function getExtractorDefaultConfig(typeOrAlias) {
-    return metadataExtractorFactory.getDefaultConfig(typeOrAlias);
-}