@juspay/neurolink 9.32.0 → 9.33.0

package/dist/client/rag/metadata/MetadataExtractorRegistry.js

@@ -1,362 +0,0 @@
-/**
- * Metadata Extractor Registry
- *
- * Centralized registry for all metadata extractor implementations with metadata
- * and discovery capabilities. Follows the BaseRegistry pattern.
- */
-import { BaseRegistry } from "../../core/infrastructure/index.js";
-import { logger } from "../../utils/logger.js";
-import { MetadataExtractionError, RAGErrorCodes } from "../errors/RAGError.js";
-/**
- * Default metadata extractor metadata entries
- */
-const DEFAULT_EXTRACTOR_METADATA = {
-    llm: {
-        description: "Full LLM-powered metadata extraction supporting all extraction types",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            temperature: 0.3,
-        },
-        supportedOptions: [
-            "provider",
-            "modelName",
-            "promptTemplate",
-            "maxTokens",
-            "temperature",
-        ],
-        useCases: [
-            "Comprehensive metadata extraction",
-            "Multi-type extraction in single pass",
-            "Custom schema extraction",
-        ],
-        aliases: ["full", "comprehensive", "all"],
-        requiresModel: true,
-        extractionTypes: ["title", "summary", "keywords", "questions", "custom"],
-    },
-    title: {
-        description: "Extracts concise, descriptive titles from document content",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            maxTokens: 100,
-        },
-        supportedOptions: ["provider", "modelName", "promptTemplate", "maxTokens"],
-        useCases: [
-            "Document indexing",
-            "Content organization",
-            "Navigation systems",
-        ],
-        aliases: ["header", "heading"],
-        requiresModel: true,
-        extractionTypes: ["title"],
-    },
-    summary: {
-        description: "Generates concise summaries of document chunks",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            maxTokens: 200,
-        },
-        supportedOptions: [
-            "provider",
-            "modelName",
-            "promptTemplate",
-            "maxTokens",
-            "maxWords",
-        ],
-        useCases: [
-            "Document previews",
-            "Search result snippets",
-            "Content condensation",
-        ],
-        aliases: ["summarize", "abstract"],
-        requiresModel: true,
-        extractionTypes: ["summary"],
-    },
-    keywords: {
-        description: "Extracts key terms and phrases from content",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            maxTokens: 100,
-        },
-        supportedOptions: [
-            "provider",
-            "modelName",
-            "promptTemplate",
-            "maxKeywords",
-        ],
-        useCases: ["Tag generation", "Topic modeling", "Search optimization"],
-        aliases: ["tags", "terms", "keyphrase"],
-        requiresModel: true,
-        extractionTypes: ["keywords"],
-    },
-    questions: {
-        description: "Generates Q&A pairs from content for training or FAQs",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            maxTokens: 500,
-        },
-        supportedOptions: [
-            "provider",
-            "modelName",
-            "promptTemplate",
-            "numQuestions",
-            "includeAnswers",
-        ],
-        useCases: [
-            "FAQ generation",
-            "Training data creation",
-            "Knowledge base building",
-        ],
-        aliases: ["qa", "faq", "questions-answers"],
-        requiresModel: true,
-        extractionTypes: ["questions"],
-    },
-    custom: {
-        description: "Extracts structured data according to custom schema",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-            maxTokens: 500,
-        },
-        supportedOptions: [
-            "provider",
-            "modelName",
-            "promptTemplate",
-            "schema",
-            "description",
-        ],
-        useCases: [
-            "Structured data extraction",
-            "Entity extraction",
-            "Custom field extraction",
-        ],
-        aliases: ["schema", "structured", "entity"],
-        requiresModel: true,
-        extractionTypes: ["custom"],
-    },
-    composite: {
-        description: "Combines multiple extraction types in a single pass",
-        defaultConfig: {
-            provider: "openai",
-            modelName: "gpt-4o-mini",
-        },
-        supportedOptions: ["provider", "modelName", "extractors"],
-        useCases: [
-            "Multi-field extraction",
-            "Complete document processing",
-            "Pipeline integration",
-        ],
-        aliases: ["multi", "combined", "batch"],
-        requiresModel: true,
-        extractionTypes: ["title", "summary", "keywords", "questions", "custom"],
-    },
-};
-/**
- * Metadata Extractor Registry
- *
- * Manages registration and discovery of all metadata extractor implementations.
- * Extends BaseRegistry for consistent lifecycle management.
- */
-export class MetadataExtractorRegistry extends BaseRegistry {
-    static instance = null;
-    aliasMap = new Map();
-    constructor() {
-        super();
-    }
-    /**
-     * Get singleton instance
-     */
-    static getInstance() {
-        if (!MetadataExtractorRegistry.instance) {
-            MetadataExtractorRegistry.instance = new MetadataExtractorRegistry();
-        }
-        return MetadataExtractorRegistry.instance;
-    }
-    /**
-     * Reset singleton (for testing)
-     */
-    static resetInstance() {
-        if (MetadataExtractorRegistry.instance) {
-            MetadataExtractorRegistry.instance.clear();
-            MetadataExtractorRegistry.instance = null;
-        }
-    }
-    /**
-     * Register all built-in extractors
-     */
-    async registerAll() {
-        const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
-        // Register all extractor types
-        for (const [type, metadata] of Object.entries(DEFAULT_EXTRACTOR_METADATA)) {
-            this.registerExtractor(type, async () => this.createExtractorInstance(LLMMetadataExtractor, type), metadata);
-        }
-        logger.debug(`[MetadataExtractorRegistry] Registered ${this.items.size} extractor types`);
-    }
-    /**
-     * Create extractor instance wrapper
-     */
-    createExtractorInstance(ExtractorClass, type) {
-        const extractor = new ExtractorClass();
-        return {
-            type,
-            async extract(chunks, params) {
-                return extractor.extract(chunks, params ?? {});
-            },
-        };
-    }
-    /**
-     * Register an extractor with aliases
-     */
-    registerExtractor(type, factory, metadata) {
-        this.register(type, factory, metadata.aliases, { metadata });
-        // Register aliases in local alias map for type resolution
-        for (const alias of metadata.aliases) {
-            this.aliasMap.set(alias.toLowerCase(), type);
-            logger.debug(`[MetadataExtractorRegistry] Registered alias '${alias}' -> '${type}'`);
-        }
-    }
-    /**
-     * Resolve type from alias
-     */
-    resolveType(nameOrAlias) {
-        const lower = nameOrAlias.toLowerCase();
-        // Check if it's a direct type
-        if (this.items.has(lower)) {
-            return lower;
-        }
-        // Check aliases
-        const resolved = this.aliasMap.get(lower);
-        if (resolved) {
-            return resolved;
-        }
-        throw new MetadataExtractionError(`Unknown metadata extractor type: '${nameOrAlias}'. Available types: ${this.getAvailableExtractors().join(", ")}`, {
-            code: RAGErrorCodes.METADATA_EXTRACTOR_NOT_FOUND,
-            extractorType: nameOrAlias,
-            details: {
-                requestedType: nameOrAlias,
-                availableTypes: this.getAvailableExtractors(),
-            },
-        });
-    }
-    /**
-     * Get an extractor by type or alias
-     */
-    async getExtractor(typeOrAlias) {
-        await this.ensureInitialized();
-        const type = this.resolveType(typeOrAlias);
-        const extractor = await this.get(type);
-        if (!extractor) {
-            throw new MetadataExtractionError(`Metadata extractor not found: ${type}`, {
-                code: RAGErrorCodes.METADATA_EXTRACTOR_NOT_FOUND,
-                extractorType: type,
-                details: { type },
-            });
-        }
-        return extractor;
-    }
-    /**
-     * Get list of available extractor types
-     */
-    getAvailableExtractors() {
-        return this.list().map((item) => item.id);
-    }
-    /**
-     * Get metadata for a specific extractor
-     */
-    getExtractorMetadata(typeOrAlias) {
-        const type = this.resolveType(typeOrAlias);
-        const entry = this.list().find((item) => item.id === type);
-        return entry?.metadata;
-    }
-    /**
-     * Get all aliases for a type
-     */
-    getAliasesForType(type) {
-        const metadata = DEFAULT_EXTRACTOR_METADATA[type];
-        return metadata?.aliases ?? [];
-    }
-    /**
-     * Get all registered aliases
-     */
-    getAllAliases() {
-        return new Map(this.aliasMap);
-    }
-    /**
-     * Check if a type or alias exists
-     */
-    hasExtractor(typeOrAlias) {
-        try {
-            this.resolveType(typeOrAlias);
-            return true;
-        }
-        catch {
-            return false;
-        }
-    }
-    /**
-     * Get extractors by use case
-     */
-    getExtractorsByUseCase(useCase) {
-        const matches = [];
-        const useCaseLower = useCase.toLowerCase();
-        for (const [type, metadata] of Object.entries(DEFAULT_EXTRACTOR_METADATA)) {
-            const hasMatchingUseCase = metadata.useCases.some((uc) => uc.toLowerCase().includes(useCaseLower));
-            if (hasMatchingUseCase) {
-                matches.push(type);
-            }
-        }
-        return matches;
-    }
-    /**
-     * Get extractors that can produce a specific extraction type
-     */
-    getExtractorsByExtractionType(extractionType) {
-        const matches = [];
-        for (const [type, metadata] of Object.entries(DEFAULT_EXTRACTOR_METADATA)) {
-            if (metadata.extractionTypes.includes(extractionType)) {
-                matches.push(type);
-            }
-        }
-        return matches;
-    }
-    /**
-     * Get default configuration for an extractor
-     */
-    getDefaultConfig(typeOrAlias) {
-        const metadata = this.getExtractorMetadata(typeOrAlias);
-        return metadata?.defaultConfig;
-    }
-    /**
-     * Clear the registry (also clears aliases)
-     */
-    clear() {
-        super.clear();
-        this.aliasMap.clear();
-    }
-}
-/**
- * Global metadata extractor registry singleton
- */
-export const metadataExtractorRegistry = MetadataExtractorRegistry.getInstance();
-/**
- * Convenience function to get available extractors
- */
-export function getAvailableExtractors() {
-    return metadataExtractorRegistry.getAvailableExtractors();
-}
-/**
- * Convenience function to get extractor by type
- */
-export async function getExtractor(typeOrAlias) {
-    return metadataExtractorRegistry.getExtractor(typeOrAlias);
-}
-/**
- * Convenience function to get extractor metadata
- */
-export function getRegisteredExtractorMetadata(typeOrAlias) {
-    return metadataExtractorRegistry.getExtractorMetadata(typeOrAlias);
-}

package/dist/client/rag/metadata/index.js

@@ -1,9 +0,0 @@
-/**
- * Metadata Extraction Module Exports
- */
-// Factory pattern exports
-export { createMetadataExtractor, getAvailableExtractorTypes, getExtractorDefaultConfig, getExtractorMetadata, MetadataExtractorFactory, metadataExtractorFactory, } from "./MetadataExtractorFactory.js";
-// Registry pattern exports
-export { getAvailableExtractors, getExtractor, getRegisteredExtractorMetadata, MetadataExtractorRegistry, metadataExtractorRegistry, } from "./MetadataExtractorRegistry.js";
-// Core metadata extractor
-export { extractMetadata, LLMMetadataExtractor } from "./metadataExtractor.js";

package/dist/client/rag/metadata/metadataExtractor.js

@@ -1,280 +0,0 @@
-/**
- * LLM-powered Metadata Extractor
- *
- * Extracts structured metadata from document chunks using language models.
- * Supports title, summary, keywords, Q&A pairs, and custom schema extraction.
- */
-import { ProviderFactory } from "../../factories/providerFactory.js";
-import { logger } from "../../utils/logger.js";
-/**
- * Default prompts for metadata extraction
- */
-const DEFAULT_PROMPTS = {
-    title: `Extract a concise, descriptive title for the following content.
-Return only the title, nothing else.
-
-Content:
-{context}
-
-Title:`,
-    summary: `Summarize the following content in {maxWords} words or less.
-Focus on the key points and main ideas.
-
-Content:
-{context}
-
-Summary:`,
-    keywords: `Extract the {maxKeywords} most important keywords or key phrases from the following content.
-Return them as a comma-separated list.
-
-Content:
-{context}
-
-Keywords:`,
-    questions: `Generate {numQuestions} questions that can be answered using the following content.
-{answerInstruction}
-
-Content:
-{context}
-
-Questions:`,
-};
-/**
- * LLM-powered metadata extractor
- * Extracts title, summary, keywords, Q&A pairs, and custom schema data
- */
-export class LLMMetadataExtractor {
-    provider;
-    modelName;
-    constructor(options) {
-        this.provider = options?.provider || "openai";
-        this.modelName = options?.modelName || "gpt-4o-mini";
-    }
-    /**
-     * Extract metadata from chunks based on configuration
-     * @param chunks - Array of chunks to extract metadata from
-     * @param params - Extraction parameters
-     * @returns Array of extraction results, one per chunk
-     */
-    async extract(chunks, params) {
-        const results = [];
-        // Group chunks by documentId for title extraction
-        const chunksByDocument = this.groupByDocument(chunks);
-        // Cache titles by document to avoid re-extraction
-        const titleCache = new Map();
-        for (const chunk of chunks) {
-            const result = {};
-            try {
-                // Extract title (shared across chunks with same documentId)
-                if (params.title) {
-                    const docId = chunk.metadata.documentId;
-                    if (!titleCache.has(docId)) {
-                        const titleConfig = typeof params.title === "boolean" ? {} : params.title;
-                        const title = await this.extractTitle(chunksByDocument.get(docId) || [chunk], titleConfig);
-                        titleCache.set(docId, title);
-                    }
-                    result.title = titleCache.get(docId);
-                }
-                // Extract summary
-                if (params.summary) {
-                    const summaryConfig = typeof params.summary === "boolean" ? {} : params.summary;
-                    result.summary = await this.extractSummary(chunk, summaryConfig);
-                }
-                // Extract keywords
-                if (params.keywords) {
-                    const keywordConfig = typeof params.keywords === "boolean" ? {} : params.keywords;
-                    result.keywords = await this.extractKeywords(chunk, keywordConfig);
-                }
-                // Generate Q&A pairs
-                if (params.questions) {
-                    const questionConfig = typeof params.questions === "boolean" ? {} : params.questions;
-                    result.questions = await this.extractQuestions(chunk, questionConfig);
-                }
-                // Custom schema extraction
-                if (params.custom) {
-                    result.custom = await this.extractCustom(chunk, params.custom);
-                }
-                results.push(result);
-            }
-            catch (error) {
-                logger.error("[MetadataExtractor] Extraction failed for chunk", {
-                    chunkId: chunk.id,
-                    error: error instanceof Error ? error.message : String(error),
-                });
-                results.push(result);
-            }
-        }
-        return results;
-    }
-    /**
-     * Group chunks by document ID
-     */
-    groupByDocument(chunks) {
-        const groups = new Map();
-        for (const chunk of chunks) {
-            const docId = chunk.metadata.documentId;
-            const group = groups.get(docId);
-            if (group) {
-                group.push(chunk);
-            }
-            else {
-                groups.set(docId, [chunk]);
-            }
-        }
-        return groups;
-    }
-    /**
-     * Extract title from document chunks
-     */
-    async extractTitle(chunks, config) {
-        const { nodes = 3, promptTemplate = DEFAULT_PROMPTS.title } = config;
-        // Use first N chunks for title extraction
-        const relevantChunks = chunks.slice(0, nodes);
-        const context = relevantChunks.map((c) => c.text).join("\n\n");
-        const prompt = promptTemplate.replace("{context}", context);
-        const response = await this.callLLM(prompt, config);
-        return response.trim();
-    }
-    /**
-     * Extract summary from a chunk
-     */
-    async extractSummary(chunk, config) {
-        const { maxWords = 100, promptTemplate = DEFAULT_PROMPTS.summary } = config;
-        const prompt = promptTemplate
-            .replace("{context}", chunk.text)
-            .replace("{maxWords}", String(maxWords));
-        const response = await this.callLLM(prompt, config);
-        return response.trim();
-    }
-    /**
-     * Extract keywords from a chunk
-     */
-    async extractKeywords(chunk, config) {
-        const { maxKeywords = 10, promptTemplate = DEFAULT_PROMPTS.keywords } = config;
-        const prompt = promptTemplate
-            .replace("{context}", chunk.text)
-            .replace("{maxKeywords}", String(maxKeywords));
-        const response = await this.callLLM(prompt, config);
-        // Parse comma-separated keywords
-        return response
-            .split(",")
-            .map((k) => k.trim())
-            .filter((k) => k.length > 0)
-            .slice(0, maxKeywords);
-    }
-    /**
-     * Extract Q&A pairs from a chunk
-     */
-    async extractQuestions(chunk, config) {
-        const { numQuestions = 3, includeAnswers = true, promptTemplate = DEFAULT_PROMPTS.questions, } = config;
-        const answerInstruction = includeAnswers
-            ? "For each question, also provide a brief answer based on the content."
-            : "Return only the questions.";
-        const prompt = promptTemplate
-            .replace("{context}", chunk.text)
-            .replace("{numQuestions}", String(numQuestions))
-            .replace("{answerInstruction}", answerInstruction);
-        const response = await this.callLLM(prompt, config);
-        // Parse Q&A pairs from response
-        return this.parseQAPairs(response, includeAnswers);
-    }
-    /**
-     * Extract custom schema data from a chunk
-     */
-    async extractCustom(chunk, config) {
-        const { description, promptTemplate } = config;
-        // Build extraction prompt
-        const prompt = promptTemplate ||
-            `Extract the following information from the content:
-${description || "Extract structured data according to the schema."}
-
-Content:
-${chunk.text}
-
-Return the extracted data as JSON.`;
-        const response = await this.callLLM(prompt, config);
-        try {
-            // Try to parse as JSON
-            const jsonMatch = response.match(/\{[\s\S]*\}/);
-            if (jsonMatch) {
-                return JSON.parse(jsonMatch[0]);
-            }
-            return JSON.parse(response);
-        }
-        catch {
-            logger.warn("[MetadataExtractor] Failed to parse custom extraction as JSON");
-            return { raw: response };
-        }
-    }
-    /**
-     * Parse Q&A pairs from LLM response
-     */
-    parseQAPairs(response, includeAnswers) {
-        const pairs = [];
-        // Try to parse numbered questions
-        const lines = response.split("\n").filter((l) => l.trim());
-        let currentQuestion = null;
-        let currentAnswer = null;
-        for (const line of lines) {
-            const trimmed = line.trim();
-            // Check if line is a question (starts with number or Q:)
-            if (/^\d+[.):]\s*/.test(trimmed) || /^Q[.:]?\s*/i.test(trimmed)) {
-                // Save previous Q&A pair
-                if (currentQuestion) {
-                    pairs.push({
-                        question: currentQuestion,
-                        ...(includeAnswers && currentAnswer
-                            ? { answer: currentAnswer }
-                            : {}),
-                    });
-                }
-                currentQuestion = trimmed
-                    .replace(/^\d+[.):]\s*/, "")
-                    .replace(/^Q[.:]?\s*/i, "");
-                currentAnswer = null;
-            }
-            else if (/^A[.:]?\s*/i.test(trimmed) && currentQuestion) {
-                currentAnswer = trimmed.replace(/^A[.:]?\s*/i, "");
-            }
-            else if (currentQuestion && !currentAnswer) {
-                // Continuation of question
-                currentQuestion += " " + trimmed;
-            }
-            else if (currentAnswer) {
-                // Continuation of answer
-                currentAnswer += " " + trimmed;
-            }
-        }
-        // Don't forget the last pair
-        if (currentQuestion) {
-            pairs.push({
-                question: currentQuestion,
-                ...(includeAnswers && currentAnswer ? { answer: currentAnswer } : {}),
-            });
-        }
-        return pairs;
-    }
-    /**
-     * Call the LLM with a prompt
-     */
-    async callLLM(prompt, config) {
-        const provider = await ProviderFactory.createProvider(config.provider || this.provider, config.modelName || this.modelName);
-        const result = await provider.generate({
-            prompt,
-            maxTokens: config.maxTokens || 500,
-            temperature: config.temperature || 0.3,
-        });
-        return result?.content || "";
-    }
-}
-/**
- * Convenience function to extract metadata from chunks
- * @param chunks - Chunks to process
- * @param params - Extraction parameters
- * @param options - Extractor options
- * @returns Extraction results
- */
-export async function extractMetadata(chunks, params, options) {
-    const extractor = new LLMMetadataExtractor(options);
-    return extractor.extract(chunks, params);
-}