@juspay/neurolink 9.30.0 → 9.31.0

package/dist/client/rag/metadata/MetadataExtractorRegistry.js

@@ -0,0 +1,362 @@
+/**
+ * Metadata Extractor Registry
+ *
+ * Centralized registry for all metadata extractor implementations with metadata
+ * and discovery capabilities. Follows the BaseRegistry pattern.
+ */
+import { BaseRegistry } from "../../core/infrastructure/index.js";
+import { logger } from "../../utils/logger.js";
+import { MetadataExtractionError, RAGErrorCodes } from "../errors/RAGError.js";
+/**
+ * Default metadata extractor metadata entries
+ */
+const DEFAULT_EXTRACTOR_METADATA = {
+    llm: {
+        description: "Full LLM-powered metadata extraction supporting all extraction types",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            temperature: 0.3,
+        },
+        supportedOptions: [
+            "provider",
+            "modelName",
+            "promptTemplate",
+            "maxTokens",
+            "temperature",
+        ],
+        useCases: [
+            "Comprehensive metadata extraction",
+            "Multi-type extraction in single pass",
+            "Custom schema extraction",
+        ],
+        aliases: ["full", "comprehensive", "all"],
+        requiresModel: true,
+        extractionTypes: ["title", "summary", "keywords", "questions", "custom"],
+    },
+    title: {
+        description: "Extracts concise, descriptive titles from document content",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            maxTokens: 100,
+        },
+        supportedOptions: ["provider", "modelName", "promptTemplate", "maxTokens"],
+        useCases: [
+            "Document indexing",
+            "Content organization",
+            "Navigation systems",
+        ],
+        aliases: ["header", "heading"],
+        requiresModel: true,
+        extractionTypes: ["title"],
+    },
+    summary: {
+        description: "Generates concise summaries of document chunks",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            maxTokens: 200,
+        },
+        supportedOptions: [
+            "provider",
+            "modelName",
+            "promptTemplate",
+            "maxTokens",
+            "maxWords",
+        ],
+        useCases: [
+            "Document previews",
+            "Search result snippets",
+            "Content condensation",
+        ],
+        aliases: ["summarize", "abstract"],
+        requiresModel: true,
+        extractionTypes: ["summary"],
+    },
+    keywords: {
+        description: "Extracts key terms and phrases from content",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            maxTokens: 100,
+        },
+        supportedOptions: [
+            "provider",
+            "modelName",
+            "promptTemplate",
+            "maxKeywords",
+        ],
+        useCases: ["Tag generation", "Topic modeling", "Search optimization"],
+        aliases: ["tags", "terms", "keyphrase"],
+        requiresModel: true,
+        extractionTypes: ["keywords"],
+    },
+    questions: {
+        description: "Generates Q&A pairs from content for training or FAQs",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            maxTokens: 500,
+        },
+        supportedOptions: [
+            "provider",
+            "modelName",
+            "promptTemplate",
+            "numQuestions",
+            "includeAnswers",
+        ],
+        useCases: [
+            "FAQ generation",
+            "Training data creation",
+            "Knowledge base building",
+        ],
+        aliases: ["qa", "faq", "questions-answers"],
+        requiresModel: true,
+        extractionTypes: ["questions"],
+    },
+    custom: {
+        description: "Extracts structured data according to custom schema",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+            maxTokens: 500,
+        },
+        supportedOptions: [
+            "provider",
+            "modelName",
+            "promptTemplate",
+            "schema",
+            "description",
+        ],
+        useCases: [
+            "Structured data extraction",
+            "Entity extraction",
+            "Custom field extraction",
+        ],
+        aliases: ["schema", "structured", "entity"],
+        requiresModel: true,
+        extractionTypes: ["custom"],
+    },
+    composite: {
+        description: "Combines multiple extraction types in a single pass",
+        defaultConfig: {
+            provider: "openai",
+            modelName: "gpt-4o-mini",
+        },
+        supportedOptions: ["provider", "modelName", "extractors"],
+        useCases: [
+            "Multi-field extraction",
+            "Complete document processing",
+            "Pipeline integration",
+        ],
+        aliases: ["multi", "combined", "batch"],
+        requiresModel: true,
+        extractionTypes: ["title", "summary", "keywords", "questions", "custom"],
+    },
+};
+/**
+ * Metadata Extractor Registry
+ *
+ * Manages registration and discovery of all metadata extractor implementations.
+ * Extends BaseRegistry for consistent lifecycle management.
+ */
+export class MetadataExtractorRegistry extends BaseRegistry {
+    static instance = null;
+    aliasMap = new Map();
+    constructor() {
+        super();
+    }
+    /**
+     * Get singleton instance
+     */
+    static getInstance() {
+        if (!MetadataExtractorRegistry.instance) {
+            MetadataExtractorRegistry.instance = new MetadataExtractorRegistry();
+        }
+        return MetadataExtractorRegistry.instance;
+    }
+    /**
+     * Reset singleton (for testing)
+     */
+    static resetInstance() {
+        if (MetadataExtractorRegistry.instance) {
+            MetadataExtractorRegistry.instance.clear();
+            MetadataExtractorRegistry.instance = null;
+        }
+    }
+    /**
+     * Register all built-in extractors
+     */
+    async registerAll() {
+        const { LLMMetadataExtractor } = await import("./metadataExtractor.js");
+        // Register all extractor types
+        for (const [type, metadata] of Object.entries(DEFAULT_EXTRACTOR_METADATA)) {
+            this.registerExtractor(type, async () => this.createExtractorInstance(LLMMetadataExtractor, type), metadata);
+        }
+        logger.debug(`[MetadataExtractorRegistry] Registered ${this.items.size} extractor types`);
+    }
+    /**
+     * Create extractor instance wrapper
+     */
+    createExtractorInstance(ExtractorClass, type) {
+        const extractor = new ExtractorClass();
+        return {
+            type,
+            async extract(chunks, params) {
+                return extractor.extract(chunks, params ?? {});
+            },
+        };
+    }
+    /**
+     * Register an extractor with aliases
+     */
+    registerExtractor(type, factory, metadata) {
+        this.register(type, factory, metadata);
+        // Register aliases
+        for (const alias of metadata.aliases) {
+            this.aliasMap.set(alias.toLowerCase(), type);
+            logger.debug(`[MetadataExtractorRegistry] Registered alias '${alias}' -> '${type}'`);
+        }
+    }
+    /**
+     * Resolve type from alias
+     */
+    resolveType(nameOrAlias) {
+        const lower = nameOrAlias.toLowerCase();
+        // Check if it's a direct type
+        if (this.items.has(lower)) {
+            return lower;
+        }
+        // Check aliases
+        const resolved = this.aliasMap.get(lower);
+        if (resolved) {
+            return resolved;
+        }
+        throw new MetadataExtractionError(`Unknown metadata extractor type: '${nameOrAlias}'. Available types: ${this.getAvailableExtractors().join(", ")}`, {
+            code: RAGErrorCodes.METADATA_EXTRACTOR_NOT_FOUND,
+            extractorType: nameOrAlias,
+            details: {
+                requestedType: nameOrAlias,
+                availableTypes: this.getAvailableExtractors(),
+            },
+        });
+    }
+    /**
+     * Get an extractor by type or alias
+     */
+    async getExtractor(typeOrAlias) {
+        await this.ensureInitialized();
+        const type = this.resolveType(typeOrAlias);
+        const extractor = await this.get(type);
+        if (!extractor) {
+            throw new MetadataExtractionError(`Metadata extractor not found: ${type}`, {
+                code: RAGErrorCodes.METADATA_EXTRACTOR_NOT_FOUND,
+                extractorType: type,
+                details: { type },
+            });
+        }
+        return extractor;
+    }
+    /**
+     * Get list of available extractor types
+     */
+    getAvailableExtractors() {
+        return this.list().map((item) => item.id);
+    }
+    /**
+     * Get metadata for a specific extractor
+     */
+    getExtractorMetadata(typeOrAlias) {
+        const type = this.resolveType(typeOrAlias);
+        const entry = this.list().find((item) => item.id === type);
+        return entry?.metadata;
+    }
+    /**
+     * Get all aliases for a type
+     */
+    getAliasesForType(type) {
+        const metadata = DEFAULT_EXTRACTOR_METADATA[type];
+        return metadata?.aliases ?? [];
+    }
+    /**
+     * Get all registered aliases
+     */
+    getAllAliases() {
+        return new Map(this.aliasMap);
+    }
+    /**
+     * Check if a type or alias exists
+     */
+    hasExtractor(typeOrAlias) {
+        try {
+            this.resolveType(typeOrAlias);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Get extractors by use case
+     */
+    getExtractorsByUseCase(useCase) {
+        const matches = [];
+        const useCaseLower = useCase.toLowerCase();
+        for (const [type, metadata] of Object.entries(DEFAULT_EXTRACTOR_METADATA)) {
+            const hasMatchingUseCase = metadata.useCases.some((uc) => uc.toLowerCase().includes(useCaseLower));
+            if (hasMatchingUseCase) {
+                matches.push(type);
+            }
+        }
+        return matches;
+    }
+    /**
+     * Get extractors that can produce a specific extraction type
+     */
+    getExtractorsByExtractionType(extractionType) {
+        const matches = [];
+        for (const [type, metadata] of Object.entries(DEFAULT_EXTRACTOR_METADATA)) {
+            if (metadata.extractionTypes.includes(extractionType)) {
+                matches.push(type);
+            }
+        }
+        return matches;
+    }
+    /**
+     * Get default configuration for an extractor
+     */
+    getDefaultConfig(typeOrAlias) {
+        const metadata = this.getExtractorMetadata(typeOrAlias);
+        return metadata?.defaultConfig;
+    }
+    /**
+     * Clear the registry (also clears aliases)
+     */
+    clear() {
+        super.clear();
+        this.aliasMap.clear();
+    }
+}
+/**
+ * Global metadata extractor registry singleton
+ */
+export const metadataExtractorRegistry = MetadataExtractorRegistry.getInstance();
+/**
+ * Convenience function to get available extractors
+ */
+export function getAvailableExtractors() {
+    return metadataExtractorRegistry.getAvailableExtractors();
+}
+/**
+ * Convenience function to get extractor by type
+ */
+export async function getExtractor(typeOrAlias) {
+    return metadataExtractorRegistry.getExtractor(typeOrAlias);
+}
+/**
+ * Convenience function to get extractor metadata
+ */
+export function getRegisteredExtractorMetadata(typeOrAlias) {
+    return metadataExtractorRegistry.getExtractorMetadata(typeOrAlias);
+}

package/dist/client/rag/metadata/index.js

@@ -0,0 +1,9 @@
+/**
+ * Metadata Extraction Module Exports
+ */
+// Factory pattern exports
+export { createMetadataExtractor, getAvailableExtractorTypes, getExtractorDefaultConfig, getExtractorMetadata, MetadataExtractorFactory, metadataExtractorFactory, } from "./MetadataExtractorFactory.js";
+// Registry pattern exports
+export { getAvailableExtractors, getExtractor, getRegisteredExtractorMetadata, MetadataExtractorRegistry, metadataExtractorRegistry, } from "./MetadataExtractorRegistry.js";
+// Core metadata extractor
+export { extractMetadata, LLMMetadataExtractor } from "./metadataExtractor.js";

package/dist/client/rag/metadata/metadataExtractor.js

@@ -0,0 +1,280 @@
+/**
+ * LLM-powered Metadata Extractor
+ *
+ * Extracts structured metadata from document chunks using language models.
+ * Supports title, summary, keywords, Q&A pairs, and custom schema extraction.
+ */
+import { ProviderFactory } from "../../factories/providerFactory.js";
+import { logger } from "../../utils/logger.js";
+/**
+ * Default prompts for metadata extraction
+ */
+const DEFAULT_PROMPTS = {
+    title: `Extract a concise, descriptive title for the following content.
+Return only the title, nothing else.
+
+Content:
+{context}
+
+Title:`,
+    summary: `Summarize the following content in {maxWords} words or less.
+Focus on the key points and main ideas.
+
+Content:
+{context}
+
+Summary:`,
+    keywords: `Extract the {maxKeywords} most important keywords or key phrases from the following content.
+Return them as a comma-separated list.
+
+Content:
+{context}
+
+Keywords:`,
+    questions: `Generate {numQuestions} questions that can be answered using the following content.
+{answerInstruction}
+
+Content:
+{context}
+
+Questions:`,
+};
+/**
+ * LLM-powered metadata extractor
+ * Extracts title, summary, keywords, Q&A pairs, and custom schema data
+ */
+export class LLMMetadataExtractor {
+    provider;
+    modelName;
+    constructor(options) {
+        this.provider = options?.provider || "openai";
+        this.modelName = options?.modelName || "gpt-4o-mini";
+    }
+    /**
+     * Extract metadata from chunks based on configuration
+     * @param chunks - Array of chunks to extract metadata from
+     * @param params - Extraction parameters
+     * @returns Array of extraction results, one per chunk
+     */
+    async extract(chunks, params) {
+        const results = [];
+        // Group chunks by documentId for title extraction
+        const chunksByDocument = this.groupByDocument(chunks);
+        // Cache titles by document to avoid re-extraction
+        const titleCache = new Map();
+        for (const chunk of chunks) {
+            const result = {};
+            try {
+                // Extract title (shared across chunks with same documentId)
+                if (params.title) {
+                    const docId = chunk.metadata.documentId;
+                    if (!titleCache.has(docId)) {
+                        const titleConfig = typeof params.title === "boolean" ? {} : params.title;
+                        const title = await this.extractTitle(chunksByDocument.get(docId) || [chunk], titleConfig);
+                        titleCache.set(docId, title);
+                    }
+                    result.title = titleCache.get(docId);
+                }
+                // Extract summary
+                if (params.summary) {
+                    const summaryConfig = typeof params.summary === "boolean" ? {} : params.summary;
+                    result.summary = await this.extractSummary(chunk, summaryConfig);
+                }
+                // Extract keywords
+                if (params.keywords) {
+                    const keywordConfig = typeof params.keywords === "boolean" ? {} : params.keywords;
+                    result.keywords = await this.extractKeywords(chunk, keywordConfig);
+                }
+                // Generate Q&A pairs
+                if (params.questions) {
+                    const questionConfig = typeof params.questions === "boolean" ? {} : params.questions;
+                    result.questions = await this.extractQuestions(chunk, questionConfig);
+                }
+                // Custom schema extraction
+                if (params.custom) {
+                    result.custom = await this.extractCustom(chunk, params.custom);
+                }
+                results.push(result);
+            }
+            catch (error) {
+                logger.error("[MetadataExtractor] Extraction failed for chunk", {
+                    chunkId: chunk.id,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+                results.push(result);
+            }
+        }
+        return results;
+    }
+    /**
+     * Group chunks by document ID
+     */
+    groupByDocument(chunks) {
+        const groups = new Map();
+        for (const chunk of chunks) {
+            const docId = chunk.metadata.documentId;
+            const group = groups.get(docId);
+            if (group) {
+                group.push(chunk);
+            }
+            else {
+                groups.set(docId, [chunk]);
+            }
+        }
+        return groups;
+    }
+    /**
+     * Extract title from document chunks
+     */
+    async extractTitle(chunks, config) {
+        const { nodes = 3, promptTemplate = DEFAULT_PROMPTS.title } = config;
+        // Use first N chunks for title extraction
+        const relevantChunks = chunks.slice(0, nodes);
+        const context = relevantChunks.map((c) => c.text).join("\n\n");
+        const prompt = promptTemplate.replace("{context}", context);
+        const response = await this.callLLM(prompt, config);
+        return response.trim();
+    }
+    /**
+     * Extract summary from a chunk
+     */
+    async extractSummary(chunk, config) {
+        const { maxWords = 100, promptTemplate = DEFAULT_PROMPTS.summary } = config;
+        const prompt = promptTemplate
+            .replace("{context}", chunk.text)
+            .replace("{maxWords}", String(maxWords));
+        const response = await this.callLLM(prompt, config);
+        return response.trim();
+    }
+    /**
+     * Extract keywords from a chunk
+     */
+    async extractKeywords(chunk, config) {
+        const { maxKeywords = 10, promptTemplate = DEFAULT_PROMPTS.keywords } = config;
+        const prompt = promptTemplate
+            .replace("{context}", chunk.text)
+            .replace("{maxKeywords}", String(maxKeywords));
+        const response = await this.callLLM(prompt, config);
+        // Parse comma-separated keywords
+        return response
+            .split(",")
+            .map((k) => k.trim())
+            .filter((k) => k.length > 0)
+            .slice(0, maxKeywords);
+    }
+    /**
+     * Extract Q&A pairs from a chunk
+     */
+    async extractQuestions(chunk, config) {
+        const { numQuestions = 3, includeAnswers = true, promptTemplate = DEFAULT_PROMPTS.questions, } = config;
+        const answerInstruction = includeAnswers
+            ? "For each question, also provide a brief answer based on the content."
+            : "Return only the questions.";
+        const prompt = promptTemplate
+            .replace("{context}", chunk.text)
+            .replace("{numQuestions}", String(numQuestions))
+            .replace("{answerInstruction}", answerInstruction);
+        const response = await this.callLLM(prompt, config);
+        // Parse Q&A pairs from response
+        return this.parseQAPairs(response, includeAnswers);
+    }
+    /**
+     * Extract custom schema data from a chunk
+     */
+    async extractCustom(chunk, config) {
+        const { description, promptTemplate } = config;
+        // Build extraction prompt
+        const prompt = promptTemplate ||
+            `Extract the following information from the content:
+${description || "Extract structured data according to the schema."}
+
+Content:
+${chunk.text}
+
+Return the extracted data as JSON.`;
+        const response = await this.callLLM(prompt, config);
+        try {
+            // Try to parse as JSON
+            const jsonMatch = response.match(/\{[\s\S]*\}/);
+            if (jsonMatch) {
+                return JSON.parse(jsonMatch[0]);
+            }
+            return JSON.parse(response);
+        }
+        catch {
+            logger.warn("[MetadataExtractor] Failed to parse custom extraction as JSON");
+            return { raw: response };
+        }
+    }
+    /**
+     * Parse Q&A pairs from LLM response
+     */
+    parseQAPairs(response, includeAnswers) {
+        const pairs = [];
+        // Try to parse numbered questions
+        const lines = response.split("\n").filter((l) => l.trim());
+        let currentQuestion = null;
+        let currentAnswer = null;
+        for (const line of lines) {
+            const trimmed = line.trim();
+            // Check if line is a question (starts with number or Q:)
+            if (/^\d+[.):]\s*/.test(trimmed) || /^Q[.:]?\s*/i.test(trimmed)) {
+                // Save previous Q&A pair
+                if (currentQuestion) {
+                    pairs.push({
+                        question: currentQuestion,
+                        ...(includeAnswers && currentAnswer
+                            ? { answer: currentAnswer }
+                            : {}),
+                    });
+                }
+                currentQuestion = trimmed
+                    .replace(/^\d+[.):]\s*/, "")
+                    .replace(/^Q[.:]?\s*/i, "");
+                currentAnswer = null;
+            }
+            else if (/^A[.:]?\s*/i.test(trimmed) && currentQuestion) {
+                currentAnswer = trimmed.replace(/^A[.:]?\s*/i, "");
+            }
+            else if (currentQuestion && !currentAnswer) {
+                // Continuation of question
+                currentQuestion += " " + trimmed;
+            }
+            else if (currentAnswer) {
+                // Continuation of answer
+                currentAnswer += " " + trimmed;
+            }
+        }
+        // Don't forget the last pair
+        if (currentQuestion) {
+            pairs.push({
+                question: currentQuestion,
+                ...(includeAnswers && currentAnswer ? { answer: currentAnswer } : {}),
+            });
+        }
+        return pairs;
+    }
+    /**
+     * Call the LLM with a prompt
+     */
+    async callLLM(prompt, config) {
+        const provider = await ProviderFactory.createProvider(config.provider || this.provider, config.modelName || this.modelName);
+        const result = await provider.generate({
+            prompt,
+            maxTokens: config.maxTokens || 500,
+            temperature: config.temperature || 0.3,
+        });
+        return result?.content || "";
+    }
+}
+/**
+ * Convenience function to extract metadata from chunks
+ * @param chunks - Chunks to process
+ * @param params - Extraction parameters
+ * @param options - Extractor options
+ * @returns Extraction results
+ */
+export async function extractMetadata(chunks, params, options) {
+    const extractor = new LLMMetadataExtractor(options);
+    return extractor.extract(chunks, params);
+}