@juspay/neurolink 11.12.0 → 11.13.1

package/dist/lib/rag/pipeline/RAGPipeline.js

@@ -1,436 +0,0 @@
-/**
- * RAG Pipeline Orchestrator
- *
- * Provides a complete end-to-end RAG pipeline that orchestrates:
- * - Document loading and preprocessing
- * - Chunking with configurable strategies
- * - Embedding generation
- * - Vector storage and retrieval
- * - Context assembly for LLM queries
- * - Response generation with citations
- *
- * @example
- * ```typescript
- * const pipeline = new RAGPipeline({
- *   vectorStore: myVectorStore,
- *   embeddingModel: { provider: 'openai', modelName: 'text-embedding-3-small' },
- *   generationModel: { provider: 'openai', modelName: 'gpt-4o-mini' }
- * });
- *
- * // Ingest documents
- * await pipeline.ingest(['/path/to/doc1.md', '/path/to/doc2.pdf']);
- *
- * // Query with RAG
- * const response = await pipeline.query('What are the key features?');
- * console.log(response.answer, response.sources);
- * ```
- */
-import { randomUUID } from "crypto";
-import { MDocument } from "../document/MDocument.js";
-import { loadDocument } from "../document/loaders.js";
-import { InMemoryVectorStore } from "../retrieval/vectorQueryTool.js";
-import { InMemoryBM25Index, createHybridSearch, } from "../retrieval/hybridSearch.js";
-import { GraphRAG } from "../graphRag/graphRAG.js";
-import { rerank } from "../reranker/reranker.js";
-import { ProviderFactory } from "../../factories/providerFactory.js";
-import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../../observability/index.js";
-import { logger } from "../../utils/logger.js";
-import { withTimeout } from "../../utils/async/withTimeout.js";
-/**
- * RAG Pipeline Orchestrator
- *
- * Complete end-to-end pipeline for Retrieval-Augmented Generation.
- */
-/** Default timeout for external provider calls (30 seconds) */
-const DEFAULT_TIMEOUT_MS = 30_000;
-export class RAGPipeline {
-    id;
-    config;
-    vectorStore;
-    bm25Index;
-    graphRAG;
-    embeddingProvider;
-    generationProvider;
-    hybridSearch;
-    documents = new Map();
-    allChunks = [];
-    constructor(config) {
-        this.id = config.id || `rag-pipeline-${randomUUID().slice(0, 8)}`;
-        this.config = {
-            indexName: "default",
-            defaultChunkingStrategy: "recursive",
-            defaultChunkSize: 1000,
-            defaultChunkOverlap: 200,
-            enableHybridSearch: false,
-            enableGraphRAG: false,
-            graphThreshold: 0.7,
-            defaultTopK: 5,
-            enableReranking: false,
-            ...config,
-        };
-        // Initialize stores
-        this.vectorStore = config.vectorStore || new InMemoryVectorStore();
-        this.bm25Index = config.bm25Index || new InMemoryBM25Index();
-        this.graphRAG = new GraphRAG({ threshold: this.config.graphThreshold });
-        logger.info("[RAGPipeline] Pipeline initialized", {
-            id: this.id,
-            indexName: this.config.indexName,
-            embeddingModel: this.config.embeddingModel,
-        });
-    }
-    /**
-     * Initialize the pipeline (lazy loading of providers)
-     */
-    async initialize() {
-        // Initialize embedding provider
-        this.embeddingProvider = await ProviderFactory.createProvider(this.config.embeddingModel.provider, this.config.embeddingModel.modelName);
-        // Initialize generation provider if configured
-        if (this.config.generationModel) {
-            this.generationProvider = await ProviderFactory.createProvider(this.config.generationModel.provider, this.config.generationModel.modelName);
-        }
-        // Initialize hybrid search if enabled
-        if (this.config.enableHybridSearch) {
-            this.hybridSearch = createHybridSearch({
-                vectorStore: this.vectorStore,
-                bm25Index: this.bm25Index,
-                indexName: this.config.indexName ?? "default",
-                embeddingModel: this.config.embeddingModel,
-            });
-        }
-        logger.info("[RAGPipeline] Pipeline initialized", { id: this.id });
-    }
-    /**
-     * Ingest documents into the pipeline
-     *
-     * @param sources - Array of file paths, URLs, or MDocument instances
-     * @param options - Ingestion options
-     */
-    async ingest(sources, options) {
-        await this.ensureInitialized();
-        const strategy = options?.strategy || this.config.defaultChunkingStrategy || "recursive";
-        const chunkSize = options?.chunkSize || this.config.defaultChunkSize || 1000;
-        const chunkOverlap = options?.chunkOverlap || this.config.defaultChunkOverlap || 200;
-        let documentsProcessed = 0;
-        let chunksCreated = 0;
-        for (const source of sources) {
-            try {
-                // Load document if string
-                const doc = source instanceof MDocument
-                    ? source
-                    : await loadDocument(source, { metadata: options?.metadata });
-                // Chunk the document
-                await doc.chunk({
-                    strategy,
-                    config: {
-                        maxSize: chunkSize,
-                        overlap: chunkOverlap,
-                        metadata: options?.metadata,
-                    },
-                });
-                // Extract metadata if requested
-                if (options?.extractMetadata) {
-                    await doc.extractMetadata({
-                        title: true,
-                        summary: true,
-                        keywords: true,
-                    });
-                }
-                // Generate embeddings
-                await doc.embed(this.config.embeddingModel.provider, this.config.embeddingModel.modelName);
-                const chunks = doc.getChunks();
-                const embeddings = doc.getEmbeddings();
-                // Store in vector store
-                await this.vectorStore.query({
-                    indexName: this.config.indexName ?? "default",
-                    queryVector: embeddings[0],
-                    topK: 1,
-                }); // Warm up
-                // Upsert into vector store
-                if ("upsert" in this.vectorStore) {
-                    await this.vectorStore.upsert(this.config.indexName ?? "default", chunks.map((chunk, i) => ({
-                        id: chunk.id,
-                        vector: embeddings[i],
-                        metadata: { ...chunk.metadata, text: chunk.text },
-                    })));
-                }
-                // Add to BM25 index
-                await this.bm25Index.addDocuments(chunks.map((chunk) => ({
-                    id: chunk.id,
-                    text: chunk.text,
-                    metadata: chunk.metadata,
-                })));
-                // Update Graph RAG if enabled
-                if (this.config.enableGraphRAG) {
-                    this.graphRAG.createGraph([...this.allChunks, ...chunks].map((c) => ({
-                        text: c.text,
-                        metadata: c.metadata,
-                    })), [...this.allChunks, ...chunks].map((c) => ({
-                        vector: c.embedding || [],
-                    })));
-                }
-                // Track documents and chunks
-                this.documents.set(doc.getId(), doc);
-                this.allChunks.push(...chunks);
-                documentsProcessed++;
-                chunksCreated += chunks.length;
-                logger.debug("[RAGPipeline] Document ingested", {
-                    documentId: doc.getId(),
-                    chunks: chunks.length,
-                });
-            }
-            catch (error) {
-                logger.error("[RAGPipeline] Failed to ingest document", {
-                    source: typeof source === "string" ? source : source.getId(),
-                    error: error instanceof Error ? error.message : String(error),
-                });
-            }
-        }
-        logger.info("[RAGPipeline] Ingestion complete", {
-            documentsProcessed,
-            chunksCreated,
-        });
-        return { documentsProcessed, chunksCreated };
-    }
-    /**
-     * Query the pipeline
-     *
-     * @param query - Search query
-     * @param options - Query options
-     * @returns RAG response with retrieved context and optional generated answer
-     */
-    async query(query, options) {
-        const span = SpanSerializer.createSpan(SpanType.RAG, "rag.pipeline", {
-            "rag.operation": "pipeline",
-            "rag.query": query.slice(0, 200),
-            "rag.topK": options?.topK ?? this.config.defaultTopK ?? 5,
-            "rag.hybrid": options?.hybrid ?? this.config.enableHybridSearch ?? false,
-            "rag.graph": options?.graph ?? this.config.enableGraphRAG ?? false,
-            "rag.rerank": options?.rerank ?? this.config.enableReranking ?? false,
-        });
-        const spanStartTime = Date.now();
-        try {
-            await this.ensureInitialized();
-            const startTime = Date.now();
-            const topK = options?.topK || this.config.defaultTopK || 5;
-            const useHybrid = options?.hybrid ?? this.config.enableHybridSearch;
-            const useGraph = options?.graph ?? this.config.enableGraphRAG;
-            const useRerank = options?.rerank ?? this.config.enableReranking;
-            let results;
-            let retrievalMethod = "vector";
-            // Generate query embedding
-            const queryEmbedding = await this.generateEmbedding(query);
-            if (useGraph && this.config.enableGraphRAG) {
-                // Graph RAG search
-                retrievalMethod = "graph";
-                const graphResults = this.graphRAG.query({
-                    query: queryEmbedding,
-                    topK: topK * 2, // Get more for potential reranking
-                });
-                results = graphResults.map((r) => ({
-                    id: r.id,
-                    text: r.content,
-                    score: r.score,
-                    metadata: r.metadata,
-                }));
-            }
-            else if (useHybrid && this.hybridSearch) {
-                // Hybrid search
-                retrievalMethod = "hybrid";
-                const hybridResults = await this.hybridSearch(query, {
-                    topK: topK * 2,
-                });
-                results = hybridResults.map((r) => ({
-                    id: r.id,
-                    text: r.text,
-                    score: r.score,
-                    metadata: r.metadata,
-                }));
-            }
-            else {
-                // Vector search
-                results = await this.vectorStore.query({
-                    indexName: this.config.indexName ?? "default",
-                    queryVector: queryEmbedding,
-                    topK: topK * 2,
-                    filter: options?.filter,
-                });
-            }
-            // Apply reranking if enabled
-            let reranked = false;
-            if (useRerank && this.config.rerankingModel && results.length > 0) {
-                const rerankModel = await ProviderFactory.createProvider(this.config.rerankingModel.provider, this.config.rerankingModel.modelName);
-                const rerankedResults = await rerank(results, query, rerankModel, {
-                    topK,
-                    queryEmbedding,
-                });
-                results = rerankedResults.map((r) => r.result);
-                reranked = true;
-            }
-            // Take top K results
-            results = results.slice(0, topK);
-            // Assemble context
-            const context = this.assembleContext(results);
-            // Format sources
-            const sources = results.map((r) => ({
-                id: r.id,
-                text: r.text || r.metadata?.text || "",
-                score: r.score || 0,
-                metadata: r.metadata,
-            }));
-            // Generate answer if requested
-            let answer;
-            if (options?.generate !== false && this.generationProvider) {
-                answer = await this.generateAnswer(query, context, options?.systemPrompt, options?.temperature);
-            }
-            const queryTime = Date.now() - startTime;
-            logger.info("[RAGPipeline] Query completed", {
-                query: query.slice(0, 50),
-                retrievalMethod,
-                resultsCount: results.length,
-                reranked,
-                queryTime,
-            });
-            const response = {
-                answer,
-                context,
-                sources,
-                metadata: {
-                    queryTime,
-                    retrievalMethod,
-                    chunksRetrieved: results.length,
-                    reranked,
-                },
-            };
-            span.durationMs = Date.now() - spanStartTime;
-            const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
-            endedSpan.attributes = {
-                ...endedSpan.attributes,
-                "rag.retrieval_method": retrievalMethod,
-                "rag.results_count": results.length,
-                "rag.reranked": reranked,
-            };
-            getMetricsAggregator().recordSpan(endedSpan);
-            return response;
-        }
-        catch (error) {
-            span.durationMs = Date.now() - spanStartTime;
-            const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR);
-            endedSpan.statusMessage =
-                error instanceof Error ? error.message : String(error);
-            getMetricsAggregator().recordSpan(endedSpan);
-            throw error;
-        }
-    }
-    /**
-     * Get pipeline statistics
-     */
-    getStats() {
-        return {
-            totalDocuments: this.documents.size,
-            totalChunks: this.allChunks.length,
-            indexName: this.config.indexName ?? "default",
-            embeddingDimension: this.allChunks[0]?.embedding?.length,
-            hybridSearchEnabled: this.config.enableHybridSearch ?? false,
-            graphRAGEnabled: this.config.enableGraphRAG ?? false,
-        };
-    }
-    /**
-     * Get pipeline ID
-     */
-    getId() {
-        return this.id;
-    }
-    /**
-     * Clear all indexed data
-     */
-    async clear() {
-        this.documents.clear();
-        this.allChunks = [];
-        this.graphRAG = new GraphRAG({ threshold: this.config.graphThreshold });
-        if ("delete" in this.vectorStore) {
-            // Clear vector store if supported
-            // Note: InMemoryVectorStore doesn't have a clear method
-        }
-        logger.info("[RAGPipeline] Pipeline cleared", { id: this.id });
-    }
-    // ============================================================================
-    // Private Methods
-    // ============================================================================
-    /**
-     * Ensure pipeline is initialized
-     */
-    async ensureInitialized() {
-        if (!this.embeddingProvider) {
-            await this.initialize();
-        }
-    }
-    /**
-     * Generate embedding for text
-     */
-    async generateEmbedding(text) {
-        if (!this.embeddingProvider) {
-            throw new Error("Embedding provider not initialized");
-        }
-        if (typeof this.embeddingProvider.embed !== "function") {
-            throw new Error(`Provider ${this.config.embeddingModel.provider} does not support embeddings`);
-        }
-        return await withTimeout(this.embeddingProvider.embed(text), DEFAULT_TIMEOUT_MS, "Embedding generation timed out");
-    }
-    /**
-     * Assemble context from results
-     */
-    assembleContext(results) {
-        return results
-            .map((r, i) => {
-            const text = r.text || r.metadata?.text || "";
-            const source = r.metadata?.source || `chunk-${i + 1}`;
-            return `[Source ${i + 1}: ${source}]\n${text}`;
-        })
-            .join("\n\n---\n\n");
-    }
-    /**
-     * Generate answer using LLM
-     */
-    async generateAnswer(query, context, customSystemPrompt, temperature) {
-        if (!this.generationProvider) {
-            throw new Error("Generation provider not configured");
-        }
-        const systemPrompt = customSystemPrompt ||
-            `You are a helpful assistant that answers questions based on the provided context.
-Use only the information from the context to answer the question.
-If the context doesn't contain relevant information, say so.
-Cite sources when possible using [Source N] format.`;
-        const prompt = `Context:\n${context}\n\nQuestion: ${query}\n\nAnswer:`;
-        const result = await withTimeout(this.generationProvider.generate({
-            prompt,
-            systemPrompt,
-            temperature: temperature ?? this.config.generationModel?.temperature ?? 0.7,
-            maxTokens: this.config.generationModel?.maxTokens ?? 1000,
-        }), DEFAULT_TIMEOUT_MS * 2, "Answer generation timed out");
-        return result?.content || "";
-    }
-}
-/**
- * Create a simple RAG pipeline with sensible defaults
- *
- * @param options - Basic configuration options
- * @returns Configured RAGPipeline instance
- */
-export function createRAGPipeline(options) {
-    const provider = options.provider || "openai";
-    return new RAGPipeline({
-        embeddingModel: {
-            provider,
-            modelName: options.embeddingModel || "text-embedding-3-small",
-        },
-        generationModel: options.generationModel
-            ? {
-                provider,
-                modelName: options.generationModel,
-            }
-            : undefined,
-        enableHybridSearch: options.enableHybrid,
-        enableGraphRAG: options.enableGraph,
-    });
-}
-//# sourceMappingURL=RAGPipeline.js.map

package/dist/lib/rag/pipeline/contextAssembly.d.ts

@@ -1,80 +0,0 @@
-/**
- * Context Assembly Utilities
- *
- * Provides utilities for assembling, formatting, and optimizing context
- * from retrieved chunks for LLM consumption.
- *
- * Features:
- * - Context window management (token-aware truncation)
- * - Citation formatting
- * - Context deduplication
- * - Relevance-based ordering
- * - Context summarization
- */
-import type { Chunk, VectorQueryResult, ContextAssemblyOptions, ContextWindow } from "../../types/index.js";
-/**
- * Assemble context from retrieved results
- *
- * Combines multiple chunks into a coherent context string
- * suitable for LLM consumption.
- *
- * @param results - Retrieved chunks or query results
- * @param options - Assembly options
- * @returns Assembled context string
- *
- * @example
- * ```typescript
- * const context = assembleContext(results, {
- *   maxTokens: 4000,
- *   citationFormat: 'numbered',
- *   deduplicate: true
- * });
- * ```
- */
-export declare function assembleContext(results: Array<Chunk | VectorQueryResult>, options?: ContextAssemblyOptions): string;
-/**
- * Format context with inline citations
- *
- * @param results - Retrieved results
- * @param options - Formatting options
- * @returns Context with citations and citation list
- */
-export declare function formatContextWithCitations(results: Array<Chunk | VectorQueryResult>, options?: ContextAssemblyOptions & {
-    returnCitations?: boolean;
-}): {
-    context: string;
-    citations: string[];
-};
-/**
- * Create a context window with detailed tracking
- *
- * @param results - Retrieved results
- * @param options - Assembly options
- * @returns Context window with metadata
- */
-export declare function createContextWindow(results: Array<Chunk | VectorQueryResult>, options?: ContextAssemblyOptions): ContextWindow;
-/**
- * Summarize context using LLM
- *
- * @param context - Context to summarize
- * @param maxLength - Maximum summary length
- * @param provider - LLM provider instance
- * @returns Summarized context
- */
-export declare function summarizeContext(context: string, maxLength?: number, provider?: {
-    generate: (params: {
-        prompt: string;
-        maxTokens: number;
-        temperature: number;
-    }) => Promise<{
-        content?: string;
-    } | null>;
-}): Promise<string>;
-/**
- * Order chunks by document structure (if available)
- */
-export declare function orderByDocumentStructure(chunks: Chunk[]): Chunk[];
-/**
- * Extract key sentences from chunks for summary
- */
-export declare function extractKeySentences(text: string, count?: number): string[];