@juspay/neurolink 9.1.1 → 9.3.0

package/dist/lib/rag/pipeline/RAGPipeline.js

@@ -0,0 +1,402 @@
+/**
+ * RAG Pipeline Orchestrator
+ *
+ * Provides a complete end-to-end RAG pipeline that orchestrates:
+ * - Document loading and preprocessing
+ * - Chunking with configurable strategies
+ * - Embedding generation
+ * - Vector storage and retrieval
+ * - Context assembly for LLM queries
+ * - Response generation with citations
+ *
+ * @example
+ * ```typescript
+ * const pipeline = new RAGPipeline({
+ *   vectorStore: myVectorStore,
+ *   embeddingModel: { provider: 'openai', modelName: 'text-embedding-3-small' },
+ *   generationModel: { provider: 'openai', modelName: 'gpt-4o-mini' }
+ * });
+ *
+ * // Ingest documents
+ * await pipeline.ingest(['/path/to/doc1.md', '/path/to/doc2.pdf']);
+ *
+ * // Query with RAG
+ * const response = await pipeline.query('What are the key features?');
+ * console.log(response.answer, response.sources);
+ * ```
+ */
+import { randomUUID } from "crypto";
+import { MDocument } from "../document/MDocument.js";
+import { loadDocument } from "../document/loaders.js";
+import { InMemoryVectorStore, } from "../retrieval/vectorQueryTool.js";
+import { InMemoryBM25Index, createHybridSearch, } from "../retrieval/hybridSearch.js";
+import { GraphRAG } from "../graphRag/graphRAG.js";
+import { rerank } from "../reranker/reranker.js";
+import { ProviderFactory } from "../../factories/providerFactory.js";
+import { logger } from "../../utils/logger.js";
+/**
+ * RAG Pipeline Orchestrator
+ *
+ * Complete end-to-end pipeline for Retrieval-Augmented Generation.
+ */
+export class RAGPipeline {
+    id;
+    config;
+    vectorStore;
+    bm25Index;
+    graphRAG;
+    embeddingProvider;
+    generationProvider;
+    hybridSearch;
+    documents = new Map();
+    allChunks = [];
+    constructor(config) {
+        this.id = config.id || `rag-pipeline-${randomUUID().slice(0, 8)}`;
+        this.config = {
+            indexName: "default",
+            defaultChunkingStrategy: "recursive",
+            defaultChunkSize: 1000,
+            defaultChunkOverlap: 200,
+            enableHybridSearch: false,
+            enableGraphRAG: false,
+            graphThreshold: 0.7,
+            defaultTopK: 5,
+            enableReranking: false,
+            ...config,
+        };
+        // Initialize stores
+        this.vectorStore = config.vectorStore || new InMemoryVectorStore();
+        this.bm25Index = config.bm25Index || new InMemoryBM25Index();
+        this.graphRAG = new GraphRAG({ threshold: this.config.graphThreshold });
+        logger.info("[RAGPipeline] Pipeline initialized", {
+            id: this.id,
+            indexName: this.config.indexName,
+            embeddingModel: this.config.embeddingModel,
+        });
+    }
+    /**
+     * Initialize the pipeline (lazy loading of providers)
+     */
+    async initialize() {
+        // Initialize embedding provider
+        this.embeddingProvider = await ProviderFactory.createProvider(this.config.embeddingModel.provider, this.config.embeddingModel.modelName);
+        // Initialize generation provider if configured
+        if (this.config.generationModel) {
+            this.generationProvider = await ProviderFactory.createProvider(this.config.generationModel.provider, this.config.generationModel.modelName);
+        }
+        // Initialize hybrid search if enabled
+        if (this.config.enableHybridSearch) {
+            this.hybridSearch = createHybridSearch({
+                vectorStore: this.vectorStore,
+                bm25Index: this.bm25Index,
+                indexName: this.config.indexName,
+                embeddingModel: this.config.embeddingModel,
+            });
+        }
+        logger.info("[RAGPipeline] Pipeline initialized", { id: this.id });
+    }
+    /**
+     * Ingest documents into the pipeline
+     *
+     * @param sources - Array of file paths, URLs, or MDocument instances
+     * @param options - Ingestion options
+     */
+    async ingest(sources, options) {
+        await this.ensureInitialized();
+        const strategy = options?.strategy || this.config.defaultChunkingStrategy;
+        const chunkSize = options?.chunkSize || this.config.defaultChunkSize;
+        const chunkOverlap = options?.chunkOverlap || this.config.defaultChunkOverlap;
+        let documentsProcessed = 0;
+        let chunksCreated = 0;
+        for (const source of sources) {
+            try {
+                // Load document if string
+                const doc = source instanceof MDocument
+                    ? source
+                    : await loadDocument(source, { metadata: options?.metadata });
+                // Chunk the document
+                await doc.chunk({
+                    strategy,
+                    config: {
+                        maxSize: chunkSize,
+                        overlap: chunkOverlap,
+                        metadata: options?.metadata,
+                    },
+                });
+                // Extract metadata if requested
+                if (options?.extractMetadata) {
+                    await doc.extractMetadata({
+                        title: true,
+                        summary: true,
+                        keywords: true,
+                    });
+                }
+                // Generate embeddings
+                await doc.embed(this.config.embeddingModel.provider, this.config.embeddingModel.modelName);
+                const chunks = doc.getChunks();
+                const embeddings = doc.getEmbeddings();
+                // Store in vector store
+                await this.vectorStore.query({
+                    indexName: this.config.indexName,
+                    queryVector: embeddings[0],
+                    topK: 1,
+                }); // Warm up
+                // Upsert into vector store
+                if ("upsert" in this.vectorStore) {
+                    await this.vectorStore.upsert(this.config.indexName, chunks.map((chunk, i) => ({
+                        id: chunk.id,
+                        vector: embeddings[i],
+                        metadata: { ...chunk.metadata, text: chunk.text },
+                    })));
+                }
+                // Add to BM25 index
+                await this.bm25Index.addDocuments(chunks.map((chunk) => ({
+                    id: chunk.id,
+                    text: chunk.text,
+                    metadata: chunk.metadata,
+                })));
+                // Update Graph RAG if enabled
+                if (this.config.enableGraphRAG) {
+                    this.graphRAG.createGraph([...this.allChunks, ...chunks].map((c) => ({
+                        text: c.text,
+                        metadata: c.metadata,
+                    })), [...this.allChunks, ...chunks].map((c) => ({
+                        vector: c.embedding || [],
+                    })));
+                }
+                // Track documents and chunks
+                this.documents.set(doc.getId(), doc);
+                this.allChunks.push(...chunks);
+                documentsProcessed++;
+                chunksCreated += chunks.length;
+                logger.debug("[RAGPipeline] Document ingested", {
+                    documentId: doc.getId(),
+                    chunks: chunks.length,
+                });
+            }
+            catch (error) {
+                logger.error("[RAGPipeline] Failed to ingest document", {
+                    source: typeof source === "string" ? source : source.getId(),
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        logger.info("[RAGPipeline] Ingestion complete", {
+            documentsProcessed,
+            chunksCreated,
+        });
+        return { documentsProcessed, chunksCreated };
+    }
+    /**
+     * Query the pipeline
+     *
+     * @param query - Search query
+     * @param options - Query options
+     * @returns RAG response with retrieved context and optional generated answer
+     */
+    async query(query, options) {
+        await this.ensureInitialized();
+        const startTime = Date.now();
+        const topK = options?.topK || this.config.defaultTopK;
+        const useHybrid = options?.hybrid ?? this.config.enableHybridSearch;
+        const useGraph = options?.graph ?? this.config.enableGraphRAG;
+        const useRerank = options?.rerank ?? this.config.enableReranking;
+        let results;
+        let retrievalMethod = "vector";
+        // Generate query embedding
+        const queryEmbedding = await this.generateEmbedding(query);
+        if (useGraph && this.config.enableGraphRAG) {
+            // Graph RAG search
+            retrievalMethod = "graph";
+            const graphResults = this.graphRAG.query({
+                query: queryEmbedding,
+                topK: topK * 2, // Get more for potential reranking
+            });
+            results = graphResults.map((r) => ({
+                id: r.id,
+                text: r.content,
+                score: r.score,
+                metadata: r.metadata,
+            }));
+        }
+        else if (useHybrid && this.hybridSearch) {
+            // Hybrid search
+            retrievalMethod = "hybrid";
+            const hybridResults = await this.hybridSearch(query, { topK: topK * 2 });
+            results = hybridResults.map((r) => ({
+                id: r.id,
+                text: r.text,
+                score: r.score,
+                metadata: r.metadata,
+            }));
+        }
+        else {
+            // Vector search
+            results = await this.vectorStore.query({
+                indexName: this.config.indexName,
+                queryVector: queryEmbedding,
+                topK: topK * 2,
+                filter: options?.filter,
+            });
+        }
+        // Apply reranking if enabled
+        let reranked = false;
+        if (useRerank && this.config.rerankingModel && results.length > 0) {
+            const rerankModel = await ProviderFactory.createProvider(this.config.rerankingModel.provider, this.config.rerankingModel.modelName);
+            const rerankedResults = await rerank(results, query, rerankModel, {
+                topK,
+                queryEmbedding,
+            });
+            results = rerankedResults.map((r) => r.result);
+            reranked = true;
+        }
+        // Take top K results
+        results = results.slice(0, topK);
+        // Assemble context
+        const context = this.assembleContext(results);
+        // Format sources
+        const sources = results.map((r) => ({
+            id: r.id,
+            text: r.text || r.metadata?.text || "",
+            score: r.score || 0,
+            metadata: r.metadata,
+        }));
+        // Generate answer if requested
+        let answer;
+        if (options?.generate !== false && this.generationProvider) {
+            answer = await this.generateAnswer(query, context, options?.systemPrompt, options?.temperature);
+        }
+        const queryTime = Date.now() - startTime;
+        logger.info("[RAGPipeline] Query completed", {
+            query: query.slice(0, 50),
+            retrievalMethod,
+            resultsCount: results.length,
+            reranked,
+            queryTime,
+        });
+        return {
+            answer,
+            context,
+            sources,
+            metadata: {
+                queryTime,
+                retrievalMethod,
+                chunksRetrieved: results.length,
+                reranked,
+            },
+        };
+    }
+    /**
+     * Get pipeline statistics
+     */
+    getStats() {
+        return {
+            totalDocuments: this.documents.size,
+            totalChunks: this.allChunks.length,
+            indexName: this.config.indexName,
+            embeddingDimension: this.allChunks[0]?.embedding?.length,
+            hybridSearchEnabled: this.config.enableHybridSearch,
+            graphRAGEnabled: this.config.enableGraphRAG,
+        };
+    }
+    /**
+     * Get pipeline ID
+     */
+    getId() {
+        return this.id;
+    }
+    /**
+     * Clear all indexed data
+     */
+    async clear() {
+        this.documents.clear();
+        this.allChunks = [];
+        this.graphRAG = new GraphRAG({ threshold: this.config.graphThreshold });
+        if ("delete" in this.vectorStore) {
+            // Clear vector store if supported
+            // Note: InMemoryVectorStore doesn't have a clear method
+        }
+        logger.info("[RAGPipeline] Pipeline cleared", { id: this.id });
+    }
+    // ============================================================================
+    // Private Methods
+    // ============================================================================
+    /**
+     * Ensure pipeline is initialized
+     */
+    async ensureInitialized() {
+        if (!this.embeddingProvider) {
+            await this.initialize();
+        }
+    }
+    /**
+     * Generate embedding for text
+     */
+    async generateEmbedding(text) {
+        if (!this.embeddingProvider) {
+            throw new Error("Embedding provider not initialized");
+        }
+        if (typeof this.embeddingProvider
+            .embed !== "function") {
+            throw new Error(`Provider ${this.config.embeddingModel.provider} does not support embeddings`);
+        }
+        return await this.embeddingProvider.embed(text);
+    }
+    /**
+     * Assemble context from results
+     */
+    assembleContext(results) {
+        return results
+            .map((r, i) => {
+            const text = r.text || r.metadata?.text || "";
+            const source = r.metadata?.source || `chunk-${i + 1}`;
+            return `[Source ${i + 1}: ${source}]\n${text}`;
+        })
+            .join("\n\n---\n\n");
+    }
+    /**
+     * Generate answer using LLM
+     */
+    async generateAnswer(query, context, customSystemPrompt, temperature) {
+        if (!this.generationProvider) {
+            throw new Error("Generation provider not configured");
+        }
+        const systemPrompt = customSystemPrompt ||
+            `You are a helpful assistant that answers questions based on the provided context.
+Use only the information from the context to answer the question.
+If the context doesn't contain relevant information, say so.
+Cite sources when possible using [Source N] format.`;
+        const prompt = `Context:\n${context}\n\nQuestion: ${query}\n\nAnswer:`;
+        const result = await this.generationProvider.generate({
+            prompt,
+            systemPrompt,
+            temperature: temperature ?? this.config.generationModel?.temperature ?? 0.7,
+            maxTokens: this.config.generationModel?.maxTokens ?? 1000,
+        });
+        return result?.content || "";
+    }
+}
+/**
+ * Create a simple RAG pipeline with sensible defaults
+ *
+ * @param options - Basic configuration options
+ * @returns Configured RAGPipeline instance
+ */
+export function createRAGPipeline(options) {
+    const provider = options.provider || "openai";
+    return new RAGPipeline({
+        embeddingModel: {
+            provider,
+            modelName: options.embeddingModel || "text-embedding-3-small",
+        },
+        generationModel: options.generationModel
+            ? {
+                provider,
+                modelName: options.generationModel,
+            }
+            : undefined,
+        enableHybridSearch: options.enableHybrid,
+        enableGraphRAG: options.enableGraph,
+    });
+}
+//# sourceMappingURL=RAGPipeline.js.map

package/dist/lib/rag/pipeline/contextAssembly.d.ts

@@ -0,0 +1,126 @@
+/**
+ * Context Assembly Utilities
+ *
+ * Provides utilities for assembling, formatting, and optimizing context
+ * from retrieved chunks for LLM consumption.
+ *
+ * Features:
+ * - Context window management (token-aware truncation)
+ * - Citation formatting
+ * - Context deduplication
+ * - Relevance-based ordering
+ * - Context summarization
+ */
+import type { Chunk, VectorQueryResult } from "../types.js";
+/**
+ * Citation format options
+ */
+export type CitationFormat = "inline" | "footnote" | "numbered" | "none";
+/**
+ * Context assembly options
+ */
+export interface ContextAssemblyOptions {
+    /** Maximum characters in assembled context */
+    maxChars?: number;
+    /** Maximum tokens (approximate, 4 chars/token) */
+    maxTokens?: number;
+    /** Citation format to use */
+    citationFormat?: CitationFormat;
+    /** Separator between chunks */
+    separator?: string;
+    /** Include chunk metadata in context */
+    includeMetadata?: boolean;
+    /** Deduplicate overlapping content */
+    deduplicate?: boolean;
+    /** Similarity threshold for deduplication (0-1) */
+    dedupeThreshold?: number;
+    /** Order by relevance score */
+    orderByRelevance?: boolean;
+    /** Include section headers */
+    includeSectionHeaders?: boolean;
+    /** Header template (use {index}, {source}, {score} placeholders) */
+    headerTemplate?: string;
+}
+/**
+ * Context window representation
+ */
+export interface ContextWindow {
+    /** Assembled context text */
+    text: string;
+    /** Number of chunks included */
+    chunkCount: number;
+    /** Total character count */
+    charCount: number;
+    /** Estimated token count */
+    tokenCount: number;
+    /** Chunks that were truncated/excluded */
+    truncatedChunks: number;
+    /** Citation map (id -> citation text) */
+    citations: Map<string, string>;
+}
+/**
+ * Assemble context from retrieved results
+ *
+ * Combines multiple chunks into a coherent context string
+ * suitable for LLM consumption.
+ *
+ * @param results - Retrieved chunks or query results
+ * @param options - Assembly options
+ * @returns Assembled context string
+ *
+ * @example
+ * ```typescript
+ * const context = assembleContext(results, {
+ *   maxTokens: 4000,
+ *   citationFormat: 'numbered',
+ *   deduplicate: true
+ * });
+ * ```
+ */
+export declare function assembleContext(results: Array<Chunk | VectorQueryResult>, options?: ContextAssemblyOptions): string;
+/**
+ * Format context with inline citations
+ *
+ * @param results - Retrieved results
+ * @param options - Formatting options
+ * @returns Context with citations and citation list
+ */
+export declare function formatContextWithCitations(results: Array<Chunk | VectorQueryResult>, options?: ContextAssemblyOptions & {
+    returnCitations?: boolean;
+}): {
+    context: string;
+    citations: string[];
+};
+/**
+ * Create a context window with detailed tracking
+ *
+ * @param results - Retrieved results
+ * @param options - Assembly options
+ * @returns Context window with metadata
+ */
+export declare function createContextWindow(results: Array<Chunk | VectorQueryResult>, options?: ContextAssemblyOptions): ContextWindow;
+/**
+ * Summarize context using LLM
+ *
+ * @param context - Context to summarize
+ * @param maxLength - Maximum summary length
+ * @param provider - LLM provider instance
+ * @returns Summarized context
+ */
+export declare function summarizeContext(context: string, maxLength?: number, provider?: {
+    generate: (params: {
+        prompt: string;
+        maxTokens: number;
+        temperature: number;
+    }) => Promise<{
+        content?: string;
+    } | null>;
+}): Promise<string>;
+/**
+ * Order chunks by document structure (if available)
+ */
+export declare function orderByDocumentStructure(chunks: Chunk[]): Chunk[];
+/**
+ * Extract key sentences from chunks for summary
+ */
+export declare function extractKeySentences(text: string, count?: number): string[];