@retrivora-ai/rag-engine 0.4.5 → 1.0.0

package/src/core/LangChainAgent.ts

@@ -0,0 +1,78 @@
+import { Pipeline } from "./Pipeline";
+import { RagConfig } from "../config/RagConfig";
+
+/**
+ * LangChainAgent — Orchestration layer that turns the RAG pipeline into an Agent.
+ * 
+ * Uses dynamic imports to remain an optional dependency.
+ */
+export class LangChainAgent {
+  private executor?: unknown;
+
+  constructor(private pipeline: Pipeline, private config: RagConfig) {}
+
+  /**
+   * Initializes the agent with the RAG pipeline as a primary tool.
+   * Dynamically imports LangChain dependencies to avoid build errors if missing.
+   */
+  async initialize(chatModel: unknown) {
+    try {
+      // Dynamic imports for optional dependency support using template strings to hide from TS static analysis
+      const { DynamicTool } = await import(`${"@langchain/core/tools"}`);
+      const { ChatPromptTemplate, MessagesPlaceholder } = await import(`${"@langchain/core/prompts"}`);
+      const { AgentExecutor, createOpenAIFunctionsAgent } = await import(`${"langchain/agents"}`);
+
+      // 1. Define the RAG Search Tool
+      const searchTool = new DynamicTool({
+        name: "document_search",
+        description: "Use this tool to search through the knowledge base and document repository. Input should be a specific search query.",
+        func: async (query: string) => {
+          const response = await this.pipeline.ask(query);
+          return `Search Results:\n${response.reply}\n\nSources Used: ${JSON.stringify(response.sources.map(s => s.id))}`;
+        },
+      });
+
+      const tools = [searchTool];
+
+      // 2. Define the Agent Prompt
+      const prompt = ChatPromptTemplate.fromMessages([
+        ["system", this.config.llm.systemPrompt || "You are a helpful AI assistant with access to a document search tool."],
+        new MessagesPlaceholder("chat_history"),
+        ["human", "{input}"],
+        new MessagesPlaceholder("agent_scratchpad"),
+      ]);
+
+      // 3. Create the Agent
+      const agent = await createOpenAIFunctionsAgent({
+        llm: chatModel,
+        tools,
+        prompt,
+      });
+
+      this.executor = new AgentExecutor({
+        agent,
+        tools,
+      });
+    } catch (error) {
+      console.error("[LangChainAgent] Failed to initialize. Ensure 'langchain' and '@langchain/core' are installed.");
+      throw error;
+    }
+  }
+
+  /**
+   * Run the agentic flow.
+   */
+  async run(input: string, chatHistory: unknown[] = []) {
+    if (!this.executor) {
+      throw new Error("[LangChainAgent] Agent not initialized. Call initialize() first.");
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const response = await (this.executor as any).invoke({
+      input,
+      chat_history: chatHistory,
+    });
+
+    return response.output;
+  }
+}

package/src/core/Pipeline.ts

@@ -1,165 +1,17 @@
 import { BaseVectorProvider } from '../providers/vectordb/BaseVectorProvider';
 import { BaseGraphProvider } from '../providers/graphdb/BaseGraphProvider';
 import { ILLMProvider, ChatMessage } from '../llm/ILLMProvider';
-import { DocumentChunker } from '../rag/DocumentChunker';
+import { DocumentChunker, Chunk } from '../rag/DocumentChunker';
 import { EntityExtractor } from '../rag/EntityExtractor';
+import { Reranker } from '../rag/Reranker';
+import { LlamaIndexIngestor } from '../rag/LlamaIndexIngestor';
+import { LangChainAgent } from './LangChainAgent';
 import { RagConfig } from '../config/RagConfig';
 import { ProviderRegistry } from './ProviderRegistry';
 import { BatchProcessor, BatchOptions } from './BatchProcessor';
 import { EmbeddingStrategyResolver } from '../config/EmbeddingStrategy';
-import { IngestDocument, ChatResponse, GraphSearchResult } from '../types';
-
-interface QueryFieldHint {
-  field?: string;
-  value: string;
-}
-
-interface QueryFilter {
-  metadata?: Record<string, string>;
-  keywords?: string[];
-  queryText?: string;
-}
-
-function normalizeHintValue(value: string): string {
-  return value.replace(/\s+/g, ' ').trim();
-}
-
-function isLikelyPromptPhrase(value: string): boolean {
-  return /^(what|which|who|where|when|why|how)\b/i.test(value.trim());
-}
-
-function extractQueryFieldHints(question: string): QueryFieldHint[] {
-  if (!question.trim()) return [];
-
-  const hints = new Map<string, QueryFieldHint>();
-
-  const addHint = (value: string, field?: string) => {
-    const normalizedValue = normalizeHintValue(value);
-    if (!normalizedValue) return;
-
-    const normalizedField = field
-      ? field
-        .toLowerCase()
-        .replace(/[^a-z0-9]+/g, ' ')
-        .trim()
-      : undefined;
-
-    const key = `${normalizedField ?? '*'}::${normalizedValue.toLowerCase()}`;
-    if (!hints.has(key)) {
-      hints.set(key, {
-        value: normalizedValue,
-        ...(normalizedField ? { field: normalizedField } : {}),
-      });
-    }
-  };
-
-  for (const match of question.matchAll(/["']([^"']{2,100})["']/g)) {
-    addHint(match[1]);
-  }
-
-  const naturalQuestionPatterns = [
-    /\b(?:what|which)\s+(?:is|are|was|were)\s+(?:the\s+)?([^?.!,]{1,60}?)\s+of\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
-    /\b(?:who|what)\s+(?:is|are|was|were)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
-    /\b(?:about|for|regarding)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
-  ];
-
-  // Targeted patterns: map common person/company question forms to a `name` metadata hint
-  const personCompanyPatterns = [
-    /\bcompany(?:\s+name)?\s+(?:of|for)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
-    /\b(?:which|what)\s+company\s+does\s+["']?([^"'\n?.!,]{2,120})["']?\s+work(?:\s+for|\s+at)?(?=[?.!,]|$)/gi,
-    /\bwhere\s+does\s+["']?([^"'\n?.!,]{2,120})["']?\s+work(?:\s+for|\s+at)?(?=[?.!,]|$)/gi,
-  ];
-
-  for (const pattern of personCompanyPatterns) {
-    for (const match of question.matchAll(pattern)) {
-      const name = match[1];
-      if (name) addHint(name, 'name');
-    }
-  }
-
-  // Universal patterns: capture common data types across domains so the system
-  // can construct metadata filters for arbitrary datasets. These are intentionally
-  // generic and provider-agnostic (e.g., `email`, `phone`, `date`, `amount`, `id`).
-  const universalPatterns: Array<{ regex: RegExp; field?: string; group?: number }> = [
-    { regex: /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/gi, field: 'email', group: 1 },
-    { regex: /(\+?\d[\d\-\.\s\(\)]{6,}\d)/g, field: 'phone', group: 1 },
-    { regex: /(\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/g, field: 'date', group: 1 },
-    { regex: /(\$\s?\d{1,3}(?:,\d{3})*(?:\.\d+)?)/g, field: 'amount', group: 1 },
-    { regex: /\b(ID|id|identifier)[: ]\s*([A-Za-z0-9\-]{3,})\b/gi, field: 'id', group: 2 },
-    // Generic quoted phrase / proper-noun sequences as keywords (already partially handled above)
-    { regex: /"([^"]{2,120})"/g, group: 1 },
-    { regex: /'([^']{2,120})'/g, group: 1 },
-  ];
-
-  for (const p of universalPatterns) {
-    for (const match of question.matchAll(p.regex)) {
-      const val = p.group ? match[p.group] ?? match[0] : match[0];
-      if (!val) continue;
-      if (p.field) addHint(val, p.field);
-      else addHint(val);
-    }
-  }
-
-  for (const pattern of naturalQuestionPatterns) {
-    for (const match of question.matchAll(pattern)) {
-      const value = match[2] ?? match[1];
-      if (value) addHint(value);
-    }
-  }
-
-  const fieldPattern = `([^\\n:=?.!,]{1,60}?)`;
-  const valuePattern = `([^\\n?.!,]{1,120}?)`;
-  const fieldValuePatterns = [
-    new RegExp(`\\b${fieldPattern}\\s*(?:=|:)\\s*["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
-    new RegExp(`\\b${fieldPattern}\\s+(?:is|are|was|were|equals?|equal to|named|called)\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
-    new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
-  ];
-
-  for (const pattern of fieldValuePatterns) {
-    for (const match of question.matchAll(pattern)) {
-      const field = normalizeHintValue(match[1] ?? '');
-      const value = match[2] ?? '';
-
-      if (field && !isLikelyPromptPhrase(field)) {
-        addHint(value, field);
-      } else {
-        addHint(value);
-      }
-    }
-  }
-
-  for (const match of question.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3}\b/g)) {
-    addHint(match[0]);
-  }
-
-  return [...hints.values()];
-}
-
-function buildQueryFilter(question: string, hints: QueryFieldHint[]): QueryFilter {
-  const filter: QueryFilter = { metadata: {}, keywords: [], queryText: question };
-
-  for (const hint of hints) {
-    if (hint.field) {
-      // prefer last-seen value for a field; providers may interpret filters differently
-      filter.metadata![hint.field] = hint.value;
-    } else {
-      // treat as keyword / named entity
-      filter.keywords!.push(hint.value);
-    }
-  }
-
-  // Also extract quoted phrases (if not already captured) as keywords
-  for (const match of question.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3}\b/g)) {
-    const term = normalizeHintValue(match[0]);
-    if (term && !filter.keywords!.includes(term)) filter.keywords!.push(term);
-  }
-
-  // Remove empty metadata object when no fields present to keep filter minimal
-  if (Object.keys(filter.metadata || {}).length === 0) delete filter.metadata;
-  if (filter.keywords && filter.keywords.length === 0) delete filter.keywords;
-
-  return filter;
-}
+import { QueryProcessor } from './QueryProcessor';
+import { IngestDocument, ChatResponse, VectorMatch, GraphSearchResult, RetrievalResult, UpsertDocument } from '../types';
 
 /**
  * Pipeline — orchestrates the RAG flow: Embed → Search → Augment → Generate.
@@ -177,16 +29,22 @@ export class Pipeline {
   private llmProvider!: ILLMProvider;
   private embeddingProvider!: ILLMProvider;
   private chunker: DocumentChunker;
+  private llamaIngestor?: LlamaIndexIngestor;
   private entityExtractor?: EntityExtractor;
-  private config: RagConfig;
+  private reranker: Reranker;
+  private agent?: LangChainAgent;
+  private embeddingCache = new Map<string, number[]>();
   private initialised = false;
 
-  constructor(config: RagConfig) {
-    this.config = config;
+  constructor(private config: RagConfig) {
     this.chunker = new DocumentChunker(
       config.rag?.chunkSize ?? 1000,
       config.rag?.chunkOverlap ?? 200
     );
+    if (config.rag?.chunkingStrategy === 'llamaindex') {
+      this.llamaIngestor = new LlamaIndexIngestor();
+    }
+    this.reranker = new Reranker();
   }
 
   async initialize(): Promise<void> {
@@ -195,9 +53,7 @@ export class Pipeline {
     // Resolve vector DB provider
     this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
 
-    // Resolve LLM + embedding providers using EmbeddingStrategyResolver.
-    // This handles all cases: integrated (same provider for chat+embed),
-    // separate (e.g. Anthropic + OpenAI embedding), and external.
+    // Resolve LLM + embedding providers
     const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
       this.config.llm,
       this.config.embedding
@@ -213,6 +69,13 @@ export class Pipeline {
     }
 
     await this.vectorDB.initialize();
+
+    // Initialize Agentic Layer if configured
+    if (this.config.rag?.architecture === 'agentic') {
+      this.agent = new LangChainAgent(this, this.config);
+      await this.agent.initialize(this.llmProvider as unknown);
+    }
+
     this.initialised = true;
   }
 
@@ -230,28 +93,9 @@ export class Pipeline {
 
     for (const doc of documents) {
       try {
-        const chunks = this.chunker.chunk(doc.content, {
-          docId: doc.docId,
-          metadata: doc.metadata,
-        });
-
-        // Batch embed with retry
-        const embedBatchOptions: BatchOptions = {
-          batchSize: 50,
-          maxRetries: 3,
-          initialDelayMs: 100,
-        };
-
-        const vectors = await BatchProcessor.mapWithRetry(
-          chunks.map(c => c.content),
-          (text) => this.embeddingProvider.embed(text, { taskType: 'document' }),
-          embedBatchOptions
-        );
-
-        if (vectors.length !== chunks.length) {
-          throw new Error(`Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks`);
-        }
-
+        const chunks = await this.prepareChunks(doc);
+        const vectors = await this.processEmbeddings(chunks);
+        
         const upsertDocs = chunks.map((chunk, i) => ({
           id: chunk.id,
           vector: vectors[i],
@@ -259,54 +103,15 @@ export class Pipeline {
           metadata: chunk.metadata,
         }));
 
-        // Batch upsert with retry
-        const upsertBatchOptions: BatchOptions = {
-          batchSize: 100,
-          maxRetries: 3,
-          initialDelayMs: 100,
-        };
-
-        const upsertResult = await BatchProcessor.processBatch(
-          upsertDocs,
-          (batch) => this.vectorDB.batchUpsert(batch, ns),
-          upsertBatchOptions
-        );
-
-        if (upsertResult.errors.length > 0) {
-          console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed for doc ${doc.docId}`);
-        }
+        const totalProcessed = await this.processUpserts(upsertDocs, ns);
 
         results.push({
           docId: doc.docId,
-          chunksIngested: upsertResult.totalProcessed,
+          chunksIngested: totalProcessed,
         });
 
-        // Graph ingestion
-        // Graph ingestion - Batched to prevent timeouts and LLM overload
         if (this.graphDB && this.entityExtractor) {
-          console.log(`[Pipeline] Extracting entities for doc ${doc.docId} (${chunks.length} chunks)...`);
-          
-          const extractionOptions: BatchOptions = {
-            batchSize: 2, // Low concurrency for LLM extraction
-            maxRetries: 1,
-            initialDelayMs: 500,
-          };
-
-          await BatchProcessor.processBatch(
-            chunks,
-            async (batch) => {
-              for (const chunk of batch) {
-                try {
-                  const { nodes, edges } = await this.entityExtractor!.extract(chunk.content);
-                  if (nodes.length > 0) await this.graphDB!.addNodes(nodes);
-                  if (edges.length > 0) await this.graphDB!.addEdges(edges);
-                } catch (err) {
-                  console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
-                }
-              }
-            },
-            extractionOptions
-          );
+          await this.processGraphIngestion(doc.docId, chunks);
         }
       } catch (error) {
         console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
@@ -317,8 +122,132 @@ export class Pipeline {
     return results;
   }
 
+  /**
+   * Step 1: Chunk the document content.
+   */
+  private async prepareChunks(doc: IngestDocument): Promise<Chunk[]> {
+    if (this.llamaIngestor) {
+      return await this.llamaIngestor.chunk(doc.content, {
+        docId: doc.docId,
+        metadata: doc.metadata,
+        chunkSize: this.config.rag?.chunkSize,
+        chunkOverlap: this.config.rag?.chunkOverlap,
+      });
+    }
+
+    return this.chunker.chunk(doc.content, {
+      docId: doc.docId,
+      metadata: doc.metadata,
+    });
+  }
+
+  /**
+   * Step 2: Generate embeddings for chunks with retry logic.
+   */
+  private async processEmbeddings(chunks: Chunk[]): Promise<number[][]> {
+    const embedBatchOptions: BatchOptions = {
+      batchSize: 50,
+      maxRetries: 3,
+      initialDelayMs: 100,
+    };
+
+    const vectors = await BatchProcessor.mapWithRetry(
+      chunks.map(c => c.content),
+      (text) => this.embeddingProvider.embed(text, { taskType: 'document' }),
+      embedBatchOptions
+    );
+
+    if (vectors.length !== chunks.length) {
+      throw new Error(`Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks`);
+    }
+
+    return vectors;
+  }
+
+  /**
+   * Step 3: Upsert chunks to vector database with retry logic.
+   */
+  private async processUpserts(upsertDocs: UpsertDocument[], namespace: string): Promise<number> {
+    const upsertBatchOptions: BatchOptions = {
+      batchSize: 100,
+      maxRetries: 3,
+      initialDelayMs: 100,
+    };
+
+    const upsertResult = await BatchProcessor.processBatch(
+      upsertDocs,
+      (batch) => this.vectorDB.batchUpsert(batch, namespace),
+      upsertBatchOptions
+    );
+
+    if (upsertResult.errors.length > 0) {
+      console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed`);
+    }
+
+    return upsertResult.totalProcessed;
+  }
+
+  /**
+   * Step 4: Optional graph-based entity extraction and ingestion.
+   */
+  private async processGraphIngestion(docId: string | number, chunks: Chunk[]): Promise<void> {
+    console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
+    
+    const extractionOptions: BatchOptions = {
+      batchSize: 2, // Low concurrency for LLM extraction
+      maxRetries: 1,
+      initialDelayMs: 500,
+    };
+
+    await BatchProcessor.processBatch(
+      chunks,
+      async (batch) => {
+        for (const chunk of batch) {
+          try {
+            const { nodes, edges } = await this.entityExtractor!.extract(chunk.content);
+            if (nodes.length > 0) await this.graphDB!.addNodes(nodes);
+            if (edges.length > 0) await this.graphDB!.addEdges(edges);
+          } catch (err) {
+            console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
+          }
+        }
+      },
+      extractionOptions
+    );
+  }
+
   async ask(question: string, history: ChatMessage[] = [], namespace?: string): Promise<ChatResponse> {
     await this.initialize();
+
+    if (this.config.rag?.architecture === 'agentic' && this.agent) {
+      console.log('[Pipeline] 🤖 Executing in Agentic Mode...');
+      const agentReply = await this.agent.run(question, history);
+      return { reply: agentReply, sources: [] };
+    }
+
+    const stream = this.askStream(question, history, namespace);
+    let reply = '';
+    let sources: VectorMatch[] = [];
+    let graphData: GraphSearchResult | undefined;
+
+    for await (const chunk of stream) {
+      if (typeof chunk === 'string') {
+        reply += chunk;
+      } else if ('sources' in chunk) {
+        sources = chunk.sources;
+        graphData = chunk.graphData;
+      }
+    }
+
+    return { reply, sources, graphData };
+  }
+
+  /**
+   * High-performance streaming RAG flow.
+   * Yields text chunks first, then the retrieval metadata at the end.
+   */
+  async *askStream(question: string, history: ChatMessage[] = [], namespace?: string): AsyncIterable<string | ChatResponse> {
+    await this.initialize();
     const ns = namespace ?? this.config.projectId;
     const topK = this.config.rag?.topK ?? 5;
     const scoreThreshold = this.config.rag?.scoreThreshold ?? 0.0;
@@ -326,24 +255,27 @@ export class Pipeline {
     try {
       let searchQuery = question;
 
-      // 1. Query Transformation
+      // 1. Query Pre-processing
       if (this.config.rag?.useQueryTransformation) {
         searchQuery = await this.rewriteQuery(question, history);
       }
 
-      // 2. Vector Retrieval
-      const queryVector = await this.embeddingProvider.embed(searchQuery, { taskType: 'query' });
-      const fieldHints = extractQueryFieldHints(question);
-      const filter = buildQueryFilter(question, fieldHints) as Record<string, unknown>;
-      filter.__entityHints = fieldHints;
+      // 2. Parallel Retrieval
+      const hints = QueryProcessor.extractQueryFieldHints(question);
+      const filter = QueryProcessor.buildQueryFilter(question, hints);
 
-      const rawMatches = await this.vectorDB.query(queryVector, topK, ns, filter);
-      const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
+      const { sources: rawSources, graphData } = await this.retrieve(searchQuery, { 
+        namespace: ns, 
+        topK: topK * 2,
+        filter
+      });
 
-      // 3. Graph Retrieval
-      let graphData: GraphSearchResult | undefined;
-      if (this.graphDB && this.config.rag?.useGraphRetrieval) {
-        graphData = await this.graphDB.query(searchQuery);
+      // 3. Reranking
+      let sources = rawSources.filter((m) => m.score >= scoreThreshold);
+      if (this.config.rag?.useReranking) {
+        sources = await this.reranker.rerank(sources, question, topK);
+      } else {
+        sources = sources.slice(0, topK);
       }
 
       // 4. Context Augmentation
@@ -358,16 +290,54 @@ export class Pipeline {
         context = `GRAPH KNOWLEDGE:\n${graphContext}\n\nVECTOR CONTEXT:\n${context}`;
       }
 
-      // 5. Generation
+      // 5. Generation (Streaming)
       const messages: ChatMessage[] = [...history, { role: 'user', content: question }];
-      const reply = await this.llmProvider.chat(messages, context);
+      
+      if (this.llmProvider.chatStream) {
+        for await (const chunk of this.llmProvider.chatStream(messages, context)) {
+          yield chunk;
+        }
+      } else {
+        const reply = await this.llmProvider.chat(messages, context);
+        yield reply;
+      }
+
+      // Yield retrieval metadata at the end so UI can show sources
+      yield { reply: '', sources, graphData };
 
-      return { reply, sources, graphData };
     } catch (error) {
-      throw new Error(`[Pipeline] Chat failed: ${error instanceof Error ? error.message : String(error)}`);
+      throw new Error(`[Pipeline] Stream failed: ${error instanceof Error ? error.message : String(error)}`);
     }
   }
 
+  /**
+   * Universal retrieval method combining all enabled providers.
+   */
+  async retrieve(query: string, options: { namespace?: string; topK?: number; filter?: Record<string, unknown> }): Promise<RetrievalResult> {
+    const ns = options.namespace ?? this.config.projectId;
+    const topK = options.topK ?? 5;
+
+    // Performance Optimization: Cache query embeddings to avoid redundant LLM calls
+    const cacheKey = `${ns}::${query}`;
+    let queryVector = this.embeddingCache.get(cacheKey);
+    
+    const [retrievedVector, graphData] = await Promise.all([
+      queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: 'query' }),
+      this.graphDB && this.config.rag?.useGraphRetrieval
+        ? this.graphDB.query(query)
+        : Promise.resolve(undefined)
+    ]);
+
+    if (!queryVector) {
+      this.embeddingCache.set(cacheKey, retrievedVector);
+      queryVector = retrievedVector;
+    }
+
+    const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
+
+    return { sources, graphData };
+  }
+
   /**
    * Rewrite the user query for better retrieval performance.
    */