@retrivora-ai/rag-engine 1.7.5 → 1.7.7

package/dist/handlers/index.js

@@ -3775,65 +3775,410 @@ var QueryProcessor = class {
   }
 };
 
-// src/core/Pipeline.ts
-var LRUEmbeddingCache = class {
-  constructor(maxSize = 500) {
-    this.cache = /* @__PURE__ */ new Map();
-    this.maxSize = maxSize;
-  }
-  get(key) {
-    const value = this.cache.get(key);
-    if (value !== void 0) {
-      this.cache.delete(key);
-      this.cache.set(key, value);
+// src/utils/UITransformer.ts
+var UITransformer = class {
+  /**
+   * Main transformation method
+   * Analyzes user query and retrieved data to determine the best visualization format
+   *
+   * @param userQuery - The original user question/query
+   * @param retrievedData - Vector database retrieval results
+   * @returns Structured JSON response for UI rendering
+   */
+  static transform(userQuery, retrievedData) {
+    if (!retrievedData || retrievedData.length === 0) {
+      return this.createTextResponse("No data available", "No relevant data found for your query.");
+    }
+    const analysis = this.analyzeData(userQuery, retrievedData);
+    switch (analysis.suggestedType) {
+      case "product_carousel":
+        return this.transformToProductCarousel(retrievedData);
+      case "pie_chart":
+        return this.transformToPieChart(retrievedData);
+      case "bar_chart":
+        return this.transformToBarChart(retrievedData);
+      case "line_chart":
+        return this.transformToLineChart(retrievedData);
+      case "table":
+        return this.transformToTable(retrievedData);
+      default:
+        return this.transformToText(retrievedData);
     }
-    return value;
   }
-  set(key, value) {
-    if (this.cache.has(key)) {
-      this.cache.delete(key);
-    } else if (this.cache.size >= this.maxSize) {
-      const oldestKey = this.cache.keys().next().value;
-      if (oldestKey !== void 0) this.cache.delete(oldestKey);
+  /**
+   * Analyzes the data and query to suggest the best visualization type
+   */
+  static analyzeData(query, data) {
+    const queryLower = query.toLowerCase();
+    const hasProducts = data.some(
+      (item) => this.isProductData(item)
+    );
+    const hasTimeSeries = data.some(
+      (item) => this.isTimeSeriesData(item)
+    );
+    const hasCategories = this.detectCategories(data).length > 0;
+    const isDistributionQuery = queryLower.includes("distribution") || queryLower.includes("breakdown") || queryLower.includes("percentage") || queryLower.includes("proportion");
+    const isComparisonQuery = queryLower.includes("compare") || queryLower.includes("vs") || queryLower.includes("difference");
+    const isTrendQuery = queryLower.includes("trend") || queryLower.includes("over time") || queryLower.includes("growth") || queryLower.includes("decline");
+    if (hasProducts && !hasTimeSeries && !isDistributionQuery) {
+      return { suggestedType: "product_carousel", confidence: 0.95, hasProducts, hasTimeSeries, hasCategories };
     }
-    this.cache.set(key, value);
+    if (isTrendQuery && hasTimeSeries) {
+      return { suggestedType: "line_chart", confidence: 0.9, hasProducts, hasTimeSeries, hasCategories };
+    }
+    if (isDistributionQuery && hasCategories) {
+      return { suggestedType: "pie_chart", confidence: 0.85, hasProducts, hasTimeSeries, hasCategories };
+    }
+    if (isComparisonQuery && hasCategories && data.length > 2) {
+      return { suggestedType: "bar_chart", confidence: 0.8, hasProducts, hasTimeSeries, hasCategories };
+    }
+    if (data.length > 5 && this.hasMultipleFields(data)) {
+      return { suggestedType: "table", confidence: 0.75, hasProducts, hasTimeSeries, hasCategories };
+    }
+    return { suggestedType: "text", confidence: 0.5, hasProducts, hasTimeSeries, hasCategories };
   }
-  clear() {
-    this.cache.clear();
+  /**
+   * Transform data to product carousel format
+   */
+  static transformToProductCarousel(data) {
+    const products = data.filter((item) => this.isProductData(item)).map((item) => this.extractProductInfo(item)).filter((p) => p !== null);
+    return {
+      type: "product_carousel",
+      title: "Recommended Products",
+      description: `Found ${products.length} relevant products`,
+      data: products
+    };
   }
-  get size() {
-    return this.cache.size;
+  /**
+   * Transform data to pie chart format
+   */
+  static transformToPieChart(data) {
+    const categories = this.detectCategories(data);
+    const categoryData = this.aggregateByCategory(data, categories);
+    const pieData = Object.entries(categoryData).map(([label, count]) => ({
+      label,
+      value: count,
+      inStock: this.checkStockStatus(label, data)
+    }));
+    return {
+      type: "pie_chart",
+      title: "Distribution by Category",
+      description: `Showing breakdown across ${categories.length} categories`,
+      data: pieData
+    };
   }
-};
-var Pipeline = class {
-  constructor(config) {
-    this.config = config;
-    /** LRU-bounded cache: avoids re-embedding identical queries within the same process. */
-    this.embeddingCache = new LRUEmbeddingCache(500);
-    this.initialised = false;
-    var _a, _b, _c, _d, _e;
-    this.chunker = new DocumentChunker(
-      (_b = (_a = config.rag) == null ? void 0 : _a.chunkSize) != null ? _b : 1e3,
-      (_d = (_c = config.rag) == null ? void 0 : _c.chunkOverlap) != null ? _d : 200
+  /**
+   * Transform data to bar chart format
+   */
+  static transformToBarChart(data) {
+    const categories = this.detectCategories(data);
+    const categoryData = this.aggregateByCategory(data, categories);
+    const barData = Object.entries(categoryData).map(([category, value]) => ({
+      category,
+      value,
+      inStock: this.checkStockStatus(category, data)
+    }));
+    return {
+      type: "bar_chart",
+      title: "Comparison by Category",
+      description: `Comparing ${categories.length} categories`,
+      data: barData
+    };
+  }
+  /**
+   * Transform data to line chart format
+   */
+  static transformToLineChart(data) {
+    const timePoints = this.extractTimeSeriesData(data);
+    const lineData = timePoints.map((point) => ({
+      timestamp: point.timestamp,
+      value: point.value,
+      label: point.label
+    }));
+    return {
+      type: "line_chart",
+      title: "Trend Over Time",
+      description: `Showing ${lineData.length} data points`,
+      data: lineData
+    };
+  }
+  /**
+   * Transform data to table format
+   */
+  static transformToTable(data) {
+    const columns = this.extractTableColumns(data);
+    const rows = data.map((item) => this.extractTableRow(item, columns));
+    const tableData = {
+      columns,
+      rows
+    };
+    return {
+      type: "table",
+      title: "Detailed Results",
+      description: `Showing ${data.length} results`,
+      data: tableData
+    };
+  }
+  /**
+   * Transform data to text format (fallback)
+   */
+  static transformToText(data) {
+    const textContent = data.map((item) => item.content).join("\n\n");
+    return this.createTextResponse(
+      "Search Results",
+      textContent,
+      `Found ${data.length} relevant results`
     );
-    if (((_e = config.rag) == null ? void 0 : _e.chunkingStrategy) === "llamaindex") {
-      this.llamaIngestor = new LlamaIndexIngestor();
+  }
+  /**
+   * Helper: Create text response
+   */
+  static createTextResponse(title, content, description) {
+    return {
+      type: "text",
+      title,
+      description,
+      data: { content }
+    };
+  }
+  /**
+   * Helper: Check if data item is product-related
+   */
+  static isProductData(item) {
+    const content = (item.content || "").toLowerCase();
+    const productKeywords = ["product", "price", "stock", "item", "sku", "brand", "model"];
+    return productKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
+      (k) => ["name", "price", "product", "sku"].includes(k.toLowerCase())
+    );
+  }
+  /**
+   * Helper: Check if data contains time series
+   */
+  static isTimeSeriesData(item) {
+    const content = (item.content || "").toLowerCase();
+    const timeKeywords = ["date", "time", "year", "month", "week", "day", "hour", "trend", "historical"];
+    return timeKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
+      (k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
+    );
+  }
+  /**
+   * Helper: Extract product information from vector match
+   */
+  static extractProductInfo(item) {
+    const meta = item.metadata || {};
+    if (meta.name || meta.product) {
+      return {
+        id: item.id,
+        name: meta.name || meta.product,
+        price: meta.price,
+        image: meta.image || meta.imageUrl,
+        brand: meta.brand,
+        description: item.content,
+        inStock: this.determineStockStatus(item)
+      };
     }
-    this.reranker = new Reranker();
+    const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
+    const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d.]+)/i);
+    if (nameMatch) {
+      return {
+        id: item.id,
+        name: nameMatch[1],
+        price: priceMatch ? parseFloat(priceMatch[1]) : void 0,
+        description: item.content,
+        inStock: this.determineStockStatus(item)
+      };
+    }
+    return null;
   }
-  async initialize() {
-    var _a, _b;
-    if (this.initialised) return;
-    const CHART_MARKER = "<!-- UI_PROTOCOL_V7 -->";
-    const chartInstruction = `
-
-${CHART_MARKER}
-### UNIVERSAL UI PROTOCOL V7 (STRICT MODE)
-
-You are responsible for returning a SINGLE structured UI block when visualization is required.
-
----
-
+  /**
+   * Helper: Detect categories in data
+   */
+  static detectCategories(data) {
+    const categories = /* @__PURE__ */ new Set();
+    data.forEach((item) => {
+      const meta = item.metadata || {};
+      if (meta.category) {
+        categories.add(String(meta.category));
+      }
+      if (meta.type) {
+        categories.add(String(meta.type));
+      }
+      if (meta.tag) {
+        const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
+        tags.forEach((t) => categories.add(String(t)));
+      }
+      const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
+      if (categoryMatch) {
+        categories.add(categoryMatch[1].trim());
+      }
+    });
+    return Array.from(categories);
+  }
+  /**
+   * Helper: Aggregate data by category
+   */
+  static aggregateByCategory(data, categories) {
+    const result = {};
+    categories.forEach((cat) => {
+      result[cat] = 0;
+    });
+    data.forEach((item) => {
+      const meta = item.metadata || {};
+      const itemCategory = meta.category || meta.type || "Other";
+      if (result.hasOwnProperty(itemCategory)) {
+        result[itemCategory]++;
+      } else {
+        result["Other"] = (result["Other"] || 0) + 1;
+      }
+    });
+    return result;
+  }
+  /**
+   * Helper: Extract time series data
+   */
+  static extractTimeSeriesData(data) {
+    return data.map((item) => {
+      const meta = item.metadata || {};
+      return {
+        timestamp: meta.timestamp || meta.date || (/* @__PURE__ */ new Date()).toISOString(),
+        value: meta.value || item.score || 0,
+        label: meta.label || item.content.substring(0, 50)
+      };
+    });
+  }
+  /**
+   * Helper: Extract table columns
+   */
+  static extractTableColumns(data) {
+    const columnSet = /* @__PURE__ */ new Set();
+    columnSet.add("Content");
+    data.forEach((item) => {
+      Object.keys(item.metadata || {}).forEach((key) => {
+        columnSet.add(key.charAt(0).toUpperCase() + key.slice(1));
+      });
+    });
+    return Array.from(columnSet);
+  }
+  /**
+   * Helper: Extract table row
+   */
+  static extractTableRow(item, columns) {
+    const meta = item.metadata || {};
+    return columns.map((col) => {
+      if (col === "Content") {
+        return item.content.substring(0, 100);
+      }
+      const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
+      const value = meta[metaKey];
+      return value !== void 0 ? String(value) : "";
+    });
+  }
+  /**
+   * Helper: Check stock status
+   */
+  static checkStockStatus(category, data) {
+    const item = data.find((d) => {
+      const meta = d.metadata || {};
+      return meta.category === category || meta.type === category;
+    });
+    return this.determineStockStatus(item || {});
+  }
+  /**
+   * Helper: Determine if item is in stock
+   */
+  static determineStockStatus(item) {
+    const meta = item.metadata || {};
+    if (meta.inStock !== void 0) {
+      return Boolean(meta.inStock);
+    }
+    if (meta.stock !== void 0) {
+      return Boolean(meta.stock);
+    }
+    if (meta.available !== void 0) {
+      return Boolean(meta.available);
+    }
+    const content = (item.content || "").toLowerCase();
+    if (content.includes("out of stock") || content.includes("unavailable")) {
+      return false;
+    }
+    if (content.includes("in stock") || content.includes("available")) {
+      return true;
+    }
+    return true;
+  }
+  /**
+   * Helper: Check if data has multiple fields
+   */
+  static hasMultipleFields(data) {
+    const fieldCount = /* @__PURE__ */ new Set();
+    data.forEach((item) => {
+      Object.keys(item.metadata || {}).forEach((key) => {
+        fieldCount.add(key);
+      });
+    });
+    return fieldCount.size > 2;
+  }
+};
+
+// src/core/Pipeline.ts
+var LRUEmbeddingCache = class {
+  constructor(maxSize = 500) {
+    this.cache = /* @__PURE__ */ new Map();
+    this.maxSize = maxSize;
+  }
+  get(key) {
+    const value = this.cache.get(key);
+    if (value !== void 0) {
+      this.cache.delete(key);
+      this.cache.set(key, value);
+    }
+    return value;
+  }
+  set(key, value) {
+    if (this.cache.has(key)) {
+      this.cache.delete(key);
+    } else if (this.cache.size >= this.maxSize) {
+      const oldestKey = this.cache.keys().next().value;
+      if (oldestKey !== void 0) this.cache.delete(oldestKey);
+    }
+    this.cache.set(key, value);
+  }
+  clear() {
+    this.cache.clear();
+  }
+  get size() {
+    return this.cache.size;
+  }
+};
+var Pipeline = class {
+  constructor(config) {
+    this.config = config;
+    /** LRU-bounded cache: avoids re-embedding identical queries within the same process. */
+    this.embeddingCache = new LRUEmbeddingCache(500);
+    this.initialised = false;
+    var _a, _b, _c, _d, _e;
+    this.chunker = new DocumentChunker(
+      (_b = (_a = config.rag) == null ? void 0 : _a.chunkSize) != null ? _b : 1e3,
+      (_d = (_c = config.rag) == null ? void 0 : _c.chunkOverlap) != null ? _d : 200
+    );
+    if (((_e = config.rag) == null ? void 0 : _e.chunkingStrategy) === "llamaindex") {
+      this.llamaIngestor = new LlamaIndexIngestor();
+    }
+    this.reranker = new Reranker();
+  }
+  async initialize() {
+    var _a, _b;
+    if (this.initialised) return;
+    const CHART_MARKER = "<!-- UI_PROTOCOL_V7 -->";
+    const chartInstruction = `
+
+${CHART_MARKER}
+### UNIVERSAL UI PROTOCOL V7 (STRICT MODE)
+
+You are responsible for returning a SINGLE structured UI block when visualization is required.
+
+---
+
 ## 1. INTENT CLASSIFICATION (MANDATORY)
 
 Classify the query into ONE of the following intents:
@@ -3926,11 +4271,14 @@ Classify the query into ONE of the following intents:
 
 ## 5. DATA RULES
 
-- NEVER hallucinate fields
+- NEVER hallucinate fields or data
 - ONLY use retrieved vector DB data
 - NO placeholders ("...", "N/A")
 - If image/link missing \u2192 REMOVE FIELD
 - Aggregate BEFORE rendering chart
+- IF NO DATA IS FOUND in the retrieved context:
+  \u2192 DO NOT output a \`\`\`ui\`\`\` block
+  \u2192 Instead, state clearly that no relevant products or data were found.
 
 ---
 
@@ -3950,6 +4298,8 @@ Classify the query into ONE of the following intents:
 - ALWAYS return EXACTLY ONE \`\`\`ui\`\`\` block
 - JSON must be VALID
 - No explanation inside block
+- NEVER use ASCII charts, Markdown tables, or text-based drawings for data.
+- If data is tabular or statistical, ALWAYS use the \`\`\`ui\`\`\` block.
 
 ---
 
@@ -4005,914 +4355,575 @@ User: "distribution of products across categories"
       { "label": "Electronics", "value": 10 }
     ]
   },
-  "insights": ["Beauty dominates inventory"]
-}
-\`\`\`
-`;
-    if (!((_a = this.config.llm.systemPrompt) == null ? void 0 : _a.includes(CHART_MARKER))) {
-      let cleanPrompt = this.config.llm.systemPrompt || "";
-      cleanPrompt = cleanPrompt.replace(/\n\n### UNIVERSAL UI PROTOCOL[\s\S]*?(?=\n\n|$)/g, "");
-      cleanPrompt = cleanPrompt.replace(/<!-- UI_PROTOCOL_V\d+ -->[\s\S]*?(?=\n\n|$)/g, "");
-      this.config.llm.systemPrompt = cleanPrompt.trim() + chartInstruction;
-    }
-    console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
-    this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
-    const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
-      this.config.llm,
-      this.config.embedding
-    );
-    this.llmProvider = llmProvider;
-    this.embeddingProvider = embeddingProvider;
-    if (this.config.graphDb) {
-      this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
-      await this.graphDB.initialize();
-      this.entityExtractor = new EntityExtractor(this.llmProvider);
-    }
-    await this.vectorDB.initialize();
-    if (((_b = this.config.rag) == null ? void 0 : _b.architecture) === "agentic") {
-      this.agent = new LangChainAgent(this, this.config);
-      await this.agent.initialize(this.llmProvider);
-    }
-    this.initialised = true;
-  }
-  /**
-   * Ingest documents with automatic chunking, embedding, and batch upsert.
-   * Handles retries for transient failures.
-   */
-  async ingest(documents, namespace) {
-    await this.initialize();
-    const ns = namespace != null ? namespace : this.config.projectId;
-    const results = [];
-    for (const doc of documents) {
-      try {
-        const chunks = await this.prepareChunks(doc);
-        const vectors = await this.processEmbeddings(chunks);
-        const upsertDocs = chunks.map((chunk, i) => ({
-          id: chunk.id,
-          vector: vectors[i],
-          content: chunk.content,
-          metadata: chunk.metadata
-        }));
-        const totalProcessed = await this.processUpserts(upsertDocs, ns);
-        results.push({
-          docId: doc.docId,
-          chunksIngested: totalProcessed
-        });
-        if (this.graphDB && this.entityExtractor) {
-          await this.processGraphIngestion(doc.docId, chunks);
-        }
-      } catch (error) {
-        console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
-        results.push({ docId: doc.docId, chunksIngested: 0 });
-      }
-    }
-    return results;
-  }
-  /**
-   * Step 1: Chunk the document content.
-   */
-  async prepareChunks(doc) {
-    var _a, _b;
-    if (this.llamaIngestor) {
-      return await this.llamaIngestor.chunk(doc.content, {
-        docId: doc.docId,
-        metadata: doc.metadata,
-        chunkSize: (_a = this.config.rag) == null ? void 0 : _a.chunkSize,
-        chunkOverlap: (_b = this.config.rag) == null ? void 0 : _b.chunkOverlap
-      });
-    }
-    return this.chunker.chunk(doc.content, {
-      docId: doc.docId,
-      metadata: doc.metadata
-    });
-  }
-  /**
-   * Step 2: Generate embeddings for chunks with retry logic.
-   * Uses batchEmbed when available for efficiency; falls back to sequential embedding.
-   */
-  async processEmbeddings(chunks) {
-    const embedBatchOptions = {
-      batchSize: 50,
-      maxRetries: 3,
-      initialDelayMs: 100
-    };
-    const vectors = await BatchProcessor.mapWithRetry(
-      chunks.map((c) => c.content),
-      (text) => this.embeddingProvider.embed(text, { taskType: "document" }),
-      embedBatchOptions
-    );
-    if (vectors.length !== chunks.length) {
-      throw new Error(
-        `[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
-      );
-    }
-    return vectors;
-  }
-  /**
-   * Step 3: Upsert chunks to vector database with retry logic.
-   */
-  async processUpserts(upsertDocs, namespace) {
-    const upsertBatchOptions = {
-      batchSize: 100,
-      maxRetries: 3,
-      initialDelayMs: 100
-    };
-    const upsertResult = await BatchProcessor.processBatch(
-      upsertDocs,
-      (batch) => this.vectorDB.batchUpsert(batch, namespace),
-      upsertBatchOptions
-    );
-    if (upsertResult.errors.length > 0) {
-      console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed`);
-    }
-    return upsertResult.totalProcessed;
-  }
-  /**
-   * Step 4: Optional graph-based entity extraction and ingestion.
-   */
-  async processGraphIngestion(docId, chunks) {
-    console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
-    const extractionOptions = {
-      batchSize: 2,
-      // Low concurrency for LLM extraction
-      maxRetries: 1,
-      initialDelayMs: 500
-    };
-    await BatchProcessor.processBatch(
-      chunks,
-      async (batch) => {
-        for (const chunk of batch) {
-          try {
-            const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
-            if (nodes.length > 0) await this.graphDB.addNodes(nodes);
-            if (edges.length > 0) await this.graphDB.addEdges(edges);
-          } catch (err) {
-            console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
-          }
-        }
-      },
-      extractionOptions
-    );
-  }
-  async ask(question, history = [], namespace) {
-    var _a;
-    await this.initialize();
-    if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic" && this.agent) {
-      console.log("[Pipeline] \u{1F916} Executing in Agentic Mode...");
-      const agentReply = await this.agent.run(question, history);
-      return { reply: agentReply, sources: [] };
-    }
-    const stream = this.askStream(question, history, namespace);
-    let reply = "";
-    let sources = [];
-    let graphData;
-    try {
-      for (var iter = __forAwait(stream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
-        const chunk = temp.value;
-        if (typeof chunk === "string") {
-          reply += chunk;
-        } else if ("sources" in chunk) {
-          sources = chunk.sources;
-          graphData = chunk.graphData;
-        }
-      }
-    } catch (temp) {
-      error = [temp];
-    } finally {
-      try {
-        more && (temp = iter.return) && await temp.call(iter);
-      } finally {
-        if (error)
-          throw error[0];
-      }
-    }
-    return { reply, sources, graphData };
-  }
-  /**
-   * High-performance streaming RAG flow.
-   * Yields text chunks first, then the retrieval metadata at the end.
-   */
-  askStream(_0) {
-    return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
-      var _a, _b, _c, _d, _e, _f, _g;
-      yield new __await(this.initialize());
-      const ns = namespace != null ? namespace : this.config.projectId;
-      const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
-      const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
-      try {
-        let searchQuery = question;
-        if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
-          searchQuery = yield new __await(this.rewriteQuery(question, history));
-        }
-        const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
-        const filter = QueryProcessor.buildQueryFilter(question, hints);
-        console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
-        const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
-          namespace: ns,
-          topK: topK * 2,
-          filter
-        }));
-        let sources = rawSources.filter((m) => m.score >= scoreThreshold);
-        if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
-          sources = yield new __await(this.reranker.rerank(sources, question, topK));
-        } else {
-          sources = sources.slice(0, topK);
-        }
-        let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
-${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
-        if (graphData && graphData.nodes.length > 0) {
-          const graphContext = graphData.nodes.map(
-            (n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
-          ).join("\n");
-          context = `GRAPH KNOWLEDGE:
-${graphContext}
-
-VECTOR CONTEXT:
-${context}`;
-        }
-        const messages = [...history, { role: "user", content: question }];
-        if (this.llmProvider.chatStream) {
-          const stream = this.llmProvider.chatStream(messages, context);
-          if (!stream) {
-            throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
-          }
-          try {
-            for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
-              const chunk = temp.value;
-              yield chunk;
-            }
-          } catch (temp) {
-            error = [temp];
-          } finally {
-            try {
-              more && (temp = iter.return) && (yield new __await(temp.call(iter)));
-            } finally {
-              if (error)
-                throw error[0];
-            }
-          }
-        } else {
-          const reply = yield new __await(this.llmProvider.chat(messages, context));
-          yield reply;
-        }
-        yield { reply: "", sources, graphData };
-      } catch (error2) {
-        throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
-      }
-    });
-  }
-  /**
-   * Universal retrieval method combining all enabled providers.
-   * Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
-   */
-  async retrieve(query, options) {
-    var _a, _b, _c;
-    const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
-    const topK = (_b = options.topK) != null ? _b : 5;
-    const cacheKey = `${ns}::${query}`;
-    let queryVector = this.embeddingCache.get(cacheKey);
-    const [retrievedVector, graphData] = await Promise.all([
-      queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: "query" }),
-      this.graphDB && ((_c = this.config.rag) == null ? void 0 : _c.useGraphRetrieval) ? this.graphDB.query(query) : Promise.resolve(void 0)
-    ]);
-    if (!queryVector) {
-      this.embeddingCache.set(cacheKey, retrievedVector);
-      queryVector = retrievedVector;
-    }
-    const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
-    return { sources, graphData };
-  }
-  /**
-   * Rewrite the user query for better retrieval performance.
-   */
-  async rewriteQuery(question, history) {
-    const prompt = `
-Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
-Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
-
-History:
-${history.map((m) => `${m.role}: ${m.content}`).join("\n")}
-
-New Question: ${question}
-
-Optimized Search Query:`;
-    const rewrite = await this.llmProvider.chat(
-      [
-        { role: "system", content: "You are an assistant that optimizes search queries for RAG systems." },
-        { role: "user", content: prompt }
-      ],
-      ""
-    );
-    return rewrite.trim() || question;
-  }
-  /**
-   * Generate 3-5 short, relevant questions based on the vector database content.
-   */
-  async getSuggestions(query, namespace) {
-    if (!query || query.trim().length < 3) {
-      return [];
-    }
-    await this.initialize();
-    const ns = namespace != null ? namespace : this.config.projectId;
-    try {
-      const { sources } = await this.retrieve(query, { namespace: ns, topK: 3 });
-      if (sources.length === 0) {
-        return [];
-      }
-      const context = sources.map((s) => s.content).join("\n\n---\n\n");
-      const prompt = `
-Based on the following snippets from a document, what are 3 short, relevant questions a user might ask? 
-Focus on questions that can be answered by the context.
-Keep each question under 10 words and make them very specific to the content.
-Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3"].
-
-Context:
-${context}
-
-Suggestions:`;
-      const response = await this.llmProvider.chat(
-        [
-          { role: "system", content: "You are a helpful assistant that generates search suggestions." },
-          { role: "user", content: prompt }
-        ],
-        ""
-      );
-      const match = response.match(/\[[\s\S]*\]/);
-      if (match) {
-        const suggestions = JSON.parse(match[0]);
-        if (Array.isArray(suggestions)) {
-          return suggestions.map((s) => String(s)).slice(0, 3);
-        }
-      }
-    } catch (error) {
-      console.error("[Pipeline] Failed to generate suggestions:", error);
-    }
-    return [];
-  }
-};
-
-// src/core/ProviderHealthCheck.ts
-var ProviderHealthCheck = class {
-  /**
-   * Validates vector database configuration before initialization.
-   */
-  static async checkVectorProvider(config) {
-    const timestamp = Date.now();
-    const configErrors = await ConfigValidator.validate({
-      projectId: "health-check",
-      vectorDb: config,
-      llm: { provider: "openai", model: "gpt-4o", apiKey: "none" },
-      embedding: { provider: "openai", model: "text-embedding-3-small", apiKey: "none" }
-    });
-    const vectorDbErrors = configErrors.filter((e) => e.field.startsWith("vectorDb"));
-    if (vectorDbErrors.length > 0) {
-      return {
-        healthy: false,
-        provider: config.provider,
-        error: `Configuration validation failed: ${vectorDbErrors.map((e) => e.message).join("; ")}`,
-        timestamp
-      };
-    }
-    try {
-      const checker = await ProviderRegistry.getVectorHealthChecker(config.provider);
-      if (checker) {
-        return await checker.check(config);
-      }
-      return await this.fallbackVectorHealthCheck(config, timestamp);
-    } catch (error) {
-      return {
-        healthy: false,
-        provider: config.provider,
-        error: error instanceof Error ? error.message : String(error),
-        timestamp
-      };
-    }
-  }
-  static async fallbackVectorHealthCheck(config, timestamp) {
-    const opts = config.options || {};
-    const baseUrl = opts.baseUrl || opts.uri;
-    if (baseUrl && baseUrl.startsWith("http")) {
-      try {
-        const response = await fetch(`${baseUrl.replace(/\/$/, "")}/health`);
-        return {
-          healthy: response.ok,
-          provider: config.provider,
-          error: response.ok ? void 0 : `Health check failed: ${response.status}`,
-          timestamp
-        };
-      } catch (e) {
-        return { healthy: false, provider: config.provider, error: "Provider unreachable", timestamp };
-      }
-    }
-    return {
-      healthy: true,
-      provider: config.provider,
-      error: "Pluggable health check not implemented; basic reachability assumed.",
-      timestamp
-    };
-  }
-  /**
-   * Validates LLM provider configuration.
-   */
-  static async checkLLMProvider(config) {
-    const timestamp = Date.now();
-    try {
-      const checker = LLMFactory.getHealthChecker(config.provider);
-      if (checker) {
-        return await checker.check(config);
-      }
-      return {
-        healthy: true,
-        provider: config.provider,
-        error: "Pluggable health check not implemented; basic reachability assumed.",
-        timestamp
-      };
-    } catch (error) {
-      return {
-        healthy: false,
-        provider: config.provider,
-        error: error instanceof Error ? error.message : String(error),
-        timestamp
-      };
-    }
-  }
-  /**
-   * Runs comprehensive health checks on all configured providers in parallel.
-   */
-  static async checkAll(vectorDbConfig, llmConfig, embeddingConfig) {
-    const [vectorDb, llm, embedding] = await Promise.all([
-      this.checkVectorProvider(vectorDbConfig),
-      this.checkLLMProvider(llmConfig),
-      embeddingConfig ? this.checkLLMProvider(embeddingConfig) : Promise.resolve(void 0)
-    ]);
-    return {
-      vectorDb,
-      llm,
-      embedding,
-      allHealthy: vectorDb.healthy && llm.healthy && (!embedding || embedding.healthy)
-    };
-  }
-};
-
-// src/core/VectorPlugin.ts
-var VectorPlugin = class {
-  constructor(hostConfig) {
-    this.config = ConfigResolver.resolve(hostConfig);
-    this.validationPromise = ConfigValidator.validateAndThrow(this.config);
-    this.pipeline = new Pipeline(this.config);
-  }
-  /**
-   * Get the current resolved configuration.
-   */
-  getConfig() {
-    return this.config;
-  }
-  /**
-   * Perform pre-flight health checks on all configured providers.
-   * Useful to verify connectivity before running operations.
-   *
-   * @returns Health status for vector DB, LLM, and embedding providers
-   */
-  async checkHealth() {
-    return ProviderHealthCheck.checkAll(
-      this.config.vectorDb,
-      this.config.llm,
-      this.config.embedding
-    );
-  }
-  /**
-   * Run a chat query.
-   */
-  async chat(message, history = [], namespace) {
-    await this.validationPromise;
-    return this.pipeline.ask(message, history, namespace);
-  }
-  /**
-   * Run a streaming chat query.
-   */
-  chatStream(_0) {
-    return __asyncGenerator(this, arguments, function* (message, history = [], namespace) {
-      yield new __await(this.validationPromise);
-      yield* __yieldStar(this.pipeline.askStream(message, history, namespace));
-    });
-  }
-  /**
-   * Ingest documents into the vector database.
-   */
-  async ingest(documents, namespace) {
-    await this.validationPromise;
-    return this.pipeline.ingest(documents, namespace);
-  }
-  /**
-   * Get auto-suggestions based on a query prefix.
-   */
-  async getSuggestions(query, namespace) {
-    await this.validationPromise;
-    return this.pipeline.getSuggestions(query, namespace);
-  }
-};
-
-// src/utils/DocumentParser.ts
-var DocumentParser = class {
-  /**
-   * Extract text from a File or Buffer based on its type.
-   */
-  static async parse(file, fileName, mimeType) {
-    var _a;
-    const extension = (_a = fileName.split(".").pop()) == null ? void 0 : _a.toLowerCase();
-    if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
-      return this.readAsText(file);
-    }
-    if (extension === "json" || mimeType === "application/json") {
-      const text = await this.readAsText(file);
-      try {
-        const obj = JSON.parse(text);
-        return JSON.stringify(obj, null, 2);
-      } catch (e) {
-        return text;
-      }
-    }
-    if (extension === "csv" || mimeType === "text/csv") {
-      return this.readAsText(file);
-    }
-    if (extension === "pdf" || mimeType === "application/pdf") {
-      try {
-        const pdf = await import("pdf-parse");
-        const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
-        const data = await pdf.default(buffer);
-        return data.text;
-      } catch (err) {
-        console.warn('[DocumentParser] PDF parsing failed. Make sure "pdf-parse" is installed.', err);
-        return `[PDF Parsing Error for ${fileName}]`;
-      }
-    }
-    if (extension === "docx" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") {
-      try {
-        const mammoth = await import("mammoth");
-        const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
-        const result = await mammoth.extractRawText({ buffer });
-        return result.value;
-      } catch (err) {
-        console.warn('[DocumentParser] DOCX parsing failed. Make sure "mammoth" is installed.', err);
-        return `[DOCX Parsing Error for ${fileName}]`;
-      }
+  "insights": ["Beauty dominates inventory"]
+}
+\`\`\`
+`;
+    if (!((_a = this.config.llm.systemPrompt) == null ? void 0 : _a.includes(CHART_MARKER))) {
+      let cleanPrompt = this.config.llm.systemPrompt || "";
+      cleanPrompt = cleanPrompt.replace(/\n\n### UNIVERSAL UI PROTOCOL[\s\S]*?(?=\n\n|$)/g, "");
+      cleanPrompt = cleanPrompt.replace(/<!-- UI_PROTOCOL_V\d+ -->[\s\S]*?(?=\n\n|$)/g, "");
+      this.config.llm.systemPrompt = cleanPrompt.trim() + chartInstruction;
     }
-    try {
-      return await this.readAsText(file);
-    } catch (e) {
-      throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
+    console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
+    this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
+    const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
+      this.config.llm,
+      this.config.embedding
+    );
+    this.llmProvider = llmProvider;
+    this.embeddingProvider = embeddingProvider;
+    if (this.config.graphDb) {
+      this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
+      await this.graphDB.initialize();
+      this.entityExtractor = new EntityExtractor(this.llmProvider);
     }
-  }
-  static async readAsText(file) {
-    if (Buffer.isBuffer(file)) {
-      return file.toString("utf-8");
+    await this.vectorDB.initialize();
+    if (((_b = this.config.rag) == null ? void 0 : _b.architecture) === "agentic") {
+      this.agent = new LangChainAgent(this, this.config);
+      await this.agent.initialize(this.llmProvider);
     }
-    return await file.text();
+    this.initialised = true;
   }
-};
-
-// src/utils/UITransformer.ts
-var UITransformer = class {
   /**
-   * Main transformation method
-   * Analyzes user query and retrieved data to determine the best visualization format
-   *
-   * @param userQuery - The original user question/query
-   * @param retrievedData - Vector database retrieval results
-   * @returns Structured JSON response for UI rendering
+   * Ingest documents with automatic chunking, embedding, and batch upsert.
+   * Handles retries for transient failures.
    */
-  static transform(userQuery, retrievedData) {
-    if (!retrievedData || retrievedData.length === 0) {
-      return this.createTextResponse("No data available", "No relevant data found for your query.");
-    }
-    const analysis = this.analyzeData(userQuery, retrievedData);
-    switch (analysis.suggestedType) {
-      case "product_carousel":
-        return this.transformToProductCarousel(retrievedData);
-      case "pie_chart":
-        return this.transformToPieChart(retrievedData);
-      case "bar_chart":
-        return this.transformToBarChart(retrievedData);
-      case "line_chart":
-        return this.transformToLineChart(retrievedData);
-      case "table":
-        return this.transformToTable(retrievedData);
-      default:
-        return this.transformToText(retrievedData);
+  async ingest(documents, namespace) {
+    await this.initialize();
+    const ns = namespace != null ? namespace : this.config.projectId;
+    const results = [];
+    for (const doc of documents) {
+      try {
+        const chunks = await this.prepareChunks(doc);
+        const vectors = await this.processEmbeddings(chunks);
+        const upsertDocs = chunks.map((chunk, i) => ({
+          id: chunk.id,
+          vector: vectors[i],
+          content: chunk.content,
+          metadata: chunk.metadata
+        }));
+        const totalProcessed = await this.processUpserts(upsertDocs, ns);
+        results.push({
+          docId: doc.docId,
+          chunksIngested: totalProcessed
+        });
+        if (this.graphDB && this.entityExtractor) {
+          await this.processGraphIngestion(doc.docId, chunks);
+        }
+      } catch (error) {
+        console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
+        results.push({ docId: doc.docId, chunksIngested: 0 });
+      }
     }
+    return results;
   }
   /**
-   * Analyzes the data and query to suggest the best visualization type
+   * Step 1: Chunk the document content.
    */
-  static analyzeData(query, data) {
-    const queryLower = query.toLowerCase();
-    const hasProducts = data.some(
-      (item) => this.isProductData(item)
-    );
-    const hasTimeSeries = data.some(
-      (item) => this.isTimeSeriesData(item)
-    );
-    const hasCategories = this.detectCategories(data).length > 0;
-    const isDistributionQuery = queryLower.includes("distribution") || queryLower.includes("breakdown") || queryLower.includes("percentage") || queryLower.includes("proportion");
-    const isComparisonQuery = queryLower.includes("compare") || queryLower.includes("vs") || queryLower.includes("difference");
-    const isTrendQuery = queryLower.includes("trend") || queryLower.includes("over time") || queryLower.includes("growth") || queryLower.includes("decline");
-    if (hasProducts && !hasTimeSeries && !isDistributionQuery) {
-      return { suggestedType: "product_carousel", confidence: 0.95, hasProducts, hasTimeSeries, hasCategories };
-    }
-    if (isTrendQuery && hasTimeSeries) {
-      return { suggestedType: "line_chart", confidence: 0.9, hasProducts, hasTimeSeries, hasCategories };
-    }
-    if (isDistributionQuery && hasCategories) {
-      return { suggestedType: "pie_chart", confidence: 0.85, hasProducts, hasTimeSeries, hasCategories };
-    }
-    if (isComparisonQuery && hasCategories && data.length > 2) {
-      return { suggestedType: "bar_chart", confidence: 0.8, hasProducts, hasTimeSeries, hasCategories };
-    }
-    if (data.length > 5 && this.hasMultipleFields(data)) {
-      return { suggestedType: "table", confidence: 0.75, hasProducts, hasTimeSeries, hasCategories };
+  async prepareChunks(doc) {
+    var _a, _b;
+    if (this.llamaIngestor) {
+      return await this.llamaIngestor.chunk(doc.content, {
+        docId: doc.docId,
+        metadata: doc.metadata,
+        chunkSize: (_a = this.config.rag) == null ? void 0 : _a.chunkSize,
+        chunkOverlap: (_b = this.config.rag) == null ? void 0 : _b.chunkOverlap
+      });
     }
-    return { suggestedType: "text", confidence: 0.5, hasProducts, hasTimeSeries, hasCategories };
+    return this.chunker.chunk(doc.content, {
+      docId: doc.docId,
+      metadata: doc.metadata
+    });
   }
   /**
-   * Transform data to product carousel format
+   * Step 2: Generate embeddings for chunks with retry logic.
+   * Uses batchEmbed when available for efficiency; falls back to sequential embedding.
    */
-  static transformToProductCarousel(data) {
-    const products = data.filter((item) => this.isProductData(item)).map((item) => this.extractProductInfo(item)).filter((p) => p !== null);
-    return {
-      type: "product_carousel",
-      title: "Recommended Products",
-      description: `Found ${products.length} relevant products`,
-      data: products
+  async processEmbeddings(chunks) {
+    const embedBatchOptions = {
+      batchSize: 50,
+      maxRetries: 3,
+      initialDelayMs: 100
     };
+    const vectors = await BatchProcessor.mapWithRetry(
+      chunks.map((c) => c.content),
+      (text) => this.embeddingProvider.embed(text, { taskType: "document" }),
+      embedBatchOptions
+    );
+    if (vectors.length !== chunks.length) {
+      throw new Error(
+        `[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
+      );
+    }
+    return vectors;
   }
   /**
-   * Transform data to pie chart format
+   * Step 3: Upsert chunks to vector database with retry logic.
    */
-  static transformToPieChart(data) {
-    const categories = this.detectCategories(data);
-    const categoryData = this.aggregateByCategory(data, categories);
-    const pieData = Object.entries(categoryData).map(([label, count]) => ({
-      label,
-      value: count,
-      inStock: this.checkStockStatus(label, data)
-    }));
-    return {
-      type: "pie_chart",
-      title: "Distribution by Category",
-      description: `Showing breakdown across ${categories.length} categories`,
-      data: pieData
+  async processUpserts(upsertDocs, namespace) {
+    const upsertBatchOptions = {
+      batchSize: 100,
+      maxRetries: 3,
+      initialDelayMs: 100
     };
+    const upsertResult = await BatchProcessor.processBatch(
+      upsertDocs,
+      (batch) => this.vectorDB.batchUpsert(batch, namespace),
+      upsertBatchOptions
+    );
+    if (upsertResult.errors.length > 0) {
+      console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed`);
+    }
+    return upsertResult.totalProcessed;
   }
   /**
-   * Transform data to bar chart format
+   * Step 4: Optional graph-based entity extraction and ingestion.
    */
-  static transformToBarChart(data) {
-    const categories = this.detectCategories(data);
-    const categoryData = this.aggregateByCategory(data, categories);
-    const barData = Object.entries(categoryData).map(([category, value]) => ({
-      category,
-      value,
-      inStock: this.checkStockStatus(category, data)
-    }));
-    return {
-      type: "bar_chart",
-      title: "Comparison by Category",
-      description: `Comparing ${categories.length} categories`,
-      data: barData
+  async processGraphIngestion(docId, chunks) {
+    console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
+    const extractionOptions = {
+      batchSize: 2,
+      // Low concurrency for LLM extraction
+      maxRetries: 1,
+      initialDelayMs: 500
     };
+    await BatchProcessor.processBatch(
+      chunks,
+      async (batch) => {
+        for (const chunk of batch) {
+          try {
+            const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
+            if (nodes.length > 0) await this.graphDB.addNodes(nodes);
+            if (edges.length > 0) await this.graphDB.addEdges(edges);
+          } catch (err) {
+            console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
+          }
+        }
+      },
+      extractionOptions
+    );
+  }
+  async ask(question, history = [], namespace) {
+    var _a;
+    await this.initialize();
+    if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic" && this.agent) {
+      console.log("[Pipeline] \u{1F916} Executing in Agentic Mode...");
+      const agentReply = await this.agent.run(question, history);
+      return { reply: agentReply, sources: [] };
+    }
+    const stream = this.askStream(question, history, namespace);
+    let reply = "";
+    let sources = [];
+    let graphData;
+    try {
+      for (var iter = __forAwait(stream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
+        const chunk = temp.value;
+        if (typeof chunk === "string") {
+          reply += chunk;
+        } else if ("sources" in chunk) {
+          sources = chunk.sources;
+          graphData = chunk.graphData;
+        }
+      }
+    } catch (temp) {
+      error = [temp];
+    } finally {
+      try {
+        more && (temp = iter.return) && await temp.call(iter);
+      } finally {
+        if (error)
+          throw error[0];
+      }
+    }
+    return { reply, sources, graphData };
   }
   /**
-   * Transform data to line chart format
+   * High-performance streaming RAG flow.
+   * Yields text chunks first, then the retrieval metadata at the end.
    */
-  static transformToLineChart(data) {
-    const timePoints = this.extractTimeSeriesData(data);
-    const lineData = timePoints.map((point) => ({
-      timestamp: point.timestamp,
-      value: point.value,
-      label: point.label
-    }));
-    return {
-      type: "line_chart",
-      title: "Trend Over Time",
-      description: `Showing ${lineData.length} data points`,
-      data: lineData
-    };
+  askStream(_0) {
+    return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
+      var _a, _b, _c, _d, _e, _f, _g;
+      yield new __await(this.initialize());
+      const ns = namespace != null ? namespace : this.config.projectId;
+      const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
+      const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
+      try {
+        let searchQuery = question;
+        if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
+          searchQuery = yield new __await(this.rewriteQuery(question, history));
+        }
+        const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
+        const filter = QueryProcessor.buildQueryFilter(question, hints);
+        console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
+        const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
+          namespace: ns,
+          topK: topK * 2,
+          filter
+        }));
+        let sources = rawSources.filter((m) => m.score >= scoreThreshold);
+        if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
+          sources = yield new __await(this.reranker.rerank(sources, question, topK));
+        } else {
+          sources = sources.slice(0, topK);
+        }
+        let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
+${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
+        if (graphData && graphData.nodes.length > 0) {
+          const graphContext = graphData.nodes.map(
+            (n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
+          ).join("\n");
+          context = `GRAPH KNOWLEDGE:
+${graphContext}
+
+VECTOR CONTEXT:
+${context}`;
+        }
+        const messages = [...history, { role: "user", content: question }];
+        if (this.llmProvider.chatStream) {
+          const stream = this.llmProvider.chatStream(messages, context);
+          if (!stream) {
+            throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
+          }
+          try {
+            for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
+              const chunk = temp.value;
+              yield chunk;
+            }
+          } catch (temp) {
+            error = [temp];
+          } finally {
+            try {
+              more && (temp = iter.return) && (yield new __await(temp.call(iter)));
+            } finally {
+              if (error)
+                throw error[0];
+            }
+          }
+        } else {
+          const reply = yield new __await(this.llmProvider.chat(messages, context));
+          yield reply;
+        }
+        const uiTransformation = UITransformer.transform(question, sources);
+        yield {
+          reply: "",
+          sources,
+          graphData,
+          ui_transformation: uiTransformation
+        };
+      } catch (error2) {
+        throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
+      }
+    });
   }
   /**
-   * Transform data to table format
+   * Universal retrieval method combining all enabled providers.
+   * Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
    */
-  static transformToTable(data) {
-    const columns = this.extractTableColumns(data);
-    const rows = data.map((item) => this.extractTableRow(item, columns));
-    const tableData = {
-      columns,
-      rows
-    };
-    return {
-      type: "table",
-      title: "Detailed Results",
-      description: `Showing ${data.length} results`,
-      data: tableData
-    };
+  async retrieve(query, options) {
+    var _a, _b, _c;
+    const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
+    const topK = (_b = options.topK) != null ? _b : 5;
+    const cacheKey = `${ns}::${query}`;
+    let queryVector = this.embeddingCache.get(cacheKey);
+    const [retrievedVector, graphData] = await Promise.all([
+      queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: "query" }),
+      this.graphDB && ((_c = this.config.rag) == null ? void 0 : _c.useGraphRetrieval) ? this.graphDB.query(query) : Promise.resolve(void 0)
+    ]);
+    if (!queryVector) {
+      this.embeddingCache.set(cacheKey, retrievedVector);
+      queryVector = retrievedVector;
+    }
+    const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
+    return { sources, graphData };
   }
   /**
-   * Transform data to text format (fallback)
+   * Rewrite the user query for better retrieval performance.
    */
-  static transformToText(data) {
-    const textContent = data.map((item) => item.content).join("\n\n");
-    return this.createTextResponse(
-      "Search Results",
-      textContent,
-      `Found ${data.length} relevant results`
+  async rewriteQuery(question, history) {
+    const prompt = `
+Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
+Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
+
+History:
+${history.map((m) => `${m.role}: ${m.content}`).join("\n")}
+
+New Question: ${question}
+
+Optimized Search Query:`;
+    const rewrite = await this.llmProvider.chat(
+      [
+        { role: "system", content: "You are an assistant that optimizes search queries for RAG systems." },
+        { role: "user", content: prompt }
+      ],
+      ""
     );
+    return rewrite.trim() || question;
   }
   /**
-   * Helper: Create text response
+   * Generate 3-5 short, relevant questions based on the vector database content.
    */
-  static createTextResponse(title, content, description) {
-    return {
-      type: "text",
-      title,
-      description,
-      data: { content }
-    };
+  async getSuggestions(query, namespace) {
+    if (!query || query.trim().length < 3) {
+      return [];
+    }
+    await this.initialize();
+    const ns = namespace != null ? namespace : this.config.projectId;
+    try {
+      const { sources } = await this.retrieve(query, { namespace: ns, topK: 3 });
+      if (sources.length === 0) {
+        return [];
+      }
+      const context = sources.map((s) => s.content).join("\n\n---\n\n");
+      const prompt = `
+Based on the following snippets from a document, what are 3 short, relevant questions a user might ask? 
+Focus on questions that can be answered by the context.
+Keep each question under 10 words and make them very specific to the content.
+Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3"].
+
+Context:
+${context}
+
+Suggestions:`;
+      const response = await this.llmProvider.chat(
+        [
+          { role: "system", content: "You are a helpful assistant that generates search suggestions." },
+          { role: "user", content: prompt }
+        ],
+        ""
+      );
+      const match = response.match(/\[[\s\S]*\]/);
+      if (match) {
+        const suggestions = JSON.parse(match[0]);
+        if (Array.isArray(suggestions)) {
+          return suggestions.map((s) => String(s)).slice(0, 3);
+        }
+      }
+    } catch (error) {
+      console.error("[Pipeline] Failed to generate suggestions:", error);
+    }
+    return [];
   }
+};
+
+// src/core/ProviderHealthCheck.ts
+var ProviderHealthCheck = class {
   /**
-   * Helper: Check if data item is product-related
+   * Validates vector database configuration before initialization.
    */
-  static isProductData(item) {
-    const content = (item.content || "").toLowerCase();
-    const productKeywords = ["product", "price", "stock", "item", "sku", "brand", "model"];
-    return productKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
-      (k) => ["name", "price", "product", "sku"].includes(k.toLowerCase())
-    );
+  static async checkVectorProvider(config) {
+    const timestamp = Date.now();
+    const configErrors = await ConfigValidator.validate({
+      projectId: "health-check",
+      vectorDb: config,
+      llm: { provider: "openai", model: "gpt-4o", apiKey: "none" },
+      embedding: { provider: "openai", model: "text-embedding-3-small", apiKey: "none" }
+    });
+    const vectorDbErrors = configErrors.filter((e) => e.field.startsWith("vectorDb"));
+    if (vectorDbErrors.length > 0) {
+      return {
+        healthy: false,
+        provider: config.provider,
+        error: `Configuration validation failed: ${vectorDbErrors.map((e) => e.message).join("; ")}`,
+        timestamp
+      };
+    }
+    try {
+      const checker = await ProviderRegistry.getVectorHealthChecker(config.provider);
+      if (checker) {
+        return await checker.check(config);
+      }
+      return await this.fallbackVectorHealthCheck(config, timestamp);
+    } catch (error) {
+      return {
+        healthy: false,
+        provider: config.provider,
+        error: error instanceof Error ? error.message : String(error),
+        timestamp
+      };
+    }
   }
-  /**
-   * Helper: Check if data contains time series
-   */
-  static isTimeSeriesData(item) {
-    const content = (item.content || "").toLowerCase();
-    const timeKeywords = ["date", "time", "year", "month", "week", "day", "hour", "trend", "historical"];
-    return timeKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
-      (k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
-    );
+  static async fallbackVectorHealthCheck(config, timestamp) {
+    const opts = config.options || {};
+    const baseUrl = opts.baseUrl || opts.uri;
+    if (baseUrl && baseUrl.startsWith("http")) {
+      try {
+        const response = await fetch(`${baseUrl.replace(/\/$/, "")}/health`);
+        return {
+          healthy: response.ok,
+          provider: config.provider,
+          error: response.ok ? void 0 : `Health check failed: ${response.status}`,
+          timestamp
+        };
+      } catch (e) {
+        return { healthy: false, provider: config.provider, error: "Provider unreachable", timestamp };
+      }
+    }
+    return {
+      healthy: true,
+      provider: config.provider,
+      error: "Pluggable health check not implemented; basic reachability assumed.",
+      timestamp
+    };
   }
   /**
-   * Helper: Extract product information from vector match
+   * Validates LLM provider configuration.
    */
-  static extractProductInfo(item) {
-    const meta = item.metadata || {};
-    if (meta.name || meta.product) {
+  static async checkLLMProvider(config) {
+    const timestamp = Date.now();
+    try {
+      const checker = LLMFactory.getHealthChecker(config.provider);
+      if (checker) {
+        return await checker.check(config);
+      }
       return {
-        id: item.id,
-        name: meta.name || meta.product,
-        price: meta.price,
-        image: meta.image || meta.imageUrl,
-        brand: meta.brand,
-        description: item.content,
-        inStock: this.determineStockStatus(item)
+        healthy: true,
+        provider: config.provider,
+        error: "Pluggable health check not implemented; basic reachability assumed.",
+        timestamp
       };
-    }
-    const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
-    const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d.]+)/i);
-    if (nameMatch) {
+    } catch (error) {
       return {
-        id: item.id,
-        name: nameMatch[1],
-        price: priceMatch ? parseFloat(priceMatch[1]) : void 0,
-        description: item.content,
-        inStock: this.determineStockStatus(item)
+        healthy: false,
+        provider: config.provider,
+        error: error instanceof Error ? error.message : String(error),
+        timestamp
       };
     }
-    return null;
   }
   /**
-   * Helper: Detect categories in data
+   * Runs comprehensive health checks on all configured providers in parallel.
    */
-  static detectCategories(data) {
-    const categories = /* @__PURE__ */ new Set();
-    data.forEach((item) => {
-      const meta = item.metadata || {};
-      if (meta.category) {
-        categories.add(String(meta.category));
-      }
-      if (meta.type) {
-        categories.add(String(meta.type));
-      }
-      if (meta.tag) {
-        const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
-        tags.forEach((t) => categories.add(String(t)));
-      }
-      const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
-      if (categoryMatch) {
-        categories.add(categoryMatch[1].trim());
-      }
-    });
-    return Array.from(categories);
+  static async checkAll(vectorDbConfig, llmConfig, embeddingConfig) {
+    const [vectorDb, llm, embedding] = await Promise.all([
+      this.checkVectorProvider(vectorDbConfig),
+      this.checkLLMProvider(llmConfig),
+      embeddingConfig ? this.checkLLMProvider(embeddingConfig) : Promise.resolve(void 0)
+    ]);
+    return {
+      vectorDb,
+      llm,
+      embedding,
+      allHealthy: vectorDb.healthy && llm.healthy && (!embedding || embedding.healthy)
+    };
+  }
+};
+
+// src/core/VectorPlugin.ts
+var VectorPlugin = class {
+  constructor(hostConfig) {
+    this.config = ConfigResolver.resolve(hostConfig);
+    this.validationPromise = ConfigValidator.validateAndThrow(this.config);
+    this.pipeline = new Pipeline(this.config);
   }
   /**
-   * Helper: Aggregate data by category
+   * Get the current resolved configuration.
    */
-  static aggregateByCategory(data, categories) {
-    const result = {};
-    categories.forEach((cat) => {
-      result[cat] = 0;
-    });
-    data.forEach((item) => {
-      const meta = item.metadata || {};
-      const itemCategory = meta.category || meta.type || "Other";
-      if (result.hasOwnProperty(itemCategory)) {
-        result[itemCategory]++;
-      } else {
-        result["Other"] = (result["Other"] || 0) + 1;
-      }
-    });
-    return result;
+  getConfig() {
+    return this.config;
   }
   /**
-   * Helper: Extract time series data
+   * Perform pre-flight health checks on all configured providers.
+   * Useful to verify connectivity before running operations.
+   *
+   * @returns Health status for vector DB, LLM, and embedding providers
    */
-  static extractTimeSeriesData(data) {
-    return data.map((item) => {
-      const meta = item.metadata || {};
-      return {
-        timestamp: meta.timestamp || meta.date || (/* @__PURE__ */ new Date()).toISOString(),
-        value: meta.value || item.score || 0,
-        label: meta.label || item.content.substring(0, 50)
-      };
-    });
+  async checkHealth() {
+    return ProviderHealthCheck.checkAll(
+      this.config.vectorDb,
+      this.config.llm,
+      this.config.embedding
+    );
   }
   /**
-   * Helper: Extract table columns
+   * Run a chat query.
    */
-  static extractTableColumns(data) {
-    const columnSet = /* @__PURE__ */ new Set();
-    columnSet.add("Content");
-    data.forEach((item) => {
-      Object.keys(item.metadata || {}).forEach((key) => {
-        columnSet.add(key.charAt(0).toUpperCase() + key.slice(1));
-      });
-    });
-    return Array.from(columnSet);
+  async chat(message, history = [], namespace) {
+    await this.validationPromise;
+    return this.pipeline.ask(message, history, namespace);
   }
   /**
-   * Helper: Extract table row
+   * Run a streaming chat query.
    */
-  static extractTableRow(item, columns) {
-    const meta = item.metadata || {};
-    return columns.map((col) => {
-      if (col === "Content") {
-        return item.content.substring(0, 100);
-      }
-      const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
-      const value = meta[metaKey];
-      return value !== void 0 ? String(value) : "";
+  chatStream(_0) {
+    return __asyncGenerator(this, arguments, function* (message, history = [], namespace) {
+      yield new __await(this.validationPromise);
+      yield* __yieldStar(this.pipeline.askStream(message, history, namespace));
     });
   }
   /**
-   * Helper: Check stock status
+   * Ingest documents into the vector database.
    */
-  static checkStockStatus(category, data) {
-    const item = data.find((d) => {
-      const meta = d.metadata || {};
-      return meta.category === category || meta.type === category;
-    });
-    return this.determineStockStatus(item || {});
+  async ingest(documents, namespace) {
+    await this.validationPromise;
+    return this.pipeline.ingest(documents, namespace);
   }
   /**
-   * Helper: Determine if item is in stock
+   * Get auto-suggestions based on a query prefix.
    */
-  static determineStockStatus(item) {
-    const meta = item.metadata || {};
-    if (meta.inStock !== void 0) {
-      return Boolean(meta.inStock);
+  async getSuggestions(query, namespace) {
+    await this.validationPromise;
+    return this.pipeline.getSuggestions(query, namespace);
+  }
+};
+
+// src/utils/DocumentParser.ts
+var DocumentParser = class {
+  /**
+   * Extract text from a File or Buffer based on its type.
+   */
+  static async parse(file, fileName, mimeType) {
+    var _a;
+    const extension = (_a = fileName.split(".").pop()) == null ? void 0 : _a.toLowerCase();
+    if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
+      return this.readAsText(file);
     }
-    if (meta.stock !== void 0) {
-      return Boolean(meta.stock);
+    if (extension === "json" || mimeType === "application/json") {
+      const text = await this.readAsText(file);
+      try {
+        const obj = JSON.parse(text);
+        return JSON.stringify(obj, null, 2);
+      } catch (e) {
+        return text;
+      }
     }
-    if (meta.available !== void 0) {
-      return Boolean(meta.available);
+    if (extension === "csv" || mimeType === "text/csv") {
+      return this.readAsText(file);
     }
-    const content = (item.content || "").toLowerCase();
-    if (content.includes("out of stock") || content.includes("unavailable")) {
-      return false;
+    if (extension === "pdf" || mimeType === "application/pdf") {
+      try {
+        const pdf = await import("pdf-parse");
+        const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
+        const data = await pdf.default(buffer);
+        return data.text;
+      } catch (err) {
+        console.warn('[DocumentParser] PDF parsing failed. Make sure "pdf-parse" is installed.', err);
+        return `[PDF Parsing Error for ${fileName}]`;
+      }
     }
-    if (content.includes("in stock") || content.includes("available")) {
-      return true;
+    if (extension === "docx" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") {
+      try {
+        const mammoth = await import("mammoth");
+        const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
+        const result = await mammoth.extractRawText({ buffer });
+        return result.value;
+      } catch (err) {
+        console.warn('[DocumentParser] DOCX parsing failed. Make sure "mammoth" is installed.', err);
+        return `[DOCX Parsing Error for ${fileName}]`;
+      }
+    }
+    try {
+      return await this.readAsText(file);
+    } catch (e) {
+      throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
     }
-    return true;
   }
-  /**
-   * Helper: Check if data has multiple fields
-   */
-  static hasMultipleFields(data) {
-    const fieldCount = /* @__PURE__ */ new Set();
-    data.forEach((item) => {
-      Object.keys(item.metadata || {}).forEach((key) => {
-        fieldCount.add(key);
-      });
-    });
-    return fieldCount.size > 2;
+  static async readAsText(file) {
+    if (Buffer.isBuffer(file)) {
+      return file.toString("utf-8");
+    }
+    return await file.text();
   }
 };