@retrivora-ai/rag-engine 1.8.1 → 1.8.3

package/dist/handlers/index.js

@@ -347,7 +347,7 @@ var init_MultiTablePostgresProvider = __esm({
     init_BaseVectorProvider();
     MultiTablePostgresProvider = class extends BaseVectorProvider {
       constructor(config) {
-        var _a, _b, _c, _d, _e;
+        var _a, _b, _c;
         super(config);
         const opts = config.options || {};
         if (!opts.connectionString) {
@@ -355,37 +355,37 @@ var init_MultiTablePostgresProvider = __esm({
         }
         this.connectionString = opts.connectionString;
         this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
-        const rawTables = (_c = (_b = opts.tables) != null ? _b : process.env.VECTOR_DB_TABLES) != null ? _c : "";
-        this.tables = typeof rawTables === "string" ? rawTables.split(",").map((t) => t.trim()).filter(Boolean) : rawTables;
-        if (this.tables.length === 0) {
-          console.warn(
-            "[MultiTablePostgresProvider] No tables configured. Set VECTOR_DB_TABLES as a comma-separated list of table names or pass options.tables."
-          );
-        }
-        const rawSearchFields = (_e = (_d = opts.searchFields) != null ? _d : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _e : ["name", "product_name", "productname", "title"];
+        this.tables = [];
+        const rawSearchFields = (_c = (_b = opts.searchFields) != null ? _b : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _c : ["name", "product_name", "productname", "title"];
         this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
+        this.uploadTable = opts.uploadTable || "document_chunks";
       }
       async initialize() {
         this.pool = new import_pg.Pool({ connectionString: this.connectionString });
         const client = await this.pool.connect();
         try {
+          await client.query("CREATE EXTENSION IF NOT EXISTS vector");
+          await client.query(`
+        CREATE TABLE IF NOT EXISTS ${this.uploadTable} (
+          id          TEXT PRIMARY KEY,
+          namespace   TEXT NOT NULL DEFAULT '',
+          content     TEXT NOT NULL,
+          metadata    JSONB,
+          embedding   VECTOR(${this.dimensions})
+        )
+      `);
+          await client.query(`
+        CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
+        ON ${this.uploadTable}
+        USING hnsw (embedding vector_cosine_ops)
+      `);
           const res = await client.query(`
         SELECT table_name 
         FROM information_schema.columns 
         WHERE column_name = 'embedding' 
         AND table_schema = 'public'
       `);
-          const discoveredTables = res.rows.map((r) => r.table_name);
-          if (this.tables.length === 0) {
-            this.tables = discoveredTables;
-          } else {
-            const staticTables = [...this.tables];
-            this.tables = staticTables.filter((t) => discoveredTables.includes(t));
-            if (this.tables.length < staticTables.length) {
-              const missing = staticTables.filter((t) => !discoveredTables.includes(t));
-              console.warn(`[MultiTablePostgresProvider] Some configured tables were skipped (missing 'embedding' column): ${missing.join(", ")}`);
-            }
-          }
+          this.tables = res.rows.map((r) => r.table_name);
           if (this.tables.length === 0) {
             console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
           } else {
@@ -398,27 +398,106 @@ var init_MultiTablePostgresProvider = __esm({
         }
       }
       /**
-       * Upsert is not supported for MultiTablePostgresProvider as it's designed for 
-       * searching across pre-existing tables with varying schemas.
+       * Upsert a document by dynamically provisioning a table and columns.
        */
-      async upsert(_doc, _namespace) {
-        throw new Error(
-          "[MultiTablePostgresProvider] upsert() is not supported in multi-table mode. Please use the standard PostgreSQLProvider for single-table managed indices."
-        );
+      async upsert(doc, namespace = "") {
+        await this.batchUpsert([doc], namespace);
       }
       /**
-       * Batch upsert is not supported for MultiTablePostgresProvider.
+       * Batch upsert documents by dynamically provisioning tables and columns based on CSV headers.
        */
-      async batchUpsert(_docs, _namespace) {
-        throw new Error(
-          "[MultiTablePostgresProvider] batchUpsert() is not supported in multi-table mode."
-        );
+      async batchUpsert(docs, namespace = "") {
+        var _a;
+        if (docs.length === 0) return;
+        const docsByFile = {};
+        for (const doc of docs) {
+          const fileName = ((_a = doc.metadata) == null ? void 0 : _a.fileName) || this.uploadTable;
+          if (!docsByFile[fileName]) docsByFile[fileName] = [];
+          docsByFile[fileName].push(doc);
+        }
+        const client = await this.pool.connect();
+        try {
+          await client.query("BEGIN");
+          for (const [fileName, fileDocs] of Object.entries(docsByFile)) {
+            let tableName = fileName;
+            if (tableName.toLowerCase().endsWith(".csv")) {
+              tableName = tableName.slice(0, -4);
+            }
+            tableName = tableName.replace(/[^a-z0-9_]/gi, "_").toLowerCase() || this.uploadTable;
+            const firstMeta = fileDocs[0].metadata || {};
+            const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding"];
+            const csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+            const columnDefs = csvHeaders.map((h) => `"${h}" TEXT`).join(",\n          ");
+            const createTableSql = `
+          CREATE TABLE IF NOT EXISTS "${tableName}" (
+            id          TEXT PRIMARY KEY,
+            ${columnDefs ? columnDefs + "," : ""}
+            namespace   TEXT NOT NULL DEFAULT '',
+            content     TEXT NOT NULL,
+            metadata    JSONB,
+            embedding   VECTOR(${this.dimensions})
+          )
+        `;
+            await client.query(createTableSql);
+            await client.query(`
+          CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
+          ON "${tableName}"
+          USING hnsw (embedding vector_cosine_ops)
+        `);
+            if (!this.tables.includes(tableName)) {
+              this.tables.push(tableName);
+            }
+            const BATCH_SIZE = 50;
+            for (let i = 0; i < fileDocs.length; i += BATCH_SIZE) {
+              const batch = fileDocs.slice(i, i + BATCH_SIZE);
+              const values = [];
+              const headerColumns = csvHeaders.map((h) => `"${h}"`).join(", ");
+              const allColumns = `id, ${headerColumns ? headerColumns + ", " : ""}namespace, content, metadata, embedding`;
+              const valuePlaceholders = batch.map((doc, idx) => {
+                var _a2, _b;
+                const offset = idx * (5 + csvHeaders.length);
+                values.push(doc.id);
+                for (const h of csvHeaders) {
+                  values.push(String(((_a2 = doc.metadata) == null ? void 0 : _a2[h]) || ""));
+                }
+                values.push(namespace, doc.content, JSON.stringify((_b = doc.metadata) != null ? _b : {}), `[${doc.vector.join(",")}]`);
+                const docPlaceholders = [];
+                for (let j = 1; j <= 5 + csvHeaders.length; j++) {
+                  if (j === 5 + csvHeaders.length) {
+                    docPlaceholders.push(`$${offset + j}::vector`);
+                  } else {
+                    docPlaceholders.push(`$${offset + j}`);
+                  }
+                }
+                return `(${docPlaceholders.join(", ")})`;
+              }).join(", ");
+              const conflictUpdates = csvHeaders.map((h) => `"${h}" = EXCLUDED."${h}"`).join(",\n              ");
+              const query = `
+            INSERT INTO "${tableName}" (${allColumns})
+            VALUES ${valuePlaceholders}
+            ON CONFLICT (id) DO UPDATE
+              SET ${conflictUpdates ? conflictUpdates + "," : ""}
+                  namespace = EXCLUDED.namespace,
+                  content   = EXCLUDED.content,
+                  metadata  = EXCLUDED.metadata,
+                  embedding = EXCLUDED.embedding
+          `;
+              await client.query(query, values);
+            }
+          }
+          await client.query("COMMIT");
+        } catch (error) {
+          await client.query("ROLLBACK");
+          throw error;
+        } finally {
+          client.release();
+        }
       }
       /**
        * Query all configured tables and merge results, sorted by cosine similarity score.
        */
       async query(vector, topK, _namespace, _filter) {
-        var _a, _b, _c;
+        var _a;
         if (!this.pool) {
           throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
         }
@@ -510,22 +589,16 @@ var init_MultiTablePostgresProvider = __esm({
         for (const tableResults of resultsArray) {
           allResults.push(...tableResults);
         }
-        const resultsByTable = {};
-        for (const res of allResults) {
-          const table = (_a = res.metadata) == null ? void 0 : _a.source_table;
-          if (!resultsByTable[table]) resultsByTable[table] = [];
-          resultsByTable[table].push(res);
-        }
-        const balancedResults = [];
-        const tables = Object.keys(resultsByTable);
-        for (const table of tables) {
-          balancedResults.push(...resultsByTable[table].slice(0, 3));
-        }
-        const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
-        if (finalSorted.length > 0) {
-          console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
-          console.log(`[MultiTablePostgresProvider] Final top match from "${(_c = (_b = finalSorted[0].metadata) == null ? void 0 : _b.source_table) != null ? _c : "unknown"}" with score ${finalSorted[0].score.toFixed(4)}`);
-        }
+        allResults.sort((a, b) => b.score - a.score);
+        if (allResults.length === 0) return [];
+        const bestMatchTable = (_a = allResults[0].metadata) == null ? void 0 : _a.source_table;
+        const finalSorted = allResults.filter((res) => {
+          var _a2;
+          return ((_a2 = res.metadata) == null ? void 0 : _a2.source_table) === bestMatchTable;
+        });
+        console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
+        console.log(`[MultiTablePostgresProvider] Final top match from "${bestMatchTable}" with score ${finalSorted[0].score.toFixed(4)}`);
+        console.log(`[MultiTablePostgresProvider] Restricting recommendations strictly to table: "${bestMatchTable}"`);
         return finalSorted.slice(0, Math.max(topK, 15));
       }
       async delete(_id, _namespace) {
@@ -1640,6 +1713,7 @@ __export(handlers_exports, {
   sseErrorFrame: () => sseErrorFrame,
   sseFrame: () => sseFrame,
   sseMetaFrame: () => sseMetaFrame,
+  sseObservabilityFrame: () => sseObservabilityFrame,
   sseTextFrame: () => sseTextFrame,
   sseUIFrame: () => sseUIFrame
 });
@@ -3306,11 +3380,70 @@ Text to extract from:
 // src/rag/Reranker.ts
 var Reranker = class {
   /**
-   * Re-ranks matches based on a secondary relevance score.
-   * In a production environment, this would call a Cross-Encoder model.
-   * Here we implement a placeholder that filters by score and limits count.
+   * Re-ranks matches based on a secondary relevance score using an LLM.
    */
-  async rerank(matches, query, limit = 5) {
+  async rerank(matches, query, limit = 5, llm) {
+    if (!llm || matches.length <= 1) {
+      const keywords = query.toLowerCase().replace(/[^\w\s]/g, "").split(/\s+/).filter((w) => w.length > 2);
+      if (keywords.length === 0) {
+        return matches.sort((a, b) => b.score - a.score).slice(0, limit);
+      }
+      const scoredMatches = matches.map((match) => {
+        const contentLower = match.content.toLowerCase();
+        let keywordScore = 0;
+        keywords.forEach((keyword) => {
+          if (contentLower.includes(keyword)) {
+            keywordScore += 1;
+          }
+        });
+        const normalizedKeywordScore = keywordScore / keywords.length;
+        const combinedScore = match.score + normalizedKeywordScore * 0.5;
+        return __spreadProps(__spreadValues({}, match), {
+          score: combinedScore
+        });
+      });
+      return scoredMatches.sort((a, b) => b.score - a.score).slice(0, limit);
+    }
+    try {
+      const topN = matches.slice(0, 10);
+      const prompt = `You are a relevance ranking expert.
+Given the following user query and a list of retrieved document chunks, rank the chunks by relevance to the query.
+Return the result as a comma-separated list of document indices in order of relevance, from most relevant to least relevant.
+Use the indices provided in brackets like [0], [1], etc.
+Only return the indices (e.g., "2,0,1"), nothing else. Do not include brackets in the output.
+
+Query: "${query}"
+
+Documents:
+${topN.map((m, i) => `[${i}] ${m.content.replace(/\n/g, " ")}`).join("\n")}`;
+      const response = await llm.chat(
+        [{ role: "user", content: prompt }],
+        "",
+        { temperature: 0, maxTokens: 50 }
+      );
+      const cleanedResponse = response.trim().replace(/[\[\]\s]/g, "");
+      const rankedIndices = cleanedResponse.split(",").map(Number).filter((n) => !isNaN(n));
+      if (rankedIndices.length > 0) {
+        const rankedMatches = [];
+        const usedIndices = /* @__PURE__ */ new Set();
+        for (const index of rankedIndices) {
+          if (index >= 0 && index < topN.length && !usedIndices.has(index)) {
+            rankedMatches.push(topN[index]);
+            usedIndices.add(index);
+          }
+        }
+        for (let i = 0; i < topN.length; i++) {
+          if (!usedIndices.has(i)) {
+            rankedMatches.push(topN[i]);
+          }
+        }
+        const rest = matches.slice(10);
+        rankedMatches.push(...rest);
+        return rankedMatches.slice(0, limit);
+      }
+    } catch (error) {
+      console.warn("[Reranker] LLM re-ranking failed, falling back to score-based sorting:", error);
+    }
     return matches.sort((a, b) => b.score - a.score).slice(0, limit);
   }
 };
@@ -3370,7 +3503,7 @@ var LangChainAgent = class {
    */
   async initialize(chatModel) {
     try {
-      const [{ DynamicTool }, { HumanMessage }, { createAgent }] = await Promise.all([
+      const [{ DynamicTool }, , { createAgent }] = await Promise.all([
         import("@langchain/core/tools"),
         import("@langchain/core/messages"),
         import("langchain")
@@ -3420,7 +3553,6 @@ When presenting structured data, statistics, or comparisons, decide if it is bes
         tools: [searchTool],
         systemPrompt: finalSystemPrompt
       });
-      void HumanMessage;
     } catch (error) {
       const isMissing = error instanceof Error && error.message.includes("Cannot find module");
       const hint = isMissing ? " Make sure 'langchain' and '@langchain/core' are installed:\n  npm install langchain @langchain/core" : "";
@@ -3898,8 +4030,129 @@ var QueryProcessor = class {
     if (filter.keywords && filter.keywords.length === 0) delete filter.keywords;
     return filter;
   }
+  /**
+   * Determines the retrieval strategy based on the query content.
+   */
+  static async determineRetrievalStrategy(query, llm, customGraphKeywords, customVectorKeywords) {
+    if (llm) {
+      try {
+        const prompt = `You are a routing expert in a RAG system.
+Given the user query, determine which retrieval mechanism is needed:
+- 'vector': For semantic search, finding specific documents or content based on similarity.
+- 'graph': For finding relationships between entities, multi-hop questions, or network structures.
+- 'both': If the query requires both specific content and relationship/structural analysis.
+
+Query: "${query}"
+
+Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
+        const response = await llm.chat(
+          [{ role: "user", content: prompt }],
+          "",
+          { temperature: 0, maxTokens: 10 }
+        );
+        const cleanResponse = response.trim().toLowerCase();
+        if (cleanResponse.includes("vector")) return "vector";
+        if (cleanResponse.includes("graph")) return "graph";
+        if (cleanResponse.includes("both")) return "both";
+      } catch (error) {
+        console.warn("[QueryProcessor] LLM strategy classification failed, falling back to keywords:", error);
+      }
+    }
+    const normalized = query.toLowerCase();
+    const graphKeywords = customGraphKeywords || [
+      "relationship",
+      "connect",
+      "link",
+      "network",
+      "friend",
+      "colleague",
+      "manager",
+      "hierarchy",
+      "multi-hop",
+      "shortest path",
+      "between",
+      "related to",
+      "associated with",
+      "belongs to",
+      "part of"
+    ];
+    const hasGraphKeyword = graphKeywords.some((kw) => normalized.includes(kw));
+    const vectorKeywords = customVectorKeywords || [
+      "find",
+      "search",
+      "tell me about",
+      "what is",
+      "how to",
+      "documents about"
+    ];
+    const hasVectorKeyword = vectorKeywords.some((kw) => normalized.includes(kw));
+    if (hasGraphKeyword && !hasVectorKeyword) {
+      return "graph";
+    } else if (hasGraphKeyword && hasVectorKeyword) {
+      return "both";
+    }
+    return "vector";
+  }
 };
 
+// src/utils/synonyms.ts
+var FIELD_SYNONYMS = {
+  name: ["product", "item", "title", "label", "heading", "subject", "product name", "item name"],
+  price: ["cost", "amount", "msrp", "price", "rate", "value", "price_usd"],
+  brand: ["manufacturer", "vendor", "make", "company", "brand_name", "supplier"],
+  image: [
+    "imageUrl",
+    "thumbnail",
+    "img",
+    "url",
+    "photo",
+    "picture",
+    "media",
+    "image_url",
+    "main_image",
+    "product_image",
+    "thumb"
+  ],
+  stock: [
+    "inventory",
+    "quantity",
+    "count",
+    "availability",
+    "stock_level",
+    "inStock",
+    "is_available",
+    "in stock",
+    "status"
+  ],
+  description: ["summary", "content", "body", "text", "info", "details"],
+  link: ["url", "href", "product_url", "page_url", "link"]
+};
+function resolveMetadataValue(meta, uiKey) {
+  var _a;
+  const synonyms = (_a = FIELD_SYNONYMS[uiKey]) != null ? _a : [];
+  const keys = Object.keys(meta);
+  const systemKeys = ["namespace", "filename", "filesize", "filetype", "docid", "uploadedat", "dimension", "chunkindex"];
+  let match = keys.find((k) => k.toLowerCase() === uiKey.toLowerCase());
+  if (match !== void 0) return meta[match];
+  match = keys.find((k) => synonyms.map((s) => s.toLowerCase()).includes(k.toLowerCase()));
+  if (match !== void 0) return meta[match];
+  const isBlacklisted = (kl) => {
+    return systemKeys.includes(kl) || kl.startsWith("option1") || kl.startsWith("option2") || kl.startsWith("option3");
+  };
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return kl.includes(uiKey.toLowerCase());
+  });
+  if (match !== void 0) return meta[match];
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return synonyms.some((sk) => kl.includes(sk.toLowerCase()) || sk.toLowerCase().includes(kl));
+  });
+  return match !== void 0 ? meta[match] : void 0;
+}
+
 // src/utils/UITransformer.ts
 var UITransformer = class {
   /**
@@ -3967,28 +4220,6 @@ var UITransformer = class {
       data: pieData
     };
   }
-  /**
-   * Transform data to bar chart format
-   */
-  static transformToBarChart(data) {
-    const categories = this.detectCategories(data);
-    const categoryData = this.aggregateByCategory(data, categories);
-    const barData = Object.entries(categoryData).map(([category, value]) => {
-      const { inStockCount, outOfStockCount } = this.calculateStockCounts(category, data);
-      return {
-        category,
-        value,
-        inStockCount,
-        outOfStockCount
-      };
-    });
-    return {
-      type: "bar_chart",
-      title: "Comparison by Category",
-      description: `Comparing ${categories.length} categories`,
-      data: barData
-    };
-  }
   /**
    * Transform data to line chart format
    */
@@ -4267,15 +4498,7 @@ var UITransformer = class {
       const trainedKey = trainedSchema[uiKey];
       if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
     }
-    const synonyms = this.SYNONYMS[uiKey] || [];
-    const searchKeys = [uiKey, ...synonyms].map((k) => k.toLowerCase());
-    const foundDirectKey = Object.keys(meta).find((k) => searchKeys.includes(k.toLowerCase()));
-    if (foundDirectKey) return meta[foundDirectKey];
-    const foundPartialKey = Object.keys(meta).find((k) => {
-      const keyLow = k.toLowerCase();
-      return searchKeys.some((sk) => keyLow.includes(sk) || sk.includes(keyLow));
-    });
-    return foundPartialKey ? meta[foundPartialKey] : void 0;
+    return resolveMetadataValue(meta, uiKey);
   }
   static extractProductInfo(item, config, trainedSchema) {
     const meta = item.metadata || {};
@@ -4345,7 +4568,7 @@ var UITransformer = class {
     data.forEach((item) => {
       const meta = item.metadata || {};
       const itemCategory = meta.category || meta.type || "Other";
-      if (result.hasOwnProperty(itemCategory)) {
+      if (Object.prototype.hasOwnProperty.call(result, itemCategory)) {
         result[itemCategory]++;
       } else {
         result["Other"] = (result["Other"] || 0) + 1;
@@ -4566,18 +4789,6 @@ IMPORTANT:
     return JSON.stringify(partial, null, 2) + "\n// ... truncated";
   }
 };
-/**
- * Central dictionary of common synonyms for UI properties.
- * This allows the system to be schema-agnostic by guessing field names.
- */
-UITransformer.SYNONYMS = {
-  name: ["product", "item", "title", "label", "heading", "subject"],
-  price: ["cost", "amount", "msrp", "price", "rate", "value", "price_usd"],
-  brand: ["manufacturer", "vendor", "make", "company", "brand_name", "supplier"],
-  image: ["imageUrl", "thumbnail", "img", "url", "photo", "picture", "media", "image_url", "main_image", "product_image", "thumb"],
-  stock: ["inventory", "quantity", "count", "availability", "stock_level", "inStock", "is_available"],
-  description: ["summary", "content", "body", "text", "info", "details"]
-};
 
 // src/utils/SchemaMapper.ts
 var SchemaMapper = class {
@@ -4698,6 +4909,57 @@ var LRUEmbeddingCache = class {
     return this.cache.size;
   }
 };
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+var MODEL_COST_PER_1K = {
+  "gpt-4o": { input: 25e-4, output: 0.01 },
+  "gpt-4o-mini": { input: 15e-5, output: 6e-4 },
+  "gpt-4-turbo": { input: 0.01, output: 0.03 },
+  "gpt-3.5-turbo": { input: 5e-4, output: 15e-4 },
+  "claude-3-5-sonnet": { input: 3e-3, output: 0.015 },
+  "claude-3-haiku": { input: 25e-5, output: 125e-5 },
+  "gemini-1.5-flash": { input: 75e-6, output: 3e-4 },
+  "gemini-1.5-pro": { input: 125e-5, output: 5e-3 }
+};
+function estimateCostUsd(promptTokens, completionTokens, model) {
+  if (!model) return void 0;
+  const key = Object.keys(MODEL_COST_PER_1K).find((k) => model.toLowerCase().includes(k));
+  if (!key) return void 0;
+  const prices = MODEL_COST_PER_1K[key];
+  return promptTokens / 1e3 * prices.input + completionTokens / 1e3 * prices.output;
+}
+async function scoreHallucination(llm, answer, context) {
+  const maxContextChars = 3e3;
+  const truncatedContext = context.length > maxContextChars ? context.slice(0, maxContextChars) + "\n...[truncated]" : context;
+  const prompt = `You are an AI quality checker. Given the CONTEXT retrieved from a knowledge base and the ANSWER generated from it, rate how well-grounded the answer is.
+
+CONTEXT:
+${truncatedContext}
+
+ANSWER:
+${answer}
+
+Return ONLY a valid JSON object with no markdown fences:
+{"score": <float 0-1>, "reason": "<one sentence>"}
+
+Where score 0 = fully grounded in context, score 1 = likely hallucinated or unsupported.`;
+  try {
+    const raw = await llm.chat(
+      [{ role: "user", content: prompt }],
+      "",
+      { temperature: 0, maxTokens: 120 }
+    );
+    const jsonMatch = raw.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return void 0;
+    const parsed = JSON.parse(jsonMatch[0]);
+    if (typeof parsed.score === "number" && typeof parsed.reason === "string") {
+      return { score: Math.min(1, Math.max(0, parsed.score)), reason: parsed.reason };
+    }
+  } catch (e) {
+  }
+  return void 0;
+}
 var Pipeline = class {
   constructor(config) {
     this.config = config;
@@ -4724,8 +4986,7 @@ var Pipeline = class {
   async initialize() {
     var _a;
     if (this.initialised) return;
-    const chartInstruction = `
-You are a helpful product assistant. Use the provided context to answer questions accurately.
+    const chartInstruction = `You are a helpful product assistant. Use the provided context to answer questions accurately.
 
 ### UI STYLE RULES (CRITICAL):
 - NEVER generate markdown tables. If you do, the UI will break.
@@ -4739,7 +5000,6 @@ You are a helpful product assistant. Use the provided context to answer question
 - Do NOT try to format product lists as tables.
 `;
     this.config.llm.systemPrompt = chartInstruction;
-    console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
     this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
     const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
       this.config.llm,
@@ -4761,7 +5021,6 @@ You are a helpful product assistant. Use the provided context to answer question
   }
   /**
    * Ingest documents with automatic chunking, embedding, and batch upsert.
-   * Handles retries for transient failures.
    */
   async ingest(documents, namespace) {
     await this.initialize();
@@ -4778,10 +5037,7 @@ You are a helpful product assistant. Use the provided context to answer question
           metadata: chunk.metadata
         }));
         const totalProcessed = await this.processUpserts(upsertDocs, ns);
-        results.push({
-          docId: doc.docId,
-          chunksIngested: totalProcessed
-        });
+        results.push({ docId: doc.docId, chunksIngested: totalProcessed });
         if (this.graphDB && this.entityExtractor) {
           await this.processGraphIngestion(doc.docId, chunks);
         }
@@ -4792,23 +5048,18 @@ You are a helpful product assistant. Use the provided context to answer question
     }
     return results;
   }
-  /**
-   * Step 1: Chunk the document content.
-   */
+  /** Step 1: Chunk the document content. */
   async prepareChunks(doc) {
     var _a, _b;
     if (this.llamaIngestor) {
-      return await this.llamaIngestor.chunk(doc.content, {
+      return this.llamaIngestor.chunk(doc.content, {
         docId: doc.docId,
         metadata: doc.metadata,
         chunkSize: (_a = this.config.rag) == null ? void 0 : _a.chunkSize,
         chunkOverlap: (_b = this.config.rag) == null ? void 0 : _b.chunkOverlap
       });
     }
-    return this.chunker.chunk(doc.content, {
-      docId: doc.docId,
-      metadata: doc.metadata
-    });
+    return this.chunker.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata });
   }
   /**
    * Step 2: Generate embeddings for chunks with retry logic.
@@ -4832,9 +5083,7 @@ You are a helpful product assistant. Use the provided context to answer question
     }
     return vectors;
   }
-  /**
-   * Step 3: Upsert chunks to vector database with retry logic.
-   */
+  /** Step 3: Upsert chunks to vector database with retry logic. */
   async processUpserts(upsertDocs, namespace) {
     const upsertBatchOptions = {
       batchSize: 100,
@@ -4851,14 +5100,10 @@ You are a helpful product assistant. Use the provided context to answer question
     }
     return upsertResult.totalProcessed;
   }
-  /**
-   * Step 4: Optional graph-based entity extraction and ingestion.
-   */
+  /** Step 4: Optional graph-based entity extraction and ingestion. */
   async processGraphIngestion(docId, chunks) {
-    console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
     const extractionOptions = {
       batchSize: 2,
-      // Low concurrency for LLM extraction
       maxRetries: 1,
       initialDelayMs: 500
     };
@@ -4871,7 +5116,7 @@ You are a helpful product assistant. Use the provided context to answer question
             if (nodes.length > 0) await this.graphDB.addNodes(nodes);
             if (edges.length > 0) await this.graphDB.addEdges(edges);
           } catch (err) {
-            console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
+            console.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}:`, err);
           }
         }
       },
@@ -4882,7 +5127,6 @@ You are a helpful product assistant. Use the provided context to answer question
     var _a;
     await this.initialize();
     if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic" && this.agent) {
-      console.log("[Pipeline] \u{1F916} Executing in Agentic Mode...");
       const agentReply = await this.agent.run(question, history);
       return { reply: agentReply, sources: [] };
     }
@@ -4891,6 +5135,7 @@ You are a helpful product assistant. Use the provided context to answer question
     let sources = [];
     let graphData;
     let uiTransformation;
+    let trace;
     try {
       for (var iter = __forAwait(stream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
         const chunk = temp.value;
@@ -4900,6 +5145,7 @@ You are a helpful product assistant. Use the provided context to answer question
           if ("sources" in chunk) sources = chunk.sources;
           if ("graphData" in chunk) graphData = chunk.graphData;
           if ("ui_transformation" in chunk) uiTransformation = chunk.ui_transformation;
+          if ("trace" in chunk) trace = chunk.trace;
         }
       }
     } catch (temp) {
@@ -4912,41 +5158,51 @@ You are a helpful product assistant. Use the provided context to answer question
           throw error[0];
       }
     }
-    return { reply, sources, graphData, ui_transformation: uiTransformation };
+    return { reply, sources, graphData, ui_transformation: uiTransformation, trace };
   }
   /**
    * High-performance streaming RAG flow.
-   * Yields text chunks first, then the retrieval metadata at the end.
+   * Yields text chunks first, then the retrieval metadata + observability trace at the end.
    */
   askStream(_0) {
     return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
-      var _a, _b, _c, _d, _e, _f, _g;
+      var _a, _b, _c, _d, _e, _f, _g, _h, _i;
       yield new __await(this.initialize());
       const ns = namespace != null ? namespace : this.config.projectId;
       const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
       const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
+      const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+      const requestStart = performance.now();
       try {
         let searchQuery = question;
+        let rewrittenQuery;
         if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
           searchQuery = yield new __await(this.rewriteQuery(question, history));
+          rewrittenQuery = searchQuery !== question ? searchQuery : void 0;
         }
         const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
         const filter = QueryProcessor.buildQueryFilter(question, hints);
-        console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
+        const embedStart = performance.now();
         const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
           namespace: ns,
           topK: topK * 2,
           filter
         }));
+        const retrieveEnd = performance.now();
+        const embedMs = retrieveEnd - embedStart;
+        const retrieveMs = retrieveEnd - embedStart;
+        const rerankStart = performance.now();
         let sources = rawSources.filter((m) => m.score >= scoreThreshold);
         if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
           sources = yield new __await(this.reranker.rerank(sources, question, topK));
         } else {
           sources = sources.slice(0, topK);
         }
+        const rerankMs = performance.now() - rerankStart;
         let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
 ${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
         if (graphData && graphData.nodes.length > 0) {
+          console.log(`[Graph Retrieval] Found ${graphData.nodes.length} relevant entities.`);
           const graphContext = graphData.nodes.map(
             (n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
           ).join("\n");
@@ -4962,6 +5218,10 @@ ${context}`;
         const hardenedHistory = [...history];
         const userQuestion = { role: "user", content: question + restrictionSuffix };
         const messages = [...hardenedHistory, userQuestion];
+        const systemPrompt = (_h = this.config.llm.systemPrompt) != null ? _h : "";
+        const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
+        let fullReply = "";
+        const generateStart = performance.now();
         if (this.llmProvider.chatStream) {
           const stream = this.llmProvider.chatStream(messages, context);
           if (!stream) {
@@ -4970,6 +5230,7 @@ ${context}`;
           try {
             for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
               const chunk = temp.value;
+              fullReply += chunk;
               yield chunk;
             }
           } catch (temp) {
@@ -4984,15 +5245,58 @@ ${context}`;
           }
         } else {
           const reply = yield new __await(this.llmProvider.chat(messages, context));
+          fullReply = reply;
           yield reply;
         }
+        const generateMs = performance.now() - generateStart;
+        const totalMs = performance.now() - requestStart;
+        const latency = {
+          embedMs: Math.round(embedMs),
+          retrieveMs: Math.round(retrieveMs),
+          rerankMs: ((_i = this.config.rag) == null ? void 0 : _i.useReranking) ? Math.round(rerankMs) : void 0,
+          generateMs: Math.round(generateMs),
+          totalMs: Math.round(totalMs)
+        };
+        const promptText = systemPrompt + "\n" + context + "\n" + userPrompt;
+        const promptTokens = estimateTokens(promptText);
+        const completionTokens = estimateTokens(fullReply);
+        const tokens = {
+          promptTokens,
+          completionTokens,
+          totalTokens: promptTokens + completionTokens,
+          estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
+        };
+        const hallucinationResult = yield new __await(scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0));
         const trainedSchema = yield new __await(trainingPromise);
-        const uiTransformation = yield new __await(this.generateUiTransformation(question, sources, context, trainedSchema));
+        const uiTransformation = yield new __await(this.generateUiTransformation(question, sources, trainedSchema));
+        const trace = {
+          requestId,
+          query: question,
+          rewrittenQuery,
+          systemPrompt,
+          userPrompt: question + restrictionSuffix,
+          chunks: sources.map((s) => {
+            var _a2;
+            return {
+              id: s.id,
+              score: s.score,
+              content: s.content,
+              metadata: (_a2 = s.metadata) != null ? _a2 : {},
+              namespace: ns
+            };
+          }),
+          latency,
+          tokens,
+          hallucinationScore: hallucinationResult == null ? void 0 : hallucinationResult.score,
+          hallucinationReason: hallucinationResult == null ? void 0 : hallucinationResult.reason,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        };
         yield {
           reply: "",
           sources,
           graphData,
-          ui_transformation: uiTransformation
+          ui_transformation: uiTransformation,
+          trace
         };
       } catch (error2) {
         throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
@@ -5003,7 +5307,7 @@ ${context}`;
    * Universal retrieval method combining all enabled providers.
    * Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
    */
-  async generateUiTransformation(question, sources, _context, trainedSchema) {
+  async generateUiTransformation(question, sources, trainedSchema) {
     if (!sources || sources.length === 0) {
       return UITransformer.transform(question, sources, this.config, trainedSchema);
     }
@@ -5020,23 +5324,24 @@ ${context}`;
     const topK = (_b = options.topK) != null ? _b : 5;
     const cacheKey = `${ns}::${query}`;
     let queryVector = this.embeddingCache.get(cacheKey);
+    const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.llmProvider);
+    console.debug(`[Pipeline] Determined retrieval strategy: ${strategy}`);
     const [retrievedVector, graphData] = await Promise.all([
-      queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: "query" }),
-      this.graphDB && ((_c = this.config.rag) == null ? void 0 : _c.useGraphRetrieval) ? this.graphDB.query(query) : Promise.resolve(void 0)
+      // Only embed if we need vector search (strategy is 'vector' or 'both')
+      strategy === "vector" || strategy === "both" ? queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: "query" }) : Promise.resolve([]),
+      // Only query graph if we need graph search (strategy is 'graph' or 'both')
+      (strategy === "graph" || strategy === "both") && this.graphDB && ((_c = this.config.rag) == null ? void 0 : _c.useGraphRetrieval) ? this.graphDB.query(query) : Promise.resolve(void 0)
     ]);
-    if (!queryVector) {
+    if ((strategy === "vector" || strategy === "both") && !queryVector && retrievedVector && retrievedVector.length > 0) {
       this.embeddingCache.set(cacheKey, retrievedVector);
       queryVector = retrievedVector;
     }
-    const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
+    const sources = (strategy === "vector" || strategy === "both") && queryVector && queryVector.length > 0 ? await this.vectorDB.query(queryVector, topK, ns, options.filter) : [];
     return { sources, graphData };
   }
-  /**
-   * Rewrite the user query for better retrieval performance.
-   */
+  /** Rewrite the user query for better retrieval performance. */
   async rewriteQuery(question, history) {
-    const prompt = `
-Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
+    const prompt = `Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
 Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
 
 History:
@@ -5054,26 +5359,19 @@ Optimized Search Query:`;
     );
     return rewrite.trim() || question;
   }
-  /**
-   * Generate 3-5 short, relevant questions based on the vector database content.
-   */
+  /** Generate 3-5 short, relevant questions based on the vector database content. */
   async getSuggestions(query, namespace) {
-    if (!query || query.trim().length < 3) {
-      return [];
-    }
+    if (!query || query.trim().length < 3) return [];
     await this.initialize();
     const ns = namespace != null ? namespace : this.config.projectId;
     try {
-      const { sources } = await this.retrieve(query, { namespace: ns, topK: 3 });
-      if (sources.length === 0) {
-        return [];
-      }
+      const { sources } = await this.retrieve(query, { namespace: ns, topK: 5 });
+      if (sources.length === 0) return [];
       const context = sources.map((s) => s.content).join("\n\n---\n\n");
-      const prompt = `
-Based on the following snippets from a document, what are 3 short, relevant questions a user might ask? 
+      const prompt = `Based on the following snippets from a document, what are at least 5 short, relevant questions a user might ask? 
 Focus on questions that can be answered by the context.
 Keep each question under 10 words and make them very specific to the content.
-Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3"].
+Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3", "Question 4", "Question 5"].
 
 Context:
 ${context}
@@ -5090,7 +5388,7 @@ Suggestions:`;
       if (match) {
         const suggestions = JSON.parse(match[0]);
         if (Array.isArray(suggestions)) {
-          return suggestions.map((s) => String(s)).slice(0, 3);
+          return suggestions.map((s) => String(s)).slice(0, 5);
         }
       }
     } catch (error) {
@@ -5347,6 +5645,11 @@ function sseMetaFrame(meta) {
 function sseUIFrame(uiTransformation) {
   return `data: ${JSON.stringify({ type: "ui_transformation", data: uiTransformation })}
 
+`;
+}
+function sseObservabilityFrame(trace) {
+  return `data: ${JSON.stringify({ type: "observability", data: trace })}
+
 `;
 }
 function sseErrorFrame(message) {
@@ -5413,6 +5716,9 @@ function createStreamHandler(configOrPlugin) {
                 enqueue(sseMetaFrame(chunk));
                 const responseChunk = chunk;
                 const sources = (responseChunk == null ? void 0 : responseChunk.sources) || [];
+                if (responseChunk == null ? void 0 : responseChunk.trace) {
+                  enqueue(sseObservabilityFrame(responseChunk.trace));
+                }
                 if (sources.length > 0) {
                   try {
                     const llmProvider = (_a = plugin.getLLMProvider) == null ? void 0 : _a.call(plugin);
@@ -5494,24 +5800,64 @@ function createUploadHandler(configOrPlugin) {
       const formData = await req.formData();
       const files = formData.getAll("files");
       const namespace = formData.get("namespace") || void 0;
+      const dimensionRaw = formData.get("dimension");
+      const dimension = dimensionRaw ? parseInt(dimensionRaw, 10) : void 0;
       if (!files || files.length === 0) {
         return import_server.NextResponse.json({ error: "No files provided" }, { status: 400 });
       }
-      const documents = await Promise.all(
-        files.map(async (file) => {
+      const documents = [];
+      for (const file of files) {
+        if (file.name.toLowerCase().endsWith(".csv") || file.type === "text/csv") {
+          const text = await file.text();
+          const Papa = await import("papaparse");
+          const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
+          if (parsed.data && parsed.data.length > 0) {
+            let i = 0;
+            let lastRowData = null;
+            for (const row of parsed.data) {
+              i++;
+              const rowData = row;
+              const groupingKey = Object.keys(rowData)[0];
+              if (lastRowData && groupingKey && rowData[groupingKey] && rowData[groupingKey] === lastRowData[groupingKey]) {
+                for (const key of Object.keys(rowData)) {
+                  if (!rowData[key] && lastRowData[key]) {
+                    rowData[key] = lastRowData[key];
+                  }
+                }
+              }
+              lastRowData = __spreadValues({}, rowData);
+              const contentParts = [];
+              for (const [key, val] of Object.entries(rowData)) {
+                if (key && val) {
+                  contentParts.push(`${key}: ${val}`);
+                }
+              }
+              documents.push({
+                docId: `${file.name}-row-${i}`,
+                content: contentParts.join(", "),
+                metadata: __spreadValues(__spreadValues({
+                  fileName: file.name,
+                  fileSize: file.size,
+                  fileType: file.type,
+                  uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
+                }, dimension ? { dimension } : {}), rowData)
+              });
+            }
+          }
+        } else {
           const content = await DocumentParser.parse(file, file.name, file.type);
-          return {
+          documents.push({
             docId: file.name,
             content,
-            metadata: {
+            metadata: __spreadValues({
               fileName: file.name,
               fileSize: file.size,
               fileType: file.type,
               uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
-            }
-          };
-        })
-      );
+            }, dimension ? { dimension } : {})
+          });
+        }
+      }
       const results = await plugin.ingest(documents, namespace);
       return import_server.NextResponse.json({ message: "Upload successful", results });
     } catch (err) {
@@ -5548,6 +5894,7 @@ function createSuggestionsHandler(configOrPlugin) {
   sseErrorFrame,
   sseFrame,
   sseMetaFrame,
+  sseObservabilityFrame,
   sseTextFrame,
   sseUIFrame
 });