@retrivora-ai/rag-engine 1.8.1 → 1.8.2

package/dist/server.js

@@ -347,7 +347,7 @@ var init_MultiTablePostgresProvider = __esm({
     init_BaseVectorProvider();
     MultiTablePostgresProvider = class extends BaseVectorProvider {
       constructor(config) {
-        var _a, _b, _c, _d, _e;
+        var _a, _b, _c;
         super(config);
         const opts = config.options || {};
         if (!opts.connectionString) {
@@ -355,37 +355,37 @@ var init_MultiTablePostgresProvider = __esm({
         }
         this.connectionString = opts.connectionString;
         this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
-        const rawTables = (_c = (_b = opts.tables) != null ? _b : process.env.VECTOR_DB_TABLES) != null ? _c : "";
-        this.tables = typeof rawTables === "string" ? rawTables.split(",").map((t) => t.trim()).filter(Boolean) : rawTables;
-        if (this.tables.length === 0) {
-          console.warn(
-            "[MultiTablePostgresProvider] No tables configured. Set VECTOR_DB_TABLES as a comma-separated list of table names or pass options.tables."
-          );
-        }
-        const rawSearchFields = (_e = (_d = opts.searchFields) != null ? _d : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _e : ["name", "product_name", "productname", "title"];
+        this.tables = [];
+        const rawSearchFields = (_c = (_b = opts.searchFields) != null ? _b : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _c : ["name", "product_name", "productname", "title"];
         this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
+        this.uploadTable = opts.uploadTable || "document_chunks";
       }
       async initialize() {
         this.pool = new import_pg.Pool({ connectionString: this.connectionString });
         const client = await this.pool.connect();
         try {
+          await client.query("CREATE EXTENSION IF NOT EXISTS vector");
+          await client.query(`
+        CREATE TABLE IF NOT EXISTS ${this.uploadTable} (
+          id          TEXT PRIMARY KEY,
+          namespace   TEXT NOT NULL DEFAULT '',
+          content     TEXT NOT NULL,
+          metadata    JSONB,
+          embedding   VECTOR(${this.dimensions})
+        )
+      `);
+          await client.query(`
+        CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
+        ON ${this.uploadTable}
+        USING hnsw (embedding vector_cosine_ops)
+      `);
           const res = await client.query(`
         SELECT table_name 
         FROM information_schema.columns 
         WHERE column_name = 'embedding' 
         AND table_schema = 'public'
       `);
-          const discoveredTables = res.rows.map((r) => r.table_name);
-          if (this.tables.length === 0) {
-            this.tables = discoveredTables;
-          } else {
-            const staticTables = [...this.tables];
-            this.tables = staticTables.filter((t) => discoveredTables.includes(t));
-            if (this.tables.length < staticTables.length) {
-              const missing = staticTables.filter((t) => !discoveredTables.includes(t));
-              console.warn(`[MultiTablePostgresProvider] Some configured tables were skipped (missing 'embedding' column): ${missing.join(", ")}`);
-            }
-          }
+          this.tables = res.rows.map((r) => r.table_name);
           if (this.tables.length === 0) {
             console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
           } else {
@@ -398,27 +398,106 @@ var init_MultiTablePostgresProvider = __esm({
         }
       }
       /**
-       * Upsert is not supported for MultiTablePostgresProvider as it's designed for 
-       * searching across pre-existing tables with varying schemas.
+       * Upsert a document by dynamically provisioning a table and columns.
        */
-      async upsert(_doc, _namespace) {
-        throw new Error(
-          "[MultiTablePostgresProvider] upsert() is not supported in multi-table mode. Please use the standard PostgreSQLProvider for single-table managed indices."
-        );
+      async upsert(doc, namespace = "") {
+        await this.batchUpsert([doc], namespace);
       }
       /**
-       * Batch upsert is not supported for MultiTablePostgresProvider.
+       * Batch upsert documents by dynamically provisioning tables and columns based on CSV headers.
        */
-      async batchUpsert(_docs, _namespace) {
-        throw new Error(
-          "[MultiTablePostgresProvider] batchUpsert() is not supported in multi-table mode."
-        );
+      async batchUpsert(docs, namespace = "") {
+        var _a;
+        if (docs.length === 0) return;
+        const docsByFile = {};
+        for (const doc of docs) {
+          const fileName = ((_a = doc.metadata) == null ? void 0 : _a.fileName) || this.uploadTable;
+          if (!docsByFile[fileName]) docsByFile[fileName] = [];
+          docsByFile[fileName].push(doc);
+        }
+        const client = await this.pool.connect();
+        try {
+          await client.query("BEGIN");
+          for (const [fileName, fileDocs] of Object.entries(docsByFile)) {
+            let tableName = fileName;
+            if (tableName.toLowerCase().endsWith(".csv")) {
+              tableName = tableName.slice(0, -4);
+            }
+            tableName = tableName.replace(/[^a-z0-9_]/gi, "_").toLowerCase() || this.uploadTable;
+            const firstMeta = fileDocs[0].metadata || {};
+            const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding"];
+            const csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+            const columnDefs = csvHeaders.map((h) => `"${h}" TEXT`).join(",\n          ");
+            const createTableSql = `
+          CREATE TABLE IF NOT EXISTS "${tableName}" (
+            id          TEXT PRIMARY KEY,
+            ${columnDefs ? columnDefs + "," : ""}
+            namespace   TEXT NOT NULL DEFAULT '',
+            content     TEXT NOT NULL,
+            metadata    JSONB,
+            embedding   VECTOR(${this.dimensions})
+          )
+        `;
+            await client.query(createTableSql);
+            await client.query(`
+          CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
+          ON "${tableName}"
+          USING hnsw (embedding vector_cosine_ops)
+        `);
+            if (!this.tables.includes(tableName)) {
+              this.tables.push(tableName);
+            }
+            const BATCH_SIZE = 50;
+            for (let i = 0; i < fileDocs.length; i += BATCH_SIZE) {
+              const batch = fileDocs.slice(i, i + BATCH_SIZE);
+              const values = [];
+              const headerColumns = csvHeaders.map((h) => `"${h}"`).join(", ");
+              const allColumns = `id, ${headerColumns ? headerColumns + ", " : ""}namespace, content, metadata, embedding`;
+              const valuePlaceholders = batch.map((doc, idx) => {
+                var _a2, _b;
+                const offset = idx * (5 + csvHeaders.length);
+                values.push(doc.id);
+                for (const h of csvHeaders) {
+                  values.push(String(((_a2 = doc.metadata) == null ? void 0 : _a2[h]) || ""));
+                }
+                values.push(namespace, doc.content, JSON.stringify((_b = doc.metadata) != null ? _b : {}), `[${doc.vector.join(",")}]`);
+                const docPlaceholders = [];
+                for (let j = 1; j <= 5 + csvHeaders.length; j++) {
+                  if (j === 5 + csvHeaders.length) {
+                    docPlaceholders.push(`$${offset + j}::vector`);
+                  } else {
+                    docPlaceholders.push(`$${offset + j}`);
+                  }
+                }
+                return `(${docPlaceholders.join(", ")})`;
+              }).join(", ");
+              const conflictUpdates = csvHeaders.map((h) => `"${h}" = EXCLUDED."${h}"`).join(",\n              ");
+              const query = `
+            INSERT INTO "${tableName}" (${allColumns})
+            VALUES ${valuePlaceholders}
+            ON CONFLICT (id) DO UPDATE
+              SET ${conflictUpdates ? conflictUpdates + "," : ""}
+                  namespace = EXCLUDED.namespace,
+                  content   = EXCLUDED.content,
+                  metadata  = EXCLUDED.metadata,
+                  embedding = EXCLUDED.embedding
+          `;
+              await client.query(query, values);
+            }
+          }
+          await client.query("COMMIT");
+        } catch (error) {
+          await client.query("ROLLBACK");
+          throw error;
+        } finally {
+          client.release();
+        }
       }
       /**
        * Query all configured tables and merge results, sorted by cosine similarity score.
        */
       async query(vector, topK, _namespace, _filter) {
-        var _a, _b, _c;
+        var _a;
         if (!this.pool) {
           throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
         }
@@ -510,22 +589,16 @@ var init_MultiTablePostgresProvider = __esm({
         for (const tableResults of resultsArray) {
           allResults.push(...tableResults);
         }
-        const resultsByTable = {};
-        for (const res of allResults) {
-          const table = (_a = res.metadata) == null ? void 0 : _a.source_table;
-          if (!resultsByTable[table]) resultsByTable[table] = [];
-          resultsByTable[table].push(res);
-        }
-        const balancedResults = [];
-        const tables = Object.keys(resultsByTable);
-        for (const table of tables) {
-          balancedResults.push(...resultsByTable[table].slice(0, 3));
-        }
-        const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
-        if (finalSorted.length > 0) {
-          console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
-          console.log(`[MultiTablePostgresProvider] Final top match from "${(_c = (_b = finalSorted[0].metadata) == null ? void 0 : _b.source_table) != null ? _c : "unknown"}" with score ${finalSorted[0].score.toFixed(4)}`);
-        }
+        allResults.sort((a, b) => b.score - a.score);
+        if (allResults.length === 0) return [];
+        const bestMatchTable = (_a = allResults[0].metadata) == null ? void 0 : _a.source_table;
+        const finalSorted = allResults.filter((res) => {
+          var _a2;
+          return ((_a2 = res.metadata) == null ? void 0 : _a2.source_table) === bestMatchTable;
+        });
+        console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
+        console.log(`[MultiTablePostgresProvider] Final top match from "${bestMatchTable}" with score ${finalSorted[0].score.toFixed(4)}`);
+        console.log(`[MultiTablePostgresProvider] Restricting recommendations strictly to table: "${bestMatchTable}"`);
         return finalSorted.slice(0, Math.max(topK, 15));
       }
       async delete(_id, _namespace) {
@@ -3978,6 +4051,64 @@ var QueryProcessor = class {
   }
 };
 
+// src/utils/synonyms.ts
+var FIELD_SYNONYMS = {
+  name: ["product", "item", "title", "label", "heading", "subject", "product name", "item name"],
+  price: ["cost", "amount", "msrp", "price", "rate", "value", "price_usd"],
+  brand: ["manufacturer", "vendor", "make", "company", "brand_name", "supplier"],
+  image: [
+    "imageUrl",
+    "thumbnail",
+    "img",
+    "url",
+    "photo",
+    "picture",
+    "media",
+    "image_url",
+    "main_image",
+    "product_image",
+    "thumb"
+  ],
+  stock: [
+    "inventory",
+    "quantity",
+    "count",
+    "availability",
+    "stock_level",
+    "inStock",
+    "is_available",
+    "in stock",
+    "status"
+  ],
+  description: ["summary", "content", "body", "text", "info", "details"],
+  link: ["url", "href", "product_url", "page_url", "link"]
+};
+function resolveMetadataValue(meta, uiKey) {
+  var _a;
+  const synonyms = (_a = FIELD_SYNONYMS[uiKey]) != null ? _a : [];
+  const keys = Object.keys(meta);
+  const systemKeys = ["namespace", "filename", "filesize", "filetype", "docid", "uploadedat", "dimension", "chunkindex"];
+  let match = keys.find((k) => k.toLowerCase() === uiKey.toLowerCase());
+  if (match !== void 0) return meta[match];
+  match = keys.find((k) => synonyms.map((s) => s.toLowerCase()).includes(k.toLowerCase()));
+  if (match !== void 0) return meta[match];
+  const isBlacklisted = (kl) => {
+    return systemKeys.includes(kl) || kl.startsWith("option1") || kl.startsWith("option2") || kl.startsWith("option3");
+  };
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return kl.includes(uiKey.toLowerCase());
+  });
+  if (match !== void 0) return meta[match];
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return synonyms.some((sk) => kl.includes(sk.toLowerCase()) || sk.toLowerCase().includes(kl));
+  });
+  return match !== void 0 ? meta[match] : void 0;
+}
+
 // src/utils/UITransformer.ts
 var UITransformer = class {
   /**
@@ -4045,28 +4176,6 @@ var UITransformer = class {
       data: pieData
     };
   }
-  /**
-   * Transform data to bar chart format
-   */
-  static transformToBarChart(data) {
-    const categories = this.detectCategories(data);
-    const categoryData = this.aggregateByCategory(data, categories);
-    const barData = Object.entries(categoryData).map(([category, value]) => {
-      const { inStockCount, outOfStockCount } = this.calculateStockCounts(category, data);
-      return {
-        category,
-        value,
-        inStockCount,
-        outOfStockCount
-      };
-    });
-    return {
-      type: "bar_chart",
-      title: "Comparison by Category",
-      description: `Comparing ${categories.length} categories`,
-      data: barData
-    };
-  }
   /**
    * Transform data to line chart format
    */
@@ -4345,15 +4454,7 @@ var UITransformer = class {
       const trainedKey = trainedSchema[uiKey];
       if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
     }
-    const synonyms = this.SYNONYMS[uiKey] || [];
-    const searchKeys = [uiKey, ...synonyms].map((k) => k.toLowerCase());
-    const foundDirectKey = Object.keys(meta).find((k) => searchKeys.includes(k.toLowerCase()));
-    if (foundDirectKey) return meta[foundDirectKey];
-    const foundPartialKey = Object.keys(meta).find((k) => {
-      const keyLow = k.toLowerCase();
-      return searchKeys.some((sk) => keyLow.includes(sk) || sk.includes(keyLow));
-    });
-    return foundPartialKey ? meta[foundPartialKey] : void 0;
+    return resolveMetadataValue(meta, uiKey);
   }
   static extractProductInfo(item, config, trainedSchema) {
     const meta = item.metadata || {};
@@ -4423,7 +4524,7 @@ var UITransformer = class {
     data.forEach((item) => {
       const meta = item.metadata || {};
       const itemCategory = meta.category || meta.type || "Other";
-      if (result.hasOwnProperty(itemCategory)) {
+      if (Object.prototype.hasOwnProperty.call(result, itemCategory)) {
         result[itemCategory]++;
       } else {
         result["Other"] = (result["Other"] || 0) + 1;
@@ -4644,18 +4745,6 @@ IMPORTANT:
     return JSON.stringify(partial, null, 2) + "\n// ... truncated";
   }
 };
-/**
- * Central dictionary of common synonyms for UI properties.
- * This allows the system to be schema-agnostic by guessing field names.
- */
-UITransformer.SYNONYMS = {
-  name: ["product", "item", "title", "label", "heading", "subject"],
-  price: ["cost", "amount", "msrp", "price", "rate", "value", "price_usd"],
-  brand: ["manufacturer", "vendor", "make", "company", "brand_name", "supplier"],
-  image: ["imageUrl", "thumbnail", "img", "url", "photo", "picture", "media", "image_url", "main_image", "product_image", "thumb"],
-  stock: ["inventory", "quantity", "count", "availability", "stock_level", "inStock", "is_available"],
-  description: ["summary", "content", "body", "text", "info", "details"]
-};
 
 // src/utils/SchemaMapper.ts
 var SchemaMapper = class {
@@ -4776,6 +4865,57 @@ var LRUEmbeddingCache = class {
     return this.cache.size;
   }
 };
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+var MODEL_COST_PER_1K = {
+  "gpt-4o": { input: 25e-4, output: 0.01 },
+  "gpt-4o-mini": { input: 15e-5, output: 6e-4 },
+  "gpt-4-turbo": { input: 0.01, output: 0.03 },
+  "gpt-3.5-turbo": { input: 5e-4, output: 15e-4 },
+  "claude-3-5-sonnet": { input: 3e-3, output: 0.015 },
+  "claude-3-haiku": { input: 25e-5, output: 125e-5 },
+  "gemini-1.5-flash": { input: 75e-6, output: 3e-4 },
+  "gemini-1.5-pro": { input: 125e-5, output: 5e-3 }
+};
+function estimateCostUsd(promptTokens, completionTokens, model) {
+  if (!model) return void 0;
+  const key = Object.keys(MODEL_COST_PER_1K).find((k) => model.toLowerCase().includes(k));
+  if (!key) return void 0;
+  const prices = MODEL_COST_PER_1K[key];
+  return promptTokens / 1e3 * prices.input + completionTokens / 1e3 * prices.output;
+}
+async function scoreHallucination(llm, answer, context) {
+  const maxContextChars = 3e3;
+  const truncatedContext = context.length > maxContextChars ? context.slice(0, maxContextChars) + "\n...[truncated]" : context;
+  const prompt = `You are an AI quality checker. Given the CONTEXT retrieved from a knowledge base and the ANSWER generated from it, rate how well-grounded the answer is.
+
+CONTEXT:
+${truncatedContext}
+
+ANSWER:
+${answer}
+
+Return ONLY a valid JSON object with no markdown fences:
+{"score": <float 0-1>, "reason": "<one sentence>"}
+
+Where score 0 = fully grounded in context, score 1 = likely hallucinated or unsupported.`;
+  try {
+    const raw = await llm.chat(
+      [{ role: "user", content: prompt }],
+      "",
+      { temperature: 0, maxTokens: 120 }
+    );
+    const jsonMatch = raw.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return void 0;
+    const parsed = JSON.parse(jsonMatch[0]);
+    if (typeof parsed.score === "number" && typeof parsed.reason === "string") {
+      return { score: Math.min(1, Math.max(0, parsed.score)), reason: parsed.reason };
+    }
+  } catch (e) {
+  }
+  return void 0;
+}
 var Pipeline = class {
   constructor(config) {
     this.config = config;
@@ -4802,8 +4942,7 @@ var Pipeline = class {
   async initialize() {
     var _a;
     if (this.initialised) return;
-    const chartInstruction = `
-You are a helpful product assistant. Use the provided context to answer questions accurately.
+    const chartInstruction = `You are a helpful product assistant. Use the provided context to answer questions accurately.
 
 ### UI STYLE RULES (CRITICAL):
 - NEVER generate markdown tables. If you do, the UI will break.
@@ -4817,7 +4956,6 @@ You are a helpful product assistant. Use the provided context to answer question
 - Do NOT try to format product lists as tables.
 `;
     this.config.llm.systemPrompt = chartInstruction;
-    console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
     this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
     const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
       this.config.llm,
@@ -4839,7 +4977,6 @@ You are a helpful product assistant. Use the provided context to answer question
   }
   /**
    * Ingest documents with automatic chunking, embedding, and batch upsert.
-   * Handles retries for transient failures.
    */
   async ingest(documents, namespace) {
     await this.initialize();
@@ -4856,10 +4993,7 @@ You are a helpful product assistant. Use the provided context to answer question
           metadata: chunk.metadata
         }));
         const totalProcessed = await this.processUpserts(upsertDocs, ns);
-        results.push({
-          docId: doc.docId,
-          chunksIngested: totalProcessed
-        });
+        results.push({ docId: doc.docId, chunksIngested: totalProcessed });
         if (this.graphDB && this.entityExtractor) {
           await this.processGraphIngestion(doc.docId, chunks);
         }
@@ -4870,23 +5004,18 @@ You are a helpful product assistant. Use the provided context to answer question
     }
     return results;
   }
-  /**
-   * Step 1: Chunk the document content.
-   */
+  /** Step 1: Chunk the document content. */
   async prepareChunks(doc) {
     var _a, _b;
     if (this.llamaIngestor) {
-      return await this.llamaIngestor.chunk(doc.content, {
+      return this.llamaIngestor.chunk(doc.content, {
         docId: doc.docId,
         metadata: doc.metadata,
         chunkSize: (_a = this.config.rag) == null ? void 0 : _a.chunkSize,
         chunkOverlap: (_b = this.config.rag) == null ? void 0 : _b.chunkOverlap
       });
     }
-    return this.chunker.chunk(doc.content, {
-      docId: doc.docId,
-      metadata: doc.metadata
-    });
+    return this.chunker.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata });
   }
   /**
    * Step 2: Generate embeddings for chunks with retry logic.
@@ -4910,9 +5039,7 @@ You are a helpful product assistant. Use the provided context to answer question
     }
     return vectors;
   }
-  /**
-   * Step 3: Upsert chunks to vector database with retry logic.
-   */
+  /** Step 3: Upsert chunks to vector database with retry logic. */
   async processUpserts(upsertDocs, namespace) {
     const upsertBatchOptions = {
       batchSize: 100,
@@ -4929,14 +5056,10 @@ You are a helpful product assistant. Use the provided context to answer question
     }
     return upsertResult.totalProcessed;
   }
-  /**
-   * Step 4: Optional graph-based entity extraction and ingestion.
-   */
+  /** Step 4: Optional graph-based entity extraction and ingestion. */
   async processGraphIngestion(docId, chunks) {
-    console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
     const extractionOptions = {
       batchSize: 2,
-      // Low concurrency for LLM extraction
       maxRetries: 1,
       initialDelayMs: 500
     };
@@ -4949,7 +5072,7 @@ You are a helpful product assistant. Use the provided context to answer question
             if (nodes.length > 0) await this.graphDB.addNodes(nodes);
             if (edges.length > 0) await this.graphDB.addEdges(edges);
           } catch (err) {
-            console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
+            console.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}:`, err);
           }
         }
       },
@@ -4960,7 +5083,6 @@ You are a helpful product assistant. Use the provided context to answer question
     var _a;
     await this.initialize();
     if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic" && this.agent) {
-      console.log("[Pipeline] \u{1F916} Executing in Agentic Mode...");
       const agentReply = await this.agent.run(question, history);
       return { reply: agentReply, sources: [] };
     }
@@ -4969,6 +5091,7 @@ You are a helpful product assistant. Use the provided context to answer question
     let sources = [];
     let graphData;
     let uiTransformation;
+    let trace;
     try {
       for (var iter = __forAwait(stream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
         const chunk = temp.value;
@@ -4978,6 +5101,7 @@ You are a helpful product assistant. Use the provided context to answer question
           if ("sources" in chunk) sources = chunk.sources;
           if ("graphData" in chunk) graphData = chunk.graphData;
           if ("ui_transformation" in chunk) uiTransformation = chunk.ui_transformation;
+          if ("trace" in chunk) trace = chunk.trace;
         }
       }
     } catch (temp) {
@@ -4990,38 +5114,47 @@ You are a helpful product assistant. Use the provided context to answer question
           throw error[0];
       }
     }
-    return { reply, sources, graphData, ui_transformation: uiTransformation };
+    return { reply, sources, graphData, ui_transformation: uiTransformation, trace };
   }
   /**
    * High-performance streaming RAG flow.
-   * Yields text chunks first, then the retrieval metadata at the end.
+   * Yields text chunks first, then the retrieval metadata + observability trace at the end.
    */
   askStream(_0) {
     return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
-      var _a, _b, _c, _d, _e, _f, _g;
+      var _a, _b, _c, _d, _e, _f, _g, _h, _i;
       yield new __await(this.initialize());
       const ns = namespace != null ? namespace : this.config.projectId;
       const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
       const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
+      const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+      const requestStart = performance.now();
       try {
         let searchQuery = question;
+        let rewrittenQuery;
         if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
           searchQuery = yield new __await(this.rewriteQuery(question, history));
+          rewrittenQuery = searchQuery !== question ? searchQuery : void 0;
         }
         const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
         const filter = QueryProcessor.buildQueryFilter(question, hints);
-        console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
+        const embedStart = performance.now();
         const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
           namespace: ns,
           topK: topK * 2,
           filter
         }));
+        const retrieveEnd = performance.now();
+        const embedMs = retrieveEnd - embedStart;
+        const retrieveMs = retrieveEnd - embedStart;
+        const rerankStart = performance.now();
         let sources = rawSources.filter((m) => m.score >= scoreThreshold);
         if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
           sources = yield new __await(this.reranker.rerank(sources, question, topK));
         } else {
           sources = sources.slice(0, topK);
         }
+        const rerankMs = performance.now() - rerankStart;
         let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
 ${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
         if (graphData && graphData.nodes.length > 0) {
@@ -5040,6 +5173,10 @@ ${context}`;
         const hardenedHistory = [...history];
         const userQuestion = { role: "user", content: question + restrictionSuffix };
         const messages = [...hardenedHistory, userQuestion];
+        const systemPrompt = (_h = this.config.llm.systemPrompt) != null ? _h : "";
+        const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
+        let fullReply = "";
+        const generateStart = performance.now();
         if (this.llmProvider.chatStream) {
           const stream = this.llmProvider.chatStream(messages, context);
           if (!stream) {
@@ -5048,6 +5185,7 @@ ${context}`;
           try {
             for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
               const chunk = temp.value;
+              fullReply += chunk;
               yield chunk;
             }
           } catch (temp) {
@@ -5062,15 +5200,58 @@ ${context}`;
           }
         } else {
           const reply = yield new __await(this.llmProvider.chat(messages, context));
+          fullReply = reply;
           yield reply;
         }
+        const generateMs = performance.now() - generateStart;
+        const totalMs = performance.now() - requestStart;
+        const latency = {
+          embedMs: Math.round(embedMs),
+          retrieveMs: Math.round(retrieveMs),
+          rerankMs: ((_i = this.config.rag) == null ? void 0 : _i.useReranking) ? Math.round(rerankMs) : void 0,
+          generateMs: Math.round(generateMs),
+          totalMs: Math.round(totalMs)
+        };
+        const promptText = systemPrompt + "\n" + context + "\n" + userPrompt;
+        const promptTokens = estimateTokens(promptText);
+        const completionTokens = estimateTokens(fullReply);
+        const tokens = {
+          promptTokens,
+          completionTokens,
+          totalTokens: promptTokens + completionTokens,
+          estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
+        };
+        const hallucinationResult = yield new __await(scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0));
         const trainedSchema = yield new __await(trainingPromise);
         const uiTransformation = yield new __await(this.generateUiTransformation(question, sources, context, trainedSchema));
+        const trace = {
+          requestId,
+          query: question,
+          rewrittenQuery,
+          systemPrompt,
+          userPrompt: question + restrictionSuffix,
+          chunks: sources.map((s) => {
+            var _a2;
+            return {
+              id: s.id,
+              score: s.score,
+              content: s.content,
+              metadata: (_a2 = s.metadata) != null ? _a2 : {},
+              namespace: ns
+            };
+          }),
+          latency,
+          tokens,
+          hallucinationScore: hallucinationResult == null ? void 0 : hallucinationResult.score,
+          hallucinationReason: hallucinationResult == null ? void 0 : hallucinationResult.reason,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        };
         yield {
           reply: "",
           sources,
           graphData,
-          ui_transformation: uiTransformation
+          ui_transformation: uiTransformation,
+          trace
         };
       } catch (error2) {
         throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
@@ -5109,12 +5290,9 @@ ${context}`;
     const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
     return { sources, graphData };
   }
-  /**
-   * Rewrite the user query for better retrieval performance.
-   */
+  /** Rewrite the user query for better retrieval performance. */
   async rewriteQuery(question, history) {
-    const prompt = `
-Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
+    const prompt = `Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
 Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
 
 History:
@@ -5132,23 +5310,16 @@ Optimized Search Query:`;
     );
     return rewrite.trim() || question;
   }
-  /**
-   * Generate 3-5 short, relevant questions based on the vector database content.
-   */
+  /** Generate 3-5 short, relevant questions based on the vector database content. */
   async getSuggestions(query, namespace) {
-    if (!query || query.trim().length < 3) {
-      return [];
-    }
+    if (!query || query.trim().length < 3) return [];
     await this.initialize();
     const ns = namespace != null ? namespace : this.config.projectId;
     try {
       const { sources } = await this.retrieve(query, { namespace: ns, topK: 3 });
-      if (sources.length === 0) {
-        return [];
-      }
+      if (sources.length === 0) return [];
       const context = sources.map((s) => s.content).join("\n\n---\n\n");
-      const prompt = `
-Based on the following snippets from a document, what are 3 short, relevant questions a user might ask? 
+      const prompt = `Based on the following snippets from a document, what are 3 short, relevant questions a user might ask? 
 Focus on questions that can be answered by the context.
 Keep each question under 10 words and make them very specific to the content.
 Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3"].
@@ -5808,6 +5979,11 @@ function sseMetaFrame(meta) {
 function sseUIFrame(uiTransformation) {
   return `data: ${JSON.stringify({ type: "ui_transformation", data: uiTransformation })}
 
+`;
+}
+function sseObservabilityFrame(trace) {
+  return `data: ${JSON.stringify({ type: "observability", data: trace })}
+
 `;
 }
 function sseErrorFrame(message) {
@@ -5874,6 +6050,9 @@ function createStreamHandler(configOrPlugin) {
                 enqueue(sseMetaFrame(chunk));
                 const responseChunk = chunk;
                 const sources = (responseChunk == null ? void 0 : responseChunk.sources) || [];
+                if (responseChunk == null ? void 0 : responseChunk.trace) {
+                  enqueue(sseObservabilityFrame(responseChunk.trace));
+                }
                 if (sources.length > 0) {
                   try {
                     const llmProvider = (_a = plugin.getLLMProvider) == null ? void 0 : _a.call(plugin);
@@ -5955,24 +6134,64 @@ function createUploadHandler(configOrPlugin) {
       const formData = await req.formData();
       const files = formData.getAll("files");
       const namespace = formData.get("namespace") || void 0;
+      const dimensionRaw = formData.get("dimension");
+      const dimension = dimensionRaw ? parseInt(dimensionRaw, 10) : void 0;
       if (!files || files.length === 0) {
         return import_server.NextResponse.json({ error: "No files provided" }, { status: 400 });
       }
-      const documents = await Promise.all(
-        files.map(async (file) => {
+      const documents = [];
+      for (const file of files) {
+        if (file.name.toLowerCase().endsWith(".csv") || file.type === "text/csv") {
+          const text = await file.text();
+          const Papa = await import("papaparse");
+          const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
+          if (parsed.data && parsed.data.length > 0) {
+            let i = 0;
+            let lastRowData = null;
+            for (const row of parsed.data) {
+              i++;
+              const rowData = row;
+              const groupingKey = Object.keys(rowData)[0];
+              if (lastRowData && groupingKey && rowData[groupingKey] && rowData[groupingKey] === lastRowData[groupingKey]) {
+                for (const key of Object.keys(rowData)) {
+                  if (!rowData[key] && lastRowData[key]) {
+                    rowData[key] = lastRowData[key];
+                  }
+                }
+              }
+              lastRowData = __spreadValues({}, rowData);
+              const contentParts = [];
+              for (const [key, val] of Object.entries(rowData)) {
+                if (key && val) {
+                  contentParts.push(`${key}: ${val}`);
+                }
+              }
+              documents.push({
+                docId: `${file.name}-row-${i}`,
+                content: contentParts.join(", "),
+                metadata: __spreadValues(__spreadValues({
+                  fileName: file.name,
+                  fileSize: file.size,
+                  fileType: file.type,
+                  uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
+                }, dimension ? { dimension } : {}), rowData)
+              });
+            }
+          }
+        } else {
           const content = await DocumentParser.parse(file, file.name, file.type);
-          return {
+          documents.push({
             docId: file.name,
             content,
-            metadata: {
+            metadata: __spreadValues({
               fileName: file.name,
               fileSize: file.size,
               fileType: file.type,
               uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
-            }
-          };
-        })
-      );
+            }, dimension ? { dimension } : {})
+          });
+        }
+      }
       const results = await plugin.ingest(documents, namespace);
       return import_server.NextResponse.json({ message: "Upload successful", results });
     } catch (err) {