@retrivora-ai/rag-engine 1.8.0 → 1.8.2

package/dist/handlers/index.js

@@ -26,6 +26,18 @@ var __spreadValues = (a, b) => {
   return a;
 };
 var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
+var __objRest = (source, exclude) => {
+  var target = {};
+  for (var prop in source)
+    if (__hasOwnProp.call(source, prop) && exclude.indexOf(prop) < 0)
+      target[prop] = source[prop];
+  if (source != null && __getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(source)) {
+      if (exclude.indexOf(prop) < 0 && __propIsEnum.call(source, prop))
+        target[prop] = source[prop];
+    }
+  return target;
+};
 var __esm = (fn, res) => function __init() {
   return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
 };
@@ -322,81 +334,31 @@ var init_PineconeProvider = __esm({
   }
 });
 
-// src/providers/vectordb/PostgreSQLProvider.ts
-var PostgreSQLProvider_exports = {};
-__export(PostgreSQLProvider_exports, {
-  PostgreSQLProvider: () => PostgreSQLProvider
+// src/providers/vectordb/MultiTablePostgresProvider.ts
+var MultiTablePostgresProvider_exports = {};
+__export(MultiTablePostgresProvider_exports, {
+  MultiTablePostgresProvider: () => MultiTablePostgresProvider
 });
-var import_pg, PostgreSQLProvider;
-var init_PostgreSQLProvider = __esm({
-  "src/providers/vectordb/PostgreSQLProvider.ts"() {
+var import_pg, MultiTablePostgresProvider;
+var init_MultiTablePostgresProvider = __esm({
+  "src/providers/vectordb/MultiTablePostgresProvider.ts"() {
     "use strict";
     import_pg = require("pg");
     init_BaseVectorProvider();
-    PostgreSQLProvider = class extends BaseVectorProvider {
+    MultiTablePostgresProvider = class extends BaseVectorProvider {
       constructor(config) {
-        var _a;
+        var _a, _b, _c;
         super(config);
-        this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
-        const opts = config.options;
-        if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
+        const opts = config.options || {};
+        if (!opts.connectionString) {
+          throw new Error("[MultiTablePostgresProvider] options.connectionString is required");
+        }
         this.connectionString = opts.connectionString;
-        this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
-      }
-      static getValidator() {
-        return {
-          validate(config) {
-            const errors = [];
-            const opts = config.options || {};
-            if (!opts.connectionString) {
-              errors.push({
-                field: "vectorDb.options.connectionString",
-                message: "PostgreSQL connection string is required",
-                suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
-                severity: "error"
-              });
-            }
-            if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
-              errors.push({
-                field: "vectorDb.options.tables",
-                message: "PostgreSQL tables must be a string or a string array",
-                severity: "error"
-              });
-            }
-            return errors;
-          }
-        };
-      }
-      static getHealthChecker() {
-        return {
-          async check(config) {
-            const opts = config.options || {};
-            const timestamp = Date.now();
-            try {
-              const { Client } = await import("pg");
-              const client = new Client({ connectionString: opts.connectionString });
-              await client.connect();
-              const result = await client.query(`
-            SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
-          `);
-              const hasVector = result.rows[0].exists;
-              await client.end();
-              return {
-                healthy: true,
-                provider: "postgresql",
-                capabilities: { pgvectorInstalled: hasVector },
-                timestamp
-              };
-            } catch (error) {
-              return {
-                healthy: false,
-                provider: "postgresql",
-                error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
-                timestamp
-              };
-            }
-          }
-        };
+        this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
+        this.tables = [];
+        const rawSearchFields = (_c = (_b = opts.searchFields) != null ? _b : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _c : ["name", "product_name", "productname", "title"];
+        this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
+        this.uploadTable = opts.uploadTable || "document_chunks";
       }
       async initialize() {
         this.pool = new import_pg.Pool({ connectionString: this.connectionString });
@@ -404,7 +366,7 @@ var init_PostgreSQLProvider = __esm({
         try {
           await client.query("CREATE EXTENSION IF NOT EXISTS vector");
           await client.query(`
-        CREATE TABLE IF NOT EXISTS ${this.tableName} (
+        CREATE TABLE IF NOT EXISTS ${this.uploadTable} (
           id          TEXT PRIMARY KEY,
           namespace   TEXT NOT NULL DEFAULT '',
           content     TEXT NOT NULL,
@@ -413,53 +375,115 @@ var init_PostgreSQLProvider = __esm({
         )
       `);
           await client.query(`
-        CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
-        ON ${this.tableName}
+        CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
+        ON ${this.uploadTable}
         USING hnsw (embedding vector_cosine_ops)
       `);
+          const res = await client.query(`
+        SELECT table_name 
+        FROM information_schema.columns 
+        WHERE column_name = 'embedding' 
+        AND table_schema = 'public'
+      `);
+          this.tables = res.rows.map((r) => r.table_name);
+          if (this.tables.length === 0) {
+            console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
+          } else {
+            console.log(
+              `[MultiTablePostgresProvider] Connected. Searching across ${this.tables.length} table(s): ${this.tables.join(", ")}`
+            );
+          }
         } finally {
           client.release();
         }
       }
+      /**
+       * Upsert a document by dynamically provisioning a table and columns.
+       */
       async upsert(doc, namespace = "") {
-        var _a;
-        const vectorLiteral = `[${doc.vector.join(",")}]`;
-        await this.pool.query(
-          `INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
-       VALUES ($1, $2, $3, $4, $5::vector)
-       ON CONFLICT (id) DO UPDATE
-         SET namespace = EXCLUDED.namespace,
-             content   = EXCLUDED.content,
-             metadata  = EXCLUDED.metadata,
-             embedding = EXCLUDED.embedding`,
-          [doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
-        );
+        await this.batchUpsert([doc], namespace);
       }
+      /**
+       * Batch upsert documents by dynamically provisioning tables and columns based on CSV headers.
+       */
       async batchUpsert(docs, namespace = "") {
+        var _a;
         if (docs.length === 0) return;
+        const docsByFile = {};
+        for (const doc of docs) {
+          const fileName = ((_a = doc.metadata) == null ? void 0 : _a.fileName) || this.uploadTable;
+          if (!docsByFile[fileName]) docsByFile[fileName] = [];
+          docsByFile[fileName].push(doc);
+        }
         const client = await this.pool.connect();
         try {
           await client.query("BEGIN");
-          const BATCH_SIZE = 50;
-          for (let i = 0; i < docs.length; i += BATCH_SIZE) {
-            const batch = docs.slice(i, i + BATCH_SIZE);
-            const values = [];
-            const valuePlaceholders = batch.map((doc, idx) => {
-              var _a;
-              const offset = idx * 5;
-              values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
-              return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
-            }).join(", ");
-            const query = `
-          INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
-          VALUES ${valuePlaceholders}
-          ON CONFLICT (id) DO UPDATE
-            SET namespace = EXCLUDED.namespace,
-                content   = EXCLUDED.content,
-                metadata  = EXCLUDED.metadata,
-                embedding = EXCLUDED.embedding
+          for (const [fileName, fileDocs] of Object.entries(docsByFile)) {
+            let tableName = fileName;
+            if (tableName.toLowerCase().endsWith(".csv")) {
+              tableName = tableName.slice(0, -4);
+            }
+            tableName = tableName.replace(/[^a-z0-9_]/gi, "_").toLowerCase() || this.uploadTable;
+            const firstMeta = fileDocs[0].metadata || {};
+            const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding"];
+            const csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+            const columnDefs = csvHeaders.map((h) => `"${h}" TEXT`).join(",\n          ");
+            const createTableSql = `
+          CREATE TABLE IF NOT EXISTS "${tableName}" (
+            id          TEXT PRIMARY KEY,
+            ${columnDefs ? columnDefs + "," : ""}
+            namespace   TEXT NOT NULL DEFAULT '',
+            content     TEXT NOT NULL,
+            metadata    JSONB,
+            embedding   VECTOR(${this.dimensions})
+          )
         `;
-            await client.query(query, values);
+            await client.query(createTableSql);
+            await client.query(`
+          CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
+          ON "${tableName}"
+          USING hnsw (embedding vector_cosine_ops)
+        `);
+            if (!this.tables.includes(tableName)) {
+              this.tables.push(tableName);
+            }
+            const BATCH_SIZE = 50;
+            for (let i = 0; i < fileDocs.length; i += BATCH_SIZE) {
+              const batch = fileDocs.slice(i, i + BATCH_SIZE);
+              const values = [];
+              const headerColumns = csvHeaders.map((h) => `"${h}"`).join(", ");
+              const allColumns = `id, ${headerColumns ? headerColumns + ", " : ""}namespace, content, metadata, embedding`;
+              const valuePlaceholders = batch.map((doc, idx) => {
+                var _a2, _b;
+                const offset = idx * (5 + csvHeaders.length);
+                values.push(doc.id);
+                for (const h of csvHeaders) {
+                  values.push(String(((_a2 = doc.metadata) == null ? void 0 : _a2[h]) || ""));
+                }
+                values.push(namespace, doc.content, JSON.stringify((_b = doc.metadata) != null ? _b : {}), `[${doc.vector.join(",")}]`);
+                const docPlaceholders = [];
+                for (let j = 1; j <= 5 + csvHeaders.length; j++) {
+                  if (j === 5 + csvHeaders.length) {
+                    docPlaceholders.push(`$${offset + j}::vector`);
+                  } else {
+                    docPlaceholders.push(`$${offset + j}`);
+                  }
+                }
+                return `(${docPlaceholders.join(", ")})`;
+              }).join(", ");
+              const conflictUpdates = csvHeaders.map((h) => `"${h}" = EXCLUDED."${h}"`).join(",\n              ");
+              const query = `
+            INSERT INTO "${tableName}" (${allColumns})
+            VALUES ${valuePlaceholders}
+            ON CONFLICT (id) DO UPDATE
+              SET ${conflictUpdates ? conflictUpdates + "," : ""}
+                  namespace = EXCLUDED.namespace,
+                  content   = EXCLUDED.content,
+                  metadata  = EXCLUDED.metadata,
+                  embedding = EXCLUDED.embedding
+          `;
+              await client.query(query, values);
+            }
           }
           await client.query("COMMIT");
         } catch (error) {
@@ -469,49 +493,119 @@ var init_PostgreSQLProvider = __esm({
           client.release();
         }
       }
-      async query(vector, topK, namespace, filter) {
-        const vectorLiteral = `[${vector.join(",")}]`;
-        let whereClause = namespace ? `WHERE namespace = $3` : "";
-        const params = [vectorLiteral, topK];
-        if (namespace) params.push(namespace);
-        const publicFilter = this.sanitizeFilter(filter);
-        if (Object.keys(publicFilter).length > 0) {
-          const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
-            const paramIdx = params.length + 1;
-            params.push(JSON.stringify(val));
-            return `metadata->>'${key}' = $${paramIdx}`;
-          }).join(" AND ");
-          whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
+      /**
+       * Query all configured tables and merge results, sorted by cosine similarity score.
+       */
+      async query(vector, topK, _namespace, _filter) {
+        var _a;
+        if (!this.pool) {
+          throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
         }
-        const client = await this.pool.connect();
-        try {
-          const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
-          await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
-          const result = await client.query(
-            `SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
-         FROM ${this.tableName}
-         ${whereClause}
-         ORDER BY embedding <=> $1::vector
-         LIMIT $2`,
-            params
-          );
-          return result.rows.map((row) => ({
-            id: String(row["id"]),
-            score: parseFloat(String(row["score"])),
-            content: String(row["content"]),
-            metadata: row["metadata"]
-          }));
-        } finally {
-          client.release();
+        const vectorLiteral = `[${vector.join(",")}]`;
+        const allResults = [];
+        console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
+        const queryText = _filter == null ? void 0 : _filter.queryText;
+        const entityHints = Array.isArray(_filter == null ? void 0 : _filter.__entityHints) ? _filter.__entityHints.filter(
+          (hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
+        ).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
+        console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
+        console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
+        const getDynamicKeywordQuery = () => {
+          if (entityHints.length > 0) {
+            return entityHints.map((h) => h.includes(" ") ? `"${h}"` : h).join(" & ");
+          }
+          if (queryText) {
+            return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
+          }
+          return "";
+        };
+        const dynamicKeywordQuery = getDynamicKeywordQuery();
+        const queryPromises = this.tables.map(async (table) => {
+          try {
+            let sqlQuery = "";
+            let params = [];
+            if (queryText) {
+              const hasEntityHints = entityHints.length > 0;
+              const exactNameScoreExpr = hasEntityHints ? `+ (
+                 SELECT COALESCE(MAX(
+                   CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(", ")})
+                   THEN 1.0 ELSE 0.0 END
+                 ), 0)
+                 FROM jsonb_each_text(to_jsonb(t)) AS kv(key, val)
+                 WHERE key IN (${this.searchFields.map((f) => `'${f}'`).join(", ")})
+               ) * 3.0` : "";
+              sqlQuery = `
+            SELECT *, 
+             (1 - (embedding <=> $1::vector)) AS vector_score,
+             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
+             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
+            FROM "${table}" t
+            ORDER BY hybrid_score DESC
+            LIMIT 50
+          `;
+              params = [vectorLiteral, dynamicKeywordQuery];
+            } else {
+              sqlQuery = `
+            SELECT *, 
+             (1 - (embedding <=> $1::vector)) AS hybrid_score
+            FROM "${table}" t
+            ORDER BY hybrid_score DESC
+            LIMIT 50
+          `;
+              params = [vectorLiteral];
+            }
+            const result = await this.pool.query(sqlQuery, params);
+            if (result.rowCount && result.rowCount > 0) {
+              console.log(`   \u2705 Table "${table}": Found ${result.rowCount} potential matches. Top score: ${result.rows[0].hybrid_score.toFixed(4)}`);
+            } else {
+              console.log(`   \u26AA Table "${table}": No relevant matches found.`);
+            }
+            const tableResults = [];
+            for (const row of result.rows) {
+              const _a2 = row, { hybrid_score, id } = _a2, rest = __objRest(_a2, ["hybrid_score", "id"]);
+              delete rest.embedding;
+              delete rest.vector_score;
+              delete rest.keyword_score;
+              delete rest.exact_name_score;
+              const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
+` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
+              tableResults.push({
+                id: `${table}-${id}`,
+                score: parseFloat(String(hybrid_score)),
+                content,
+                metadata: __spreadProps(__spreadValues({}, rest), { source_table: table })
+              });
+            }
+            return tableResults;
+          } catch (err) {
+            console.error(
+              `[MultiTablePostgresProvider] Error querying table "${table}":`,
+              err.message
+            );
+            return [];
+          }
+        });
+        const resultsArray = await Promise.all(queryPromises);
+        for (const tableResults of resultsArray) {
+          allResults.push(...tableResults);
         }
+        allResults.sort((a, b) => b.score - a.score);
+        if (allResults.length === 0) return [];
+        const bestMatchTable = (_a = allResults[0].metadata) == null ? void 0 : _a.source_table;
+        const finalSorted = allResults.filter((res) => {
+          var _a2;
+          return ((_a2 = res.metadata) == null ? void 0 : _a2.source_table) === bestMatchTable;
+        });
+        console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
+        console.log(`[MultiTablePostgresProvider] Final top match from "${bestMatchTable}" with score ${finalSorted[0].score.toFixed(4)}`);
+        console.log(`[MultiTablePostgresProvider] Restricting recommendations strictly to table: "${bestMatchTable}"`);
+        return finalSorted.slice(0, Math.max(topK, 15));
       }
-      async delete(id, namespace) {
-        const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
-        const params = namespace ? [id, namespace] : [id];
-        await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
+      async delete(_id, _namespace) {
+        console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
       }
-      async deleteNamespace(namespace) {
-        await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
+      async deleteNamespace(_namespace) {
+        console.warn("[MultiTablePostgresProvider] deleteNamespace() is a no-op for multi-table mode.");
       }
       async ping() {
         try {
@@ -522,7 +616,9 @@ var init_PostgreSQLProvider = __esm({
         }
       }
       async disconnect() {
-        await this.pool.end();
+        if (this.pool) {
+          await this.pool.end();
+        }
       }
     };
   }
@@ -1617,6 +1713,7 @@ __export(handlers_exports, {
   sseErrorFrame: () => sseErrorFrame,
   sseFrame: () => sseFrame,
   sseMetaFrame: () => sseMetaFrame,
+  sseObservabilityFrame: () => sseObservabilityFrame,
   sseTextFrame: () => sseTextFrame,
   sseUIFrame: () => sseUIFrame
 });
@@ -1691,7 +1788,7 @@ function readEnum(env, name, fallback, allowed) {
   throw new Error(`[getRagConfig] ${name} must be one of: ${allowed.join(", ")}`);
 }
 function getEnvConfig(env = process.env, base) {
-  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va;
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa;
   const projectId = (_c = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : base == null ? void 0 : base.projectId) != null ? _c : "__default__";
   const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
   const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
@@ -1702,33 +1799,35 @@ function getEnvConfig(env = process.env, base) {
     vectorDbOptions.apiKey = (_g = (_f = readString(env, "PINECONE_API_KEY")) != null ? _f : (_e = (_d = base == null ? void 0 : base.vectorDb) == null ? void 0 : _d.options) == null ? void 0 : _e.apiKey) != null ? _g : "";
     vectorDbOptions.indexName = (_l = (_i = readString(env, "PINECONE_INDEX")) != null ? _i : (_h = base == null ? void 0 : base.vectorDb) == null ? void 0 : _h.indexName) != null ? _l : (_k = (_j = base == null ? void 0 : base.vectorDb) == null ? void 0 : _j.options) == null ? void 0 : _k.indexName;
   } else if (vectorProvider === "pgvector" || vectorProvider === "postgresql") {
-    vectorDbOptions.connectionString = (_p = (_o = readString(env, "PGVECTOR_CONNECTION_STRING")) != null ? _o : (_n = (_m = base == null ? void 0 : base.vectorDb) == null ? void 0 : _m.options) == null ? void 0 : _n.connectionString) != null ? _p : "";
+    vectorDbOptions.connectionString = (_q = (_p = (_m = readString(env, "PGVECTOR_CONNECTION_STRING")) != null ? _m : readString(env, "POSTGRES_URL")) != null ? _p : (_o = (_n = base == null ? void 0 : base.vectorDb) == null ? void 0 : _n.options) == null ? void 0 : _o.connectionString) != null ? _q : "";
+    vectorDbOptions.tables = (_v = (_s = (_r = readString(env, "VECTOR_DB_TABLES")) != null ? _r : readString(env, "POSTGRES_TABLES")) == null ? void 0 : _s.split(",").map((t) => t.trim())) != null ? _v : (_u = (_t = base == null ? void 0 : base.vectorDb) == null ? void 0 : _t.options) == null ? void 0 : _u.tables;
+    vectorDbOptions.searchFields = (_z = (_w = readString(env, "POSTGRES_SEARCH_FIELDS")) == null ? void 0 : _w.split(",").map((f) => f.trim())) != null ? _z : (_y = (_x = base == null ? void 0 : base.vectorDb) == null ? void 0 : _x.options) == null ? void 0 : _y.searchFields;
     vectorDbOptions.dimensions = embeddingDimensions;
   } else if (vectorProvider === "mongodb") {
-    vectorDbOptions.uri = (_t = (_s = readString(env, "MONGODB_URI")) != null ? _s : (_r = (_q = base == null ? void 0 : base.vectorDb) == null ? void 0 : _q.options) == null ? void 0 : _r.uri) != null ? _t : "";
-    vectorDbOptions.database = (_x = (_w = readString(env, "MONGODB_DB")) != null ? _w : (_v = (_u = base == null ? void 0 : base.vectorDb) == null ? void 0 : _u.options) == null ? void 0 : _v.database) != null ? _x : "";
-    vectorDbOptions.collection = (_B = (_A = readString(env, "MONGODB_COLLECTION")) != null ? _A : (_z = (_y = base == null ? void 0 : base.vectorDb) == null ? void 0 : _y.options) == null ? void 0 : _z.collection) != null ? _B : "";
-    vectorDbOptions.indexName = (_H = (_G = (_D = readString(env, "MONGODB_INDEX_NAME")) != null ? _D : (_C = base == null ? void 0 : base.vectorDb) == null ? void 0 : _C.indexName) != null ? _G : (_F = (_E = base == null ? void 0 : base.vectorDb) == null ? void 0 : _E.options) == null ? void 0 : _F.indexName) != null ? _H : "vector_index";
+    vectorDbOptions.uri = (_D = (_C = readString(env, "MONGODB_URI")) != null ? _C : (_B = (_A = base == null ? void 0 : base.vectorDb) == null ? void 0 : _A.options) == null ? void 0 : _B.uri) != null ? _D : "";
+    vectorDbOptions.database = (_H = (_G = readString(env, "MONGODB_DB")) != null ? _G : (_F = (_E = base == null ? void 0 : base.vectorDb) == null ? void 0 : _E.options) == null ? void 0 : _F.database) != null ? _H : "";
+    vectorDbOptions.collection = (_L = (_K = readString(env, "MONGODB_COLLECTION")) != null ? _K : (_J = (_I = base == null ? void 0 : base.vectorDb) == null ? void 0 : _I.options) == null ? void 0 : _J.collection) != null ? _L : "";
+    vectorDbOptions.indexName = (_R = (_Q = (_N = readString(env, "MONGODB_INDEX_NAME")) != null ? _N : (_M = base == null ? void 0 : base.vectorDb) == null ? void 0 : _M.indexName) != null ? _Q : (_P = (_O = base == null ? void 0 : base.vectorDb) == null ? void 0 : _O.options) == null ? void 0 : _P.indexName) != null ? _R : "vector_index";
   } else if (vectorProvider === "qdrant") {
-    vectorDbOptions.baseUrl = (_L = (_K = readString(env, "QDRANT_URL")) != null ? _K : (_J = (_I = base == null ? void 0 : base.vectorDb) == null ? void 0 : _I.options) == null ? void 0 : _J.baseUrl) != null ? _L : "http://localhost:6333";
-    vectorDbOptions.apiKey = (_O = readString(env, "QDRANT_API_KEY")) != null ? _O : (_N = (_M = base == null ? void 0 : base.vectorDb) == null ? void 0 : _M.options) == null ? void 0 : _N.apiKey;
+    vectorDbOptions.baseUrl = (_V = (_U = readString(env, "QDRANT_URL")) != null ? _U : (_T = (_S = base == null ? void 0 : base.vectorDb) == null ? void 0 : _S.options) == null ? void 0 : _T.baseUrl) != null ? _V : "http://localhost:6333";
+    vectorDbOptions.apiKey = (_Y = readString(env, "QDRANT_API_KEY")) != null ? _Y : (_X = (_W = base == null ? void 0 : base.vectorDb) == null ? void 0 : _W.options) == null ? void 0 : _X.apiKey;
     vectorDbOptions.dimensions = embeddingDimensions;
   } else if (vectorProvider === "milvus") {
-    vectorDbOptions.baseUrl = (_P = readString(env, "MILVUS_URL")) != null ? _P : "http://localhost:19530";
+    vectorDbOptions.baseUrl = (_Z = readString(env, "MILVUS_URL")) != null ? _Z : "http://localhost:19530";
     vectorDbOptions.apiKey = readString(env, "MILVUS_API_KEY");
   } else if (vectorProvider === "chromadb") {
-    vectorDbOptions.baseUrl = (_Q = readString(env, "CHROMADB_URL")) != null ? _Q : "http://localhost:8000";
+    vectorDbOptions.baseUrl = (__ = readString(env, "CHROMADB_URL")) != null ? __ : "http://localhost:8000";
   } else if (vectorProvider === "weaviate") {
-    vectorDbOptions.baseUrl = (_R = readString(env, "WEAVIATE_URL")) != null ? _R : "http://localhost:8080";
+    vectorDbOptions.baseUrl = (_$ = readString(env, "WEAVIATE_URL")) != null ? _$ : "http://localhost:8080";
     vectorDbOptions.apiKey = readString(env, "WEAVIATE_API_KEY");
   } else if (vectorProvider === "redis") {
-    vectorDbOptions.baseUrl = (_S = readString(env, "REDIS_URL")) != null ? _S : "";
+    vectorDbOptions.baseUrl = (_aa = readString(env, "REDIS_URL")) != null ? _aa : "";
     vectorDbOptions.apiKey = readString(env, "REDIS_API_KEY");
   } else if (vectorProvider === "rest") {
-    vectorDbOptions.baseUrl = (_T = readString(env, "VECTOR_DB_REST_URL")) != null ? _T : "";
+    vectorDbOptions.baseUrl = (_ba = readString(env, "VECTOR_DB_REST_URL")) != null ? _ba : "";
     vectorDbOptions.headers = readString(env, "VECTOR_DB_REST_API_KEY") ? { "api-key": readString(env, "VECTOR_DB_REST_API_KEY") } : {};
   } else if (vectorProvider === "universal_rest") {
-    vectorDbOptions.baseUrl = (_V = (_U = readString(env, "VECTOR_BASE_URL")) != null ? _U : readString(env, "VECTOR_DB_REST_URL")) != null ? _V : "";
+    vectorDbOptions.baseUrl = (_da = (_ca = readString(env, "VECTOR_BASE_URL")) != null ? _ca : readString(env, "VECTOR_DB_REST_URL")) != null ? _da : "";
     vectorDbOptions.profile = readString(env, "VECTOR_UNIVERSAL_PROFILE");
     vectorDbOptions.headers = readString(env, "VECTOR_DB_REST_API_KEY") ? { Authorization: `Bearer ${readString(env, "VECTOR_DB_REST_API_KEY")}` } : {};
   }
@@ -1745,20 +1844,20 @@ function getEnvConfig(env = process.env, base) {
     openai: readString(env, "OPENAI_API_KEY"),
     gemini: readString(env, "GEMINI_API_KEY"),
     ollama: void 0,
-    universal_rest: (_W = readString(env, "EMBEDDING_API_KEY")) != null ? _W : readString(env, "OPENAI_API_KEY"),
-    custom: (_X = readString(env, "EMBEDDING_API_KEY")) != null ? _X : readString(env, "OPENAI_API_KEY")
+    universal_rest: (_ea = readString(env, "EMBEDDING_API_KEY")) != null ? _ea : readString(env, "OPENAI_API_KEY"),
+    custom: (_fa = readString(env, "EMBEDDING_API_KEY")) != null ? _fa : readString(env, "OPENAI_API_KEY")
   };
   return {
     projectId,
     vectorDb: {
       provider: vectorProvider,
-      indexName: (_Z = (_Y = readString(env, "VECTOR_DB_INDEX")) != null ? _Y : vectorDbOptions.indexName) != null ? _Z : "rag-index",
+      indexName: (_ha = (_ga = readString(env, "VECTOR_DB_INDEX")) != null ? _ga : vectorDbOptions.indexName) != null ? _ha : "rag-index",
       options: vectorDbOptions
     },
     llm: {
       provider: llmProvider,
-      model: (__ = readString(env, "LLM_MODEL")) != null ? __ : "gpt-4o",
-      apiKey: (_$ = llmApiKeyByProvider[llmProvider]) != null ? _$ : "",
+      model: (_ia = readString(env, "LLM_MODEL")) != null ? _ia : "gpt-4o",
+      apiKey: (_ja = llmApiKeyByProvider[llmProvider]) != null ? _ja : "",
       baseUrl: readString(env, "LLM_BASE_URL"),
       systemPrompt: readString(env, "LLM_SYSTEM_PROMPT"),
       maxTokens: readNumber(env, "LLM_MAX_TOKENS", 4096),
@@ -1769,7 +1868,7 @@ function getEnvConfig(env = process.env, base) {
     },
     embedding: {
       provider: embeddingProvider,
-      model: (_aa = readString(env, "EMBEDDING_MODEL")) != null ? _aa : "text-embedding-3-small",
+      model: (_ka = readString(env, "EMBEDDING_MODEL")) != null ? _ka : "text-embedding-3-small",
       apiKey: embeddingApiKeyByProvider[embeddingProvider],
       baseUrl: readString(env, "EMBEDDING_BASE_URL"),
       dimensions: embeddingDimensions,
@@ -1780,17 +1879,17 @@ function getEnvConfig(env = process.env, base) {
       }
     },
     ui: {
-      title: (_ca = (_ba = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _ba : readString(env, "UI_TITLE")) != null ? _ca : "AI Assistant",
-      subtitle: (_ea = (_da = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _da : readString(env, "UI_SUBTITLE")) != null ? _ea : "Powered by RAG",
-      primaryColor: (_ga = (_fa = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _fa : readString(env, "UI_PRIMARY_COLOR")) != null ? _ga : "#10b981",
-      accentColor: (_ia = (_ha = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _ha : readString(env, "UI_ACCENT_COLOR")) != null ? _ia : "#3b82f6",
-      logoUrl: (_ja = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _ja : readString(env, "UI_LOGO_URL"),
-      placeholder: (_la = (_ka = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _ka : readString(env, "UI_PLACEHOLDER")) != null ? _la : "Ask me anything\u2026",
-      showSources: ((_na = (_ma = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _ma : readString(env, "UI_SHOW_SOURCES")) != null ? _na : "true") !== "false",
-      welcomeMessage: (_pa = (_oa = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _oa : readString(env, "UI_WELCOME_MESSAGE")) != null ? _pa : "Hello! I'm your AI assistant. Ask me anything about your documents.",
-      visualStyle: (_ra = (_qa = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _qa : readString(env, "UI_VISUAL_STYLE")) != null ? _ra : "glass",
-      borderRadius: (_ta = (_sa = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _sa : readString(env, "UI_BORDER_RADIUS")) != null ? _ta : "xl",
-      allowUpload: ((_va = (_ua = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _ua : readString(env, "UI_ALLOW_UPLOAD")) != null ? _va : "false") === "true"
+      title: (_ma = (_la = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _la : readString(env, "UI_TITLE")) != null ? _ma : "AI Assistant",
+      subtitle: (_oa = (_na = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _na : readString(env, "UI_SUBTITLE")) != null ? _oa : "Powered by RAG",
+      primaryColor: (_qa = (_pa = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _pa : readString(env, "UI_PRIMARY_COLOR")) != null ? _qa : "#10b981",
+      accentColor: (_sa = (_ra = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _ra : readString(env, "UI_ACCENT_COLOR")) != null ? _sa : "#3b82f6",
+      logoUrl: (_ta = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _ta : readString(env, "UI_LOGO_URL"),
+      placeholder: (_va = (_ua = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _ua : readString(env, "UI_PLACEHOLDER")) != null ? _va : "Ask me anything\u2026",
+      showSources: ((_xa = (_wa = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _wa : readString(env, "UI_SHOW_SOURCES")) != null ? _xa : "true") !== "false",
+      welcomeMessage: (_za = (_ya = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _ya : readString(env, "UI_WELCOME_MESSAGE")) != null ? _za : "Hello! I'm your AI assistant. Ask me anything about your documents.",
+      visualStyle: (_Ba = (_Aa = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _Aa : readString(env, "UI_VISUAL_STYLE")) != null ? _Ba : "glass",
+      borderRadius: (_Da = (_Ca = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _Ca : readString(env, "UI_BORDER_RADIUS")) != null ? _Da : "xl",
+      allowUpload: ((_Fa = (_Ea = readString(env, "NEXT_PUBLIC_ALLOW_UPLOAD")) != null ? _Ea : readString(env, "UI_ALLOW_UPLOAD")) != null ? _Fa : "false") === "true"
     },
     rag: {
       topK: readNumber(env, "RAG_TOP_K", 5),
@@ -1907,14 +2006,14 @@ var OpenAIProvider = class {
     };
   }
   async chat(messages, context, options) {
-    var _a, _b, _c, _d, _e, _f, _g, _h;
-    const systemContent = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Answer questions based on the provided context.
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i;
+    const resolvedSystemPrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : `You are a helpful assistant. Answer questions based on the provided context.
 
 Context:
 ${context}`;
     const systemMessage = {
       role: "system",
-      content: systemContent.includes("{{context}}") ? systemContent.replace("{{context}}", context) : `${systemContent}
+      content: resolvedSystemPrompt.includes("{{context}}") ? resolvedSystemPrompt.replace("{{context}}", context) : (options == null ? void 0 : options.systemPrompt) ? resolvedSystemPrompt : `${resolvedSystemPrompt}
 
 Context:
 ${context}`
@@ -1929,22 +2028,22 @@ ${context}`
     const completion = await this.client.chat.completions.create({
       model: this.llmConfig.model,
       messages: formattedMessages,
-      max_tokens: (_c = (_b = options == null ? void 0 : options.maxTokens) != null ? _b : this.llmConfig.maxTokens) != null ? _c : 1024,
-      temperature: (_e = (_d = options == null ? void 0 : options.temperature) != null ? _d : this.llmConfig.temperature) != null ? _e : 0.7,
+      max_tokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024,
+      temperature: (_f = (_e = options == null ? void 0 : options.temperature) != null ? _e : this.llmConfig.temperature) != null ? _f : 0.7,
       stop: options == null ? void 0 : options.stop
     });
-    return (_h = (_g = (_f = completion.choices[0]) == null ? void 0 : _f.message) == null ? void 0 : _g.content) != null ? _h : "";
+    return (_i = (_h = (_g = completion.choices[0]) == null ? void 0 : _g.message) == null ? void 0 : _h.content) != null ? _i : "";
   }
   chatStream(messages, context, options) {
     return __asyncGenerator(this, null, function* () {
-      var _a, _b, _c, _d, _e, _f, _g;
-      const systemContent = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Answer questions based on the provided context.
+      var _a, _b, _c, _d, _e, _f, _g, _h;
+      const resolvedSystemPrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : `You are a helpful assistant. Answer questions based on the provided context.
 
 Context:
 ${context}`;
       const systemMessage = {
         role: "system",
-        content: systemContent.includes("{{context}}") ? systemContent.replace("{{context}}", context) : `${systemContent}
+        content: resolvedSystemPrompt.includes("{{context}}") ? resolvedSystemPrompt.replace("{{context}}", context) : (options == null ? void 0 : options.systemPrompt) ? resolvedSystemPrompt : `${resolvedSystemPrompt}
 
 Context:
 ${context}`
@@ -1959,8 +2058,8 @@ ${context}`
       const stream = yield new __await(this.client.chat.completions.create({
         model: this.llmConfig.model,
         messages: formattedMessages,
-        max_tokens: (_c = (_b = options == null ? void 0 : options.maxTokens) != null ? _b : this.llmConfig.maxTokens) != null ? _c : 1024,
-        temperature: (_e = (_d = options == null ? void 0 : options.temperature) != null ? _d : this.llmConfig.temperature) != null ? _e : 0.7,
+        max_tokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024,
+        temperature: (_f = (_e = options == null ? void 0 : options.temperature) != null ? _e : this.llmConfig.temperature) != null ? _f : 0.7,
         stop: options == null ? void 0 : options.stop,
         stream: true
       }));
@@ -1970,7 +2069,7 @@ ${context}`
       try {
         for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
           const chunk = temp.value;
-          const content = ((_g = (_f = chunk.choices[0]) == null ? void 0 : _f.delta) == null ? void 0 : _g.content) || "";
+          const content = ((_h = (_g = chunk.choices[0]) == null ? void 0 : _g.delta) == null ? void 0 : _h.content) || "";
           if (content) yield content;
         }
       } catch (temp) {
@@ -2068,12 +2167,12 @@ var AnthropicProvider = class {
     };
   }
   async chat(messages, context, options) {
-    var _a, _b, _c;
-    const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Use the context below to answer the user's question accurately.
+    var _a, _b, _c, _d;
+    const resolvedPrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : `You are a helpful assistant. Use the context below to answer the user's question accurately.
 
 Context:
 ${context}`;
-    const system = systemPrompt.includes("{{context}}") ? systemPrompt.replace("{{context}}", context) : `${systemPrompt}
+    const system = resolvedPrompt.includes("{{context}}") ? resolvedPrompt.replace("{{context}}", context) : (options == null ? void 0 : options.systemPrompt) ? resolvedPrompt : `${resolvedPrompt}
 
 Context:
 ${context}`;
@@ -2083,7 +2182,7 @@ ${context}`;
     }));
     const response = await this.client.messages.create({
       model: this.llmConfig.model,
-      max_tokens: (_c = (_b = options == null ? void 0 : options.maxTokens) != null ? _b : this.llmConfig.maxTokens) != null ? _c : 1024,
+      max_tokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024,
       system,
       messages: anthropicMessages
     });
@@ -2092,12 +2191,12 @@ ${context}`;
   }
   chatStream(messages, context, options) {
     return __asyncGenerator(this, null, function* () {
-      var _a, _b, _c;
-      const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Use the context below to answer the user's question accurately.
+      var _a, _b, _c, _d;
+      const resolvedPrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : `You are a helpful assistant. Use the context below to answer the user's question accurately.
 
 Context:
 ${context}`;
-      const system = systemPrompt.includes("{{context}}") ? systemPrompt.replace("{{context}}", context) : `${systemPrompt}
+      const system = resolvedPrompt.includes("{{context}}") ? resolvedPrompt.replace("{{context}}", context) : (options == null ? void 0 : options.systemPrompt) ? resolvedPrompt : `${resolvedPrompt}
 
 Context:
 ${context}`;
@@ -2107,7 +2206,7 @@ ${context}`;
       }));
       const stream = yield new __await(this.client.messages.create({
         model: this.llmConfig.model,
-        max_tokens: (_c = (_b = options == null ? void 0 : options.maxTokens) != null ? _b : this.llmConfig.maxTokens) != null ? _c : 1024,
+        max_tokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024,
         system,
         messages: anthropicMessages,
         stream: true
@@ -2222,7 +2321,7 @@ var OllamaProvider = class {
   }
   async chat(messages, context, options) {
     var _a, _b, _c, _d;
-    const system = this.buildSystemPrompt(context);
+    const system = this.buildSystemPrompt(context, options == null ? void 0 : options.systemPrompt);
     const { data } = await this.http.post("/api/chat", {
       model: this.llmConfig.model,
       stream: false,
@@ -2240,7 +2339,7 @@ var OllamaProvider = class {
   chatStream(messages, context, options) {
     return __asyncGenerator(this, null, function* () {
       var _a, _b, _c, _d, _e, _f;
-      const system = this.buildSystemPrompt(context);
+      const system = this.buildSystemPrompt(context, options == null ? void 0 : options.systemPrompt);
       const response = yield new __await(this.http.post("/api/chat", {
         model: this.llmConfig.model,
         stream: true,
@@ -2296,8 +2395,11 @@ var OllamaProvider = class {
       }
     });
   }
-  buildSystemPrompt(context) {
+  buildSystemPrompt(context, overridePrompt) {
     var _a;
+    if (overridePrompt) {
+      return overridePrompt;
+    }
     const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Use the provided context to answer the user's question.
 
 Context:
@@ -2349,20 +2451,41 @@ ${context}`;
 };
 
 // src/llm/providers/GeminiProvider.ts
-var import_genai = require("@google/genai");
+var import_generative_ai = require("@google/generative-ai");
+var DEFAULT_EMBEDDING_MODEL = "text-embedding-004";
+var DEFAULT_TEMPERATURE = 0.7;
+var DEFAULT_MAX_TOKENS = 1024;
+function sanitizeModel(model) {
+  return model ? model.split(":")[0] : model;
+}
+function buildClient(apiKey) {
+  return new import_generative_ai.GoogleGenerativeAI(apiKey);
+}
+function applyPrefix(text, taskType, queryPrefix, docPrefix) {
+  if (taskType === "query" && queryPrefix && !text.startsWith(queryPrefix)) {
+    return `${queryPrefix}${text}`;
+  }
+  if (taskType === "document" && docPrefix && !text.startsWith(docPrefix)) {
+    return `${docPrefix}${text}`;
+  }
+  return text;
+}
 var GeminiProvider = class {
   constructor(llmConfig, embeddingConfig) {
-    if (!llmConfig.apiKey) throw new Error("[GeminiProvider] llmConfig.apiKey is required");
-    this.client = new import_genai.GoogleGenAI({ apiKey: llmConfig.apiKey });
-    this.llmConfig = __spreadProps(__spreadValues({}, llmConfig), {
-      model: this.sanitizeModel(llmConfig.model)
-    });
+    if (!llmConfig.apiKey) {
+      throw new Error("[GeminiProvider] llmConfig.apiKey is required");
+    }
+    this.llmConfig = __spreadProps(__spreadValues({}, llmConfig), { model: sanitizeModel(llmConfig.model) });
+    this.client = buildClient(this.llmConfig.apiKey);
     if (embeddingConfig) {
       this.embeddingConfig = __spreadProps(__spreadValues({}, embeddingConfig), {
-        model: this.sanitizeModel(embeddingConfig.model)
+        model: sanitizeModel(embeddingConfig.model)
       });
     }
   }
+  // -------------------------------------------------------------------------
+  // Static factory helpers
+  // -------------------------------------------------------------------------
   static getValidator() {
     return {
       validate(config) {
@@ -2376,7 +2499,11 @@ var GeminiProvider = class {
           });
         }
         if (!config.model) {
-          errors.push({ field: "llm.model", message: "Gemini model name is required", severity: "error" });
+          errors.push({
+            field: "llm.model",
+            message: "Gemini model name is required",
+            severity: "error"
+          });
         }
         return errors;
       }
@@ -2385,13 +2512,15 @@ var GeminiProvider = class {
   static getHealthChecker() {
     return {
       async check(config) {
+        var _a, _b;
         const timestamp = Date.now();
-        const apiKey = config.apiKey || process.env.GOOGLE_GENAI_API_KEY || "";
-        const modelName = config.model;
+        const apiKey = (_b = (_a = config.apiKey) != null ? _a : process.env.GOOGLE_GENAI_API_KEY) != null ? _b : "";
+        const modelName = sanitizeModel(config.model);
         try {
-          const { GoogleGenAI: GoogleGenAI2 } = await import("@google/genai");
-          const genAI = new GoogleGenAI2({ apiKey });
-          await genAI.models.get({ model: modelName });
+          const { GoogleGenerativeAI: GoogleGenerativeAI2 } = await import("@google/generative-ai");
+          const ai = new GoogleGenerativeAI2(apiKey);
+          const model = ai.getGenerativeModel({ model: DEFAULT_EMBEDDING_MODEL });
+          await model.embedContent("health-check");
           return {
             healthy: true,
             provider: "gemini",
@@ -2409,70 +2538,90 @@ var GeminiProvider = class {
       }
     };
   }
-  sanitizeModel(model) {
-    if (!model) return model;
-    return model.split(":")[0];
+  // -------------------------------------------------------------------------
+  // Private utilities
+  // -------------------------------------------------------------------------
+  /** Resolve the embedding client — uses a separate client when the embedding
+   *  API key differs from the LLM API key. */
+  get embeddingClient() {
+    var _a;
+    if (((_a = this.embeddingConfig) == null ? void 0 : _a.apiKey) && this.embeddingConfig.apiKey !== this.llmConfig.apiKey) {
+      return buildClient(this.embeddingConfig.apiKey);
+    }
+    return this.client;
   }
-  async chat(messages, context, options) {
-    var _a, _b, _c, _d, _e, _f;
-    const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Answer questions based on the provided context.
+  /** Resolve the embedding model to use, in order of specificity. */
+  resolveEmbeddingModel(optionsModel) {
+    var _a, _b;
+    return sanitizeModel((_b = optionsModel != null ? optionsModel : (_a = this.embeddingConfig) == null ? void 0 : _a.model) != null ? _b : DEFAULT_EMBEDDING_MODEL);
+  }
+  /**
+   * Build the system instruction string, inserting the RAG context either via
+   * the `{{context}}` placeholder or by appending it.
+   */
+  buildSystemInstruction(context) {
+    var _a;
+    const base = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Answer questions based on the provided context.
 
 Context:
 ${context}`;
-    const systemInstruction = systemPrompt.includes("{{context}}") ? systemPrompt.replace("{{context}}", context) : `${systemPrompt}
+    return base.includes("{{context}}") ? base.replace("{{context}}", context) : `${base}
 
 Context:
 ${context}`;
-    const geminiMessages = messages.map((m) => ({
+  }
+  /**
+   * Convert ChatMessage[] to the Gemini contents format.
+   *
+   * Rules enforced by the Gemini API:
+   *   - Only `user` and `model` roles are allowed in `contents`.
+   *   - System messages must be passed via `systemInstruction`, not here.
+   *   - Messages must strictly alternate between `user` and `model`.
+   */
+  buildGeminiContents(messages) {
+    return messages.filter((m) => m.role !== "system").map((m) => ({
       role: m.role === "assistant" ? "model" : "user",
       parts: [{ text: m.content }]
     }));
-    const response = await this.client.models.generateContent({
+  }
+  // -------------------------------------------------------------------------
+  // ILLMProvider — chat
+  // -------------------------------------------------------------------------
+  async chat(messages, context, options) {
+    var _a, _b, _c, _d;
+    const model = this.client.getGenerativeModel({
       model: this.llmConfig.model,
-      contents: geminiMessages,
-      config: {
-        systemInstruction,
-        temperature: (_c = (_b = options == null ? void 0 : options.temperature) != null ? _b : this.llmConfig.temperature) != null ? _c : 0.7,
-        maxOutputTokens: (_e = (_d = options == null ? void 0 : options.maxTokens) != null ? _d : this.llmConfig.maxTokens) != null ? _e : 1024,
+      systemInstruction: this.buildSystemInstruction(context)
+    });
+    const result = await model.generateContent({
+      contents: this.buildGeminiContents(messages),
+      generationConfig: {
+        temperature: (_b = (_a = options == null ? void 0 : options.temperature) != null ? _a : this.llmConfig.temperature) != null ? _b : DEFAULT_TEMPERATURE,
+        maxOutputTokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : DEFAULT_MAX_TOKENS,
         stopSequences: options == null ? void 0 : options.stop
       }
     });
-    const text = typeof response.text === "function" ? response.text() : (_f = response.text) != null ? _f : "";
-    return text;
+    return result.response.text();
   }
   chatStream(messages, context, options) {
     return __asyncGenerator(this, null, function* () {
-      var _a, _b, _c, _d, _e, _f;
-      const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Answer questions based on the provided context.
-
-Context:
-${context}`;
-      const systemInstruction = systemPrompt.includes("{{context}}") ? systemPrompt.replace("{{context}}", context) : `${systemPrompt}
-
-Context:
-${context}`;
-      const geminiMessages = messages.map((m) => ({
-        role: m.role === "assistant" ? "model" : "user",
-        parts: [{ text: m.content }]
-      }));
-      const result = yield new __await(this.client.models.generateContentStream({
+      var _a, _b, _c, _d;
+      const model = this.client.getGenerativeModel({
         model: this.llmConfig.model,
-        contents: geminiMessages,
-        config: {
-          systemInstruction,
-          temperature: (_c = (_b = options == null ? void 0 : options.temperature) != null ? _b : this.llmConfig.temperature) != null ? _c : 0.7,
-          maxOutputTokens: (_e = (_d = options == null ? void 0 : options.maxTokens) != null ? _d : this.llmConfig.maxTokens) != null ? _e : 1024,
+        systemInstruction: this.buildSystemInstruction(context)
+      });
+      const result = yield new __await(model.generateContentStream({
+        contents: this.buildGeminiContents(messages),
+        generationConfig: {
+          temperature: (_b = (_a = options == null ? void 0 : options.temperature) != null ? _a : this.llmConfig.temperature) != null ? _b : DEFAULT_TEMPERATURE,
+          maxOutputTokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : DEFAULT_MAX_TOKENS,
           stopSequences: options == null ? void 0 : options.stop
         }
       }));
-      const stream = (_f = result == null ? void 0 : result.stream) != null ? _f : result;
-      if (!stream) {
-        throw new Error("[GeminiProvider] generateContentStream returned undefined");
-      }
       try {
-        for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
+        for (var iter = __forAwait(result.stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
           const chunk = temp.value;
-          const text = typeof chunk.text === "function" ? chunk.text() : chunk.text;
+          const text = chunk.text();
           if (text) yield text;
         }
       } catch (temp) {
@@ -2487,69 +2636,69 @@ ${context}`;
       }
     });
   }
+  // -------------------------------------------------------------------------
+  // ILLMProvider — embeddings
+  // -------------------------------------------------------------------------
   async embed(text, options) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
-    const model = this.sanitizeModel(
-      (_c = (_b = options == null ? void 0 : options.model) != null ? _b : (_a = this.embeddingConfig) == null ? void 0 : _a.model) != null ? _c : "text-embedding-004"
+    var _a, _b;
+    const content = applyPrefix(
+      text,
+      options == null ? void 0 : options.taskType,
+      (_a = this.embeddingConfig) == null ? void 0 : _a.queryPrefix,
+      (_b = this.embeddingConfig) == null ? void 0 : _b.documentPrefix
     );
-    const apiKey = (_e = (_d = this.embeddingConfig) == null ? void 0 : _d.apiKey) != null ? _e : this.llmConfig.apiKey;
-    const client = apiKey !== this.llmConfig.apiKey ? new import_genai.GoogleGenAI({ apiKey }) : this.client;
-    let content = text;
-    const queryPrefix = (_f = this.embeddingConfig) == null ? void 0 : _f.queryPrefix;
-    const docPrefix = (_g = this.embeddingConfig) == null ? void 0 : _g.documentPrefix;
-    if ((options == null ? void 0 : options.taskType) === "query" && queryPrefix) {
-      if (!content.startsWith(queryPrefix)) {
-        content = `${queryPrefix}${text}`;
-      }
-    } else if ((options == null ? void 0 : options.taskType) === "document" && docPrefix) {
-      if (!content.startsWith(docPrefix)) {
-        content = `${docPrefix}${text}`;
-      }
-    }
-    const response = await client.models.embedContent({
-      model,
-      contents: content,
-      config: ((_h = this.embeddingConfig) == null ? void 0 : _h.dimensions) ? { outputDimensionality: this.embeddingConfig.dimensions } : void 0
+    const modelName = this.resolveEmbeddingModel(options == null ? void 0 : options.model);
+    const model = this.embeddingClient.getGenerativeModel({ model: modelName });
+    const response = await model.embedContent({
+      content: { role: "user", parts: [{ text: content }] }
     }).catch((err) => {
-      console.error(`[GeminiProvider] Embedding failed for model "${model}":`, err.message);
+      console.error(`[GeminiProvider] Embedding failed for model "${modelName}":`, err.message);
       throw err;
     });
-    return (_k = (_j = (_i = response.embeddings) == null ? void 0 : _i[0]) == null ? void 0 : _j.values) != null ? _k : [];
+    return response.embedding.values;
   }
   async batchEmbed(texts, options) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
-    const model = this.sanitizeModel(
-      (_c = (_b = options == null ? void 0 : options.model) != null ? _b : (_a = this.embeddingConfig) == null ? void 0 : _a.model) != null ? _c : "text-embedding-004"
-    );
-    const apiKey = (_e = (_d = this.embeddingConfig) == null ? void 0 : _d.apiKey) != null ? _e : this.llmConfig.apiKey;
-    const client = apiKey !== this.llmConfig.apiKey ? new import_genai.GoogleGenAI({ apiKey }) : this.client;
-    let contents = texts;
-    const queryPrefix = (_f = this.embeddingConfig) == null ? void 0 : _f.queryPrefix;
-    const docPrefix = (_g = this.embeddingConfig) == null ? void 0 : _g.documentPrefix;
-    if ((options == null ? void 0 : options.taskType) === "query" && queryPrefix) {
-      contents = texts.map((text) => text.startsWith(queryPrefix) ? text : `${queryPrefix}${text}`);
-    } else if ((options == null ? void 0 : options.taskType) === "document" && docPrefix) {
-      contents = texts.map((text) => text.startsWith(docPrefix) ? text : `${docPrefix}${text}`);
-    }
-    const response = await client.models.embedContent({
-      model,
-      contents,
-      config: ((_h = this.embeddingConfig) == null ? void 0 : _h.dimensions) ? { outputDimensionality: this.embeddingConfig.dimensions } : void 0
+    const modelName = this.resolveEmbeddingModel(options == null ? void 0 : options.model);
+    const model = this.embeddingClient.getGenerativeModel({ model: modelName });
+    const requests = texts.map((text) => {
+      var _a, _b;
+      return {
+        content: {
+          role: "user",
+          parts: [{
+            text: applyPrefix(
+              text,
+              options == null ? void 0 : options.taskType,
+              (_a = this.embeddingConfig) == null ? void 0 : _a.queryPrefix,
+              (_b = this.embeddingConfig) == null ? void 0 : _b.documentPrefix
+            )
+          }]
+        }
+      };
+    });
+    const response = await model.batchEmbedContents({
+      requests: requests.map((r) => ({
+        model: `models/${modelName}`,
+        content: r.content
+      }))
     }).catch((err) => {
-      console.error(`[GeminiProvider] Batch embedding failed for model "${model}":`, err.message);
+      console.error(`[GeminiProvider] Batch embedding failed for model "${modelName}":`, err.message);
       throw err;
     });
-    return (_j = (_i = response.embeddings) == null ? void 0 : _i.map((e) => {
-      var _a2;
-      return (_a2 = e.values) != null ? _a2 : [];
-    })) != null ? _j : [];
+    return response.embeddings.map((e) => {
+      var _a;
+      return (_a = e.values) != null ? _a : [];
+    });
   }
+  // -------------------------------------------------------------------------
+  // ILLMProvider — health
+  // -------------------------------------------------------------------------
   async ping() {
     try {
-      await this.client.models.embedContent({
-        model: "text-embedding-004",
-        contents: "ping"
+      const model = this.embeddingClient.getGenerativeModel({
+        model: this.resolveEmbeddingModel()
       });
+      await model.embedContent("ping");
       return true;
     } catch (err) {
       console.error("[GeminiProvider] Ping failed:", err);
@@ -2706,9 +2855,51 @@ ${context != null ? context : "None"}` },
 };
 
 // src/llm/LLMFactory.ts
+var customProviders = /* @__PURE__ */ new Map();
 var LLMFactory = class _LLMFactory {
+  /**
+   * Register a custom LLM provider factory at runtime.
+   *
+   * Use this to add support for any LLM backend (Azure OpenAI, Cohere, Mistral,
+   * Bedrock, etc.) without modifying this library's source code.
+   *
+   * @example
+   * // In your Next.js app initialization:
+   * import { LLMFactory } from '@retrivora-ai/rag-engine/server';
+   * import { MyCustomProvider } from './providers/MyCustomProvider';
+   *
+   * LLMFactory.register('my-provider', (config) => new MyCustomProvider(config));
+   *
+   * // Then set in your .env.local:
+   * // LLM_PROVIDER=my-provider
+   */
+  static register(name, factory) {
+    customProviders.set(name.toLowerCase(), factory);
+    console.log(`[LLMFactory] Registered custom provider: "${name}"`);
+  }
+  /**
+   * Unregister a previously registered custom provider.
+   */
+  static unregister(name) {
+    customProviders.delete(name.toLowerCase());
+  }
+  /**
+   * List all registered provider names (built-in + custom).
+   */
+  static listProviders() {
+    return [
+      "openai",
+      "anthropic",
+      "ollama",
+      "gemini",
+      "rest",
+      "universal_rest",
+      "custom",
+      ...Array.from(customProviders.keys())
+    ];
+  }
   static create(llmConfig, embeddingConfig) {
-    var _a;
+    var _a, _b;
     switch (llmConfig.provider) {
       case "openai":
         return new OpenAIProvider(llmConfig, embeddingConfig);
@@ -2722,11 +2913,19 @@ var LLMFactory = class _LLMFactory {
       case "universal_rest":
       case "custom":
         return new UniversalLLMAdapter(llmConfig);
-      default:
-        if (llmConfig.baseUrl || ((_a = llmConfig.options) == null ? void 0 : _a.baseUrl)) {
+      default: {
+        const providerName = String((_a = llmConfig.provider) != null ? _a : "").toLowerCase();
+        const customFactory = customProviders.get(providerName);
+        if (customFactory) {
+          return customFactory(llmConfig);
+        }
+        if (llmConfig.baseUrl || ((_b = llmConfig.options) == null ? void 0 : _b.baseUrl)) {
           return new UniversalLLMAdapter(llmConfig);
         }
-        throw new Error(`[LLMFactory] Unknown provider "${llmConfig.provider}"`);
+        throw new Error(
+          `[LLMFactory] Unknown provider "${llmConfig.provider}". Built-in providers: ${_LLMFactory.listProviders().join(", ")}. Register a custom provider with LLMFactory.register().`
+        );
+      }
     }
   }
   static getValidator(provider) {
@@ -2816,8 +3015,8 @@ var ProviderRegistry = class {
       }
       case "pgvector":
       case "postgresql": {
-        const { PostgreSQLProvider: PostgreSQLProvider2 } = await Promise.resolve().then(() => (init_PostgreSQLProvider(), PostgreSQLProvider_exports));
-        return PostgreSQLProvider2;
+        const { MultiTablePostgresProvider: MultiTablePostgresProvider2 } = await Promise.resolve().then(() => (init_MultiTablePostgresProvider(), MultiTablePostgresProvider_exports));
+        return MultiTablePostgresProvider2;
       }
       case "mongodb": {
         const { MongoDBProvider: MongoDBProvider2 } = await Promise.resolve().then(() => (init_MongoDBProvider(), MongoDBProvider_exports));
@@ -3775,73 +3974,102 @@ var QueryProcessor = class {
   }
 };
 
+// src/utils/synonyms.ts
+var FIELD_SYNONYMS = {
+  name: ["product", "item", "title", "label", "heading", "subject", "product name", "item name"],
+  price: ["cost", "amount", "msrp", "price", "rate", "value", "price_usd"],
+  brand: ["manufacturer", "vendor", "make", "company", "brand_name", "supplier"],
+  image: [
+    "imageUrl",
+    "thumbnail",
+    "img",
+    "url",
+    "photo",
+    "picture",
+    "media",
+    "image_url",
+    "main_image",
+    "product_image",
+    "thumb"
+  ],
+  stock: [
+    "inventory",
+    "quantity",
+    "count",
+    "availability",
+    "stock_level",
+    "inStock",
+    "is_available",
+    "in stock",
+    "status"
+  ],
+  description: ["summary", "content", "body", "text", "info", "details"],
+  link: ["url", "href", "product_url", "page_url", "link"]
+};
+function resolveMetadataValue(meta, uiKey) {
+  var _a;
+  const synonyms = (_a = FIELD_SYNONYMS[uiKey]) != null ? _a : [];
+  const keys = Object.keys(meta);
+  const systemKeys = ["namespace", "filename", "filesize", "filetype", "docid", "uploadedat", "dimension", "chunkindex"];
+  let match = keys.find((k) => k.toLowerCase() === uiKey.toLowerCase());
+  if (match !== void 0) return meta[match];
+  match = keys.find((k) => synonyms.map((s) => s.toLowerCase()).includes(k.toLowerCase()));
+  if (match !== void 0) return meta[match];
+  const isBlacklisted = (kl) => {
+    return systemKeys.includes(kl) || kl.startsWith("option1") || kl.startsWith("option2") || kl.startsWith("option3");
+  };
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return kl.includes(uiKey.toLowerCase());
+  });
+  if (match !== void 0) return meta[match];
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return synonyms.some((sk) => kl.includes(sk.toLowerCase()) || sk.toLowerCase().includes(kl));
+  });
+  return match !== void 0 ? meta[match] : void 0;
+}
+
 // src/utils/UITransformer.ts
 var UITransformer = class {
   /**
    * Main transformation method
-   * Analyzes user query and retrieved data to determine the best visualization format
-   *
-   * @param userQuery - The original user question/query
-   * @param retrievedData - Vector database retrieval results
-   * @returns Structured JSON response for UI rendering
+   * Analyzes user query and retrieved data to determine if a product carousel is needed.
    */
-  static transform(userQuery, retrievedData) {
+  static transform(userQuery, retrievedData, config, trainedSchema) {
     if (!retrievedData || retrievedData.length === 0) {
       return this.createTextResponse("No data available", "No relevant data found for your query.");
     }
-    const analysis = this.analyzeData(userQuery, retrievedData);
-    switch (analysis.suggestedType) {
-      case "product_carousel":
-        return this.transformToProductCarousel(retrievedData);
-      case "pie_chart":
-        return this.transformToPieChart(retrievedData);
-      case "bar_chart":
-        return this.transformToBarChart(retrievedData);
-      case "line_chart":
-        return this.transformToLineChart(retrievedData);
-      case "table":
-        return this.transformToTable(retrievedData);
-      default:
-        return this.transformToText(retrievedData);
+    const isStockRequest = this.isStockQuery(userQuery);
+    const filteredData = isStockRequest ? retrievedData.filter((item) => this.determineStockStatus(item)) : retrievedData;
+    const categories = this.detectCategories(filteredData);
+    const hasProducts = filteredData.some((item) => this.isProductData(item));
+    const isTimeSeries = filteredData.some((item) => this.isTimeSeriesData(item));
+    const isTrendQuery = this.isTrendQuery(userQuery);
+    if (isTrendQuery && isTimeSeries) {
+      return this.transformToLineChart(filteredData);
     }
-  }
-  /**
-   * Analyzes the data and query to suggest the best visualization type
-   */
-  static analyzeData(query, data) {
-    const queryLower = query.toLowerCase();
-    const hasProducts = data.some(
-      (item) => this.isProductData(item)
-    );
-    const hasTimeSeries = data.some(
-      (item) => this.isTimeSeriesData(item)
-    );
-    const hasCategories = this.detectCategories(data).length > 0;
-    const isDistributionQuery = queryLower.includes("distribution") || queryLower.includes("breakdown") || queryLower.includes("percentage") || queryLower.includes("proportion");
-    const isComparisonQuery = queryLower.includes("compare") || queryLower.includes("vs") || queryLower.includes("difference");
-    const isTrendQuery = queryLower.includes("trend") || queryLower.includes("over time") || queryLower.includes("growth") || queryLower.includes("decline");
-    if (hasProducts && !hasTimeSeries && !isDistributionQuery) {
-      return { suggestedType: "product_carousel", confidence: 0.95, hasProducts, hasTimeSeries, hasCategories };
-    }
-    if (isTrendQuery && hasTimeSeries) {
-      return { suggestedType: "line_chart", confidence: 0.9, hasProducts, hasTimeSeries, hasCategories };
+    if (hasProducts && !this.shouldShowCategoryChart(userQuery, categories)) {
+      return this.transformToProductCarousel(filteredData, config, trainedSchema);
     }
-    if (isDistributionQuery && hasCategories) {
-      return { suggestedType: "pie_chart", confidence: 0.85, hasProducts, hasTimeSeries, hasCategories };
+    if (categories.length > 1 && this.shouldShowCategoryChart(userQuery, categories)) {
+      return this.transformToPieChart(filteredData);
     }
-    if (isComparisonQuery && hasCategories && data.length > 2) {
-      return { suggestedType: "bar_chart", confidence: 0.8, hasProducts, hasTimeSeries, hasCategories };
+    if (hasProducts) {
+      return this.transformToProductCarousel(filteredData, config, trainedSchema);
     }
-    if (data.length > 5 && this.hasMultipleFields(data)) {
-      return { suggestedType: "table", confidence: 0.75, hasProducts, hasTimeSeries, hasCategories };
+    if (this.hasMultipleFields(filteredData)) {
+      return this.transformToTable(filteredData);
     }
-    return { suggestedType: "text", confidence: 0.5, hasProducts, hasTimeSeries, hasCategories };
+    return this.transformToText(filteredData);
   }
   /**
    * Transform data to product carousel format
    */
-  static transformToProductCarousel(data) {
-    const products = data.filter((item) => this.isProductData(item)).map((item) => this.extractProductInfo(item)).filter((p) => p !== null);
+  static transformToProductCarousel(data, config, trainedSchema) {
+    const products = data.filter((item) => this.isProductData(item)).map((item) => this.extractProductInfo(item, config, trainedSchema)).filter((p) => p !== null);
     return {
       type: "product_carousel",
       title: "Recommended Products",
@@ -3871,28 +4099,6 @@ var UITransformer = class {
       data: pieData
     };
   }
-  /**
-   * Transform data to bar chart format
-   */
-  static transformToBarChart(data) {
-    const categories = this.detectCategories(data);
-    const categoryData = this.aggregateByCategory(data, categories);
-    const barData = Object.entries(categoryData).map(([category, value]) => {
-      const { inStockCount, outOfStockCount } = this.calculateStockCounts(category, data);
-      return {
-        category,
-        value,
-        inStockCount,
-        outOfStockCount
-      };
-    });
-    return {
-      type: "bar_chart",
-      title: "Comparison by Category",
-      description: `Comparing ${categories.length} categories`,
-      data: barData
-    };
-  }
   /**
    * Transform data to line chart format
    */
@@ -3949,49 +4155,253 @@ var UITransformer = class {
       data: { content }
     };
   }
+  static parseTransformationResponse(raw) {
+    const payloadText = this.extractJsonCandidate(raw);
+    if (!payloadText) return null;
+    try {
+      const parsed = JSON.parse(payloadText);
+      return this.normalizeTransformation(parsed);
+    } catch (e) {
+      return null;
+    }
+  }
+  static extractJsonCandidate(raw) {
+    if (!raw) return null;
+    const cleaned = raw.replace(/```(?:json|chart|ui)?\s*/gi, "").replace(/```/g, "").trim();
+    const start = cleaned.indexOf("{");
+    if (start === -1) return null;
+    let depth = 0;
+    let inString = false;
+    let escape = false;
+    for (let i = start; i < cleaned.length; i += 1) {
+      const char = cleaned[i];
+      if (escape) {
+        escape = false;
+        continue;
+      }
+      if (char === "\\") {
+        escape = true;
+        continue;
+      }
+      if (char === '"') {
+        inString = !inString;
+        continue;
+      }
+      if (inString) continue;
+      if (char === "{") depth += 1;
+      if (char === "}") {
+        depth -= 1;
+        if (depth === 0) {
+          return cleaned.slice(start, i + 1);
+        }
+      }
+    }
+    return null;
+  }
+  static normalizeTransformation(payload) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
+    if (!payload || typeof payload !== "object") return null;
+    const payloadObj = payload;
+    const type = this.normalizeVisualizationType(String((_c = (_b = (_a = payloadObj.type) != null ? _a : payloadObj.view) != null ? _b : payloadObj.chartType) != null ? _c : ""));
+    if (!type) return null;
+    const title = String((_e = (_d = payloadObj.title) != null ? _d : payloadObj.heading) != null ? _e : "Visualization");
+    const description = payloadObj.description ? String(payloadObj.description) : void 0;
+    const rawData = (_j = (_i = (_h = (_g = (_f = payloadObj.data) != null ? _f : payloadObj.table) != null ? _g : payloadObj.rows) != null ? _h : payloadObj.items) != null ? _i : payloadObj.content) != null ? _j : null;
+    const data = type === "text" && typeof rawData === "string" ? { content: rawData } : rawData;
+    const transformation = {
+      type,
+      title,
+      description,
+      data
+    };
+    return this.validateTransformation(transformation) ? transformation : null;
+  }
+  static normalizeVisualizationType(type) {
+    var _a;
+    const mapping = {
+      pie: "pie_chart",
+      pie_chart: "pie_chart",
+      bar: "bar_chart",
+      bar_chart: "bar_chart",
+      line: "line_chart",
+      line_chart: "line_chart",
+      radar: "radar_chart",
+      radar_chart: "radar_chart",
+      table: "table",
+      text: "text",
+      product_carousel: "product_carousel",
+      carousel: "carousel"
+    };
+    return (_a = mapping[type.toLowerCase()]) != null ? _a : null;
+  }
+  static validateTransformation(transformation) {
+    const { type, data } = transformation;
+    switch (type) {
+      case "pie_chart":
+      case "bar_chart":
+        return Array.isArray(data) && data.every(
+          (item) => item !== null && typeof item === "object" && (typeof item.value === "number" || !Number.isNaN(Number(item.value)))
+        );
+      case "line_chart":
+        return Array.isArray(data) && data.every(
+          (item) => item !== null && typeof item === "object" && "timestamp" in item && (typeof item.value === "number" || !Number.isNaN(Number(item.value)))
+        );
+      case "radar_chart":
+        return Array.isArray(data) && data.every(
+          (item) => item !== null && typeof item === "object" && "attribute" in item
+        );
+      case "table":
+        return typeof data === "object" && data !== null && Array.isArray(data.columns) && Array.isArray(data.rows);
+      case "text":
+        return typeof data === "object" && data !== null && typeof data.content === "string";
+      case "product_carousel":
+      case "carousel":
+        return Array.isArray(data) && data.every(
+          (item) => item !== null && typeof item === "object" && ("id" in item || "name" in item)
+        );
+      default:
+        return false;
+    }
+  }
   /**
    * Helper: Check if data item is product-related
    */
   static isProductData(item) {
     const content = (item.content || "").toLowerCase();
-    const productKeywords = ["product", "price", "stock", "item", "sku", "brand", "model"];
-    return productKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
-      (k) => ["name", "price", "product", "sku"].includes(k.toLowerCase())
+    const productKeywords = [
+      "product",
+      "price",
+      "stock",
+      "item",
+      "sku",
+      "brand",
+      "model",
+      "msrp",
+      "inventory",
+      "buy",
+      "shop",
+      "beauty",
+      "care",
+      "cosmetic",
+      "facial",
+      "cream",
+      "serum",
+      "mask",
+      "makeup",
+      "fragrance"
+    ];
+    const hasKeywords = productKeywords.some((kw) => content.includes(kw));
+    const hasMetadataKey = Object.keys(item.metadata || {}).some(
+      (k) => ["name", "price", "product", "sku", "brand", "model", "cost", "item"].includes(k.toLowerCase())
     );
+    const hasPricePattern = /\$\s*\d+/.test(content);
+    return hasKeywords || hasMetadataKey || hasPricePattern;
   }
   /**
    * Helper: Check if data contains time series
    */
   static isTimeSeriesData(item) {
     const content = (item.content || "").toLowerCase();
-    const timeKeywords = ["date", "time", "year", "month", "week", "day", "hour", "trend", "historical"];
-    return timeKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
+    const timeKeywords = ["trend", "historical", "growth", "decline", "change", "increase", "decrease"];
+    const hasTimeKeyword = timeKeywords.some((kw) => content.includes(kw));
+    const metadata = item.metadata || {};
+    const maybeDateKeys = Object.keys(metadata).filter(
       (k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
     );
+    const hasValidDateValue = maybeDateKeys.some((key) => {
+      const value = metadata[key];
+      if (typeof value === "string") {
+        return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
+      }
+      return typeof value === "number";
+    });
+    return hasTimeKeyword || hasValidDateValue;
+  }
+  static shouldShowCategoryChart(query, categories) {
+    if (categories.length < 2) {
+      return false;
+    }
+    const normalized = query.toLowerCase();
+    const chartKeywords = [
+      "distribution",
+      "breakdown",
+      "by category",
+      "by type",
+      "compare",
+      "share",
+      "percentage",
+      "segmentation",
+      "split",
+      "category breakdown",
+      "category distribution"
+    ];
+    return chartKeywords.some((keyword) => normalized.includes(keyword));
+  }
+  static isTrendQuery(query) {
+    const normalized = query.toLowerCase();
+    const trendKeywords = [
+      "trend",
+      "over time",
+      "historical",
+      "growth",
+      "decline",
+      "increase",
+      "decrease",
+      "year",
+      "month",
+      "week",
+      "day",
+      "comparison",
+      "compare",
+      "changes",
+      "timeline"
+    ];
+    return trendKeywords.some((keyword) => normalized.includes(keyword));
+  }
+  static isStockQuery(query) {
+    const normalized = query.toLowerCase();
+    return normalized.includes("in stock") || normalized.includes("available") || normalized.includes("availability") || normalized.includes("inventory") || normalized.includes("stock status");
   }
   /**
-   * Helper: Extract product information from vector match
+   * Helper: Extract property from metadata using mapping, AI training, case-insensitivity, and synonyms.
    */
-  static extractProductInfo(item) {
-    const meta = item.metadata || {};
-    if (meta.name || meta.product) {
-      return {
-        id: item.id,
-        name: meta.name || meta.product,
-        price: meta.price,
-        image: meta.image || meta.imageUrl,
-        brand: meta.brand,
-        description: item.content,
-        inStock: this.determineStockStatus(item)
-      };
+  static getDynamicVal(meta, uiKey, config, trainedSchema) {
+    var _a;
+    if (!meta) return void 0;
+    const mapping = (_a = config == null ? void 0 : config.rag) == null ? void 0 : _a.uiMapping;
+    if (mapping && mapping[uiKey]) {
+      const mappedKey = mapping[uiKey];
+      if (meta[mappedKey] !== void 0) return meta[mappedKey];
+    }
+    if (trainedSchema && typeof trainedSchema === "object" && trainedSchema !== null) {
+      const trainedKey = trainedSchema[uiKey];
+      if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
     }
-    const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
-    const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d.]+)/i);
-    if (nameMatch) {
+    return resolveMetadataValue(meta, uiKey);
+  }
+  static extractProductInfo(item, config, trainedSchema) {
+    const meta = item.metadata || {};
+    const name = this.getDynamicVal(meta, "name", config, trainedSchema);
+    const price = this.getDynamicVal(meta, "price", config, trainedSchema);
+    const brand = this.getDynamicVal(meta, "brand", config, trainedSchema);
+    if (name || this.isProductData(item)) {
+      let finalName = name ? String(name) : void 0;
+      if (!finalName) {
+        const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
+        finalName = nameMatch ? nameMatch[1].trim() : item.content.split("\n")[0].substring(0, 60);
+      }
+      let finalPrice = typeof price === "number" || typeof price === "string" ? price : void 0;
+      if (!finalPrice) {
+        const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || item.content.match(/\$\s*([\d,.]+)/);
+        if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, "");
+      }
+      const imageValue = this.getDynamicVal(meta, "image", config, trainedSchema);
       return {
         id: item.id,
-        name: nameMatch[1],
-        price: priceMatch ? parseFloat(priceMatch[1]) : void 0,
+        name: finalName,
+        price: finalPrice,
+        image: typeof imageValue === "string" ? imageValue : void 0,
+        brand: brand ? String(brand) : void 0,
         description: item.content,
         inStock: this.determineStockStatus(item)
       };
@@ -4015,6 +4425,10 @@ var UITransformer = class {
         const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
         tags.forEach((t) => categories.add(String(t)));
       }
+      const contentCategories = Array.from(new Set(
+        Array.from(item.content.matchAll(/^\s*([^:\n]+):\s*(?:•|\*|-)*/gm)).map((match) => match[1].trim()).filter(Boolean)
+      ));
+      contentCategories.forEach((category) => categories.add(category));
       const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
       if (categoryMatch) {
         categories.add(categoryMatch[1].trim());
@@ -4033,7 +4447,7 @@ var UITransformer = class {
     data.forEach((item) => {
       const meta = item.metadata || {};
       const itemCategory = meta.category || meta.type || "Other";
-      if (result.hasOwnProperty(itemCategory)) {
+      if (Object.prototype.hasOwnProperty.call(result, itemCategory)) {
         result[itemCategory]++;
       } else {
         result["Other"] = (result["Other"] || 0) + 1;
@@ -4135,6 +4549,213 @@ var UITransformer = class {
     });
     return fieldCount.size > 2;
   }
+  // ─── LLM-Driven Visualization Decision ────────────────────────────────────
+  /**
+   * analyzeAndDecide — sends user question + RAG data to the LLM with a
+   * structured system prompt and parses the JSON response into a
+   * UITransformationResponse.
+   *
+   * This is the recommended entry point for production use. The heuristic
+   * `transform()` method is used as a fallback if the LLM call fails.
+   *
+   * System prompt instructs the LLM to:
+   *   - Analyze the question and retrieved data
+   *   - Choose the best visualization: bar_chart | line_chart | pie_chart | table | text
+   *   - Return a strict JSON object — no prose, no markdown fences
+   *
+   * @param query   - the original user question
+   * @param sources - vector DB matches returned by RAG retrieval
+   * @param llm     - any ILLMProvider instance (OpenAI, Anthropic, Ollama, Gemini, REST…)
+   * @returns       - a validated UITransformationResponse (type + title + description + data)
+   */
+  static async analyzeAndDecide(query, sources, llm) {
+    try {
+      const context = this.buildContextSummary(sources);
+      const systemPrompt = this.buildVisualizationSystemPrompt();
+      const userPrompt = [
+        `USER QUESTION: ${query}`,
+        "",
+        "RETRIEVED DATA (JSON):",
+        context
+      ].join("\n");
+      const rawResponse = await llm.chat(
+        [{ role: "user", content: userPrompt }],
+        "",
+        { systemPrompt, temperature: 0 }
+      );
+      const parsed = this.parseTransformationResponse(rawResponse);
+      if (parsed) {
+        console.debug("[UITransformer] LLM chose visualization type:", parsed.type);
+        return parsed;
+      }
+      console.warn("[UITransformer] LLM returned unparseable response; falling back to heuristic.");
+    } catch (err) {
+      console.warn("[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.", err);
+    }
+    return this.transform(query, sources);
+  }
+  /**
+   * Build the system prompt that instructs the LLM to return a visualization JSON.
+   */
+  static buildVisualizationSystemPrompt() {
+    return `You are a data visualization expert embedded in a RAG chat system.
+You will receive a user question and structured data retrieved from a vector database.
+Your ONLY job is to analyze this information and return a single JSON object that tells
+the frontend how to visualize it.
+
+Return ONLY a valid JSON object \u2014 no markdown code fences, no explanation, no prose.
+
+The JSON must have this exact shape:
+{
+  "type": "bar_chart" | "line_chart" | "pie_chart" | "radar_chart" | "table" | "text",
+  "title": "A concise, descriptive title for the visualization",
+  "description": "One sentence describing what the visualization shows",
+  "data": <structured data \u2014 see rules below>
+}
+
+DATA SHAPE per type:
+- bar_chart:  array of { "category": string, "value": number }
+- line_chart: array of { "timestamp": string, "value": number, "label": string }
+- pie_chart:  array of { "label": string, "value": number }
+- radar_chart: array of { "attribute": string, "[series1]": number, "[series2]": number, ... }
+  Example for radar_chart:
+  [
+    { "attribute": "Longevity", "Dolce Shine": 4, "CK One": 3 },
+    { "attribute": "Sillage", "Dolce Shine": 3, "CK One": 4 },
+    { "attribute": "Freshness", "Dolce Shine": 5, "CK One": 4 }
+  ]
+- table:      { "columns": string[], "rows": (string|number)[][] }
+- text:       { "content": "<prose answer>" }
+
+DECISION RULES (follow strictly):
+1. bar_chart  \u2192 comparing quantities across categories (e.g. sales by region, price by product). Use this when there is only ONE value per category.
+2. line_chart \u2192 trends or changes over time (dates, months, years, sequential events)
+3. pie_chart  \u2192 proportional breakdown or percentage distribution
+4. radar_chart \u2192 comparing 2 or more products across MULTIPLE attributes (e.g. comparing features, ratings, or characteristics of 2 specific products). If the user asks to "compare" products and the data contains multiple dimensions or ratings for them, you MUST use this.
+5. table      \u2192 multi-field structured records where each item has \u2265 3 attributes
+6. text       \u2192 conversational or free-form answers where no chart adds value
+
+IMPORTANT:
+- Aggregate numeric values from the raw data \u2014 do not pass raw object arrays as data.
+- Ensure all "value" fields are numbers, not strings.
+- For bar/line/pie, keep at most 12 data points for readability.
+- Never include nested objects or arrays inside bar_chart / line_chart / pie_chart data items.`;
+  }
+  /**
+   * Serialize retrieved vector matches into a compact JSON context string.
+   * Limits the total character count to avoid exceeding LLM context windows.
+   */
+  static buildContextSummary(sources, maxChars = 6e3) {
+    const items = sources.map((s, i) => {
+      var _a, _b, _c, _d;
+      return {
+        index: i + 1,
+        content: (_b = (_a = s.content) == null ? void 0 : _a.substring(0, 400)) != null ? _b : "",
+        metadata: (_c = s.metadata) != null ? _c : {},
+        score: (_d = s.score) != null ? _d : 0
+      };
+    });
+    const full = JSON.stringify(items, null, 2);
+    if (full.length <= maxChars) return full;
+    const partial = [];
+    let chars = 2;
+    for (const item of items) {
+      const chunk = JSON.stringify(item);
+      if (chars + chunk.length + 2 > maxChars) break;
+      partial.push(item);
+      chars += chunk.length + 2;
+    }
+    return JSON.stringify(partial, null, 2) + "\n// ... truncated";
+  }
+};
+
+// src/utils/SchemaMapper.ts
+var SchemaMapper = class {
+  /**
+   * Trains the plugin on a set of keys.
+   * This is done once per schema and cached.
+   */
+  static async train(llm, projectId, keys) {
+    const cacheKey = `${projectId}:${keys.sort().join(",")}`;
+    if (this.cache.has(cacheKey)) {
+      return this.cache.get(cacheKey);
+    }
+    console.log(`[SchemaMapper] \u{1F9E0} Training AI on new schema keys: ${keys.join(", ")}`);
+    const propertyList = Object.entries(this.TARGET_PROPERTIES).map(([prop, desc]) => `- ${prop} (${desc})`).join("\n");
+    const prompt = `
+Given these metadata keys from a database: [${keys.join(", ")}]
+
+Identify which keys best correspond to these standard UI properties:
+${propertyList}
+
+Return ONLY a JSON object where the keys are the UI properties and the values are the matching database keys. 
+If no good match is found for a property, omit it.
+
+Example:
+{
+  "name": "Product_Title",
+  "price": "MSRP_USD",
+  "brand": "VendorName"
+}
+`;
+    try {
+      const response = await llm.chat(
+        [
+          { role: "system", content: "You are a database schema expert. You ONLY respond with valid JSON." },
+          { role: "user", content: prompt }
+        ],
+        ""
+      );
+      const startIdx = response.indexOf("{");
+      if (startIdx !== -1) {
+        let braceCount = 0;
+        let endIdx = -1;
+        for (let i = startIdx; i < response.length; i++) {
+          if (response[i] === "{") braceCount++;
+          else if (response[i] === "}") {
+            braceCount--;
+            if (braceCount === 0) {
+              endIdx = i;
+              break;
+            }
+          }
+        }
+        if (endIdx !== -1) {
+          const jsonContent = response.substring(startIdx, endIdx + 1);
+          const cleanJson = this.sanitizeJson(jsonContent);
+          const mapping = JSON.parse(cleanJson);
+          this.cache.set(cacheKey, mapping);
+          return mapping;
+        }
+      }
+    } catch (error) {
+      console.warn("[SchemaMapper] AI training failed, falling back to heuristics:", error);
+    }
+    return {};
+  }
+  /**
+   * Forgiving JSON parser that fixes common AI formatting mistakes.
+   */
+  static sanitizeJson(s) {
+    return s.replace(/\/\*[\s\S]*?\*\/|([^:]|^)\/\/.*$/gm, "").replace(/[\u201C\u201D\u2018\u2019]/g, '"').replace(/'/g, '"').replace(/([{,]\s*)([a-zA-Z0-9_]+?)\s*:/g, '$1"$2":').replace(/,\s*([}\]])/g, "$1").trim();
+  }
+  static getCached(projectId, keys) {
+    const cacheKey = `${projectId}:${keys.sort().join(",")}`;
+    return this.cache.get(cacheKey);
+  }
+};
+SchemaMapper.cache = /* @__PURE__ */ new Map();
+/**
+ * Descriptions of standard UI properties to help the AI map fields accurately.
+ */
+SchemaMapper.TARGET_PROPERTIES = {
+  name: "The primary title, name, or label of the item",
+  price: "The numeric cost, price, MSRP, or amount",
+  brand: "The manufacturer, vendor, brand, or maker",
+  image: "The URL to an image, thumbnail, photo, or picture",
+  stock: "The availability, inventory count, quantity, or stock level",
+  description: "The detailed text content, summary, or body info",
+  category: "The group, department, type, or category name"
 };
 
 // src/core/Pipeline.ts
@@ -4167,6 +4788,57 @@ var LRUEmbeddingCache = class {
     return this.cache.size;
   }
 };
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+var MODEL_COST_PER_1K = {
+  "gpt-4o": { input: 25e-4, output: 0.01 },
+  "gpt-4o-mini": { input: 15e-5, output: 6e-4 },
+  "gpt-4-turbo": { input: 0.01, output: 0.03 },
+  "gpt-3.5-turbo": { input: 5e-4, output: 15e-4 },
+  "claude-3-5-sonnet": { input: 3e-3, output: 0.015 },
+  "claude-3-haiku": { input: 25e-5, output: 125e-5 },
+  "gemini-1.5-flash": { input: 75e-6, output: 3e-4 },
+  "gemini-1.5-pro": { input: 125e-5, output: 5e-3 }
+};
+function estimateCostUsd(promptTokens, completionTokens, model) {
+  if (!model) return void 0;
+  const key = Object.keys(MODEL_COST_PER_1K).find((k) => model.toLowerCase().includes(k));
+  if (!key) return void 0;
+  const prices = MODEL_COST_PER_1K[key];
+  return promptTokens / 1e3 * prices.input + completionTokens / 1e3 * prices.output;
+}
+async function scoreHallucination(llm, answer, context) {
+  const maxContextChars = 3e3;
+  const truncatedContext = context.length > maxContextChars ? context.slice(0, maxContextChars) + "\n...[truncated]" : context;
+  const prompt = `You are an AI quality checker. Given the CONTEXT retrieved from a knowledge base and the ANSWER generated from it, rate how well-grounded the answer is.
+
+CONTEXT:
+${truncatedContext}
+
+ANSWER:
+${answer}
+
+Return ONLY a valid JSON object with no markdown fences:
+{"score": <float 0-1>, "reason": "<one sentence>"}
+
+Where score 0 = fully grounded in context, score 1 = likely hallucinated or unsupported.`;
+  try {
+    const raw = await llm.chat(
+      [{ role: "user", content: prompt }],
+      "",
+      { temperature: 0, maxTokens: 120 }
+    );
+    const jsonMatch = raw.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return void 0;
+    const parsed = JSON.parse(jsonMatch[0]);
+    if (typeof parsed.score === "number" && typeof parsed.reason === "string") {
+      return { score: Math.min(1, Math.max(0, parsed.score)), reason: parsed.reason };
+    }
+  } catch (e) {
+  }
+  return void 0;
+}
 var Pipeline = class {
   constructor(config) {
     this.config = config;
@@ -4183,206 +4855,30 @@ var Pipeline = class {
     }
     this.reranker = new Reranker();
   }
+  /**
+   * Expose the underlying LLM provider (set after initialize()).
+   * Used by the stream handler to pass to UITransformer.analyzeAndDecide().
+   */
+  getLLMProvider() {
+    return this.initialised ? this.llmProvider : void 0;
+  }
   async initialize() {
-    var _a, _b;
+    var _a;
     if (this.initialised) return;
-    const CHART_MARKER = "<!-- UI_PROTOCOL_V7 -->";
-    const chartInstruction = `
-
-${CHART_MARKER}
-### UNIVERSAL UI PROTOCOL V7 (STRICT MODE)
-
-You are responsible for returning a SINGLE structured UI block when visualization is required.
-
----
-
-## 1. INTENT CLASSIFICATION (MANDATORY)
-
-Classify the query into ONE of the following intents:
-
-- "explore" \u2192 user wants to browse items (e.g., "show me products")
-- "analyze" \u2192 user wants aggregation/distribution (e.g., "distribution of products")
-- "compare" \u2192 user wants side-by-side comparison
-- "detail" \u2192 user wants structured/tabular data
-
----
-
-## 2. VIEW SELECTION (DYNAMIC)
-
-Choose the best primary "view" ("carousel", "chart", or "table") based on the data:
-- TRENDS/GROWTH \u2192 "chart" (type: "line")
-- DISTRIBUTIONS/RATIOS \u2192 "chart" (type: "pie")
-- COMPARISONS \u2192 "chart" (type: "bar")
-- PRODUCT LISTS \u2192 "carousel"
-- RAW DATA/SPECS \u2192 "table"
-
-\u26A0\uFE0F Dynamic Overrides:
-- If query is "show products", ALWAYS use carousel.
-- If data is statistical, ALWAYS use a chart.
-- NEVER explain why you chose a view.
-
----
-
-## 3. RESPONSE FORMAT (STRICT JSON ONLY INSIDE BLOCK)
-
-\`\`\`ui
-{
-  "view": "carousel" | "chart" | "table",
-  "title": "string",
-  "description": "string",
-  "metadata": {
-    "intent": "explore" | "analyze" | "compare" | "detail",
-    "confidence": 0.0-1.0
-  },
-
-  // CHART ONLY
-  "chart": {
-    "type": "pie" | "bar" | "line",
-    "xKey": "label",
-    "yKeys": ["value"],
-    "data": []
-  },
-
-  // TABLE ONLY
-  "table": {
-    "columns": [],
-    "rows": []
-  },
-
-  // CAROUSEL ONLY
-  "carousel": {
-    "items": []
-  },
-
-  "insights": []
-}
-\`\`\`
-
----
-
-## 4. DATA CONTRACT (CRITICAL)
-
-### \u{1F539} CAROUSEL ITEM FORMAT
-{
-  "id": "string",
-  "name": "string",
-  "brand": "string",
-  "price": number,
-  "image": "url",
-  "link": "url",
-  "inStock": boolean
-}
-
-### \u{1F539} CHART DATA FORMAT
-{
-  "label": "string",
-  "value": number,
-  "inStockCount": number (optional),
-  "outOfStockCount": number (optional)
-}
-
-### \u{1F539} TABLE FORMAT
-- columns MUST match row keys
-- rows MUST be flat objects
-
----
-
-## 5. DATA RULES
-
-- NEVER hallucinate fields or data.
-- ONLY use data provided in the retrieved context.
-- STRICT DATA ISOLATION: The labels and numbers in the EXAMPLES section below are placeholders. NEVER use them.
-- IF NO DATA IS FOUND: You MUST NOT generate a chart. You MUST NOT say "This example assumes" or provide hypothetical numbers. Simply state "No relevant data found in the database." and STOP.
-- NO META-COMMENTARY: NEVER talk about the task itself (e.g., "I've used JSON", "In a real-world app", "Dedicated library"). 
-- NEVER explain YOUR formatting choice.
-- Aggregate BEFORE rendering chart.
-- DYNAMIC INTELLIGENCE: Autonomously extract relevant categories/values from the context and calculate totals for the "data" array based on the user query.
-
----
-
-## 6. SMART RULES (IMPORTANT)
-
-- If user asks BOTH:
-  "show products AND distribution"
-  \u2192 PRIORITIZE "explore" \u2192 carousel
-
-- If dataset size > 20:
-  \u2192 aggregate \u2192 chart or table
-
----
-
-## 7. OUTPUT RULES
-
-- ALWAYS return EXACTLY ONE \`\`\`ui\`\`\` block.
-- JSON must be VALID.
-- FIELD MAPPING (MANDATORY): Always use "label" for the name/category and "value" for the numeric count.
-- NO xKey or yKeys: Do not include these fields; the UI uses "label" and "value" by default.
-- SILENT DATA: If a chart is provided, DO NOT say "Here is the JSON" or "Below is the data". Simply output the block.
-- TEXT OUTSIDE THE BLOCK: One brief sentence maximum.
-
----
-
-## 8. EXAMPLES
-
-User: "show me beauty products"
-
-\`\`\`ui
-{
-  "view": "carousel",
-  "title": "TITLE",
-  "description": "DESCRIPTION",
-  "metadata": {
-    "intent": "explore",
-    "confidence": 0.95
-  },
-  "carousel": {
-    "items": [
-      {
-        "id": "ID",
-        "name": "PRODUCT_NAME",
-        "brand": "BRAND",
-        "price": 0,
-        "image": "URL",
-        "link": "URL",
-        "inStock": true
-      }
-    ]
-  },
-  "insights": ["INSIGHT"]
-}
-\`\`\`
-
----
-
+    const chartInstruction = `You are a helpful product assistant. Use the provided context to answer questions accurately.
 
+### UI STYLE RULES (CRITICAL):
+- NEVER generate markdown tables. If you do, the UI will break.
+- NEVER generate HTML tags like <figure>, <tbody>, <tr>, etc.
+- NEVER generate text-based charts or graphs.
+- ONLY use plain text and bullet points.
 
-\`\`\`ui
-{
-  "view": "chart",
-  "title": "TITLE",
-  "description": "DESCRIPTION",
-  "metadata": {
-    "intent": "analyze",
-    "confidence": 0.98
-  },
-  "chart": {
-    "type": "pie",
-    "data": [
-      { "label": "LABEL_A", "value": 0, "inStockCount": 0, "outOfStockCount": 0 },
-      { "label": "LABEL_B", "value": 0, "inStockCount": 0, "outOfStockCount": 0 }
-    ]
-  },
-  "insights": ["INSIGHT"]
-}
-\`\`\`
+### PRODUCT DISPLAY:
+- When recommending products, simply list their names, prices, and features in a friendly, conversational manner.
+- The UI will automatically detect these products and show high-quality product cards in a carousel below your message.
+- Do NOT try to format product lists as tables.
 `;
-    if (!((_a = this.config.llm.systemPrompt) == null ? void 0 : _a.includes(CHART_MARKER))) {
-      let cleanPrompt = this.config.llm.systemPrompt || "";
-      cleanPrompt = cleanPrompt.replace(/\n\n### UNIVERSAL UI PROTOCOL[\s\S]*?(?=\n\n|$)/g, "");
-      cleanPrompt = cleanPrompt.replace(/<!-- UI_PROTOCOL_V\d+ -->[\s\S]*?(?=\n\n|$)/g, "");
-      this.config.llm.systemPrompt = cleanPrompt.trim() + chartInstruction;
-    }
-    console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
+    this.config.llm.systemPrompt = chartInstruction;
     this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
     const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
       this.config.llm,
@@ -4396,7 +4892,7 @@ User: "show me beauty products"
       this.entityExtractor = new EntityExtractor(this.llmProvider);
     }
     await this.vectorDB.initialize();
-    if (((_b = this.config.rag) == null ? void 0 : _b.architecture) === "agentic") {
+    if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic") {
       this.agent = new LangChainAgent(this, this.config);
       await this.agent.initialize(this.llmProvider);
     }
@@ -4404,7 +4900,6 @@ User: "show me beauty products"
   }
   /**
    * Ingest documents with automatic chunking, embedding, and batch upsert.
-   * Handles retries for transient failures.
    */
   async ingest(documents, namespace) {
     await this.initialize();
@@ -4421,10 +4916,7 @@ User: "show me beauty products"
           metadata: chunk.metadata
         }));
         const totalProcessed = await this.processUpserts(upsertDocs, ns);
-        results.push({
-          docId: doc.docId,
-          chunksIngested: totalProcessed
-        });
+        results.push({ docId: doc.docId, chunksIngested: totalProcessed });
         if (this.graphDB && this.entityExtractor) {
           await this.processGraphIngestion(doc.docId, chunks);
         }
@@ -4435,23 +4927,18 @@ User: "show me beauty products"
     }
     return results;
   }
-  /**
-   * Step 1: Chunk the document content.
-   */
+  /** Step 1: Chunk the document content. */
   async prepareChunks(doc) {
     var _a, _b;
     if (this.llamaIngestor) {
-      return await this.llamaIngestor.chunk(doc.content, {
+      return this.llamaIngestor.chunk(doc.content, {
         docId: doc.docId,
         metadata: doc.metadata,
         chunkSize: (_a = this.config.rag) == null ? void 0 : _a.chunkSize,
         chunkOverlap: (_b = this.config.rag) == null ? void 0 : _b.chunkOverlap
       });
     }
-    return this.chunker.chunk(doc.content, {
-      docId: doc.docId,
-      metadata: doc.metadata
-    });
+    return this.chunker.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata });
   }
   /**
    * Step 2: Generate embeddings for chunks with retry logic.
@@ -4475,9 +4962,7 @@ User: "show me beauty products"
     }
     return vectors;
   }
-  /**
-   * Step 3: Upsert chunks to vector database with retry logic.
-   */
+  /** Step 3: Upsert chunks to vector database with retry logic. */
   async processUpserts(upsertDocs, namespace) {
     const upsertBatchOptions = {
       batchSize: 100,
@@ -4494,14 +4979,10 @@ User: "show me beauty products"
     }
     return upsertResult.totalProcessed;
   }
-  /**
-   * Step 4: Optional graph-based entity extraction and ingestion.
-   */
+  /** Step 4: Optional graph-based entity extraction and ingestion. */
   async processGraphIngestion(docId, chunks) {
-    console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
     const extractionOptions = {
       batchSize: 2,
-      // Low concurrency for LLM extraction
       maxRetries: 1,
       initialDelayMs: 500
     };
@@ -4514,7 +4995,7 @@ User: "show me beauty products"
             if (nodes.length > 0) await this.graphDB.addNodes(nodes);
             if (edges.length > 0) await this.graphDB.addEdges(edges);
           } catch (err) {
-            console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
+            console.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}:`, err);
           }
         }
       },
@@ -4525,7 +5006,6 @@ User: "show me beauty products"
     var _a;
     await this.initialize();
     if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic" && this.agent) {
-      console.log("[Pipeline] \u{1F916} Executing in Agentic Mode...");
       const agentReply = await this.agent.run(question, history);
       return { reply: agentReply, sources: [] };
     }
@@ -4534,6 +5014,7 @@ User: "show me beauty products"
     let sources = [];
     let graphData;
     let uiTransformation;
+    let trace;
     try {
       for (var iter = __forAwait(stream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
         const chunk = temp.value;
@@ -4543,6 +5024,7 @@ User: "show me beauty products"
           if ("sources" in chunk) sources = chunk.sources;
           if ("graphData" in chunk) graphData = chunk.graphData;
           if ("ui_transformation" in chunk) uiTransformation = chunk.ui_transformation;
+          if ("trace" in chunk) trace = chunk.trace;
         }
       }
     } catch (temp) {
@@ -4555,38 +5037,47 @@ User: "show me beauty products"
           throw error[0];
       }
     }
-    return { reply, sources, graphData, ui_transformation: uiTransformation };
+    return { reply, sources, graphData, ui_transformation: uiTransformation, trace };
   }
   /**
    * High-performance streaming RAG flow.
-   * Yields text chunks first, then the retrieval metadata at the end.
+   * Yields text chunks first, then the retrieval metadata + observability trace at the end.
    */
   askStream(_0) {
     return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
-      var _a, _b, _c, _d, _e, _f, _g;
+      var _a, _b, _c, _d, _e, _f, _g, _h, _i;
       yield new __await(this.initialize());
       const ns = namespace != null ? namespace : this.config.projectId;
       const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
       const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
+      const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+      const requestStart = performance.now();
       try {
         let searchQuery = question;
+        let rewrittenQuery;
         if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
           searchQuery = yield new __await(this.rewriteQuery(question, history));
+          rewrittenQuery = searchQuery !== question ? searchQuery : void 0;
         }
         const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
         const filter = QueryProcessor.buildQueryFilter(question, hints);
-        console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
+        const embedStart = performance.now();
         const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
           namespace: ns,
           topK: topK * 2,
           filter
         }));
+        const retrieveEnd = performance.now();
+        const embedMs = retrieveEnd - embedStart;
+        const retrieveMs = retrieveEnd - embedStart;
+        const rerankStart = performance.now();
         let sources = rawSources.filter((m) => m.score >= scoreThreshold);
         if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
           sources = yield new __await(this.reranker.rerank(sources, question, topK));
         } else {
           sources = sources.slice(0, topK);
         }
+        const rerankMs = performance.now() - rerankStart;
         let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
 ${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
         if (graphData && graphData.nodes.length > 0) {
@@ -4599,7 +5090,16 @@ ${graphContext}
 VECTOR CONTEXT:
 ${context}`;
         }
-        const messages = [...history, { role: "user", content: question }];
+        const allMetadataKeys = Array.from(new Set(sources.flatMap((s) => Object.keys(s.metadata || {}))));
+        const trainingPromise = allMetadataKeys.length > 0 ? SchemaMapper.train(this.llmProvider, ns, allMetadataKeys) : Promise.resolve(void 0);
+        const restrictionSuffix = "\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. If listing products, use simple bullet points.)";
+        const hardenedHistory = [...history];
+        const userQuestion = { role: "user", content: question + restrictionSuffix };
+        const messages = [...hardenedHistory, userQuestion];
+        const systemPrompt = (_h = this.config.llm.systemPrompt) != null ? _h : "";
+        const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
+        let fullReply = "";
+        const generateStart = performance.now();
         if (this.llmProvider.chatStream) {
           const stream = this.llmProvider.chatStream(messages, context);
           if (!stream) {
@@ -4608,6 +5108,7 @@ ${context}`;
           try {
             for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
               const chunk = temp.value;
+              fullReply += chunk;
               yield chunk;
             }
           } catch (temp) {
@@ -4622,14 +5123,58 @@ ${context}`;
           }
         } else {
           const reply = yield new __await(this.llmProvider.chat(messages, context));
+          fullReply = reply;
           yield reply;
         }
-        const uiTransformation = UITransformer.transform(question, sources);
+        const generateMs = performance.now() - generateStart;
+        const totalMs = performance.now() - requestStart;
+        const latency = {
+          embedMs: Math.round(embedMs),
+          retrieveMs: Math.round(retrieveMs),
+          rerankMs: ((_i = this.config.rag) == null ? void 0 : _i.useReranking) ? Math.round(rerankMs) : void 0,
+          generateMs: Math.round(generateMs),
+          totalMs: Math.round(totalMs)
+        };
+        const promptText = systemPrompt + "\n" + context + "\n" + userPrompt;
+        const promptTokens = estimateTokens(promptText);
+        const completionTokens = estimateTokens(fullReply);
+        const tokens = {
+          promptTokens,
+          completionTokens,
+          totalTokens: promptTokens + completionTokens,
+          estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
+        };
+        const hallucinationResult = yield new __await(scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0));
+        const trainedSchema = yield new __await(trainingPromise);
+        const uiTransformation = yield new __await(this.generateUiTransformation(question, sources, context, trainedSchema));
+        const trace = {
+          requestId,
+          query: question,
+          rewrittenQuery,
+          systemPrompt,
+          userPrompt: question + restrictionSuffix,
+          chunks: sources.map((s) => {
+            var _a2;
+            return {
+              id: s.id,
+              score: s.score,
+              content: s.content,
+              metadata: (_a2 = s.metadata) != null ? _a2 : {},
+              namespace: ns
+            };
+          }),
+          latency,
+          tokens,
+          hallucinationScore: hallucinationResult == null ? void 0 : hallucinationResult.score,
+          hallucinationReason: hallucinationResult == null ? void 0 : hallucinationResult.reason,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        };
         yield {
           reply: "",
           sources,
           graphData,
-          ui_transformation: uiTransformation
+          ui_transformation: uiTransformation,
+          trace
         };
       } catch (error2) {
         throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
@@ -4640,6 +5185,17 @@ ${context}`;
    * Universal retrieval method combining all enabled providers.
    * Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
    */
+  async generateUiTransformation(question, sources, _context, trainedSchema) {
+    if (!sources || sources.length === 0) {
+      return UITransformer.transform(question, sources, this.config, trainedSchema);
+    }
+    try {
+      return await UITransformer.analyzeAndDecide(question, sources, this.llmProvider);
+    } catch (err) {
+      console.warn("[Pipeline] generateUiTransformation failed, using heuristic fallback:", err);
+      return UITransformer.transform(question, sources, this.config, trainedSchema);
+    }
+  }
   async retrieve(query, options) {
     var _a, _b, _c;
     const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
@@ -4657,12 +5213,9 @@ ${context}`;
     const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
     return { sources, graphData };
   }
-  /**
-   * Rewrite the user query for better retrieval performance.
-   */
+  /** Rewrite the user query for better retrieval performance. */
   async rewriteQuery(question, history) {
-    const prompt = `
-Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
+    const prompt = `Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
 Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
 
 History:
@@ -4680,23 +5233,16 @@ Optimized Search Query:`;
     );
     return rewrite.trim() || question;
   }
-  /**
-   * Generate 3-5 short, relevant questions based on the vector database content.
-   */
+  /** Generate 3-5 short, relevant questions based on the vector database content. */
   async getSuggestions(query, namespace) {
-    if (!query || query.trim().length < 3) {
-      return [];
-    }
+    if (!query || query.trim().length < 3) return [];
     await this.initialize();
     const ns = namespace != null ? namespace : this.config.projectId;
     try {
       const { sources } = await this.retrieve(query, { namespace: ns, topK: 3 });
-      if (sources.length === 0) {
-        return [];
-      }
+      if (sources.length === 0) return [];
       const context = sources.map((s) => s.content).join("\n\n---\n\n");
-      const prompt = `
-Based on the following snippets from a document, what are 3 short, relevant questions a user might ask? 
+      const prompt = `Based on the following snippets from a document, what are 3 short, relevant questions a user might ask? 
 Focus on questions that can be answered by the context.
 Keep each question under 10 words and make them very specific to the content.
 Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3"].
@@ -4842,6 +5388,14 @@ var VectorPlugin = class {
   getConfig() {
     return this.config;
   }
+  /**
+   * Get the initialized LLM provider (available after the first request).
+   * Used by handlers to pass to UITransformer.analyzeAndDecide() for
+   * LLM-driven visualization decisions.
+   */
+  getLLMProvider() {
+    return this.pipeline.getLLMProvider();
+  }
   /**
    * Perform pre-flight health checks on all configured providers.
    * Useful to verify connectivity before running operations.
@@ -4963,7 +5517,12 @@ function sseMetaFrame(meta) {
 `;
 }
 function sseUIFrame(uiTransformation) {
-  return `data: ${JSON.stringify(__spreadValues({ type: "ui_transformation" }, uiTransformation != null ? uiTransformation : {}))}
+  return `data: ${JSON.stringify({ type: "ui_transformation", data: uiTransformation })}
+
+`;
+}
+function sseObservabilityFrame(trace) {
+  return `data: ${JSON.stringify({ type: "observability", data: trace })}
 
 `;
 }
@@ -5018,6 +5577,7 @@ function createStreamHandler(configOrPlugin) {
     const encoder = new TextEncoder();
     const stream = new ReadableStream({
       async start(controller) {
+        var _a, _b;
         const enqueue = (text) => controller.enqueue(encoder.encode(text));
         try {
           const pipelineStream = plugin.chatStream(message, history, namespace);
@@ -5028,13 +5588,25 @@ function createStreamHandler(configOrPlugin) {
                 enqueue(sseTextFrame(chunk));
               } else {
                 enqueue(sseMetaFrame(chunk));
-                const sources = (chunk == null ? void 0 : chunk.sources) || [];
-                if (sources && sources.length > 0) {
+                const responseChunk = chunk;
+                const sources = (responseChunk == null ? void 0 : responseChunk.sources) || [];
+                if (responseChunk == null ? void 0 : responseChunk.trace) {
+                  enqueue(sseObservabilityFrame(responseChunk.trace));
+                }
+                if (sources.length > 0) {
                   try {
-                    const uiTransformation = UITransformer.transform(message, sources);
-                    enqueue(sseUIFrame(uiTransformation));
+                    const llmProvider = (_a = plugin.getLLMProvider) == null ? void 0 : _a.call(plugin);
+                    const uiTransformation = (_b = responseChunk == null ? void 0 : responseChunk.ui_transformation) != null ? _b : llmProvider ? await UITransformer.analyzeAndDecide(message, sources, llmProvider) : UITransformer.transform(message, sources, plugin.getConfig());
+                    if (uiTransformation) {
+                      enqueue(sseUIFrame(uiTransformation));
+                    }
                   } catch (transformError) {
                     console.warn("[createStreamHandler] UI transformation warning:", transformError);
+                    try {
+                      const fallback = UITransformer.transform(message, sources, plugin.getConfig());
+                      if (fallback) enqueue(sseUIFrame(fallback));
+                    } catch (e) {
+                    }
                   }
                 }
               }
@@ -5102,24 +5674,64 @@ function createUploadHandler(configOrPlugin) {
       const formData = await req.formData();
       const files = formData.getAll("files");
       const namespace = formData.get("namespace") || void 0;
+      const dimensionRaw = formData.get("dimension");
+      const dimension = dimensionRaw ? parseInt(dimensionRaw, 10) : void 0;
       if (!files || files.length === 0) {
         return import_server.NextResponse.json({ error: "No files provided" }, { status: 400 });
       }
-      const documents = await Promise.all(
-        files.map(async (file) => {
+      const documents = [];
+      for (const file of files) {
+        if (file.name.toLowerCase().endsWith(".csv") || file.type === "text/csv") {
+          const text = await file.text();
+          const Papa = await import("papaparse");
+          const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
+          if (parsed.data && parsed.data.length > 0) {
+            let i = 0;
+            let lastRowData = null;
+            for (const row of parsed.data) {
+              i++;
+              const rowData = row;
+              const groupingKey = Object.keys(rowData)[0];
+              if (lastRowData && groupingKey && rowData[groupingKey] && rowData[groupingKey] === lastRowData[groupingKey]) {
+                for (const key of Object.keys(rowData)) {
+                  if (!rowData[key] && lastRowData[key]) {
+                    rowData[key] = lastRowData[key];
+                  }
+                }
+              }
+              lastRowData = __spreadValues({}, rowData);
+              const contentParts = [];
+              for (const [key, val] of Object.entries(rowData)) {
+                if (key && val) {
+                  contentParts.push(`${key}: ${val}`);
+                }
+              }
+              documents.push({
+                docId: `${file.name}-row-${i}`,
+                content: contentParts.join(", "),
+                metadata: __spreadValues(__spreadValues({
+                  fileName: file.name,
+                  fileSize: file.size,
+                  fileType: file.type,
+                  uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
+                }, dimension ? { dimension } : {}), rowData)
+              });
+            }
+          }
+        } else {
           const content = await DocumentParser.parse(file, file.name, file.type);
-          return {
+          documents.push({
             docId: file.name,
             content,
-            metadata: {
+            metadata: __spreadValues({
               fileName: file.name,
               fileSize: file.size,
               fileType: file.type,
               uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
-            }
-          };
-        })
-      );
+            }, dimension ? { dimension } : {})
+          });
+        }
+      }
       const results = await plugin.ingest(documents, namespace);
       return import_server.NextResponse.json({ message: "Upload successful", results });
     } catch (err) {
@@ -5156,6 +5768,7 @@ function createSuggestionsHandler(configOrPlugin) {
   sseErrorFrame,
   sseFrame,
   sseMetaFrame,
+  sseObservabilityFrame,
   sseTextFrame,
   sseUIFrame
 });