@retrivora-ai/rag-engine 1.9.2 → 1.9.6

package/dist/handlers/index.js

@@ -661,6 +661,11 @@ var init_MultiTablePostgresProvider = __esm({
         CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
         ON ${this.uploadTable}
         USING hnsw (embedding vector_cosine_ops)
+      `);
+          await client.query(`
+        CREATE INDEX IF NOT EXISTS "${this.uploadTable}_fts_idx"
+        ON "${this.uploadTable}"
+        USING gin (to_tsvector('english', content))
       `);
           const res = await client.query(`
         SELECT table_name 
@@ -731,6 +736,11 @@ var init_MultiTablePostgresProvider = __esm({
           CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
           ON "${tableName}"
           USING hnsw (embedding vector_cosine_ops)
+        `);
+            await client.query(`
+          CREATE INDEX IF NOT EXISTS "${tableName}_fts_idx"
+          ON "${tableName}"
+          USING gin (to_tsvector('english', content))
         `);
             if (!this.tables.includes(tableName)) {
               this.tables.push(tableName);
@@ -822,8 +832,11 @@ var init_MultiTablePostgresProvider = __esm({
                 const paramIdx = baseOffset + filterParams.length;
                 const synonyms = (_a3 = FIELD_SYNONYMS[key]) != null ? _a3 : [];
                 const keysToCheck = [key, ...synonyms];
-                const coalesceExprs = keysToCheck.map((k) => `LOWER(metadata->>'${k}')`);
-                return `COALESCE(${coalesceExprs.join(", ")}) = LOWER($${paramIdx})`;
+                const coalesceExprs = keysToCheck.flatMap((k) => [
+                  `metadata->>'${k}'`,
+                  `to_jsonb(t)->>'${k}'`
+                ]);
+                return `COALESCE(${coalesceExprs.map((expr) => `LOWER(${expr})`).join(", ")}) = LOWER($${paramIdx})`;
               });
               whereClause = `WHERE ${conditions.join(" AND ")}`;
             }
@@ -834,16 +847,35 @@ var init_MultiTablePostgresProvider = __esm({
                    CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(", ")})
                    THEN 1.0 ELSE 0.0 END
                  ), 0)
-                 FROM jsonb_each_text(to_jsonb(t)) AS kv(key, val)
+                 FROM jsonb_each_text(COALESCE(v.metadata, k.metadata)) AS kv(key, val)
                  WHERE key IN (${this.searchFields.map((f) => `'${f}'`).join(", ")})
                ) * 3.0` : "";
               sqlQuery = `
-            SELECT *, 
-             (1 - (embedding <=> $1::vector)) AS vector_score,
-             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
-             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
-            FROM "${table}" t
-            ${whereClause}
+            WITH vector_search AS (
+              SELECT *, (1 - (embedding <=> $1::vector)) AS vector_score
+              FROM "${table}" t
+              ${whereClause}
+              ORDER BY embedding <=> $1::vector ASC
+              LIMIT ${tableLimit}
+            ),
+            keyword_search AS (
+              SELECT *, COALESCE(ts_rank(to_tsvector('english', content), to_tsquery('english', $2)), 0) AS keyword_score
+              FROM "${table}" t
+              WHERE ${whereClause ? whereClause.replace("WHERE", "") : "TRUE"}
+                AND to_tsvector('english', content) @@ to_tsquery('english', $2)
+              ORDER BY keyword_score DESC
+              LIMIT ${tableLimit}
+            )
+            SELECT 
+              COALESCE(v.id, k.id) AS id,
+              COALESCE(v.namespace, k.namespace) AS namespace,
+              COALESCE(v.content, k.content) AS content,
+              COALESCE(v.metadata, k.metadata) AS metadata,
+              COALESCE(v.vector_score, 0) AS vector_score,
+              COALESCE(k.keyword_score, 0) AS keyword_score,
+              (COALESCE(v.vector_score, 0) + COALESCE(k.keyword_score, 0) * 2.0 ${exactNameScoreExpr}) AS hybrid_score
+            FROM vector_search v
+            FULL OUTER JOIN keyword_search k ON v.id = k.id
             ORDER BY hybrid_score DESC
             LIMIT ${tableLimit}
           `;
@@ -854,7 +886,7 @@ var init_MultiTablePostgresProvider = __esm({
             (1 - (embedding <=> $1::vector)) AS hybrid_score
             FROM "${table}" t
             ${whereClause}
-            ORDER BY hybrid_score DESC
+            ORDER BY embedding <=> $1::vector ASC
             LIMIT ${tableLimit}
           `;
               params = [vectorLiteral, ...filterParams];
@@ -1060,7 +1092,11 @@ var init_MongoDBProvider = __esm({
           if (key === "namespace") {
             vectorSearchFilter.namespace = value;
           } else {
-            matchFilter[key] = value;
+            if (typeof value === "string") {
+              matchFilter[key] = { $regex: new RegExp(`^${value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`, "i") };
+            } else {
+              matchFilter[key] = value;
+            }
           }
         }
         const pipeline = [
@@ -1090,12 +1126,15 @@ var init_MongoDBProvider = __esm({
           }
         );
         const results = await this.collection.aggregate(pipeline).toArray();
-        return results.map((res) => ({
-          id: res._id,
-          content: res[this.contentKey],
-          metadata: res[this.metadataKey] || {},
-          score: res.score
-        }));
+        return results.map((res) => {
+          const normalizedScore = res.score * 2 - 1;
+          return {
+            id: res._id,
+            content: res[this.contentKey],
+            metadata: res[this.metadataKey] || {},
+            score: normalizedScore
+          };
+        });
       }
       async delete(id, namespace) {
         await this.collection.deleteOne(__spreadValues({ _id: id }, namespace ? { namespace } : {}));
@@ -2146,7 +2185,7 @@ function readEnum(env, name, fallback, allowed) {
   throw new Error(`[getRagConfig] ${name} must be one of: ${allowed.join(", ")}`);
 }
 function getEnvConfig(env = process.env, base) {
-  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa, _Ga;
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K, _L, _M, _N, _O, _P, _Q, _R, _S, _T, _U, _V, _W, _X, _Y, _Z, __, _$, _aa, _ba, _ca, _da, _ea, _fa, _ga, _ha, _ia, _ja, _ka, _la, _ma, _na, _oa, _pa, _qa, _ra, _sa, _ta, _ua, _va, _wa, _xa, _ya, _za, _Aa, _Ba, _Ca, _Da, _Ea, _Fa, _Ga, _Ha, _Ia, _Ja;
   const projectId = (_c = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : base == null ? void 0 : base.projectId) != null ? _c : "__default__";
   const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
   const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
@@ -2216,7 +2255,7 @@ function getEnvConfig(env = process.env, base) {
     rest: "default",
     custom: "default"
   };
-  return {
+  return __spreadValues({
     projectId,
     vectorDb: {
       provider: vectorProvider,
@@ -2232,7 +2271,9 @@ function getEnvConfig(env = process.env, base) {
       maxTokens: readNumber(env, "LLM_MAX_TOKENS", 4096),
       temperature: readNumber(env, "LLM_TEMPERATURE", 0.7),
       options: {
-        profile: readString(env, "LLM_UNIVERSAL_PROFILE")
+        profile: readString(env, "LLM_UNIVERSAL_PROFILE"),
+        thinking: readString(env, "LLM_THINKING") === "true" || readString(env, "LLM_THINKING") === "enabled" ? true : readString(env, "LLM_THINKING") === "false" ? false : void 0,
+        thinkingBudget: env.LLM_THINKING_BUDGET ? parseInt(env.LLM_THINKING_BUDGET, 10) : void 0
       }
     },
     embedding: {
@@ -2264,11 +2305,56 @@ function getEnvConfig(env = process.env, base) {
       topK: readNumber(env, "RAG_TOP_K", 5),
       scoreThreshold: readNumber(env, "RAG_SCORE_THRESHOLD", 0),
       chunkSize: readNumber(env, "RAG_CHUNK_SIZE", 1e3),
-      chunkOverlap: readNumber(env, "RAG_CHUNK_OVERLAP", 200)
+      chunkOverlap: readNumber(env, "RAG_CHUNK_OVERLAP", 200),
+      filterableFields: (_Ha = readString(env, "RAG_FILTERABLE_FIELDS")) == null ? void 0 : _Ha.split(",").map((f) => f.trim()),
+      // Query pipeline toggles — read from .env.local
+      useQueryTransformation: readString(env, "RAG_USE_QUERY_TRANSFORMATION") === "true",
+      useReranking: readString(env, "RAG_USE_RERANKING") === "true",
+      useGraphRetrieval: readString(env, "RAG_USE_GRAPH_RETRIEVAL") === "true",
+      architecture: (_Ia = readString(env, "RAG_ARCHITECTURE")) != null ? _Ia : "simple",
+      chunkingStrategy: (_Ja = readString(env, "RAG_CHUNKING_STRATEGY")) != null ? _Ja : "recursive",
+      uiMapping: (() => {
+        const raw = readString(env, "RAG_UI_MAPPING");
+        if (!raw) return void 0;
+        try {
+          return JSON.parse(raw);
+        } catch (e) {
+          return void 0;
+        }
+      })()
     }
-  };
+  }, readString(env, "GRAPH_DB_PROVIDER") ? {
+    graphDb: {
+      provider: readString(env, "GRAPH_DB_PROVIDER"),
+      options: {
+        uri: readString(env, "GRAPH_DB_URI"),
+        username: readString(env, "GRAPH_DB_USERNAME"),
+        password: readString(env, "GRAPH_DB_PASSWORD")
+      }
+    }
+  } : {});
 }
 
+// src/exceptions/index.ts
+var RetrivoraError = class extends Error {
+  constructor(message, code, details) {
+    super(message);
+    this.name = new.target.name;
+    this.code = code;
+    this.details = details;
+  }
+};
+var ProviderNotFoundException = class extends RetrivoraError {
+  constructor(providerType, provider, details) {
+    super(`Unsupported ${providerType} provider: ${provider}`, "PROVIDER_NOT_FOUND", details);
+  }
+};
+var ConfigurationException = class extends RetrivoraError {
+  constructor(message, details) {
+    super(message, "CONFIGURATION_ERROR", details);
+  }
+};
+
 // src/core/ConfigResolver.ts
 var ConfigResolver = class {
   /**
@@ -2295,24 +2381,81 @@ var ConfigResolver = class {
       rag: hostConfig.rag ? __spreadValues(__spreadValues({}, envConfig.rag), hostConfig.rag) : envConfig.rag
     });
   }
+  /**
+   * Resolves the public SDK config shape used by `new Retrivora({...})`.
+   * Supports aliases from the product prompt while preserving existing env
+   * fallback behavior.
+   */
+  static resolveUniversal(hostConfig, env = process.env) {
+    var _a;
+    if (!hostConfig) return this.resolve(void 0, env);
+    const normalized = __spreadProps(__spreadValues({}, hostConfig), {
+      vectorDb: (_a = hostConfig.vectorDb) != null ? _a : hostConfig.vectorDatabase,
+      rag: this.mergeRetrievalWorkflow(hostConfig.rag, hostConfig.retrieval, hostConfig.workflow)
+    });
+    return this.resolve(normalized, env);
+  }
   /**
    * Validates the configuration for required fields.
    */
   static validate(config) {
     if (!config.projectId) {
-      throw new Error("[ConfigResolver] projectId is required");
+      throw new ConfigurationException("[ConfigResolver] projectId is required");
     }
     if (!config.vectorDb.provider) {
-      throw new Error("[ConfigResolver] vectorDb.provider is required");
+      throw new ConfigurationException("[ConfigResolver] vectorDb.provider is required");
     }
     if (!config.llm.provider) {
-      throw new Error("[ConfigResolver] llm.provider is required");
+      throw new ConfigurationException("[ConfigResolver] llm.provider is required");
     }
   }
+  static mergeRetrievalWorkflow(rag, retrieval, workflow) {
+    var _a, _b, _c, _d, _e;
+    if (!rag && !retrieval && !workflow) return void 0;
+    const normalized = __spreadValues({}, rag != null ? rag : {});
+    if (retrieval) {
+      normalized.topK = (_a = retrieval.topK) != null ? _a : normalized.topK;
+      normalized.scoreThreshold = (_b = retrieval.scoreThreshold) != null ? _b : normalized.scoreThreshold;
+      normalized.useReranking = (_c = retrieval.useReranking) != null ? _c : normalized.useReranking;
+      if (retrieval.strategy === "hybrid") normalized.architecture = (_d = normalized.architecture) != null ? _d : "hybrid";
+      if (retrieval.strategy === "agentic") normalized.architecture = "agentic";
+      if (retrieval.strategy === "contextual-compression") normalized.useReranking = true;
+    }
+    if (workflow == null ? void 0 : workflow.type) {
+      const type = workflow.type;
+      if (type === "agentic" || type === "agentic-rag") normalized.architecture = "agentic";
+      else if (type === "hybrid-rag") normalized.architecture = "hybrid";
+      else if (type === "graph-rag") {
+        normalized.architecture = "graph";
+        normalized.useGraphRetrieval = true;
+      } else if (type === "simple-rag" || type === "rag") {
+        normalized.architecture = (_e = normalized.architecture) != null ? _e : "simple";
+      }
+    }
+    return normalized;
+  }
 };
 
 // src/llm/providers/OpenAIProvider.ts
 var import_openai = __toESM(require("openai"));
+
+// src/llm/utils.ts
+function buildSystemContent(systemPrompt, context) {
+  const noContext = !context || context.trim() === "" || context.trim() === "No relevant context found.";
+  if (systemPrompt.includes("{{context}}")) {
+    return systemPrompt.replace("{{context}}", noContext ? "" : context);
+  }
+  if (noContext) {
+    return systemPrompt;
+  }
+  return `${systemPrompt}
+
+---
+Context:
+${context}`;
+}
+
+// src/llm/providers/OpenAIProvider.ts
 var OpenAIProvider = class {
   constructor(llmConfig, embeddingConfig) {
     if (!llmConfig.apiKey) throw new Error("[OpenAIProvider] llmConfig.apiKey is required");
@@ -2376,16 +2519,10 @@ var OpenAIProvider = class {
   }
   async chat(messages, context, options) {
     var _a, _b, _c, _d, _e, _f, _g, _h, _i;
-    const resolvedSystemPrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : `You are a helpful assistant. Answer questions based on the provided context.
-
-Context:
-${context}`;
+    const basePrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Answer questions based on the provided context.";
     const systemMessage = {
       role: "system",
-      content: resolvedSystemPrompt.includes("{{context}}") ? resolvedSystemPrompt.replace("{{context}}", context) : (options == null ? void 0 : options.systemPrompt) ? resolvedSystemPrompt : `${resolvedSystemPrompt}
-
-Context:
-${context}`
+      content: buildSystemContent(basePrompt, context)
     };
     const formattedMessages = [
       systemMessage,
@@ -2406,16 +2543,10 @@ ${context}`
   chatStream(messages, context, options) {
     return __asyncGenerator(this, null, function* () {
       var _a, _b, _c, _d, _e, _f, _g, _h;
-      const resolvedSystemPrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : `You are a helpful assistant. Answer questions based on the provided context.
-
-Context:
-${context}`;
+      const basePrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Answer questions based on the provided context.";
       const systemMessage = {
         role: "system",
-        content: resolvedSystemPrompt.includes("{{context}}") ? resolvedSystemPrompt.replace("{{context}}", context) : (options == null ? void 0 : options.systemPrompt) ? resolvedSystemPrompt : `${resolvedSystemPrompt}
-
-Context:
-${context}`
+        content: buildSystemContent(basePrompt, context)
       };
       const formattedMessages = [
         systemMessage,
@@ -2536,58 +2667,87 @@ var AnthropicProvider = class {
     };
   }
   async chat(messages, context, options) {
-    var _a, _b, _c, _d;
-    const resolvedPrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : `You are a helpful assistant. Use the context below to answer the user's question accurately.
-
-Context:
-${context}`;
-    const system = resolvedPrompt.includes("{{context}}") ? resolvedPrompt.replace("{{context}}", context) : (options == null ? void 0 : options.systemPrompt) ? resolvedPrompt : `${resolvedPrompt}
-
-Context:
-${context}`;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
+    const basePrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Use the context below to answer the user's question accurately.";
+    const system = buildSystemContent(basePrompt, context);
     const anthropicMessages = messages.map((m) => ({
       role: m.role === "assistant" ? "assistant" : "user",
       content: m.content
     }));
-    const response = await this.client.messages.create({
+    const isClaude37 = this.llmConfig.model.includes("claude-3-7-sonnet");
+    const isThinkingEnabled = ((_c = this.llmConfig.options) == null ? void 0 : _c.thinking) === true || ((_d = this.llmConfig.options) == null ? void 0 : _d.thinking) === "enabled" || isClaude37 && ((_e = this.llmConfig.options) == null ? void 0 : _e.thinking) !== false;
+    const extraParams = {};
+    if (isThinkingEnabled && isClaude37) {
+      extraParams.betas = ["interleaved-thinking-2025-05-14"];
+      const maxTokens = (_g = (_f = options == null ? void 0 : options.maxTokens) != null ? _f : this.llmConfig.maxTokens) != null ? _g : 4096;
+      const budget = Math.min(
+        typeof ((_h = this.llmConfig.options) == null ? void 0 : _h.thinkingBudget) === "number" ? (_i = this.llmConfig.options) == null ? void 0 : _i.thinkingBudget : 2048,
+        maxTokens - 1
+      );
+      extraParams.thinking = {
+        type: "enabled",
+        budget_tokens: budget
+      };
+      extraParams.max_tokens = Math.max(maxTokens, budget + 1024);
+    }
+    const response = await this.client.messages.create(__spreadValues({
       model: this.llmConfig.model,
-      max_tokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024,
+      max_tokens: (_k = (_j = options == null ? void 0 : options.maxTokens) != null ? _j : this.llmConfig.maxTokens) != null ? _k : 1024,
       system,
       messages: anthropicMessages
-    });
-    const block = response.content[0];
-    return block.type === "text" ? block.text : "";
+    }, extraParams));
+    let reply = "";
+    for (const block of response.content) {
+      if (block.type === "text") {
+        reply += block.text;
+      }
+    }
+    return reply;
   }
   chatStream(messages, context, options) {
     return __asyncGenerator(this, null, function* () {
-      var _a, _b, _c, _d;
-      const resolvedPrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : `You are a helpful assistant. Use the context below to answer the user's question accurately.
-
-Context:
-${context}`;
-      const system = resolvedPrompt.includes("{{context}}") ? resolvedPrompt.replace("{{context}}", context) : (options == null ? void 0 : options.systemPrompt) ? resolvedPrompt : `${resolvedPrompt}
-
-Context:
-${context}`;
+      var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
+      const basePrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Use the context below to answer the user's question accurately.";
+      const system = buildSystemContent(basePrompt, context);
       const anthropicMessages = messages.map((m) => ({
         role: m.role === "assistant" ? "assistant" : "user",
         content: m.content
       }));
-      const stream = yield new __await(this.client.messages.create({
+      const isClaude37 = this.llmConfig.model.includes("claude-3-7-sonnet");
+      const isThinkingEnabled = ((_c = this.llmConfig.options) == null ? void 0 : _c.thinking) === true || ((_d = this.llmConfig.options) == null ? void 0 : _d.thinking) === "enabled" || isClaude37 && ((_e = this.llmConfig.options) == null ? void 0 : _e.thinking) !== false;
+      const extraParams = {};
+      if (isThinkingEnabled && isClaude37) {
+        extraParams.betas = ["interleaved-thinking-2025-05-14"];
+        const maxTokens = (_g = (_f = options == null ? void 0 : options.maxTokens) != null ? _f : this.llmConfig.maxTokens) != null ? _g : 4096;
+        const budget = Math.min(
+          typeof ((_h = this.llmConfig.options) == null ? void 0 : _h.thinkingBudget) === "number" ? (_i = this.llmConfig.options) == null ? void 0 : _i.thinkingBudget : 2048,
+          maxTokens - 1
+        );
+        extraParams.thinking = {
+          type: "enabled",
+          budget_tokens: budget
+        };
+        extraParams.max_tokens = Math.max(maxTokens, budget + 1024);
+      }
+      const stream = yield new __await(this.client.messages.create(__spreadValues({
         model: this.llmConfig.model,
-        max_tokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024,
+        max_tokens: (_k = (_j = options == null ? void 0 : options.maxTokens) != null ? _j : this.llmConfig.maxTokens) != null ? _k : 1024,
         system,
         messages: anthropicMessages,
         stream: true
-      }));
+      }, extraParams)));
       if (!stream) {
         throw new Error("[AnthropicProvider] messages.create stream is undefined");
       }
       try {
         for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
           const chunk = temp.value;
-          if (chunk.type === "content_block_delta" && chunk.delta.type === "text_delta") {
-            yield chunk.delta.text;
+          if (chunk.type === "content_block_delta") {
+            if (chunk.delta.type === "text_delta") {
+              yield chunk.delta.text;
+            } else if (chunk.delta.type === "thinking_delta") {
+              yield `\0THINKING\0${chunk.delta.thinking}`;
+            }
           }
         }
       } catch (temp) {
@@ -2631,9 +2791,10 @@ ${context}`;
 var import_axios = __toESM(require("axios"));
 var OllamaProvider = class {
   constructor(llmConfig, embeddingConfig) {
-    var _a;
+    var _a, _b;
     const baseURL = (_a = llmConfig.baseUrl) != null ? _a : "http://localhost:11434";
-    this.http = import_axios.default.create({ baseURL, timeout: 12e4 });
+    const timeout = Number((_b = llmConfig.options) == null ? void 0 : _b.timeout) || 3e5;
+    this.http = import_axios.default.create({ baseURL, timeout });
     this.llmConfig = llmConfig;
     this.embeddingConfig = embeddingConfig;
   }
@@ -2689,14 +2850,15 @@ var OllamaProvider = class {
     };
   }
   async chat(messages, context, options) {
-    var _a, _b, _c, _d;
-    const system = this.buildSystemPrompt(context, options == null ? void 0 : options.systemPrompt);
+    var _a, _b, _c, _d, _e, _f;
+    const basePrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Use the provided context to answer the user's question.";
+    const system = buildSystemContent(basePrompt, context);
     const { data } = await this.http.post("/api/chat", {
       model: this.llmConfig.model,
       stream: false,
       options: {
-        temperature: (_b = (_a = options == null ? void 0 : options.temperature) != null ? _a : this.llmConfig.temperature) != null ? _b : 0.7,
-        num_predict: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024
+        temperature: (_d = (_c = options == null ? void 0 : options.temperature) != null ? _c : this.llmConfig.temperature) != null ? _d : 0.7,
+        num_predict: (_f = (_e = options == null ? void 0 : options.maxTokens) != null ? _e : this.llmConfig.maxTokens) != null ? _f : 1024
       },
       messages: [
         { role: "system", content: system },
@@ -2707,14 +2869,15 @@ var OllamaProvider = class {
   }
   chatStream(messages, context, options) {
     return __asyncGenerator(this, null, function* () {
-      var _a, _b, _c, _d, _e, _f;
-      const system = this.buildSystemPrompt(context, options == null ? void 0 : options.systemPrompt);
+      var _a, _b, _c, _d, _e, _f, _g, _h;
+      const basePrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Use the provided context to answer the user's question.";
+      const system = buildSystemContent(basePrompt, context);
       const response = yield new __await(this.http.post("/api/chat", {
         model: this.llmConfig.model,
         stream: true,
         options: {
-          temperature: (_b = (_a = options == null ? void 0 : options.temperature) != null ? _a : this.llmConfig.temperature) != null ? _b : 0.7,
-          num_predict: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024
+          temperature: (_d = (_c = options == null ? void 0 : options.temperature) != null ? _c : this.llmConfig.temperature) != null ? _d : 0.7,
+          num_predict: (_f = (_e = options == null ? void 0 : options.maxTokens) != null ? _e : this.llmConfig.maxTokens) != null ? _f : 1024
         },
         messages: [
           { role: "system", content: system },
@@ -2736,7 +2899,7 @@ var OllamaProvider = class {
             if (!line.trim()) continue;
             try {
               const json = JSON.parse(line);
-              if ((_e = json.message) == null ? void 0 : _e.content) {
+              if ((_g = json.message) == null ? void 0 : _g.content) {
                 yield json.message.content;
               }
               if (json.done) return;
@@ -2758,26 +2921,12 @@ var OllamaProvider = class {
       if (lineBuffer.trim()) {
         try {
           const json = JSON.parse(lineBuffer);
-          if ((_f = json.message) == null ? void 0 : _f.content) yield json.message.content;
+          if ((_h = json.message) == null ? void 0 : _h.content) yield json.message.content;
         } catch (e) {
         }
       }
     });
   }
-  buildSystemPrompt(context, overridePrompt) {
-    var _a;
-    if (overridePrompt) {
-      return overridePrompt;
-    }
-    const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Use the provided context to answer the user's question.
-
-Context:
-${context}`;
-    return systemPrompt.includes("{{context}}") ? systemPrompt.replace("{{context}}", context) : `${systemPrompt}
-
-Context:
-${context}`;
-  }
   async embed(text, options) {
     var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
     const model = (_c = (_b = options == null ? void 0 : options.model) != null ? _b : (_a = this.embeddingConfig) == null ? void 0 : _a.model) != null ? _c : "nomic-embed-text";
@@ -2924,21 +3073,6 @@ var GeminiProvider = class {
     var _a, _b;
     return sanitizeModel((_b = optionsModel != null ? optionsModel : (_a = this.embeddingConfig) == null ? void 0 : _a.model) != null ? _b : DEFAULT_EMBEDDING_MODEL);
   }
-  /**
-   * Build the system instruction string, inserting the RAG context either via
-   * the `{{context}}` placeholder or by appending it.
-   */
-  buildSystemInstruction(context) {
-    var _a;
-    const base = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Answer questions based on the provided context.
-
-Context:
-${context}`;
-    return base.includes("{{context}}") ? base.replace("{{context}}", context) : `${base}
-
-Context:
-${context}`;
-  }
   /**
    * Convert ChatMessage[] to the Gemini contents format.
    *
@@ -2957,16 +3091,17 @@ ${context}`;
   // ILLMProvider — chat
   // -------------------------------------------------------------------------
   async chat(messages, context, options) {
-    var _a, _b, _c, _d;
+    var _a, _b, _c, _d, _e, _f;
+    const basePrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Answer questions based on the provided context.";
     const model = this.client.getGenerativeModel({
       model: this.llmConfig.model,
-      systemInstruction: this.buildSystemInstruction(context)
+      systemInstruction: buildSystemContent(basePrompt, context)
     });
     const result = await model.generateContent({
       contents: this.buildGeminiContents(messages),
       generationConfig: {
-        temperature: (_b = (_a = options == null ? void 0 : options.temperature) != null ? _a : this.llmConfig.temperature) != null ? _b : DEFAULT_TEMPERATURE,
-        maxOutputTokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : DEFAULT_MAX_TOKENS,
+        temperature: (_d = (_c = options == null ? void 0 : options.temperature) != null ? _c : this.llmConfig.temperature) != null ? _d : DEFAULT_TEMPERATURE,
+        maxOutputTokens: (_f = (_e = options == null ? void 0 : options.maxTokens) != null ? _e : this.llmConfig.maxTokens) != null ? _f : DEFAULT_MAX_TOKENS,
         stopSequences: options == null ? void 0 : options.stop
       }
     });
@@ -2974,16 +3109,17 @@ ${context}`;
   }
   chatStream(messages, context, options) {
     return __asyncGenerator(this, null, function* () {
-      var _a, _b, _c, _d;
+      var _a, _b, _c, _d, _e, _f;
+      const basePrompt = (_b = (_a = options == null ? void 0 : options.systemPrompt) != null ? _a : this.llmConfig.systemPrompt) != null ? _b : "You are a helpful assistant. Answer questions based on the provided context.";
       const model = this.client.getGenerativeModel({
         model: this.llmConfig.model,
-        systemInstruction: this.buildSystemInstruction(context)
+        systemInstruction: buildSystemContent(basePrompt, context)
       });
       const result = yield new __await(model.generateContentStream({
         contents: this.buildGeminiContents(messages),
         generationConfig: {
-          temperature: (_b = (_a = options == null ? void 0 : options.temperature) != null ? _a : this.llmConfig.temperature) != null ? _b : DEFAULT_TEMPERATURE,
-          maxOutputTokens: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : DEFAULT_MAX_TOKENS,
+          temperature: (_d = (_c = options == null ? void 0 : options.temperature) != null ? _c : this.llmConfig.temperature) != null ? _d : DEFAULT_TEMPERATURE,
+          maxOutputTokens: (_f = (_e = options == null ? void 0 : options.maxTokens) != null ? _e : this.llmConfig.maxTokens) != null ? _f : DEFAULT_MAX_TOKENS,
           stopSequences: options == null ? void 0 : options.stop
         }
       }));
@@ -3379,7 +3515,9 @@ var LLMFactory = class _LLMFactory {
         if (llmConfig.baseUrl || ((_b = llmConfig.options) == null ? void 0 : _b.baseUrl)) {
           return new UniversalLLMAdapter(llmConfig);
         }
-        throw new Error(
+        throw new ProviderNotFoundException(
+          "LLM",
+          String(llmConfig.provider),
           `[LLMFactory] Unknown provider "${llmConfig.provider}". Built-in providers: ${_LLMFactory.listProviders().join(", ")}. Register a custom provider with LLMFactory.register().`
         );
       }
@@ -3511,7 +3649,7 @@ var ProviderRegistry = class {
         return UniversalVectorProvider2;
       }
       default:
-        throw new Error(`Unsupported vector provider: ${provider}`);
+        throw new ProviderNotFoundException("vector", provider);
     }
   }
   static async createVectorProvider(config) {
@@ -3529,12 +3667,18 @@ var ProviderRegistry = class {
         return new SimpleGraphProvider2(config);
       }
       default:
-        throw new Error(`Unsupported graph provider: ${provider}`);
+        throw new ProviderNotFoundException("graph", provider);
     }
   }
   static createLLMProvider(llmConfig, embeddingConfig) {
     return LLMFactory.create(llmConfig, embeddingConfig);
   }
+  static registerLLMProvider(name, factory) {
+    LLMFactory.register(name, factory);
+  }
+  static createEmbeddingProvider(embeddingConfig) {
+    return LLMFactory.createEmbeddingProvider(embeddingConfig);
+  }
 };
 ProviderRegistry.vectorProviders = {};
 ProviderRegistry.graphProviders = {};
@@ -3681,8 +3825,8 @@ var ConfigValidator = class {
     const errorItems = errors.filter((e) => e.severity === "error");
     if (errorItems.length > 0) {
       const message = errorItems.map((e) => `${e.field}: ${e.message}${e.suggestion ? ` (${e.suggestion})` : ""}`).join("\n  ");
-      throw new Error(`[ConfigValidator] Configuration validation failed:
-  ${message}`);
+      throw new ConfigurationException(`[ConfigValidator] Configuration validation failed:
+  ${message}`, errorItems);
     }
   }
 };
@@ -3869,20 +4013,17 @@ var Reranker = class {
     }
     try {
       const topN = matches.slice(0, 10);
-      const prompt = `You are a relevance ranking expert.
-Given the following user query and a list of retrieved document chunks, rank the chunks by relevance to the query.
-Return the result as a comma-separated list of document indices in order of relevance, from most relevant to least relevant.
-Use the indices provided in brackets like [0], [1], etc.
-Only return the indices (e.g., "2,0,1"), nothing else. Do not include brackets in the output.
-
-Query: "${query}"
+      const response = await llm.chat(
+        [{ role: "user", content: `Query: "${query}"
 
 Documents:
-${topN.map((m, i) => `[${i}] ${m.content.replace(/\n/g, " ")}`).join("\n")}`;
-      const response = await llm.chat(
-        [{ role: "user", content: prompt }],
+${topN.map((m, i) => `[${i}] ${m.content.replace(/\n/g, " ")}`).join("\n")}` }],
         "",
-        { temperature: 0, maxTokens: 50 }
+        {
+          systemPrompt: 'You are a relevance ranking expert. Given the user query and a list of retrieved document chunks, rank the chunks by relevance to the query. Return the result as a comma-separated list of document indices in order of relevance, from most relevant to least relevant. Use the indices provided in brackets like [0], [1], etc. Only return the indices (e.g., "2,0,1"), nothing else. Do not include brackets in the output.',
+          temperature: 0,
+          maxTokens: 50
+        }
       );
       const cleanedResponse = response.trim().replace(/[\[\]\s]/g, "");
       const rankedIndices = cleanedResponse.split(",").map(Number).filter((n) => !isNaN(n));
@@ -4388,6 +4529,18 @@ var QueryProcessor = class {
         }, normalizedField ? { field: normalizedField } : {}));
       }
     };
+    const activeValidFields = validFields.length > 0 ? validFields : ["brand", "category", "price", "rating", "stock", "stock_quantity", "status", "name", "title"];
+    const resolveValidField = (fieldStr) => {
+      const cleaned = this.normalizeHintValue(fieldStr).replace(/\b(?:provide|show|get|give|list|all|the|a|an|of|organizations?|companies?|records?|items?|whose|with|having|where|that|which|have|has|is|are|was|were)\b/gi, " ").replace(/\s+/g, " ").trim();
+      const comparable = (v) => v.toLowerCase().replace(/[^a-z0-9]/g, "");
+      const cleanedComparable = comparable(cleaned);
+      if (!cleanedComparable) return null;
+      const matchedField = activeValidFields.find((fieldName) => {
+        const candidate = comparable(fieldName);
+        return candidate === cleanedComparable || candidate.includes(cleanedComparable) || cleanedComparable.includes(candidate);
+      });
+      return matchedField != null ? matchedField : null;
+    };
     for (const match of question.matchAll(/["']([^"']{2,100})["']/g)) {
       addHint(match[1]);
     }
@@ -4412,7 +4565,13 @@ var QueryProcessor = class {
       const field = match[1];
       const value = match[2];
       if (field && !this.isLikelyPromptPhrase(field)) {
-        addHint(value, field);
+        const resolvedField = resolveValidField(field);
+        if (resolvedField) {
+          addHint(value, resolvedField);
+        } else {
+          addHint(value);
+          addHint(field);
+        }
       }
     }
     if (validFields.length > 0) {
@@ -4458,12 +4617,16 @@ var QueryProcessor = class {
     ];
     for (const pattern of fieldValuePatterns) {
       for (const match of question.matchAll(pattern)) {
-        const field = this.normalizeHintValue((_c = match[1]) != null ? _c : "");
+        const field = (_c = match[1]) != null ? _c : "";
         const value = (_d = match[2]) != null ? _d : "";
-        if (field && !this.isLikelyPromptPhrase(field)) {
-          addHint(value, field);
+        const resolvedField = resolveValidField(field);
+        if (resolvedField) {
+          addHint(value, resolvedField);
         } else {
           addHint(value);
+          if (field && !this.isLikelyPromptPhrase(field)) {
+            addHint(field);
+          }
         }
       }
     }
@@ -4690,7 +4853,7 @@ var LLMRouter = class {
 
 // src/utils/UITransformer.ts
 init_synonyms();
-var UITransformer = class {
+var UITransformer = class _UITransformer {
   // ─── Public Entry Points ─────────────────────────────────────────────────
   /**
    * Heuristic-only transform (no LLM required).
@@ -4752,7 +4915,7 @@ var UITransformer = class {
   static async analyzeAndDecide(query, sources, llm) {
     let intent;
     try {
-      intent = await this.detectIntent(query, llm);
+      intent = await this.detectIntent(query, llm, sources);
       console.debug("[UITransformer] Detected intent:", intent);
     } catch (err) {
       console.warn("[UITransformer] Intent detection failed; using heuristic.", err);
@@ -4770,11 +4933,23 @@ var UITransformer = class {
     try {
       const context = this.buildContextSummary(sources);
       const systemPrompt = this.buildVisualizationSystemPrompt();
+      const profile = this.profileData(sources);
+      const schemaContext = `
+RETRIEVED DATA SCHEMA:
+- Numeric fields (measures): ${profile.numericFields.map((f) => `${f.key} (unique values: ${f.uniqueCount})`).join(", ") || "None"}
+- Category fields (dimensions): ${profile.categoricalFields.map((f) => `${f.key} (unique values: ${f.uniqueCount})`).join(", ") || "None"}
+- Date fields (temporal): ${profile.dateFields.map((f) => `${f.key} (unique values: ${f.uniqueCount})`).join(", ") || "None"}
+- Boolean fields: ${profile.booleanFields.map((f) => `${f.key} (unique values: ${f.uniqueCount})`).join(", ") || "None"}
+- Text/Other fields: ${profile.fields.filter((f) => f.kind === "text").map((f) => f.key).join(", ") || "None"}
+`;
       const userPrompt = [
         `USER QUESTION: ${query}`,
         "",
         `DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
         "",
+        `RETRIEVED DATA SCHEMA:`,
+        schemaContext,
+        "",
         "RETRIEVED DATA (JSON):",
         context
       ].join("\n");
@@ -4785,6 +4960,20 @@ var UITransformer = class {
       );
       const parsed = this.parseTransformationResponse(rawResponse);
       if (parsed) {
+        let isCompatible = true;
+        if (parsed.type === "line_chart" && (profile.dateFields.length === 0 || profile.numericFields.length === 0)) {
+          isCompatible = false;
+        } else if (["bar_chart", "horizontal_bar", "pie_chart", "donut_chart"].includes(parsed.type) && (profile.categoricalFields.length === 0 && profile.numericFields.length === 0)) {
+          isCompatible = false;
+        } else if (parsed.type === "scatter_plot" && profile.numericFields.length < 2) {
+          isCompatible = false;
+        } else if (parsed.type === "metric_card" && profile.numericFields.length === 0) {
+          isCompatible = false;
+        }
+        if (!isCompatible) {
+          console.warn(`[UITransformer] LLM chose incompatible visualization type "${parsed.type}" for retrieved schema. Falling back to heuristic.`);
+          return this.transform(query, sources, void 0, void 0, intent);
+        }
         const intentAllowsTable = intent.wantsExplicitTable || ["tabular", "table", "geographic"].includes(intent.visualizationHint);
         const intentWantsPieLikeChart = ["pie_chart", "donut_chart"].includes(intent.recommendedChart) || ["composition", "category_breakdown"].includes(intent.visualizationHint);
         if (parsed.type === "table" && !intentAllowsTable) {
@@ -4814,7 +5003,26 @@ var UITransformer = class {
    * - The intent object can be reused across both the heuristic and LLM paths.
    * - It is easy to unit-test intent detection in isolation.
    */
-  static async detectIntent(query, llm) {
+  static async detectIntent(query, llm, sources) {
+    let schemaProfileText = "";
+    if (sources && sources.length > 0) {
+      const profile = this.profileData(sources);
+      const hasProducts = sources.some((item) => this.isProductData(item));
+      schemaProfileText = `
+RETRIEVED DATA SCHEMA:
+- Numeric fields (measures): ${profile.numericFields.map((f) => `${f.key} (unique values: ${f.uniqueCount})`).join(", ") || "None"}
+- Category fields (dimensions): ${profile.categoricalFields.map((f) => `${f.key} (unique values: ${f.uniqueCount})`).join(", ") || "None"}
+- Date fields (temporal): ${profile.dateFields.map((f) => `${f.key} (unique values: ${f.uniqueCount})`).join(", ") || "None"}
+- Boolean fields: ${profile.booleanFields.map((f) => `${f.key} (unique values: ${f.uniqueCount})`).join(", ") || "None"}
+- Text fields: ${profile.fields.filter((f) => f.kind === "text").map((f) => f.key).join(", ") || "None"}
+
+Please choose visualizationHint and recommendedChart dynamically based on the available schema.
+- Do NOT recommend "trend" or "line_chart" if there are no Date fields.
+- Do NOT recommend numeric-dependent visualizations (like bar_chart, line_chart, pie_chart, scatter_plot, histogram, metric_card) if there are no Numeric fields. Fall back to tabular/table or text.
+- If there are multiple metadata fields but no clear categories or numbers, "tabular"/"table" is a highly accurate default.
+${hasProducts ? `- Note: The retrieved data matches product/catalog structure. If the user query is looking for, exploring, searching, browsing, or listing products/items, you MUST recommend visualizationHint: "product_browse" and recommendedChart: "text".` : ""}
+`;
+    }
     const systemPrompt = `You are an intent classifier for a product-search RAG system.
 Given a user query, return ONLY a valid JSON object with this exact shape \u2014 no prose, no markdown:
 
@@ -4850,8 +5058,11 @@ RULES:
 - isTemporal: true if time words appear (trend, historical, over time, last year, monthly, etc.)
 - isComparison: true if user compares, ranks, or contrasts entities or categories.
 - language: detect from the query text itself; default "en" if uncertain.`;
+    const userPrompt = `QUERY: ${query}${schemaProfileText ? `
+
+${schemaProfileText}` : ""}`;
     const rawResponse = await llm.chat(
-      [{ role: "user", content: `QUERY: ${query}` }],
+      [{ role: "user", content: userPrompt }],
       "",
       { systemPrompt, temperature: 0 }
     );
@@ -5294,10 +5505,21 @@ RULES:
     const hasEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
     return asksForRecords && (hasNumericPredicate || hasEntityTerms);
   }
+  /** @internal kept private for transform() internal logic */
   static isProductQuery(query) {
+    return _UITransformer._productQueryTest(query);
+  }
+  /**
+   * Public entry-point for external callers (e.g. Pipeline)
+   * to check whether a query maps to product_browse without triggering an LLM call.
+   */
+  static isProductQueryPublic(query) {
+    return _UITransformer._productQueryTest(query);
+  }
+  static _productQueryTest(query) {
     const q = query.toLowerCase();
     const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
-    const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list)\b/.test(q);
+    const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
     const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
     return productTerms && productAction && !nonProductEntityTerms;
   }
@@ -6070,23 +6292,19 @@ function estimateCostUsd(promptTokens, completionTokens, model) {
 async function scoreHallucination(llm, answer, context) {
   const maxContextChars = 3e3;
   const truncatedContext = context.length > maxContextChars ? context.slice(0, maxContextChars) + "\n...[truncated]" : context;
-  const prompt = `You are an AI quality checker. Given the CONTEXT retrieved from a knowledge base and the ANSWER generated from it, rate how well-grounded the answer is.
-
-CONTEXT:
+  try {
+    const raw = await llm.chat(
+      [{ role: "user", content: `CONTEXT:
 ${truncatedContext}
 
 ANSWER:
-${answer}
-
-Return ONLY a valid JSON object with no markdown fences:
-{"score": <float 0-1>, "reason": "<one sentence>"}
-
-Where score 0 = fully grounded in context, score 1 = likely hallucinated or unsupported.`;
-  try {
-    const raw = await llm.chat(
-      [{ role: "user", content: prompt }],
+${answer}` }],
       "",
-      { temperature: 0, maxTokens: 120 }
+      {
+        systemPrompt: 'You are an AI quality checker. Given the CONTEXT retrieved from a knowledge base and the ANSWER generated from it, rate how well-grounded the answer is. Return ONLY a valid JSON object with no markdown fences: {"score": <float 0-1>, "reason": "<one sentence>"} where score 0 = fully grounded in context, score 1 = likely hallucinated or unsupported.',
+        temperature: 0,
+        maxTokens: 120
+      }
     );
     const jsonMatch = raw.match(/\{[\s\S]*\}/);
     if (!jsonMatch) return void 0;
@@ -6146,7 +6364,10 @@ var Pipeline = class {
 - For product descriptions, summarize customer-facing details only. Do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels.
 - Do NOT try to format product lists as tables.
 `;
-    this.config.llm.systemPrompt = chartInstruction;
+    const userPromptPrefix = this.config.llm.systemPrompt ? `${this.config.llm.systemPrompt}
+
+` : "";
+    this.config.llm.systemPrompt = `${userPromptPrefix}${chartInstruction}`;
     this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
     this.llmRouter = new LLMRouter(this.config);
     const { llmProvider: resolvedLLM, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
@@ -6324,11 +6545,11 @@ var Pipeline = class {
    */
   askStream(_0) {
     return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
-      var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
+      var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v;
       yield new __await(this.initialize());
       const ns = namespace != null ? namespace : this.config.projectId;
       const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
-      const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
+      const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0.3;
       const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
       const requestStart = performance.now();
       try {
@@ -6347,12 +6568,13 @@ var Pipeline = class {
         const embedStart = performance.now();
         const cacheKey = `${ns}::${searchQuery}`;
         const cachedVector = this.embeddingCache.get(cacheKey);
+        const useGraph = this.graphDB && ((_h = this.config.rag) == null ? void 0 : _h.useGraphRetrieval);
         const [strategyResult, embeddedVector] = yield new __await(Promise.all([
           QueryProcessor.determineRetrievalStrategy(
             searchQuery,
-            this.llmRouter.get("fast"),
-            (_h = this.config.rag) == null ? void 0 : _h.graphKeywords,
-            (_i = this.config.rag) == null ? void 0 : _i.vectorKeywords
+            useGraph ? this.llmRouter.get("fast") : void 0,
+            (_i = this.config.rag) == null ? void 0 : _i.graphKeywords,
+            (_j = this.config.rag) == null ? void 0 : _j.vectorKeywords
           ),
           // Embed immediately regardless of strategy — costs nothing if cached.
           // If strategy turns out to be 'graph'-only we just won't use the vector.
@@ -6362,7 +6584,7 @@ var Pipeline = class {
         if (!cachedVector && queryVector.length > 0) {
           this.embeddingCache.set(cacheKey, queryVector);
         }
-        const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
+        const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_k = this.config.rag) == null ? void 0 : _k.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
         const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
         const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
         const wantsExhaustiveList = hasNumericPredicates || hasMetadataFilter || /\b(list|all|show|provide|give|get|browse|find|search|display)\b/i.test(question);
@@ -6375,7 +6597,7 @@ var Pipeline = class {
         const structuredSources = this.applyStructuredFilters(rawSources, filter);
         let fullSources = hasNumericPredicates ? structuredSources : structuredSources.filter((m) => m.score >= scoreThreshold);
         const rerankLimit = wantsExhaustiveList ? retrievalLimit : topK;
-        if (!hasNumericPredicates && ((_k = this.config.rag) == null ? void 0 : _k.useReranking)) {
+        if (!hasNumericPredicates && ((_l = this.config.rag) == null ? void 0 : _l.useReranking)) {
           fullSources = yield new __await(this.reranker.rerank(fullSources, question, rerankLimit));
         } else if (!wantsExhaustiveList) {
           fullSources = fullSources.slice(0, topK);
@@ -6387,7 +6609,8 @@ ${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
         if (hasMetadataFilter) {
           displayCount = fullSources.length;
         } else {
-          const highlyRelevant = fullSources.filter((m) => m.score >= 0.4);
+          const baseThreshold = Math.max(scoreThreshold, 0.4);
+          const highlyRelevant = fullSources.filter((m) => m.score >= baseThreshold);
           displayCount = Math.max(highlyRelevant.length, topK);
           if (displayCount > 15) {
             displayCount = 15;
@@ -6405,34 +6628,117 @@ VECTOR CONTEXT:
 ${context}`;
         }
         const allMetadataKeys = Array.from(new Set(sources.flatMap((s) => Object.keys(s.metadata || {}))));
-        const trainedSchemaPromise = allMetadataKeys.length > 0 ? SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => void 0) : Promise.resolve(void 0);
-        const uiTransformationPromise = trainedSchemaPromise.then(
-          (trainedSchema) => this.generateUiTransformation(
+        const cachedSchema = allMetadataKeys.length > 0 ? SchemaMapper.getCached(ns, allMetadataKeys) : void 0;
+        if (allMetadataKeys.length > 0 && !cachedSchema) {
+          SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => {
+          });
+        }
+        const isProductQ = UITransformer.isProductQueryPublic(question);
+        const uiTransformationPromise = isProductQ ? Promise.resolve(
+          UITransformer.transform(
             question,
             sources,
-            trainedSchema,
-            hasNumericPredicates
+            this.config,
+            cachedSchema,
+            { visualizationHint: "product_browse", recommendedChart: "text", filterInStockOnly: false, wantsExplicitTable: false, isTemporal: false, isComparison: false, language: "en" }
           )
+        ) : this.generateUiTransformation(
+          question,
+          sources,
+          cachedSchema,
+          hasNumericPredicates
         ).catch((uiError) => {
           console.warn("[Pipeline] UI transformation failed concurrently:", uiError);
-          return UITransformer.transform(question, sources, this.config);
+          return UITransformer.transform(question, sources, this.config, cachedSchema);
         });
+        let hallucinationScoringPromise = Promise.resolve(void 0);
         const restrictionSuffix = "\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
+        const isClaude37 = this.config.llm.model.includes("claude-3-7-sonnet");
+        const isNativeThinking = isClaude37 && (((_m = this.config.llm.options) == null ? void 0 : _m.thinking) === true || ((_n = this.config.llm.options) == null ? void 0 : _n.thinking) === "enabled" || ((_o = this.config.llm.options) == null ? void 0 : _o.thinking) !== false);
+        const modelNameLower = this.config.llm.model.toLowerCase();
+        const isReasoningModel = modelNameLower.includes("-r1") || modelNameLower.includes("deepseek-r1") || modelNameLower.includes("reasoning") || modelNameLower.includes("thinking");
+        const isSimulatedThinking = !isNativeThinking && (((_p = this.config.llm.options) == null ? void 0 : _p.thinking) === true || ((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === "enabled" || isReasoningModel && ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) !== false);
+        let finalRestrictionSuffix = restrictionSuffix;
+        if (isSimulatedThinking) {
+          finalRestrictionSuffix += "\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)";
+        }
         const hardenedHistory = [...history];
-        const userQuestion = { role: "user", content: question + restrictionSuffix };
+        const userQuestion = { role: "user", content: question + finalRestrictionSuffix };
         const messages = [...hardenedHistory, userQuestion];
-        const systemPrompt = (_l = this.config.llm.systemPrompt) != null ? _l : "";
+        const systemPrompt = (_s = this.config.llm.systemPrompt) != null ? _s : "";
         const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
         let fullReply = "";
+        let thinkingText = "";
+        let thinkingStartMs = performance.now();
+        let thinkingDurationMs = 0;
         const generateStart = performance.now();
         if (this.llmProvider.chatStream) {
           const stream = this.llmProvider.chatStream(messages, context);
           if (!stream) throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
+          let inThinking = false;
+          let textBuffer = "";
           try {
             for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
               const chunk = temp.value;
-              fullReply += chunk;
-              yield chunk;
+              if (chunk.startsWith("\0THINKING\0")) {
+                const thinkingDelta = chunk.slice("\0THINKING\0".length);
+                thinkingText += thinkingDelta;
+                thinkingDurationMs = performance.now() - thinkingStartMs;
+                yield { type: "thinking", text: thinkingDelta };
+                continue;
+              }
+              if (isSimulatedThinking) {
+                textBuffer += chunk;
+                while (textBuffer.length > 0) {
+                  if (!inThinking) {
+                    const thinkIndex = textBuffer.indexOf("<think>");
+                    if (thinkIndex !== -1) {
+                      const before = textBuffer.slice(0, thinkIndex);
+                      if (before) {
+                        fullReply += before;
+                        yield before;
+                      }
+                      inThinking = true;
+                      thinkingStartMs = performance.now();
+                      textBuffer = textBuffer.slice(thinkIndex + "<think>".length);
+                    } else {
+                      const maxSafeLength = textBuffer.length - "<think>".length;
+                      if (maxSafeLength > 0) {
+                        const safeText = textBuffer.slice(0, maxSafeLength);
+                        fullReply += safeText;
+                        yield safeText;
+                        textBuffer = textBuffer.slice(maxSafeLength);
+                      }
+                      break;
+                    }
+                  } else {
+                    const endThinkIndex = textBuffer.indexOf("</think>");
+                    if (endThinkIndex !== -1) {
+                      const thinkingDelta = textBuffer.slice(0, endThinkIndex);
+                      if (thinkingDelta) {
+                        thinkingText += thinkingDelta;
+                        thinkingDurationMs = performance.now() - thinkingStartMs;
+                        yield { type: "thinking", text: thinkingDelta };
+                      }
+                      inThinking = false;
+                      textBuffer = textBuffer.slice(endThinkIndex + "</think>".length);
+                    } else {
+                      const maxSafeLength = textBuffer.length - "</think>".length;
+                      if (maxSafeLength > 0) {
+                        const thinkingDelta = textBuffer.slice(0, maxSafeLength);
+                        thinkingText += thinkingDelta;
+                        thinkingDurationMs = performance.now() - thinkingStartMs;
+                        yield { type: "thinking", text: thinkingDelta };
+                        textBuffer = textBuffer.slice(maxSafeLength);
+                      }
+                      break;
+                    }
+                  }
+                }
+              } else {
+                fullReply += chunk;
+                yield chunk;
+              }
             }
           } catch (temp) {
             error = [temp];
@@ -6444,17 +6750,42 @@ ${context}`;
                 throw error[0];
             }
           }
+          if (isSimulatedThinking && textBuffer.length > 0) {
+            if (inThinking) {
+              thinkingText += textBuffer;
+              yield { type: "thinking", text: textBuffer };
+            } else {
+              fullReply += textBuffer;
+              yield textBuffer;
+            }
+          }
         } else {
           const reply = yield new __await(this.llmProvider.chat(messages, context));
-          fullReply = reply;
-          yield reply;
+          if (isSimulatedThinking) {
+            const thinkStart = reply.indexOf("<think>");
+            const thinkEnd = reply.indexOf("</think>");
+            if (thinkStart !== -1 && thinkEnd !== -1 && thinkEnd > thinkStart) {
+              thinkingText = reply.slice(thinkStart + "<think>".length, thinkEnd);
+              fullReply = reply.slice(0, thinkStart) + reply.slice(thinkEnd + "</think>".length);
+              thinkingDurationMs = 100;
+            } else {
+              fullReply = reply;
+            }
+          } else {
+            fullReply = reply;
+          }
+          yield fullReply;
+        }
+        const runHallucination = ((_t = this.config.llm.options) == null ? void 0 : _t.hallucinationScoring) === true || this.config.llm.provider !== "ollama" && ((_u = this.config.llm.options) == null ? void 0 : _u.hallucinationScoring) !== false;
+        if (runHallucination) {
+          hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0);
         }
         const generateMs = performance.now() - generateStart;
         const totalMs = performance.now() - requestStart;
         const latency = {
           embedMs: Math.round(embedMs),
           retrieveMs: Math.round(retrieveMs),
-          rerankMs: ((_m = this.config.rag) == null ? void 0 : _m.useReranking) ? Math.round(rerankMs) : void 0,
+          rerankMs: ((_v = this.config.rag) == null ? void 0 : _v.useReranking) ? Math.round(rerankMs) : void 0,
           generateMs: Math.round(generateMs),
           totalMs: Math.round(totalMs)
         };
@@ -6467,12 +6798,16 @@ ${context}`;
           totalTokens: promptTokens + completionTokens,
           estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
         };
-        const trace = {
+        const [uiTransformation, hallucinationResult] = yield new __await(Promise.all([
+          uiTransformationPromise,
+          hallucinationScoringPromise
+        ]));
+        const trace = __spreadValues({
           requestId,
           query: question,
           rewrittenQuery,
           systemPrompt,
-          userPrompt: question + restrictionSuffix,
+          userPrompt: question + finalRestrictionSuffix,
           chunks: sources.map((s) => {
             var _a2;
             return {
@@ -6486,22 +6821,19 @@ ${context}`;
           latency,
           tokens,
           timestamp: (/* @__PURE__ */ new Date()).toISOString()
-        };
-        const uiTransformation = yield new __await(uiTransformationPromise);
+        }, hallucinationResult ? {
+          hallucinationScore: hallucinationResult.score,
+          hallucinationReason: hallucinationResult.reason
+        } : {});
         yield {
           reply: "",
           sources,
           graphData,
           ui_transformation: uiTransformation,
-          trace
+          trace,
+          thinking: thinkingText || void 0,
+          thinkingMs: thinkingDurationMs > 0 ? Math.round(thinkingDurationMs) : void 0
         };
-        scoreHallucination(this.llmProvider, fullReply, context).then((hallucinationResult) => {
-          if (hallucinationResult) {
-            trace.hallucinationScore = hallucinationResult.score;
-            trace.hallucinationReason = hallucinationResult.reason;
-          }
-        }).catch(() => {
-        });
       } catch (error2) {
         throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
       }
@@ -6511,18 +6843,30 @@ ${context}`;
    * Universal retrieval method combining all enabled providers.
    * Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
    */
-  async generateUiTransformation(question, sources, trainedSchema, forceDeterministic = false) {
+  async generateUiTransformation(question, sources, cachedSchema, forceDeterministic = false) {
+    var _a;
     if (!sources || sources.length === 0) {
-      return UITransformer.transform(question, sources, this.config, trainedSchema);
+      return UITransformer.transform(question, sources, this.config, cachedSchema);
     }
-    if (forceDeterministic) {
-      return UITransformer.transform(question, sources, this.config, trainedSchema);
+    if (UITransformer.isProductQueryPublic(question)) {
+      return UITransformer.transform(
+        question,
+        sources,
+        this.config,
+        cachedSchema,
+        { visualizationHint: "product_browse", recommendedChart: "text", filterInStockOnly: false, wantsExplicitTable: false, isTemporal: false, isComparison: false, language: "en" }
+      );
+    }
+    const isLocalProvider = this.config.llm.provider === "ollama";
+    const disableLlmUiTransform = ((_a = this.config.llm.options) == null ? void 0 : _a.disableLlmUiTransform) === true;
+    if (forceDeterministic || isLocalProvider || disableLlmUiTransform) {
+      return UITransformer.transform(question, sources, this.config, cachedSchema);
     }
     try {
       return await UITransformer.analyzeAndDecide(question, sources, this.llmRouter.get("fast"));
     } catch (err) {
       console.warn("[Pipeline] generateUiTransformation failed, using heuristic fallback:", err);
-      return UITransformer.transform(question, sources, this.config, trainedSchema);
+      return UITransformer.transform(question, sources, this.config, cachedSchema);
     }
   }
   applyStructuredFilters(sources, filter) {
@@ -6606,25 +6950,29 @@ ${context}`;
     return Number.isFinite(numeric) ? numeric : null;
   }
   async retrieve(query, options) {
-    var _a, _b, _c, _d, _e, _f;
+    var _a, _b, _c, _d, _e, _f, _g;
     const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
     const topK = (_b = options.topK) != null ? _b : 5;
     const cacheKey = `${ns}::${query}`;
     let queryVector = this.embeddingCache.get(cacheKey);
-    const strategy = await QueryProcessor.determineRetrievalStrategy(query, this.llmRouter.get("fast"));
+    const useGraph = this.graphDB && ((_c = this.config.rag) == null ? void 0 : _c.useGraphRetrieval);
+    const strategy = await QueryProcessor.determineRetrievalStrategy(
+      query,
+      useGraph ? this.llmRouter.get("fast") : void 0
+    );
     console.debug(`[Pipeline] Determined retrieval strategy: ${strategy}`);
     const [retrievedVector, graphData] = await Promise.all([
       // Only embed if we need vector search (strategy is 'vector' or 'both')
       strategy === "vector" || strategy === "both" ? queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: "query" }) : Promise.resolve([]),
       // Only query graph if we need graph search (strategy is 'graph' or 'both')
-      (strategy === "graph" || strategy === "both") && this.graphDB && ((_c = this.config.rag) == null ? void 0 : _c.useGraphRetrieval) ? this.graphDB.query(query) : Promise.resolve(void 0)
+      (strategy === "graph" || strategy === "both") && this.graphDB && ((_d = this.config.rag) == null ? void 0 : _d.useGraphRetrieval) ? this.graphDB.query(query) : Promise.resolve(void 0)
     ]);
     if ((strategy === "vector" || strategy === "both") && !queryVector && retrievedVector && retrievedVector.length > 0) {
       this.embeddingCache.set(cacheKey, retrievedVector);
       queryVector = retrievedVector;
     }
-    const baseFilter = __spreadProps(__spreadValues({}, (_d = options.filter) != null ? _d : {}), { queryText: query });
-    const numericPredicates = QueryProcessor.extractNumericPredicates(query, (_e = this.config.rag) == null ? void 0 : _e.filterableFields);
+    const baseFilter = __spreadProps(__spreadValues({}, (_e = options.filter) != null ? _e : {}), { queryText: query });
+    const numericPredicates = QueryProcessor.extractNumericPredicates(query, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
     if (numericPredicates.length > 0) {
       baseFilter.__numericPredicates = numericPredicates;
     }
@@ -6635,7 +6983,7 @@ ${context}`;
     ) : [];
     const resolvedSources = [];
     for (const source of sources) {
-      const parentId = (_f = source.metadata) == null ? void 0 : _f.parent_id;
+      const parentId = (_g = source.metadata) == null ? void 0 : _g.parent_id;
       if (parentId) {
         console.log(`[Pipeline] Multi-Vector: Found child chunk. Parent ID: ${parentId}`);
         resolvedSources.push(source);
@@ -6658,10 +7006,13 @@ New Question: ${question}
 Optimized Search Query:`;
     const rewrite = await this.llmProvider.chat(
       [
-        { role: "system", content: "You are an assistant that optimizes search queries for RAG systems." },
         { role: "user", content: prompt }
       ],
-      ""
+      "",
+      {
+        systemPrompt: "You are an assistant that optimizes search queries for RAG systems. Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.",
+        temperature: 0
+      }
     );
     return rewrite.trim() || question;
   }
@@ -6685,10 +7036,13 @@ ${context}
 Suggestions:`;
       const response = await this.llmProvider.chat(
         [
-          { role: "system", content: "You are a helpful assistant that generates search suggestions." },
           { role: "user", content: prompt }
         ],
-        ""
+        "",
+        {
+          systemPrompt: "You are a helpful assistant that generates search suggestions based on the provided snippets. Focus on questions that can be answered by the context, keep each under 10 words, and return ONLY a JSON array of strings.",
+          temperature: 0
+        }
       );
       const match = response.match(/\[[\s\S]*\]/);
       if (match) {
@@ -7027,6 +7381,10 @@ function createStreamHandler(configOrPlugin) {
               if (!isActive) break;
               if (typeof chunk === "string") {
                 enqueue(sseTextFrame(chunk));
+              } else if (chunk && typeof chunk === "object" && "type" in chunk && chunk.type === "thinking") {
+                enqueue(`data: ${JSON.stringify(chunk)}
+
+`);
               } else {
                 enqueue(sseMetaFrame(chunk));
                 const responseChunk = chunk;