@tiledesk/tiledesk-server 2.19.4 → 2.19.6

package/routes/kb.js

@@ -35,12 +35,12 @@ let MAX_UPLOAD_FILE_SIZE = process.env.MAX_UPLOAD_FILE_SIZE;
 let uploadlimits = undefined;
 
 if (MAX_UPLOAD_FILE_SIZE) {
-  uploadlimits = {fileSize: parseInt(MAX_UPLOAD_FILE_SIZE)} ;
+  uploadlimits = { fileSize: parseInt(MAX_UPLOAD_FILE_SIZE) };
   winston.debug("Max upload file size is : " + MAX_UPLOAD_FILE_SIZE);
 } else {
   winston.debug("Max upload file size is infinity");
 }
-var upload = multer({limits: uploadlimits});
+var upload = multer({ limits: uploadlimits });
 
 let jobManager = new JobManager(AMQP_MANAGER_URL, {
   debug: false,
@@ -84,22 +84,23 @@ const default_engine_hybrid = require('../config/kb/engine.hybrid');
 const default_embedding = require('../config/kb/embedding');
 const PromptManager = require('../config/kb/prompt/rag/PromptManager');
 const situatedContext = require('../config/kb/situatedContext');
+const kbQuestionService = require('../services/kbQuestionService');
 
 const ragPromptManager = new PromptManager(path.join(__dirname, '../config/kb/prompt/rag'));
 
 const RAG_CONTEXT_ENV_OVERRIDES = {
-  "gpt-3.5-turbo":       process.env.GPT_3_5_CONTEXT,
-  "gpt-4":               process.env.GPT_4_CONTEXT,
+  "gpt-3.5-turbo": process.env.GPT_3_5_CONTEXT,
+  "gpt-4": process.env.GPT_4_CONTEXT,
   "gpt-4-turbo-preview": process.env.GPT_4T_CONTEXT,
-  "gpt-4o":              process.env.GPT_4O_CONTEXT,
-  "gpt-4o-mini":         process.env.GPT_4O_MINI_CONTEXT,
-  "gpt-4.1":             process.env.GPT_4_1_CONTEXT,
-  "gpt-4.1-mini":        process.env.GPT_4_1_MINI_CONTEXT,
-  "gpt-4.1-nano":        process.env.GPT_4_1_NANO_CONTEXT,
-  "gpt-5":               process.env.GPT_5_CONTEXT,
-  "gpt-5-mini":          process.env.GPT_5_MINI_CONTEXT,
-  "gpt-5-nano":          process.env.GPT_5_NANO_CONTEXT,
-  "general":             process.env.GENERAL_CONTEXT
+  "gpt-4o": process.env.GPT_4O_CONTEXT,
+  "gpt-4o-mini": process.env.GPT_4O_MINI_CONTEXT,
+  "gpt-4.1": process.env.GPT_4_1_CONTEXT,
+  "gpt-4.1-mini": process.env.GPT_4_1_MINI_CONTEXT,
+  "gpt-4.1-nano": process.env.GPT_4_1_NANO_CONTEXT,
+  "gpt-5": process.env.GPT_5_CONTEXT,
+  "gpt-5-mini": process.env.GPT_5_MINI_CONTEXT,
+  "gpt-5-nano": process.env.GPT_5_NANO_CONTEXT,
+  "general": process.env.GENERAL_CONTEXT
 };
 
 /** RAG system prompt per modello: file in config/kb/prompt/rag, sovrascrivibili via env (come prima). */
@@ -114,13 +115,6 @@ function getRagContextTemplate(modelName) {
   return ragPromptManager.getPrompt(modelName);
 }
 
-function normalizeEmbedding(embedding) {
-  const normalizedEmbedding = (embedding && typeof embedding.toObject === 'function')
-    ? embedding.toObject()
-    : (embedding || default_embedding);
-  return { ...normalizedEmbedding };
-}
-
 function normalizeSituatedContext(enable = false) {
   situatedContext.enable = enable;
   return situatedContext.enable
@@ -142,15 +136,15 @@ router.post('/scrape/single', async (req, res) => {
 
   let data = req.body;
   winston.debug("/scrape/single data: ", data)
-  
+
   let namespace;
   try {
     namespace = await aiManager.checkNamespace(project_id, data.namespace);
   } catch (err) {
     let errorCode = err?.errorCode ?? 500;
     return res.status(errorCode).send({ success: false, error: err.error });
-  } 
-  
+  }
+
   if (data.type === "sitemap") {
 
     const urls = await aiManager.fetchSitemap(data.source).catch((err) => {
@@ -166,18 +160,18 @@ router.post('/scrape/single', async (req, res) => {
     try {
       sitemapKb = await KB.findById(data.id);
     } catch (err) {
-      winston.error("Error finding sitemap content with id " +  data.id);
+      winston.error("Error finding sitemap content with id " + data.id);
       return res.status(500).send({ success: false, error: "Error finding sitemap content with id " + data.id });
     }
 
     if (!sitemapKb) {
       return res.status(404).send({ success: false, error: "Content not found with id " + data.id });
     }
-    
+
     let existingKbs;
     try {
       existingKbs = await KB.find({ id_project: project_id, namespace: data.namespace, sitemap_origin_id: data.id }).lean().exec();
-    } catch(err) {
+    } catch (err) {
       winston.error("Error finding existing contents: ", err);
       return res.status(500).send({ success: false, error: "Error finding existing sitemap contents" });
     }
@@ -263,8 +257,7 @@ router.post('/scrape/single', async (req, res) => {
       }
 
       json.engine = namespace.engine || default_engine;
-      json.embedding = normalizeEmbedding(namespace.embedding);
-      json.embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
+      json.embedding = aiManager.normalizeEmbedding(namespace.embedding);
 
       if (namespace.hybrid === true) {
         json.hybrid = true;
@@ -396,7 +389,7 @@ router.post('/qa', async (req, res) => {
   if (publicKey === true) {
     let isAvailable = await quoteManager.checkQuote(req.project, obj, 'tokens');
     if (isAvailable === false) {
-      return res.status(403).send({ success: false, message: "Tokens quota exceeded", error_code: 13001})
+      return res.status(403).send({ success: false, message: "Tokens quota exceeded", error_code: 13001 })
     }
   }
 
@@ -411,15 +404,14 @@ router.post('/qa', async (req, res) => {
   }
 
   data.engine = namespace.engine || default_engine;
-  data.embedding = normalizeEmbedding(namespace.embedding);
-  data.embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
+  data.embedding = aiManager.normalizeEmbedding(namespace.embedding);
 
   if (namespace.hybrid === true) {
     data.search_type = 'hybrid';
-    
+
     if (data.reranking === true) {
       data.reranker_model = process.env.RERANKING_MODEL || "cross-encoder/ms-marco-MiniLM-L-6-v2";
-      
+
       if (!data.reranking_multiplier) {
         data.reranking_multiplier = 3;
       }
@@ -432,10 +424,10 @@ router.post('/qa', async (req, res) => {
         data.reranking_multiplier = calculatedRerankingMultiplier;
       }
     }
-  } 
+  }
 
   if (!namespace.hybrid && data.reranking === true && PINECONE_RERANKING) {
-    
+
     data.reranking = {
       "provider": "pinecone",
       "api_key": process.env.PINECONE_API_KEY,
@@ -476,7 +468,7 @@ router.post('/qa', async (req, res) => {
     const sendError = (message) => {
       try {
         res.write('data: ' + JSON.stringify({ error: message }) + '\n\n');
-      } catch (_) {}
+      } catch (_) { }
       res.end();
     };
 
@@ -518,6 +510,32 @@ router.post('/qa', async (req, res) => {
       return false;
     }
 
+    let streamQaResult = null;
+    const qaStartedAt = Date.now();
+    let streamDurationSeconds = null;
+
+    function captureStreamDurationFromFirstChunk(obj) {
+      if (streamDurationSeconds != null) return;
+      const content = extractContent(obj);
+      if (content != null && content !== '') {
+        streamDurationSeconds = (Date.now() - qaStartedAt) / 1000;
+      }
+    }
+
+    function withMeasuredDuration(payload) {
+      if (streamDurationSeconds != null && payload && typeof payload === 'object') {
+        payload.duration = streamDurationSeconds;
+      }
+      return payload;
+    }
+
+    function captureStreamQaResult(obj) {
+      const normalized = normalizeKbQaPayload(obj);
+      if (normalized && typeof normalized === 'object' && typeof normalized.success === 'boolean') {
+        streamQaResult = normalized;
+      }
+    }
+
     function forwardSsePayload(payload) {
       if (payload === '[DONE]') return;
       let obj;
@@ -527,16 +545,19 @@ router.post('/qa', async (req, res) => {
         return;
       }
 
+      captureStreamQaResult(obj);
+      captureStreamDurationFromFirstChunk(obj);
+
       if (obj.status === 'started') {
         return;
       }
       if (isKbStreamCompletedSummary(obj)) {
-        res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
+        res.write('data: ' + JSON.stringify(withMeasuredDuration(normalizeKbQaPayload(obj))) + '\n\n');
         return;
       }
 
       if (obj.type === 'metadata' || obj.event === 'metadata') {
-        res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
+        res.write('data: ' + JSON.stringify(withMeasuredDuration(normalizeKbQaPayload(obj))) + '\n\n');
         return;
       }
       const content = extractContent(obj);
@@ -546,7 +567,7 @@ router.post('/qa', async (req, res) => {
       }
       const normalized = normalizeKbQaPayload(obj);
       if (isMetadataPayload(normalized, content)) {
-        res.write('data: ' + JSON.stringify(normalized) + '\n\n');
+        res.write('data: ' + JSON.stringify(withMeasuredDuration(normalized)) + '\n\n');
       }
     }
 
@@ -567,7 +588,7 @@ router.post('/qa', async (req, res) => {
         }
       });
 
-      stream.on('end', () => {
+      stream.on('end', async () => {
         const tail = buffer.trim();
         if (tail) {
           for (const line of tail.split('\n')) {
@@ -576,6 +597,26 @@ router.post('/qa', async (req, res) => {
             forwardSsePayload(trimmed.slice(6));
           }
         }
+        if (streamQaResult) {
+          withMeasuredDuration(streamQaResult);
+          const tokens = await aiManager.getTrackingTokens({
+            qaResult: streamQaResult,
+            publicKey,
+            model: data.model,
+            obj,
+            quoteManager,
+            project: req.project,
+          });
+          kbQuestionService.trackKbQaResult(
+            id_project,
+            { namespace: data.namespace, question: data.question },
+            streamQaResult,
+            tokens
+          );
+          if (publicKey === true && typeof streamQaResult.legacy_prompt_token_size === 'number') {
+            res.write('data: ' + JSON.stringify(streamQaResult) + '\n\n');
+          }
+        }
         res.write('data: [DONE]\n\n');
         res.end();
       });
@@ -604,35 +645,29 @@ router.post('/qa', async (req, res) => {
     return;
   }
 
-  let duration = Date.now();
+  const qaStartedAt = Date.now();
 
-  aiService.askNamespace(data).then((resp) => {
+  aiService.askNamespace(data).then(async (resp) => {
     winston.debug("qa resp: ", resp.data);
     let answer = resp.data;
 
-    answer.duration = (Date.now() - duration) / 1000;
-
-    if (publicKey === true) {
-      let modelKey;
-      if (typeof data.model === 'string') {
-        modelKey = data.model;
-      } else if (data.model && typeof data.model.name === 'string') {
-        modelKey = data.model.name;
-      }
-
-      let multiplier = MODELS_MULTIPLIER[modelKey];
-      if (!multiplier) {
-        multiplier = 1;
-        winston.info("No multiplier found for AI model (qa) " + modelKey);
-      }
+    answer.duration = (Date.now() - qaStartedAt) / 1000;
 
-      obj.multiplier = multiplier;
-      obj.tokens = answer.prompt_token_size;
+    const tokens = await aiManager.getTrackingTokens({
+      qaResult: answer,
+      publicKey,
+      model: data.model,
+      obj,
+      quoteManager,
+      project: req.project,
+    });
+    kbQuestionService.trackKbQaResult(
+      id_project,
+      { namespace: data.namespace, question: data.question },
+      answer,
+      tokens
+    );
 
-      let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
-      winston.verbose("incremented_key: ", incremented_key);
-    }
-  
     return res.status(200).send(answer);
 
   }).catch((err) => {
@@ -738,7 +773,7 @@ router.post('/qa', async (req, res) => {
 
 //   if (namespace.hybrid === true) {
 //     data.search_type = 'hybrid';
-    
+
 //     if (data.reranking === true) {
 //       data.reranking_multiplier = 3;
 //       data.reranker_model = "cross-encoder/ms-marco-MiniLM-L-6-v2";
@@ -771,7 +806,7 @@ router.post('/qa', async (req, res) => {
 
 //   delete data.advancedPrompt;
 //   winston.verbose("ask data: ", data);
-  
+
 //   if (process.env.NODE_ENV === 'test') {
 //     return res.status(200).send({ success: true, message: "Question skipped in test environment", data: data });
 //   }
@@ -794,7 +829,7 @@ router.post('/qa', async (req, res) => {
 //       let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
 //       winston.verbose("incremented_key: ", incremented_key);
 //     }
-  
+
 //     return res.status(200).send(answer);
 
 //   }).catch((err) => {
@@ -886,7 +921,7 @@ router.get('/namespace/all', async (req, res) => {
 
   let project_id = req.projectid;
 
-  Namespace.find({ id_project: project_id }).lean().exec( async (err, namespaces) => {
+  Namespace.find({ id_project: project_id }).lean().exec(async (err, namespaces) => {
 
     if (err) {
       winston.error("find namespaces error: ", err);
@@ -935,7 +970,7 @@ router.get('/namespace/all', async (req, res) => {
       } else {
         namespaceObjArray = namespaces.map(({ _id, __v, ...keepAttrs }) => keepAttrs)
       }
-        
+
       winston.debug("namespaceObjArray: ", namespaceObjArray);
       return res.status(200).send(namespaceObjArray);
     }
@@ -961,14 +996,14 @@ router.get('/namespace/:id/chunks/:content_id', async (req, res) => {
     return res.status(403).send({ success: false, error: err })
   })
 
-  if(!content) {
+  if (!content) {
     return res.status(403).send({ success: false, error: "Not allowed. The content does not belong to the current namespace." })
   }
 
   let engine = namespace.engine || default_engine;
 
   if (process.env.NODE_ENV === 'test') {
-    return res.status(200).send({ success: true, message: "Get chunks skipped in test environment"});
+    return res.status(200).send({ success: true, message: "Get chunks skipped in test environment" });
   }
 
   aiService.getContentChunks(namespace_id, content_id, engine).then((resp) => {
@@ -989,7 +1024,7 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
 
   let project_id = req.projectid;
   let namespace_id = req.params.id;
-  
+
   let chatbotsArray = [];
 
   let namespace;
@@ -1031,12 +1066,12 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
   await Promise.all(chatbotPromises);
 
   winston.debug("chatbotsArray: ", chatbotsArray);
-  
+
   res.status(200).send(chatbotsArray);
 })
 
 router.get('/namespace/export/:id', async (req, res) => {
-  
+
   let id_project = req.projectid;
   let namespace_id = req.params.id;
 
@@ -1048,7 +1083,7 @@ router.get('/namespace/export/:id', async (req, res) => {
     query.status = parseInt(req.query.status)
   }
 
-  query.type = { $in: [ kbTypes.URL, kbTypes.TEXT, kbTypes.FAQ ] };
+  query.type = { $in: [kbTypes.URL, kbTypes.TEXT, kbTypes.FAQ] };
 
   let namespace;
   try {
@@ -1078,11 +1113,11 @@ router.get('/namespace/export/:id', async (req, res) => {
     let json_string = JSON.stringify(json);
     res.set({ "Content-Disposition": `attachment; filename="${filename}.json"` });
     return res.send(json_string);
-  } catch(err) {
+  } catch (err) {
     winston.error("Error genereting json ", err);
     return res.status(500).send({ success: false, error: "Error genereting json file" })
   }
-  
+
 })
 
 router.post('/namespace', async (req, res) => {
@@ -1212,7 +1247,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
   }
 
   let addingContents = [];
-  contents.forEach( async (e) => {
+  contents.forEach(async (e) => {
     let content = {
       id_project: id_project,
       name: e.name,
@@ -1254,8 +1289,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
   //          import operation the content's limit is respected
   let ns = namespaces.find(n => n.id === namespace_id);
   let engine = ns.engine || default_engine;
-  let embedding = normalizeEmbedding(ns.embedding);
-  embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
+  let embedding = aiManager.normalizeEmbedding(ns.embedding);
   let hybrid = ns.hybrid;
   const situated_context = normalizeSituatedContext();
 
@@ -1295,26 +1329,31 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
 
   let resources = new_contents.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
   resources = resources.map(({ _id, scrape_options, ...rest }) => {
-    return { 
-      id: _id, 
-      parameters_scrape_type_4: scrape_options, 
-      embedding: embedding, 
-      engine: engine, 
-      ...(situated_context && { situated_context: situated_context }), 
-      ...rest}
+    return {
+      id: _id,
+      parameters_scrape_type_4: scrape_options,
+      embedding: embedding,
+      engine: engine,
+      ...(situated_context && { situated_context: situated_context }),
+      ...rest
+    }
   });
-  
+
   winston.verbose("resources to be sent to worker: ", resources);
 
   if (process.env.NODE_ENV !== "test") {
     aiManager.scheduleScrape(resources, hybrid);
   }
 
+  if (process.env.NODE_ENV === "test") {
+    return res.status(200).send({ success: true, message: "Contents imported successfully", schedule_json: resources });
+  }
+
   res.status(200).send({ success: true, message: "Contents imported successfully" });
 
 
   // saveBulk(operations, addingContents, id_project).then((result) => {
-    
+
   //   let ns = namespaces.find(n => n.id === namespace_id);
   //   let engine = ns.engine || default_engine;
 
@@ -1328,7 +1367,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
   //   if (process.env.NODE_ENV !== 'test') {
   //     scheduleScrape(resources);
   //   }
-    
+
   //   //res.status(200).send(result);
   //   res.status(200).send({ success: true, message: "Contents imported successfully"});
 
@@ -1336,7 +1375,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
   //   winston.error("Unable to save kbs in bulk ", err)
   //   res.status(500).send(err);
   // })
-  
+
 })
 
 router.put('/namespace/:id', async (req, res) => {
@@ -1614,7 +1653,7 @@ router.post('/', async (req, res) => {
 
   const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags, situated_context, chunk_regex } = req.body;
   const namespace_id = req.body?.namespace;
-  
+
   if (!namespace_id) {
     return res.status(400).send({ success: false, error: "parameter 'namespace' is not defined" });
   }
@@ -1626,11 +1665,11 @@ router.post('/', async (req, res) => {
     let errorCode = err?.errorCode ?? 500;
     return res.status(errorCode).send({ success: false, error: err.error });
   }
-  
+
   const quoteManager = req.app.get('quote_manager');
   try {
     await aiManager.checkQuotaAvailability(quoteManager, project, 1)
-  } catch(err) {
+  } catch (err) {
     let errorCode = err?.errorCode ?? 500;
     return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
   }
@@ -1683,11 +1722,10 @@ router.post('/', async (req, res) => {
       delete raw_content.ok;
       delete raw_content.$clusterTime;
       delete raw_content.operationTime;
-      
+
       const saved_kb = raw_content.value;
       const webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
-      const embedding = normalizeEmbedding(namespace.embedding);
-      embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
+      const embedding = aiManager.normalizeEmbedding(namespace.embedding);
 
       const situated_context_obj = normalizeSituatedContext(saved_kb.situated_context);
 
@@ -1713,7 +1751,7 @@ router.post('/', async (req, res) => {
       if (process.env.NODE_ENV === 'test') {
         return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: raw_content, schedule_json: json });
       }
-        
+
       aiManager.scheduleScrape([json], namespace.hybrid);
       return res.status(200).send(raw_content);
 
@@ -1763,7 +1801,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
   const quoteManager = req.app.get('quote_manager');
   try {
     await aiManager.checkQuotaAvailability(quoteManager, project, list.length)
-  } catch(err) {
+  } catch (err) {
     let errorCode = err?.errorCode ?? 500;
     return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
   }
@@ -1844,7 +1882,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
       const quoteManager = req.app.get('quote_manager');
       try {
         await aiManager.checkQuotaAvailability(quoteManager, req.project, kbs.length)
-      } catch(err) {
+      } catch (err) {
         let errorCode = err?.errorCode ?? 500;
         return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
       }
@@ -1863,8 +1901,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
       aiManager.saveBulk(operations, kbs, project_id, namespace_id).then((result) => {
 
         let engine = namespace.engine || default_engine;
-        let embedding = normalizeEmbedding(namespace.embedding);
-        embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
+        let embedding = aiManager.normalizeEmbedding(namespace.embedding);
         let hybrid = namespace.hybrid;
 
         let situated_context_obj;
@@ -1872,15 +1909,15 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
           situated_context_obj = normalizeSituatedContext(situated_context);
         }
 
-        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context,  ...keepAttrs }) => keepAttrs)
-        resources = resources.map(({ _id, ...rest}) => {
-          return { 
-            id: _id, 
-            webhook: webhook, 
-            embedding: embedding, 
-            engine: engine, 
-            ...(situated_context_obj && { situated_context: situated_context_obj }), 
-            ...rest 
+        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context, ...keepAttrs }) => keepAttrs)
+        resources = resources.map(({ _id, ...rest }) => {
+          return {
+            id: _id,
+            webhook: webhook,
+            embedding: embedding,
+            engine: engine,
+            ...(situated_context_obj && { situated_context: situated_context_obj }),
+            ...rest
           };
         })
         winston.verbose("resources to be sent to worker: ", resources);
@@ -1888,7 +1925,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
         if (process.env.NODE_ENV === 'test') {
           return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: result, schedule_json: resources });
         }
-        
+
         aiManager.scheduleScrape(resources, hybrid);
         return res.status(200).send(result);
 
@@ -1920,7 +1957,7 @@ router.post('/sitemap', async (req, res) => {
     winston.debug("data: ", data);
     res.status(200).send(data);
   }).catch((err) => {
-    console.error("err ", err)
+    winston.error("Error fetching sitemap: ", err)
     res.status(500).send({ success: false, error: err });
   })
 
@@ -1937,13 +1974,13 @@ router.post('/sitemap/import', async (req, res) => {
   }
 
   if (type !== "sitemap") {
-    return res.status(403).send({success: false, error: "Endpoint available for sitemap type only." });
+    return res.status(403).send({ success: false, error: "Endpoint available for sitemap type only." });
   }
 
   if (!namespace_id) {
     return res.status(400).send({ success: false, error: "queryParam 'namespace' is not defined" })
   }
-  
+
   let namespace;
   try {
     namespace = await aiManager.checkNamespace(id_project, namespace_id);
@@ -1974,7 +2011,7 @@ router.post('/sitemap/import', async (req, res) => {
 
   if (data.errors && data.errors.length > 0) {
     winston.error("An error occurred during sitemap fetch: ", data.errors[0])
-    return res.status(500).send({ success: false, error: "Unable to fecth sitemap due to an error: " + data.errors[0].message})
+    return res.status(500).send({ success: false, error: "Unable to fecth sitemap due to an error: " + data.errors[0].message })
   }
 
   const urls = Array.isArray(data.sites) ? data.sites : [];
@@ -2018,7 +2055,7 @@ router.post('/sitemap/import', async (req, res) => {
     ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {}),
     ...(situated_context && situated_context === true && scrape_type === 0 ? { situated_context: true } : {}),
   }
-  
+
   try {
     await aiManager.scheduleSitemap(namespace, saved_content, options);
     let result = await aiManager.addMultipleUrls(namespace, urls, options);
@@ -2030,8 +2067,8 @@ router.post('/sitemap/import', async (req, res) => {
     return res.status(200).send(result);
   } catch (err) {
     return res.status(500).send({ success: false, error: "Unable to add multiple urls from sitemap due to an error." });
-  }  
-  
+  }
+
 })
 
 router.put('/:kb_id', async (req, res) => {
@@ -2039,7 +2076,7 @@ router.put('/:kb_id', async (req, res) => {
   const id_project = req.projectid;
   const project = req.project;
   const kb_id = req.params.kb_id;
-  
+
   const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags } = req.body;
   const namespace_id = req.body.namespace;
   let situated_context = req.body.situated_context;
@@ -2157,8 +2194,7 @@ router.put('/:kb_id', async (req, res) => {
     return res.status(500).send({ success: false, error: err });
   }
 
-  const embedding = normalizeEmbedding(namespace.embedding);
-  embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
+  const embedding = aiManager.normalizeEmbedding(namespace.embedding);
   let webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
   const situated_context_obj = normalizeSituatedContext(updated_content.situated_context);
 
@@ -2183,7 +2219,7 @@ router.put('/:kb_id', async (req, res) => {
   if (process.env.NODE_ENV === 'test') {
     return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: updated_content, schedule_json: json });
   }
-    
+
   aiManager.scheduleScrape([json], namespace.hybrid);
   return res.status(200).send(updated_content);
 
@@ -2195,11 +2231,11 @@ router.delete('/:kb_id', async (req, res) => {
   let kb_id = req.params.kb_id;
   winston.verbose("delete kb_id " + kb_id);
 
-  let kb = await KB.findOne({ id_project: project_id, _id: kb_id}).catch((err) => {
+  let kb = await KB.findOne({ id_project: project_id, _id: kb_id }).catch((err) => {
     winston.warn("Unable to find kb. Error: ", err);
     return res.status(500).send({ success: false, error: err })
   })
-  
+
   if (!kb) {
     winston.error("Unable to delete kb. Kb not found...")
     return res.status(404).send({ success: false, error: "Content not found" })
@@ -2218,7 +2254,7 @@ router.delete('/:kb_id', async (req, res) => {
   if (kb.type === 'sitemap') {
     try {
       await aiManager.deleteSitemap(kb, namespace);
-      console.log("Scheduled jobs for deleting sitemap");
+      winston.info("Scheduled jobs for deleting sitemap");
       //return res.status(200).send({ success: true, message: "Scheduled jobs for deleting sitemap" });
     } catch (err) {
       winston.error("Error deleting sitemap: ", err);
@@ -2234,7 +2270,7 @@ router.delete('/:kb_id', async (req, res) => {
   if (!data.namespace) {
     data.namespace = project_id;
   }
-  
+
   data.engine = namespace.engine || default_engine;
   winston.verbose("/:delete_id data: ", data);