@tiledesk/tiledesk-server 2.19.3 → 2.19.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/routes/kb.js CHANGED
@@ -35,12 +35,12 @@ let MAX_UPLOAD_FILE_SIZE = process.env.MAX_UPLOAD_FILE_SIZE;
35
35
  let uploadlimits = undefined;
36
36
 
37
37
  if (MAX_UPLOAD_FILE_SIZE) {
38
- uploadlimits = {fileSize: parseInt(MAX_UPLOAD_FILE_SIZE)} ;
38
+ uploadlimits = { fileSize: parseInt(MAX_UPLOAD_FILE_SIZE) };
39
39
  winston.debug("Max upload file size is : " + MAX_UPLOAD_FILE_SIZE);
40
40
  } else {
41
41
  winston.debug("Max upload file size is infinity");
42
42
  }
43
- var upload = multer({limits: uploadlimits});
43
+ var upload = multer({ limits: uploadlimits });
44
44
 
45
45
  let jobManager = new JobManager(AMQP_MANAGER_URL, {
46
46
  debug: false,
@@ -84,22 +84,23 @@ const default_engine_hybrid = require('../config/kb/engine.hybrid');
84
84
  const default_embedding = require('../config/kb/embedding');
85
85
  const PromptManager = require('../config/kb/prompt/rag/PromptManager');
86
86
  const situatedContext = require('../config/kb/situatedContext');
87
+ const kbQuestionService = require('../services/kbQuestionService');
87
88
 
88
89
  const ragPromptManager = new PromptManager(path.join(__dirname, '../config/kb/prompt/rag'));
89
90
 
90
91
  const RAG_CONTEXT_ENV_OVERRIDES = {
91
- "gpt-3.5-turbo": process.env.GPT_3_5_CONTEXT,
92
- "gpt-4": process.env.GPT_4_CONTEXT,
92
+ "gpt-3.5-turbo": process.env.GPT_3_5_CONTEXT,
93
+ "gpt-4": process.env.GPT_4_CONTEXT,
93
94
  "gpt-4-turbo-preview": process.env.GPT_4T_CONTEXT,
94
- "gpt-4o": process.env.GPT_4O_CONTEXT,
95
- "gpt-4o-mini": process.env.GPT_4O_MINI_CONTEXT,
96
- "gpt-4.1": process.env.GPT_4_1_CONTEXT,
97
- "gpt-4.1-mini": process.env.GPT_4_1_MINI_CONTEXT,
98
- "gpt-4.1-nano": process.env.GPT_4_1_NANO_CONTEXT,
99
- "gpt-5": process.env.GPT_5_CONTEXT,
100
- "gpt-5-mini": process.env.GPT_5_MINI_CONTEXT,
101
- "gpt-5-nano": process.env.GPT_5_NANO_CONTEXT,
102
- "general": process.env.GENERAL_CONTEXT
95
+ "gpt-4o": process.env.GPT_4O_CONTEXT,
96
+ "gpt-4o-mini": process.env.GPT_4O_MINI_CONTEXT,
97
+ "gpt-4.1": process.env.GPT_4_1_CONTEXT,
98
+ "gpt-4.1-mini": process.env.GPT_4_1_MINI_CONTEXT,
99
+ "gpt-4.1-nano": process.env.GPT_4_1_NANO_CONTEXT,
100
+ "gpt-5": process.env.GPT_5_CONTEXT,
101
+ "gpt-5-mini": process.env.GPT_5_MINI_CONTEXT,
102
+ "gpt-5-nano": process.env.GPT_5_NANO_CONTEXT,
103
+ "general": process.env.GENERAL_CONTEXT
103
104
  };
104
105
 
105
106
  /** RAG system prompt per modello: file in config/kb/prompt/rag, sovrascrivibili via env (come prima). */
@@ -142,15 +143,15 @@ router.post('/scrape/single', async (req, res) => {
142
143
 
143
144
  let data = req.body;
144
145
  winston.debug("/scrape/single data: ", data)
145
-
146
+
146
147
  let namespace;
147
148
  try {
148
149
  namespace = await aiManager.checkNamespace(project_id, data.namespace);
149
150
  } catch (err) {
150
151
  let errorCode = err?.errorCode ?? 500;
151
152
  return res.status(errorCode).send({ success: false, error: err.error });
152
- }
153
-
153
+ }
154
+
154
155
  if (data.type === "sitemap") {
155
156
 
156
157
  const urls = await aiManager.fetchSitemap(data.source).catch((err) => {
@@ -166,18 +167,18 @@ router.post('/scrape/single', async (req, res) => {
166
167
  try {
167
168
  sitemapKb = await KB.findById(data.id);
168
169
  } catch (err) {
169
- winston.error("Error finding sitemap content with id " + data.id);
170
+ winston.error("Error finding sitemap content with id " + data.id);
170
171
  return res.status(500).send({ success: false, error: "Error finding sitemap content with id " + data.id });
171
172
  }
172
173
 
173
174
  if (!sitemapKb) {
174
175
  return res.status(404).send({ success: false, error: "Content not found with id " + data.id });
175
176
  }
176
-
177
+
177
178
  let existingKbs;
178
179
  try {
179
180
  existingKbs = await KB.find({ id_project: project_id, namespace: data.namespace, sitemap_origin_id: data.id }).lean().exec();
180
- } catch(err) {
181
+ } catch (err) {
181
182
  winston.error("Error finding existing contents: ", err);
182
183
  return res.status(500).send({ success: false, error: "Error finding existing sitemap contents" });
183
184
  }
@@ -276,6 +277,8 @@ router.post('/scrape/single', async (req, res) => {
276
277
 
277
278
  winston.verbose("/scrape/single json: ", json);
278
279
 
280
+ json.id_project = project_id;
281
+
279
282
  if (process.env.NODE_ENV === "test") {
280
283
  res.status(200).send({ success: true, message: "Skip indexing in test environment", data: json })
281
284
  }
@@ -362,6 +365,7 @@ router.post('/qa', async (req, res) => {
362
365
  let id_project = req.projectid;
363
366
  let publicKey = false;
364
367
  let data = req.body;
368
+ data.id_project = id_project;
365
369
 
366
370
  let namespace;
367
371
  try {
@@ -393,7 +397,7 @@ router.post('/qa', async (req, res) => {
393
397
  if (publicKey === true) {
394
398
  let isAvailable = await quoteManager.checkQuote(req.project, obj, 'tokens');
395
399
  if (isAvailable === false) {
396
- return res.status(403).send({ success: false, message: "Tokens quota exceeded", error_code: 13001})
400
+ return res.status(403).send({ success: false, message: "Tokens quota exceeded", error_code: 13001 })
397
401
  }
398
402
  }
399
403
 
@@ -413,10 +417,10 @@ router.post('/qa', async (req, res) => {
413
417
 
414
418
  if (namespace.hybrid === true) {
415
419
  data.search_type = 'hybrid';
416
-
420
+
417
421
  if (data.reranking === true) {
418
422
  data.reranker_model = process.env.RERANKING_MODEL || "cross-encoder/ms-marco-MiniLM-L-6-v2";
419
-
423
+
420
424
  if (!data.reranking_multiplier) {
421
425
  data.reranking_multiplier = 3;
422
426
  }
@@ -429,10 +433,10 @@ router.post('/qa', async (req, res) => {
429
433
  data.reranking_multiplier = calculatedRerankingMultiplier;
430
434
  }
431
435
  }
432
- }
436
+ }
433
437
 
434
438
  if (!namespace.hybrid && data.reranking === true && PINECONE_RERANKING) {
435
-
439
+
436
440
  data.reranking = {
437
441
  "provider": "pinecone",
438
442
  "api_key": process.env.PINECONE_API_KEY,
@@ -473,7 +477,7 @@ router.post('/qa', async (req, res) => {
473
477
  const sendError = (message) => {
474
478
  try {
475
479
  res.write('data: ' + JSON.stringify({ error: message }) + '\n\n');
476
- } catch (_) {}
480
+ } catch (_) { }
477
481
  res.end();
478
482
  };
479
483
 
@@ -515,6 +519,32 @@ router.post('/qa', async (req, res) => {
515
519
  return false;
516
520
  }
517
521
 
522
+ let streamQaResult = null;
523
+ const qaStartedAt = Date.now();
524
+ let streamDurationSeconds = null;
525
+
526
+ function captureStreamDurationFromFirstChunk(obj) {
527
+ if (streamDurationSeconds != null) return;
528
+ const content = extractContent(obj);
529
+ if (content != null && content !== '') {
530
+ streamDurationSeconds = (Date.now() - qaStartedAt) / 1000;
531
+ }
532
+ }
533
+
534
+ function withMeasuredDuration(payload) {
535
+ if (streamDurationSeconds != null && payload && typeof payload === 'object') {
536
+ payload.duration = streamDurationSeconds;
537
+ }
538
+ return payload;
539
+ }
540
+
541
+ function captureStreamQaResult(obj) {
542
+ const normalized = normalizeKbQaPayload(obj);
543
+ if (normalized && typeof normalized === 'object' && typeof normalized.success === 'boolean') {
544
+ streamQaResult = normalized;
545
+ }
546
+ }
547
+
518
548
  function forwardSsePayload(payload) {
519
549
  if (payload === '[DONE]') return;
520
550
  let obj;
@@ -524,16 +554,19 @@ router.post('/qa', async (req, res) => {
524
554
  return;
525
555
  }
526
556
 
557
+ captureStreamQaResult(obj);
558
+ captureStreamDurationFromFirstChunk(obj);
559
+
527
560
  if (obj.status === 'started') {
528
561
  return;
529
562
  }
530
563
  if (isKbStreamCompletedSummary(obj)) {
531
- res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
564
+ res.write('data: ' + JSON.stringify(withMeasuredDuration(normalizeKbQaPayload(obj))) + '\n\n');
532
565
  return;
533
566
  }
534
567
 
535
568
  if (obj.type === 'metadata' || obj.event === 'metadata') {
536
- res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
569
+ res.write('data: ' + JSON.stringify(withMeasuredDuration(normalizeKbQaPayload(obj))) + '\n\n');
537
570
  return;
538
571
  }
539
572
  const content = extractContent(obj);
@@ -543,7 +576,7 @@ router.post('/qa', async (req, res) => {
543
576
  }
544
577
  const normalized = normalizeKbQaPayload(obj);
545
578
  if (isMetadataPayload(normalized, content)) {
546
- res.write('data: ' + JSON.stringify(normalized) + '\n\n');
579
+ res.write('data: ' + JSON.stringify(withMeasuredDuration(normalized)) + '\n\n');
547
580
  }
548
581
  }
549
582
 
@@ -564,7 +597,7 @@ router.post('/qa', async (req, res) => {
564
597
  }
565
598
  });
566
599
 
567
- stream.on('end', () => {
600
+ stream.on('end', async () => {
568
601
  const tail = buffer.trim();
569
602
  if (tail) {
570
603
  for (const line of tail.split('\n')) {
@@ -573,6 +606,26 @@ router.post('/qa', async (req, res) => {
573
606
  forwardSsePayload(trimmed.slice(6));
574
607
  }
575
608
  }
609
+ if (streamQaResult) {
610
+ withMeasuredDuration(streamQaResult);
611
+ const tokens = await aiManager.getTrackingTokens({
612
+ qaResult: streamQaResult,
613
+ publicKey,
614
+ model: data.model,
615
+ obj,
616
+ quoteManager,
617
+ project: req.project,
618
+ });
619
+ kbQuestionService.trackKbQaResult(
620
+ id_project,
621
+ { namespace: data.namespace, question: data.question },
622
+ streamQaResult,
623
+ tokens
624
+ );
625
+ if (publicKey === true && typeof streamQaResult.legacy_prompt_token_size === 'number') {
626
+ res.write('data: ' + JSON.stringify(streamQaResult) + '\n\n');
627
+ }
628
+ }
576
629
  res.write('data: [DONE]\n\n');
577
630
  res.end();
578
631
  });
@@ -601,31 +654,29 @@ router.post('/qa', async (req, res) => {
601
654
  return;
602
655
  }
603
656
 
604
- aiService.askNamespace(data).then((resp) => {
657
+ const qaStartedAt = Date.now();
658
+
659
+ aiService.askNamespace(data).then(async (resp) => {
605
660
  winston.debug("qa resp: ", resp.data);
606
661
  let answer = resp.data;
607
662
 
608
- if (publicKey === true) {
609
- let modelKey;
610
- if (typeof data.model === 'string') {
611
- modelKey = data.model;
612
- } else if (data.model && typeof data.model.name === 'string') {
613
- modelKey = data.model.name;
614
- }
663
+ answer.duration = (Date.now() - qaStartedAt) / 1000;
615
664
 
616
- let multiplier = MODELS_MULTIPLIER[modelKey];
617
- if (!multiplier) {
618
- multiplier = 1;
619
- winston.info("No multiplier found for AI model (qa) " + modelKey);
620
- }
621
-
622
- obj.multiplier = multiplier;
623
- obj.tokens = answer.prompt_token_size;
665
+ const tokens = await aiManager.getTrackingTokens({
666
+ qaResult: answer,
667
+ publicKey,
668
+ model: data.model,
669
+ obj,
670
+ quoteManager,
671
+ project: req.project,
672
+ });
673
+ kbQuestionService.trackKbQaResult(
674
+ id_project,
675
+ { namespace: data.namespace, question: data.question },
676
+ answer,
677
+ tokens
678
+ );
624
679
 
625
- let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
626
- winston.verbose("incremented_key: ", incremented_key);
627
- }
628
-
629
680
  return res.status(200).send(answer);
630
681
 
631
682
  }).catch((err) => {
@@ -731,7 +782,7 @@ router.post('/qa', async (req, res) => {
731
782
 
732
783
  // if (namespace.hybrid === true) {
733
784
  // data.search_type = 'hybrid';
734
-
785
+
735
786
  // if (data.reranking === true) {
736
787
  // data.reranking_multiplier = 3;
737
788
  // data.reranker_model = "cross-encoder/ms-marco-MiniLM-L-6-v2";
@@ -764,7 +815,7 @@ router.post('/qa', async (req, res) => {
764
815
 
765
816
  // delete data.advancedPrompt;
766
817
  // winston.verbose("ask data: ", data);
767
-
818
+
768
819
  // if (process.env.NODE_ENV === 'test') {
769
820
  // return res.status(200).send({ success: true, message: "Question skipped in test environment", data: data });
770
821
  // }
@@ -787,7 +838,7 @@ router.post('/qa', async (req, res) => {
787
838
  // let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
788
839
  // winston.verbose("incremented_key: ", incremented_key);
789
840
  // }
790
-
841
+
791
842
  // return res.status(200).send(answer);
792
843
 
793
844
  // }).catch((err) => {
@@ -879,7 +930,7 @@ router.get('/namespace/all', async (req, res) => {
879
930
 
880
931
  let project_id = req.projectid;
881
932
 
882
- Namespace.find({ id_project: project_id }).lean().exec( async (err, namespaces) => {
933
+ Namespace.find({ id_project: project_id }).lean().exec(async (err, namespaces) => {
883
934
 
884
935
  if (err) {
885
936
  winston.error("find namespaces error: ", err);
@@ -928,7 +979,7 @@ router.get('/namespace/all', async (req, res) => {
928
979
  } else {
929
980
  namespaceObjArray = namespaces.map(({ _id, __v, ...keepAttrs }) => keepAttrs)
930
981
  }
931
-
982
+
932
983
  winston.debug("namespaceObjArray: ", namespaceObjArray);
933
984
  return res.status(200).send(namespaceObjArray);
934
985
  }
@@ -954,14 +1005,14 @@ router.get('/namespace/:id/chunks/:content_id', async (req, res) => {
954
1005
  return res.status(403).send({ success: false, error: err })
955
1006
  })
956
1007
 
957
- if(!content) {
1008
+ if (!content) {
958
1009
  return res.status(403).send({ success: false, error: "Not allowed. The content does not belong to the current namespace." })
959
1010
  }
960
1011
 
961
1012
  let engine = namespace.engine || default_engine;
962
1013
 
963
1014
  if (process.env.NODE_ENV === 'test') {
964
- return res.status(200).send({ success: true, message: "Get chunks skipped in test environment"});
1015
+ return res.status(200).send({ success: true, message: "Get chunks skipped in test environment" });
965
1016
  }
966
1017
 
967
1018
  aiService.getContentChunks(namespace_id, content_id, engine).then((resp) => {
@@ -982,7 +1033,7 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
982
1033
 
983
1034
  let project_id = req.projectid;
984
1035
  let namespace_id = req.params.id;
985
-
1036
+
986
1037
  let chatbotsArray = [];
987
1038
 
988
1039
  let namespace;
@@ -1024,12 +1075,12 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
1024
1075
  await Promise.all(chatbotPromises);
1025
1076
 
1026
1077
  winston.debug("chatbotsArray: ", chatbotsArray);
1027
-
1078
+
1028
1079
  res.status(200).send(chatbotsArray);
1029
1080
  })
1030
1081
 
1031
1082
  router.get('/namespace/export/:id', async (req, res) => {
1032
-
1083
+
1033
1084
  let id_project = req.projectid;
1034
1085
  let namespace_id = req.params.id;
1035
1086
 
@@ -1041,7 +1092,7 @@ router.get('/namespace/export/:id', async (req, res) => {
1041
1092
  query.status = parseInt(req.query.status)
1042
1093
  }
1043
1094
 
1044
- query.type = { $in: [ kbTypes.URL, kbTypes.TEXT, kbTypes.FAQ ] };
1095
+ query.type = { $in: [kbTypes.URL, kbTypes.TEXT, kbTypes.FAQ] };
1045
1096
 
1046
1097
  let namespace;
1047
1098
  try {
@@ -1071,11 +1122,11 @@ router.get('/namespace/export/:id', async (req, res) => {
1071
1122
  let json_string = JSON.stringify(json);
1072
1123
  res.set({ "Content-Disposition": `attachment; filename="${filename}.json"` });
1073
1124
  return res.send(json_string);
1074
- } catch(err) {
1125
+ } catch (err) {
1075
1126
  winston.error("Error genereting json ", err);
1076
1127
  return res.status(500).send({ success: false, error: "Error genereting json file" })
1077
1128
  }
1078
-
1129
+
1079
1130
  })
1080
1131
 
1081
1132
  router.post('/namespace', async (req, res) => {
@@ -1205,7 +1256,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1205
1256
  }
1206
1257
 
1207
1258
  let addingContents = [];
1208
- contents.forEach( async (e) => {
1259
+ contents.forEach(async (e) => {
1209
1260
  let content = {
1210
1261
  id_project: id_project,
1211
1262
  name: e.name,
@@ -1288,15 +1339,16 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1288
1339
 
1289
1340
  let resources = new_contents.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
1290
1341
  resources = resources.map(({ _id, scrape_options, ...rest }) => {
1291
- return {
1292
- id: _id,
1293
- parameters_scrape_type_4: scrape_options,
1294
- embedding: embedding,
1295
- engine: engine,
1296
- ...(situated_context && { situated_context: situated_context }),
1297
- ...rest}
1342
+ return {
1343
+ id: _id,
1344
+ parameters_scrape_type_4: scrape_options,
1345
+ embedding: embedding,
1346
+ engine: engine,
1347
+ ...(situated_context && { situated_context: situated_context }),
1348
+ ...rest
1349
+ }
1298
1350
  });
1299
-
1351
+
1300
1352
  winston.verbose("resources to be sent to worker: ", resources);
1301
1353
 
1302
1354
  if (process.env.NODE_ENV !== "test") {
@@ -1307,7 +1359,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1307
1359
 
1308
1360
 
1309
1361
  // saveBulk(operations, addingContents, id_project).then((result) => {
1310
-
1362
+
1311
1363
  // let ns = namespaces.find(n => n.id === namespace_id);
1312
1364
  // let engine = ns.engine || default_engine;
1313
1365
 
@@ -1321,7 +1373,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1321
1373
  // if (process.env.NODE_ENV !== 'test') {
1322
1374
  // scheduleScrape(resources);
1323
1375
  // }
1324
-
1376
+
1325
1377
  // //res.status(200).send(result);
1326
1378
  // res.status(200).send({ success: true, message: "Contents imported successfully"});
1327
1379
 
@@ -1329,7 +1381,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1329
1381
  // winston.error("Unable to save kbs in bulk ", err)
1330
1382
  // res.status(500).send(err);
1331
1383
  // })
1332
-
1384
+
1333
1385
  })
1334
1386
 
1335
1387
  router.put('/namespace/:id', async (req, res) => {
@@ -1607,7 +1659,7 @@ router.post('/', async (req, res) => {
1607
1659
 
1608
1660
  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags, situated_context, chunk_regex } = req.body;
1609
1661
  const namespace_id = req.body?.namespace;
1610
-
1662
+
1611
1663
  if (!namespace_id) {
1612
1664
  return res.status(400).send({ success: false, error: "parameter 'namespace' is not defined" });
1613
1665
  }
@@ -1619,11 +1671,11 @@ router.post('/', async (req, res) => {
1619
1671
  let errorCode = err?.errorCode ?? 500;
1620
1672
  return res.status(errorCode).send({ success: false, error: err.error });
1621
1673
  }
1622
-
1674
+
1623
1675
  const quoteManager = req.app.get('quote_manager');
1624
1676
  try {
1625
1677
  await aiManager.checkQuotaAvailability(quoteManager, project, 1)
1626
- } catch(err) {
1678
+ } catch (err) {
1627
1679
  let errorCode = err?.errorCode ?? 500;
1628
1680
  return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
1629
1681
  }
@@ -1676,7 +1728,7 @@ router.post('/', async (req, res) => {
1676
1728
  delete raw_content.ok;
1677
1729
  delete raw_content.$clusterTime;
1678
1730
  delete raw_content.operationTime;
1679
-
1731
+
1680
1732
  const saved_kb = raw_content.value;
1681
1733
  const webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
1682
1734
  const embedding = normalizeEmbedding(namespace.embedding);
@@ -1706,7 +1758,7 @@ router.post('/', async (req, res) => {
1706
1758
  if (process.env.NODE_ENV === 'test') {
1707
1759
  return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: raw_content, schedule_json: json });
1708
1760
  }
1709
-
1761
+
1710
1762
  aiManager.scheduleScrape([json], namespace.hybrid);
1711
1763
  return res.status(200).send(raw_content);
1712
1764
 
@@ -1756,7 +1808,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
1756
1808
  const quoteManager = req.app.get('quote_manager');
1757
1809
  try {
1758
1810
  await aiManager.checkQuotaAvailability(quoteManager, project, list.length)
1759
- } catch(err) {
1811
+ } catch (err) {
1760
1812
  let errorCode = err?.errorCode ?? 500;
1761
1813
  return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
1762
1814
  }
@@ -1837,7 +1889,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
1837
1889
  const quoteManager = req.app.get('quote_manager');
1838
1890
  try {
1839
1891
  await aiManager.checkQuotaAvailability(quoteManager, req.project, kbs.length)
1840
- } catch(err) {
1892
+ } catch (err) {
1841
1893
  let errorCode = err?.errorCode ?? 500;
1842
1894
  return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
1843
1895
  }
@@ -1865,15 +1917,15 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
1865
1917
  situated_context_obj = normalizeSituatedContext(situated_context);
1866
1918
  }
1867
1919
 
1868
- let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context, ...keepAttrs }) => keepAttrs)
1869
- resources = resources.map(({ _id, ...rest}) => {
1870
- return {
1871
- id: _id,
1872
- webhook: webhook,
1873
- embedding: embedding,
1874
- engine: engine,
1875
- ...(situated_context_obj && { situated_context: situated_context_obj }),
1876
- ...rest
1920
+ let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context, ...keepAttrs }) => keepAttrs)
1921
+ resources = resources.map(({ _id, ...rest }) => {
1922
+ return {
1923
+ id: _id,
1924
+ webhook: webhook,
1925
+ embedding: embedding,
1926
+ engine: engine,
1927
+ ...(situated_context_obj && { situated_context: situated_context_obj }),
1928
+ ...rest
1877
1929
  };
1878
1930
  })
1879
1931
  winston.verbose("resources to be sent to worker: ", resources);
@@ -1881,7 +1933,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
1881
1933
  if (process.env.NODE_ENV === 'test') {
1882
1934
  return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: result, schedule_json: resources });
1883
1935
  }
1884
-
1936
+
1885
1937
  aiManager.scheduleScrape(resources, hybrid);
1886
1938
  return res.status(200).send(result);
1887
1939
 
@@ -1930,13 +1982,13 @@ router.post('/sitemap/import', async (req, res) => {
1930
1982
  }
1931
1983
 
1932
1984
  if (type !== "sitemap") {
1933
- return res.status(403).send({success: false, error: "Endpoint available for sitemap type only." });
1985
+ return res.status(403).send({ success: false, error: "Endpoint available for sitemap type only." });
1934
1986
  }
1935
1987
 
1936
1988
  if (!namespace_id) {
1937
1989
  return res.status(400).send({ success: false, error: "queryParam 'namespace' is not defined" })
1938
1990
  }
1939
-
1991
+
1940
1992
  let namespace;
1941
1993
  try {
1942
1994
  namespace = await aiManager.checkNamespace(id_project, namespace_id);
@@ -1967,7 +2019,7 @@ router.post('/sitemap/import', async (req, res) => {
1967
2019
 
1968
2020
  if (data.errors && data.errors.length > 0) {
1969
2021
  winston.error("An error occurred during sitemap fetch: ", data.errors[0])
1970
- return res.status(500).send({ success: false, error: "Unable to fecth sitemap due to an error: " + data.errors[0].message})
2022
+ return res.status(500).send({ success: false, error: "Unable to fecth sitemap due to an error: " + data.errors[0].message })
1971
2023
  }
1972
2024
 
1973
2025
  const urls = Array.isArray(data.sites) ? data.sites : [];
@@ -2011,7 +2063,7 @@ router.post('/sitemap/import', async (req, res) => {
2011
2063
  ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {}),
2012
2064
  ...(situated_context && situated_context === true && scrape_type === 0 ? { situated_context: true } : {}),
2013
2065
  }
2014
-
2066
+
2015
2067
  try {
2016
2068
  await aiManager.scheduleSitemap(namespace, saved_content, options);
2017
2069
  let result = await aiManager.addMultipleUrls(namespace, urls, options);
@@ -2023,8 +2075,8 @@ router.post('/sitemap/import', async (req, res) => {
2023
2075
  return res.status(200).send(result);
2024
2076
  } catch (err) {
2025
2077
  return res.status(500).send({ success: false, error: "Unable to add multiple urls from sitemap due to an error." });
2026
- }
2027
-
2078
+ }
2079
+
2028
2080
  })
2029
2081
 
2030
2082
  router.put('/:kb_id', async (req, res) => {
@@ -2032,7 +2084,7 @@ router.put('/:kb_id', async (req, res) => {
2032
2084
  const id_project = req.projectid;
2033
2085
  const project = req.project;
2034
2086
  const kb_id = req.params.kb_id;
2035
-
2087
+
2036
2088
  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags } = req.body;
2037
2089
  const namespace_id = req.body.namespace;
2038
2090
  let situated_context = req.body.situated_context;
@@ -2176,7 +2228,7 @@ router.put('/:kb_id', async (req, res) => {
2176
2228
  if (process.env.NODE_ENV === 'test') {
2177
2229
  return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: updated_content, schedule_json: json });
2178
2230
  }
2179
-
2231
+
2180
2232
  aiManager.scheduleScrape([json], namespace.hybrid);
2181
2233
  return res.status(200).send(updated_content);
2182
2234
 
@@ -2188,11 +2240,11 @@ router.delete('/:kb_id', async (req, res) => {
2188
2240
  let kb_id = req.params.kb_id;
2189
2241
  winston.verbose("delete kb_id " + kb_id);
2190
2242
 
2191
- let kb = await KB.findOne({ id_project: project_id, _id: kb_id}).catch((err) => {
2243
+ let kb = await KB.findOne({ id_project: project_id, _id: kb_id }).catch((err) => {
2192
2244
  winston.warn("Unable to find kb. Error: ", err);
2193
2245
  return res.status(500).send({ success: false, error: err })
2194
2246
  })
2195
-
2247
+
2196
2248
  if (!kb) {
2197
2249
  winston.error("Unable to delete kb. Kb not found...")
2198
2250
  return res.status(404).send({ success: false, error: "Content not found" })
@@ -2227,7 +2279,7 @@ router.delete('/:kb_id', async (req, res) => {
2227
2279
  if (!data.namespace) {
2228
2280
  data.namespace = project_id;
2229
2281
  }
2230
-
2282
+
2231
2283
  data.engine = namespace.engine || default_engine;
2232
2284
  winston.verbose("/:delete_id data: ", data);
2233
2285