@tiledesk/tiledesk-server 2.19.4 → 2.19.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/routes/kb.js CHANGED
@@ -35,12 +35,12 @@ let MAX_UPLOAD_FILE_SIZE = process.env.MAX_UPLOAD_FILE_SIZE;
35
35
  let uploadlimits = undefined;
36
36
 
37
37
  if (MAX_UPLOAD_FILE_SIZE) {
38
- uploadlimits = {fileSize: parseInt(MAX_UPLOAD_FILE_SIZE)} ;
38
+ uploadlimits = { fileSize: parseInt(MAX_UPLOAD_FILE_SIZE) };
39
39
  winston.debug("Max upload file size is : " + MAX_UPLOAD_FILE_SIZE);
40
40
  } else {
41
41
  winston.debug("Max upload file size is infinity");
42
42
  }
43
- var upload = multer({limits: uploadlimits});
43
+ var upload = multer({ limits: uploadlimits });
44
44
 
45
45
  let jobManager = new JobManager(AMQP_MANAGER_URL, {
46
46
  debug: false,
@@ -84,22 +84,23 @@ const default_engine_hybrid = require('../config/kb/engine.hybrid');
84
84
  const default_embedding = require('../config/kb/embedding');
85
85
  const PromptManager = require('../config/kb/prompt/rag/PromptManager');
86
86
  const situatedContext = require('../config/kb/situatedContext');
87
+ const kbQuestionService = require('../services/kbQuestionService');
87
88
 
88
89
  const ragPromptManager = new PromptManager(path.join(__dirname, '../config/kb/prompt/rag'));
89
90
 
90
91
  const RAG_CONTEXT_ENV_OVERRIDES = {
91
- "gpt-3.5-turbo": process.env.GPT_3_5_CONTEXT,
92
- "gpt-4": process.env.GPT_4_CONTEXT,
92
+ "gpt-3.5-turbo": process.env.GPT_3_5_CONTEXT,
93
+ "gpt-4": process.env.GPT_4_CONTEXT,
93
94
  "gpt-4-turbo-preview": process.env.GPT_4T_CONTEXT,
94
- "gpt-4o": process.env.GPT_4O_CONTEXT,
95
- "gpt-4o-mini": process.env.GPT_4O_MINI_CONTEXT,
96
- "gpt-4.1": process.env.GPT_4_1_CONTEXT,
97
- "gpt-4.1-mini": process.env.GPT_4_1_MINI_CONTEXT,
98
- "gpt-4.1-nano": process.env.GPT_4_1_NANO_CONTEXT,
99
- "gpt-5": process.env.GPT_5_CONTEXT,
100
- "gpt-5-mini": process.env.GPT_5_MINI_CONTEXT,
101
- "gpt-5-nano": process.env.GPT_5_NANO_CONTEXT,
102
- "general": process.env.GENERAL_CONTEXT
95
+ "gpt-4o": process.env.GPT_4O_CONTEXT,
96
+ "gpt-4o-mini": process.env.GPT_4O_MINI_CONTEXT,
97
+ "gpt-4.1": process.env.GPT_4_1_CONTEXT,
98
+ "gpt-4.1-mini": process.env.GPT_4_1_MINI_CONTEXT,
99
+ "gpt-4.1-nano": process.env.GPT_4_1_NANO_CONTEXT,
100
+ "gpt-5": process.env.GPT_5_CONTEXT,
101
+ "gpt-5-mini": process.env.GPT_5_MINI_CONTEXT,
102
+ "gpt-5-nano": process.env.GPT_5_NANO_CONTEXT,
103
+ "general": process.env.GENERAL_CONTEXT
103
104
  };
104
105
 
105
106
  /** RAG system prompt per modello: file in config/kb/prompt/rag, sovrascrivibili via env (come prima). */
@@ -142,15 +143,15 @@ router.post('/scrape/single', async (req, res) => {
142
143
 
143
144
  let data = req.body;
144
145
  winston.debug("/scrape/single data: ", data)
145
-
146
+
146
147
  let namespace;
147
148
  try {
148
149
  namespace = await aiManager.checkNamespace(project_id, data.namespace);
149
150
  } catch (err) {
150
151
  let errorCode = err?.errorCode ?? 500;
151
152
  return res.status(errorCode).send({ success: false, error: err.error });
152
- }
153
-
153
+ }
154
+
154
155
  if (data.type === "sitemap") {
155
156
 
156
157
  const urls = await aiManager.fetchSitemap(data.source).catch((err) => {
@@ -166,18 +167,18 @@ router.post('/scrape/single', async (req, res) => {
166
167
  try {
167
168
  sitemapKb = await KB.findById(data.id);
168
169
  } catch (err) {
169
- winston.error("Error finding sitemap content with id " + data.id);
170
+ winston.error("Error finding sitemap content with id " + data.id);
170
171
  return res.status(500).send({ success: false, error: "Error finding sitemap content with id " + data.id });
171
172
  }
172
173
 
173
174
  if (!sitemapKb) {
174
175
  return res.status(404).send({ success: false, error: "Content not found with id " + data.id });
175
176
  }
176
-
177
+
177
178
  let existingKbs;
178
179
  try {
179
180
  existingKbs = await KB.find({ id_project: project_id, namespace: data.namespace, sitemap_origin_id: data.id }).lean().exec();
180
- } catch(err) {
181
+ } catch (err) {
181
182
  winston.error("Error finding existing contents: ", err);
182
183
  return res.status(500).send({ success: false, error: "Error finding existing sitemap contents" });
183
184
  }
@@ -396,7 +397,7 @@ router.post('/qa', async (req, res) => {
396
397
  if (publicKey === true) {
397
398
  let isAvailable = await quoteManager.checkQuote(req.project, obj, 'tokens');
398
399
  if (isAvailable === false) {
399
- return res.status(403).send({ success: false, message: "Tokens quota exceeded", error_code: 13001})
400
+ return res.status(403).send({ success: false, message: "Tokens quota exceeded", error_code: 13001 })
400
401
  }
401
402
  }
402
403
 
@@ -416,10 +417,10 @@ router.post('/qa', async (req, res) => {
416
417
 
417
418
  if (namespace.hybrid === true) {
418
419
  data.search_type = 'hybrid';
419
-
420
+
420
421
  if (data.reranking === true) {
421
422
  data.reranker_model = process.env.RERANKING_MODEL || "cross-encoder/ms-marco-MiniLM-L-6-v2";
422
-
423
+
423
424
  if (!data.reranking_multiplier) {
424
425
  data.reranking_multiplier = 3;
425
426
  }
@@ -432,10 +433,10 @@ router.post('/qa', async (req, res) => {
432
433
  data.reranking_multiplier = calculatedRerankingMultiplier;
433
434
  }
434
435
  }
435
- }
436
+ }
436
437
 
437
438
  if (!namespace.hybrid && data.reranking === true && PINECONE_RERANKING) {
438
-
439
+
439
440
  data.reranking = {
440
441
  "provider": "pinecone",
441
442
  "api_key": process.env.PINECONE_API_KEY,
@@ -476,7 +477,7 @@ router.post('/qa', async (req, res) => {
476
477
  const sendError = (message) => {
477
478
  try {
478
479
  res.write('data: ' + JSON.stringify({ error: message }) + '\n\n');
479
- } catch (_) {}
480
+ } catch (_) { }
480
481
  res.end();
481
482
  };
482
483
 
@@ -518,6 +519,32 @@ router.post('/qa', async (req, res) => {
518
519
  return false;
519
520
  }
520
521
 
522
+ let streamQaResult = null;
523
+ const qaStartedAt = Date.now();
524
+ let streamDurationSeconds = null;
525
+
526
+ function captureStreamDurationFromFirstChunk(obj) {
527
+ if (streamDurationSeconds != null) return;
528
+ const content = extractContent(obj);
529
+ if (content != null && content !== '') {
530
+ streamDurationSeconds = (Date.now() - qaStartedAt) / 1000;
531
+ }
532
+ }
533
+
534
+ function withMeasuredDuration(payload) {
535
+ if (streamDurationSeconds != null && payload && typeof payload === 'object') {
536
+ payload.duration = streamDurationSeconds;
537
+ }
538
+ return payload;
539
+ }
540
+
541
+ function captureStreamQaResult(obj) {
542
+ const normalized = normalizeKbQaPayload(obj);
543
+ if (normalized && typeof normalized === 'object' && typeof normalized.success === 'boolean') {
544
+ streamQaResult = normalized;
545
+ }
546
+ }
547
+
521
548
  function forwardSsePayload(payload) {
522
549
  if (payload === '[DONE]') return;
523
550
  let obj;
@@ -527,16 +554,19 @@ router.post('/qa', async (req, res) => {
527
554
  return;
528
555
  }
529
556
 
557
+ captureStreamQaResult(obj);
558
+ captureStreamDurationFromFirstChunk(obj);
559
+
530
560
  if (obj.status === 'started') {
531
561
  return;
532
562
  }
533
563
  if (isKbStreamCompletedSummary(obj)) {
534
- res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
564
+ res.write('data: ' + JSON.stringify(withMeasuredDuration(normalizeKbQaPayload(obj))) + '\n\n');
535
565
  return;
536
566
  }
537
567
 
538
568
  if (obj.type === 'metadata' || obj.event === 'metadata') {
539
- res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
569
+ res.write('data: ' + JSON.stringify(withMeasuredDuration(normalizeKbQaPayload(obj))) + '\n\n');
540
570
  return;
541
571
  }
542
572
  const content = extractContent(obj);
@@ -546,7 +576,7 @@ router.post('/qa', async (req, res) => {
546
576
  }
547
577
  const normalized = normalizeKbQaPayload(obj);
548
578
  if (isMetadataPayload(normalized, content)) {
549
- res.write('data: ' + JSON.stringify(normalized) + '\n\n');
579
+ res.write('data: ' + JSON.stringify(withMeasuredDuration(normalized)) + '\n\n');
550
580
  }
551
581
  }
552
582
 
@@ -567,7 +597,7 @@ router.post('/qa', async (req, res) => {
567
597
  }
568
598
  });
569
599
 
570
- stream.on('end', () => {
600
+ stream.on('end', async () => {
571
601
  const tail = buffer.trim();
572
602
  if (tail) {
573
603
  for (const line of tail.split('\n')) {
@@ -576,6 +606,26 @@ router.post('/qa', async (req, res) => {
576
606
  forwardSsePayload(trimmed.slice(6));
577
607
  }
578
608
  }
609
+ if (streamQaResult) {
610
+ withMeasuredDuration(streamQaResult);
611
+ const tokens = await aiManager.getTrackingTokens({
612
+ qaResult: streamQaResult,
613
+ publicKey,
614
+ model: data.model,
615
+ obj,
616
+ quoteManager,
617
+ project: req.project,
618
+ });
619
+ kbQuestionService.trackKbQaResult(
620
+ id_project,
621
+ { namespace: data.namespace, question: data.question },
622
+ streamQaResult,
623
+ tokens
624
+ );
625
+ if (publicKey === true && typeof streamQaResult.legacy_prompt_token_size === 'number') {
626
+ res.write('data: ' + JSON.stringify(streamQaResult) + '\n\n');
627
+ }
628
+ }
579
629
  res.write('data: [DONE]\n\n');
580
630
  res.end();
581
631
  });
@@ -604,35 +654,29 @@ router.post('/qa', async (req, res) => {
604
654
  return;
605
655
  }
606
656
 
607
- let duration = Date.now();
657
+ const qaStartedAt = Date.now();
608
658
 
609
- aiService.askNamespace(data).then((resp) => {
659
+ aiService.askNamespace(data).then(async (resp) => {
610
660
  winston.debug("qa resp: ", resp.data);
611
661
  let answer = resp.data;
612
662
 
613
- answer.duration = (Date.now() - duration) / 1000;
614
-
615
- if (publicKey === true) {
616
- let modelKey;
617
- if (typeof data.model === 'string') {
618
- modelKey = data.model;
619
- } else if (data.model && typeof data.model.name === 'string') {
620
- modelKey = data.model.name;
621
- }
663
+ answer.duration = (Date.now() - qaStartedAt) / 1000;
622
664
 
623
- let multiplier = MODELS_MULTIPLIER[modelKey];
624
- if (!multiplier) {
625
- multiplier = 1;
626
- winston.info("No multiplier found for AI model (qa) " + modelKey);
627
- }
628
-
629
- obj.multiplier = multiplier;
630
- obj.tokens = answer.prompt_token_size;
665
+ const tokens = await aiManager.getTrackingTokens({
666
+ qaResult: answer,
667
+ publicKey,
668
+ model: data.model,
669
+ obj,
670
+ quoteManager,
671
+ project: req.project,
672
+ });
673
+ kbQuestionService.trackKbQaResult(
674
+ id_project,
675
+ { namespace: data.namespace, question: data.question },
676
+ answer,
677
+ tokens
678
+ );
631
679
 
632
- let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
633
- winston.verbose("incremented_key: ", incremented_key);
634
- }
635
-
636
680
  return res.status(200).send(answer);
637
681
 
638
682
  }).catch((err) => {
@@ -738,7 +782,7 @@ router.post('/qa', async (req, res) => {
738
782
 
739
783
  // if (namespace.hybrid === true) {
740
784
  // data.search_type = 'hybrid';
741
-
785
+
742
786
  // if (data.reranking === true) {
743
787
  // data.reranking_multiplier = 3;
744
788
  // data.reranker_model = "cross-encoder/ms-marco-MiniLM-L-6-v2";
@@ -771,7 +815,7 @@ router.post('/qa', async (req, res) => {
771
815
 
772
816
  // delete data.advancedPrompt;
773
817
  // winston.verbose("ask data: ", data);
774
-
818
+
775
819
  // if (process.env.NODE_ENV === 'test') {
776
820
  // return res.status(200).send({ success: true, message: "Question skipped in test environment", data: data });
777
821
  // }
@@ -794,7 +838,7 @@ router.post('/qa', async (req, res) => {
794
838
  // let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
795
839
  // winston.verbose("incremented_key: ", incremented_key);
796
840
  // }
797
-
841
+
798
842
  // return res.status(200).send(answer);
799
843
 
800
844
  // }).catch((err) => {
@@ -886,7 +930,7 @@ router.get('/namespace/all', async (req, res) => {
886
930
 
887
931
  let project_id = req.projectid;
888
932
 
889
- Namespace.find({ id_project: project_id }).lean().exec( async (err, namespaces) => {
933
+ Namespace.find({ id_project: project_id }).lean().exec(async (err, namespaces) => {
890
934
 
891
935
  if (err) {
892
936
  winston.error("find namespaces error: ", err);
@@ -935,7 +979,7 @@ router.get('/namespace/all', async (req, res) => {
935
979
  } else {
936
980
  namespaceObjArray = namespaces.map(({ _id, __v, ...keepAttrs }) => keepAttrs)
937
981
  }
938
-
982
+
939
983
  winston.debug("namespaceObjArray: ", namespaceObjArray);
940
984
  return res.status(200).send(namespaceObjArray);
941
985
  }
@@ -961,14 +1005,14 @@ router.get('/namespace/:id/chunks/:content_id', async (req, res) => {
961
1005
  return res.status(403).send({ success: false, error: err })
962
1006
  })
963
1007
 
964
- if(!content) {
1008
+ if (!content) {
965
1009
  return res.status(403).send({ success: false, error: "Not allowed. The content does not belong to the current namespace." })
966
1010
  }
967
1011
 
968
1012
  let engine = namespace.engine || default_engine;
969
1013
 
970
1014
  if (process.env.NODE_ENV === 'test') {
971
- return res.status(200).send({ success: true, message: "Get chunks skipped in test environment"});
1015
+ return res.status(200).send({ success: true, message: "Get chunks skipped in test environment" });
972
1016
  }
973
1017
 
974
1018
  aiService.getContentChunks(namespace_id, content_id, engine).then((resp) => {
@@ -989,7 +1033,7 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
989
1033
 
990
1034
  let project_id = req.projectid;
991
1035
  let namespace_id = req.params.id;
992
-
1036
+
993
1037
  let chatbotsArray = [];
994
1038
 
995
1039
  let namespace;
@@ -1031,12 +1075,12 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
1031
1075
  await Promise.all(chatbotPromises);
1032
1076
 
1033
1077
  winston.debug("chatbotsArray: ", chatbotsArray);
1034
-
1078
+
1035
1079
  res.status(200).send(chatbotsArray);
1036
1080
  })
1037
1081
 
1038
1082
  router.get('/namespace/export/:id', async (req, res) => {
1039
-
1083
+
1040
1084
  let id_project = req.projectid;
1041
1085
  let namespace_id = req.params.id;
1042
1086
 
@@ -1048,7 +1092,7 @@ router.get('/namespace/export/:id', async (req, res) => {
1048
1092
  query.status = parseInt(req.query.status)
1049
1093
  }
1050
1094
 
1051
- query.type = { $in: [ kbTypes.URL, kbTypes.TEXT, kbTypes.FAQ ] };
1095
+ query.type = { $in: [kbTypes.URL, kbTypes.TEXT, kbTypes.FAQ] };
1052
1096
 
1053
1097
  let namespace;
1054
1098
  try {
@@ -1078,11 +1122,11 @@ router.get('/namespace/export/:id', async (req, res) => {
1078
1122
  let json_string = JSON.stringify(json);
1079
1123
  res.set({ "Content-Disposition": `attachment; filename="${filename}.json"` });
1080
1124
  return res.send(json_string);
1081
- } catch(err) {
1125
+ } catch (err) {
1082
1126
  winston.error("Error genereting json ", err);
1083
1127
  return res.status(500).send({ success: false, error: "Error genereting json file" })
1084
1128
  }
1085
-
1129
+
1086
1130
  })
1087
1131
 
1088
1132
  router.post('/namespace', async (req, res) => {
@@ -1212,7 +1256,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1212
1256
  }
1213
1257
 
1214
1258
  let addingContents = [];
1215
- contents.forEach( async (e) => {
1259
+ contents.forEach(async (e) => {
1216
1260
  let content = {
1217
1261
  id_project: id_project,
1218
1262
  name: e.name,
@@ -1295,15 +1339,16 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1295
1339
 
1296
1340
  let resources = new_contents.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
1297
1341
  resources = resources.map(({ _id, scrape_options, ...rest }) => {
1298
- return {
1299
- id: _id,
1300
- parameters_scrape_type_4: scrape_options,
1301
- embedding: embedding,
1302
- engine: engine,
1303
- ...(situated_context && { situated_context: situated_context }),
1304
- ...rest}
1342
+ return {
1343
+ id: _id,
1344
+ parameters_scrape_type_4: scrape_options,
1345
+ embedding: embedding,
1346
+ engine: engine,
1347
+ ...(situated_context && { situated_context: situated_context }),
1348
+ ...rest
1349
+ }
1305
1350
  });
1306
-
1351
+
1307
1352
  winston.verbose("resources to be sent to worker: ", resources);
1308
1353
 
1309
1354
  if (process.env.NODE_ENV !== "test") {
@@ -1314,7 +1359,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1314
1359
 
1315
1360
 
1316
1361
  // saveBulk(operations, addingContents, id_project).then((result) => {
1317
-
1362
+
1318
1363
  // let ns = namespaces.find(n => n.id === namespace_id);
1319
1364
  // let engine = ns.engine || default_engine;
1320
1365
 
@@ -1328,7 +1373,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1328
1373
  // if (process.env.NODE_ENV !== 'test') {
1329
1374
  // scheduleScrape(resources);
1330
1375
  // }
1331
-
1376
+
1332
1377
  // //res.status(200).send(result);
1333
1378
  // res.status(200).send({ success: true, message: "Contents imported successfully"});
1334
1379
 
@@ -1336,7 +1381,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1336
1381
  // winston.error("Unable to save kbs in bulk ", err)
1337
1382
  // res.status(500).send(err);
1338
1383
  // })
1339
-
1384
+
1340
1385
  })
1341
1386
 
1342
1387
  router.put('/namespace/:id', async (req, res) => {
@@ -1614,7 +1659,7 @@ router.post('/', async (req, res) => {
1614
1659
 
1615
1660
  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags, situated_context, chunk_regex } = req.body;
1616
1661
  const namespace_id = req.body?.namespace;
1617
-
1662
+
1618
1663
  if (!namespace_id) {
1619
1664
  return res.status(400).send({ success: false, error: "parameter 'namespace' is not defined" });
1620
1665
  }
@@ -1626,11 +1671,11 @@ router.post('/', async (req, res) => {
1626
1671
  let errorCode = err?.errorCode ?? 500;
1627
1672
  return res.status(errorCode).send({ success: false, error: err.error });
1628
1673
  }
1629
-
1674
+
1630
1675
  const quoteManager = req.app.get('quote_manager');
1631
1676
  try {
1632
1677
  await aiManager.checkQuotaAvailability(quoteManager, project, 1)
1633
- } catch(err) {
1678
+ } catch (err) {
1634
1679
  let errorCode = err?.errorCode ?? 500;
1635
1680
  return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
1636
1681
  }
@@ -1683,7 +1728,7 @@ router.post('/', async (req, res) => {
1683
1728
  delete raw_content.ok;
1684
1729
  delete raw_content.$clusterTime;
1685
1730
  delete raw_content.operationTime;
1686
-
1731
+
1687
1732
  const saved_kb = raw_content.value;
1688
1733
  const webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
1689
1734
  const embedding = normalizeEmbedding(namespace.embedding);
@@ -1713,7 +1758,7 @@ router.post('/', async (req, res) => {
1713
1758
  if (process.env.NODE_ENV === 'test') {
1714
1759
  return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: raw_content, schedule_json: json });
1715
1760
  }
1716
-
1761
+
1717
1762
  aiManager.scheduleScrape([json], namespace.hybrid);
1718
1763
  return res.status(200).send(raw_content);
1719
1764
 
@@ -1763,7 +1808,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
1763
1808
  const quoteManager = req.app.get('quote_manager');
1764
1809
  try {
1765
1810
  await aiManager.checkQuotaAvailability(quoteManager, project, list.length)
1766
- } catch(err) {
1811
+ } catch (err) {
1767
1812
  let errorCode = err?.errorCode ?? 500;
1768
1813
  return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
1769
1814
  }
@@ -1844,7 +1889,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
1844
1889
  const quoteManager = req.app.get('quote_manager');
1845
1890
  try {
1846
1891
  await aiManager.checkQuotaAvailability(quoteManager, req.project, kbs.length)
1847
- } catch(err) {
1892
+ } catch (err) {
1848
1893
  let errorCode = err?.errorCode ?? 500;
1849
1894
  return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
1850
1895
  }
@@ -1872,15 +1917,15 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
1872
1917
  situated_context_obj = normalizeSituatedContext(situated_context);
1873
1918
  }
1874
1919
 
1875
- let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context, ...keepAttrs }) => keepAttrs)
1876
- resources = resources.map(({ _id, ...rest}) => {
1877
- return {
1878
- id: _id,
1879
- webhook: webhook,
1880
- embedding: embedding,
1881
- engine: engine,
1882
- ...(situated_context_obj && { situated_context: situated_context_obj }),
1883
- ...rest
1920
+ let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context, ...keepAttrs }) => keepAttrs)
1921
+ resources = resources.map(({ _id, ...rest }) => {
1922
+ return {
1923
+ id: _id,
1924
+ webhook: webhook,
1925
+ embedding: embedding,
1926
+ engine: engine,
1927
+ ...(situated_context_obj && { situated_context: situated_context_obj }),
1928
+ ...rest
1884
1929
  };
1885
1930
  })
1886
1931
  winston.verbose("resources to be sent to worker: ", resources);
@@ -1888,7 +1933,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
1888
1933
  if (process.env.NODE_ENV === 'test') {
1889
1934
  return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: result, schedule_json: resources });
1890
1935
  }
1891
-
1936
+
1892
1937
  aiManager.scheduleScrape(resources, hybrid);
1893
1938
  return res.status(200).send(result);
1894
1939
 
@@ -1937,13 +1982,13 @@ router.post('/sitemap/import', async (req, res) => {
1937
1982
  }
1938
1983
 
1939
1984
  if (type !== "sitemap") {
1940
- return res.status(403).send({success: false, error: "Endpoint available for sitemap type only." });
1985
+ return res.status(403).send({ success: false, error: "Endpoint available for sitemap type only." });
1941
1986
  }
1942
1987
 
1943
1988
  if (!namespace_id) {
1944
1989
  return res.status(400).send({ success: false, error: "queryParam 'namespace' is not defined" })
1945
1990
  }
1946
-
1991
+
1947
1992
  let namespace;
1948
1993
  try {
1949
1994
  namespace = await aiManager.checkNamespace(id_project, namespace_id);
@@ -1974,7 +2019,7 @@ router.post('/sitemap/import', async (req, res) => {
1974
2019
 
1975
2020
  if (data.errors && data.errors.length > 0) {
1976
2021
  winston.error("An error occurred during sitemap fetch: ", data.errors[0])
1977
- return res.status(500).send({ success: false, error: "Unable to fecth sitemap due to an error: " + data.errors[0].message})
2022
+ return res.status(500).send({ success: false, error: "Unable to fecth sitemap due to an error: " + data.errors[0].message })
1978
2023
  }
1979
2024
 
1980
2025
  const urls = Array.isArray(data.sites) ? data.sites : [];
@@ -2018,7 +2063,7 @@ router.post('/sitemap/import', async (req, res) => {
2018
2063
  ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {}),
2019
2064
  ...(situated_context && situated_context === true && scrape_type === 0 ? { situated_context: true } : {}),
2020
2065
  }
2021
-
2066
+
2022
2067
  try {
2023
2068
  await aiManager.scheduleSitemap(namespace, saved_content, options);
2024
2069
  let result = await aiManager.addMultipleUrls(namespace, urls, options);
@@ -2030,8 +2075,8 @@ router.post('/sitemap/import', async (req, res) => {
2030
2075
  return res.status(200).send(result);
2031
2076
  } catch (err) {
2032
2077
  return res.status(500).send({ success: false, error: "Unable to add multiple urls from sitemap due to an error." });
2033
- }
2034
-
2078
+ }
2079
+
2035
2080
  })
2036
2081
 
2037
2082
  router.put('/:kb_id', async (req, res) => {
@@ -2039,7 +2084,7 @@ router.put('/:kb_id', async (req, res) => {
2039
2084
  const id_project = req.projectid;
2040
2085
  const project = req.project;
2041
2086
  const kb_id = req.params.kb_id;
2042
-
2087
+
2043
2088
  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags } = req.body;
2044
2089
  const namespace_id = req.body.namespace;
2045
2090
  let situated_context = req.body.situated_context;
@@ -2183,7 +2228,7 @@ router.put('/:kb_id', async (req, res) => {
2183
2228
  if (process.env.NODE_ENV === 'test') {
2184
2229
  return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: updated_content, schedule_json: json });
2185
2230
  }
2186
-
2231
+
2187
2232
  aiManager.scheduleScrape([json], namespace.hybrid);
2188
2233
  return res.status(200).send(updated_content);
2189
2234
 
@@ -2195,11 +2240,11 @@ router.delete('/:kb_id', async (req, res) => {
2195
2240
  let kb_id = req.params.kb_id;
2196
2241
  winston.verbose("delete kb_id " + kb_id);
2197
2242
 
2198
- let kb = await KB.findOne({ id_project: project_id, _id: kb_id}).catch((err) => {
2243
+ let kb = await KB.findOne({ id_project: project_id, _id: kb_id }).catch((err) => {
2199
2244
  winston.warn("Unable to find kb. Error: ", err);
2200
2245
  return res.status(500).send({ success: false, error: err })
2201
2246
  })
2202
-
2247
+
2203
2248
  if (!kb) {
2204
2249
  winston.error("Unable to delete kb. Kb not found...")
2205
2250
  return res.status(404).send({ success: false, error: "Content not found" })
@@ -2234,7 +2279,7 @@ router.delete('/:kb_id', async (req, res) => {
2234
2279
  if (!data.namespace) {
2235
2280
  data.namespace = project_id;
2236
2281
  }
2237
-
2282
+
2238
2283
  data.engine = namespace.engine || default_engine;
2239
2284
  winston.verbose("/:delete_id data: ", data);
2240
2285