@tiledesk/tiledesk-server 2.15.8 → 2.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/routes/kb.js CHANGED
@@ -80,6 +80,37 @@ let default_preview_settings = {
80
80
  const default_engine = require('../config/kb/engine');
81
81
  const default_engine_hybrid = require('../config/kb/engine.hybrid');
82
82
  const default_embedding = require('../config/kb/embedding');
83
+ const PromptManager = require('../config/kb/prompt/rag/PromptManager');
84
+ const situatedContext = require('../config/kb/situatedContext');
85
+
86
+ const ragPromptManager = new PromptManager(path.join(__dirname, '../config/kb/prompt/rag'));
87
+
88
+ const RAG_CONTEXT_ENV_OVERRIDES = {
89
+ "gpt-3.5-turbo": process.env.GPT_3_5_CONTEXT,
90
+ "gpt-4": process.env.GPT_4_CONTEXT,
91
+ "gpt-4-turbo-preview": process.env.GPT_4T_CONTEXT,
92
+ "gpt-4o": process.env.GPT_4O_CONTEXT,
93
+ "gpt-4o-mini": process.env.GPT_4O_MINI_CONTEXT,
94
+ "gpt-4.1": process.env.GPT_4_1_CONTEXT,
95
+ "gpt-4.1-mini": process.env.GPT_4_1_MINI_CONTEXT,
96
+ "gpt-4.1-nano": process.env.GPT_4_1_NANO_CONTEXT,
97
+ "gpt-5": process.env.GPT_5_CONTEXT,
98
+ "gpt-5-mini": process.env.GPT_5_MINI_CONTEXT,
99
+ "gpt-5-nano": process.env.GPT_5_NANO_CONTEXT,
100
+ "general": process.env.GENERAL_CONTEXT
101
+ };
102
+
103
+ /** RAG system prompt per modello: file in config/kb/prompt/rag, sovrascrivibili via env (come prima). */
104
+ function getRagContextTemplate(modelName) {
105
+ const envOverride = RAG_CONTEXT_ENV_OVERRIDES[modelName];
106
+ if (envOverride) {
107
+ return envOverride;
108
+ }
109
+ if (!PromptManager.modelMap[modelName] && process.env.GENERAL_CONTEXT) {
110
+ return process.env.GENERAL_CONTEXT;
111
+ }
112
+ return ragPromptManager.getPrompt(modelName);
113
+ }
83
114
 
84
115
  function normalizeEmbedding(embedding) {
85
116
  const normalizedEmbedding = (embedding && typeof embedding.toObject === 'function')
@@ -88,19 +119,13 @@ function normalizeEmbedding(embedding) {
88
119
  return { ...normalizedEmbedding };
89
120
  }
90
121
 
91
- let contexts = {
92
- "gpt-3.5-turbo": process.env.GPT_3_5_CONTEXT || "You are an helpful assistant for question-answering tasks.\nUse ONLY the pieces of retrieved context delimited by #### and the chat history to answer the question.\nIf you don't know the answer, just say: \"I don't know<NOANS>\"\n\n####{context}####",
93
- "gpt-4": process.env.GPT_4_CONTEXT || "You are an helpful assistant for question-answering tasks.\nUse ONLY the pieces of retrieved context delimited by #### and the chat history to answer the question.\nIf you don't know the answer, just say that you don't know.\nIf and only if none of the retrieved context is useful for your task, add this word to the end <NOANS>\n\n####{context}####",
94
- "gpt-4-turbo-preview": process.env.GPT_4T_CONTEXT || "You are an helpful assistant for question-answering tasks.\nUse ONLY the pieces of retrieved context delimited by #### and the chat history to answer the question.\nIf you don't know the answer, just say that you don't know.\nIf and only if none of the retrieved context is useful for your task, add this word to the end <NOANS>\n\n####{context}####",
95
- "gpt-4o": process.env.GPT_4O_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, return <NOANS>\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
96
- "gpt-4o-mini": process.env.GPT_4O_MINI_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, return <NOANS>\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
97
- "gpt-4.1": process.env.GPT_4_1_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
98
- "gpt-4.1-mini": process.env.GPT_4_1_MINI_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
99
- "gpt-4.1-nano": process.env.GPT_4_1_NANO_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
100
- "gpt-5": process.env.GPT_5_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
101
- "gpt-5-mini": process.env.GPT_5_MINI_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
102
- "gpt-5-nano": process.env.GPT_5_NANO_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
103
- "general": process.env.GENERAL_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end=="
122
+ function normalizeSituatedContext() {
123
+ return situatedContext.enable
124
+ ? {
125
+ ...situatedContext,
126
+ api_key: process.env.SITUATED_CONTEXT_API_KEY || process.env.GPTKEY
127
+ }
128
+ : undefined;
104
129
  }
105
130
 
106
131
  /**
@@ -236,6 +261,11 @@ router.post('/scrape/single', async (req, res) => {
236
261
  json.hybrid = true;
237
262
  }
238
263
 
264
+ const situated_context = normalizeSituatedContext();
265
+ if (situated_context) {
266
+ json.situated_context = situated_context;
267
+ }
268
+
239
269
  winston.verbose("/scrape/single json: ", json);
240
270
 
241
271
  if (process.env.NODE_ENV === "test") {
@@ -361,7 +391,7 @@ router.post('/qa', async (req, res) => {
361
391
 
362
392
  // Check if "Advanced Mode" is active. In such case the default_context must be not appended
363
393
  if (!data.advancedPrompt) {
364
- const contextTemplate = contexts[data.model.name] || contexts["general"];
394
+ const contextTemplate = getRagContextTemplate(data.model.name);
365
395
  if (data.system_context) {
366
396
  data.system_context = data.system_context + " \n" + contextTemplate;
367
397
  } else {
@@ -393,7 +423,7 @@ router.post('/qa', async (req, res) => {
393
423
  }
394
424
  }
395
425
 
396
- data.stream = false;
426
+ data.stream = data.stream === true;
397
427
  data.debug = true;
398
428
  delete data.advancedPrompt;
399
429
  winston.verbose("ask data: ", data);
@@ -402,16 +432,163 @@ router.post('/qa', async (req, res) => {
402
432
  return res.status(200).send({ success: true, message: "Question skipped in test environment", data: data });
403
433
  }
404
434
 
435
+ if (data.stream === true) {
436
+ // Streaming SSE: use askStream and forward only content as JSON SSE events
437
+ res.status(200);
438
+ res.setHeader('Content-Type', 'text/event-stream');
439
+ res.setHeader('Cache-Control', 'no-cache');
440
+ res.setHeader('Connection', 'keep-alive');
441
+ res.setHeader('Access-Control-Allow-Origin', '*');
442
+
443
+ const sendError = (message) => {
444
+ try {
445
+ res.write('data: ' + JSON.stringify({ error: message }) + '\n\n');
446
+ } catch (_) {}
447
+ res.end();
448
+ };
449
+
450
+ function extractContent(obj) {
451
+ if (obj.content != null) return obj.content;
452
+ if (obj.choices && obj.choices[0]) {
453
+ const c = obj.choices[0];
454
+ if (c.delta && c.delta.content != null) return c.delta.content;
455
+ if (c.message && c.message.content != null) return c.message.content;
456
+ }
457
+ return null;
458
+ }
459
+
460
+ /** Same JSON shape as non-stream /qa: stream may wrap it in model_used */
461
+ function normalizeKbQaPayload(obj) {
462
+ if (obj && typeof obj === 'object' && obj.model_used != null && typeof obj.model_used === 'object') {
463
+ return obj.model_used;
464
+ }
465
+ return obj;
466
+ }
467
+
468
+ /** Flat final payload like non-stream /qa (answer, prompt_token_size, …) */
469
+ function isMetadataPayload(obj, streamedContent) {
470
+ if (obj == null || typeof obj !== 'object') return false;
471
+ if (streamedContent != null && streamedContent !== '') return false;
472
+ if (typeof obj.prompt_token_size === 'number') return true;
473
+ if (obj.answer != null) return true;
474
+ if (obj.sources != null) return true;
475
+ if (obj.chunks != null) return true;
476
+ if (obj.content_chunks != null) return true;
477
+ return false;
478
+ }
479
+
480
+ /** KB stream summary: full_response + model_used (same info as non-stream body, plus envelope) */
481
+ function isKbStreamCompletedSummary(obj) {
482
+ if (obj == null || typeof obj !== 'object') return false;
483
+ if (obj.status === 'completed') return true;
484
+ if (obj.full_response != null && obj.model_used != null && typeof obj.model_used === 'object') return true;
485
+ return false;
486
+ }
487
+
488
+ function forwardSsePayload(payload) {
489
+ if (payload === '[DONE]') return;
490
+ let obj;
491
+ try {
492
+ obj = JSON.parse(payload);
493
+ } catch (_) {
494
+ return;
495
+ }
496
+
497
+ if (obj.status === 'started') {
498
+ return;
499
+ }
500
+ if (isKbStreamCompletedSummary(obj)) {
501
+ res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
502
+ return;
503
+ }
504
+
505
+ if (obj.type === 'metadata' || obj.event === 'metadata') {
506
+ res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
507
+ return;
508
+ }
509
+ const content = extractContent(obj);
510
+ if (content != null && content !== '') {
511
+ res.write('data: ' + JSON.stringify({ content }) + '\n\n');
512
+ return;
513
+ }
514
+ const normalized = normalizeKbQaPayload(obj);
515
+ if (isMetadataPayload(normalized, content)) {
516
+ res.write('data: ' + JSON.stringify(normalized) + '\n\n');
517
+ }
518
+ }
519
+
520
+ aiService.askStream(data).then((resp) => {
521
+ const stream = resp.data;
522
+ let buffer = '';
523
+
524
+ stream.on('data', (chunk) => {
525
+ buffer += chunk.toString();
526
+ const lines = buffer.split('\n');
527
+ buffer = lines.pop() || '';
528
+
529
+ for (const line of lines) {
530
+ const trimmed = line.trim();
531
+ if (!trimmed.startsWith('data: ')) continue;
532
+ const payload = trimmed.slice(6);
533
+ forwardSsePayload(payload);
534
+ }
535
+ });
536
+
537
+ stream.on('end', () => {
538
+ const tail = buffer.trim();
539
+ if (tail) {
540
+ for (const line of tail.split('\n')) {
541
+ const trimmed = line.trim();
542
+ if (!trimmed.startsWith('data: ')) continue;
543
+ forwardSsePayload(trimmed.slice(6));
544
+ }
545
+ }
546
+ res.write('data: [DONE]\n\n');
547
+ res.end();
548
+ });
549
+
550
+ stream.on('error', (err) => {
551
+ winston.error('qa stream err: ', err);
552
+ sendError(err.message || 'Stream error');
553
+ });
554
+
555
+ res.on('close', () => {
556
+ if (!res.writableEnded) {
557
+ stream.destroy();
558
+ }
559
+ });
560
+ }).catch((err) => {
561
+ winston.error('qa err: ', err);
562
+ winston.error('qa err.response: ', err.response);
563
+ const message = (err.response && err.response.data && typeof err.response.data.pipe !== 'function' && err.response.data.detail)
564
+ ? err.response.data.detail
565
+ : (err.response && err.response.statusText) || err.message || String(err);
566
+ if (!res.headersSent) {
567
+ res.status(err.response && err.response.status ? err.response.status : 500);
568
+ }
569
+ sendError(message);
570
+ });
571
+ return;
572
+ }
573
+
405
574
  aiService.askNamespace(data).then((resp) => {
406
575
  winston.debug("qa resp: ", resp.data);
407
576
  let answer = resp.data;
408
577
 
409
578
  if (publicKey === true) {
410
- let multiplier = MODELS_MULTIPLIER[data.model];
579
+ let modelKey;
580
+ if (typeof data.model === 'string') {
581
+ modelKey = data.model;
582
+ } else if (data.model && typeof data.model.name === 'string') {
583
+ modelKey = data.model.name;
584
+ }
585
+
586
+ let multiplier = MODELS_MULTIPLIER[modelKey];
411
587
  if (!multiplier) {
412
588
  multiplier = 1;
413
- winston.info("No multiplier found for AI model (qa) " + data.model);
589
+ winston.info("No multiplier found for AI model (qa) " + modelKey);
414
590
  }
591
+
415
592
  obj.multiplier = multiplier;
416
593
  obj.tokens = answer.prompt_token_size;
417
594
 
@@ -1039,6 +1216,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1039
1216
  let embedding = normalizeEmbedding(ns.embedding);
1040
1217
  embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
1041
1218
  let hybrid = ns.hybrid;
1219
+ const situated_context = normalizeSituatedContext();
1042
1220
 
1043
1221
 
1044
1222
  if (process.env.NODE_ENV !== "test") {
@@ -1076,7 +1254,13 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
1076
1254
 
1077
1255
  let resources = new_contents.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
1078
1256
  resources = resources.map(({ _id, scrape_options, ...rest }) => {
1079
- return { id: _id, parameters_scrape_type_4: scrape_options, embedding: embedding, engine: engine, ...rest}
1257
+ return {
1258
+ id: _id,
1259
+ parameters_scrape_type_4: scrape_options,
1260
+ embedding: embedding,
1261
+ engine: engine,
1262
+ ...(situated_context && { situated_context: situated_context }),
1263
+ ...rest}
1080
1264
  });
1081
1265
 
1082
1266
  winston.verbose("resources to be sent to worker: ", resources);
@@ -1420,13 +1604,14 @@ router.post('/', async (req, res) => {
1420
1604
  }
1421
1605
  if (type === 'url') {
1422
1606
  new_kb.refresh_rate = refresh_rate || 'never';
1423
- if (!scrape_type || scrape_type === 2) {
1424
- new_kb.scrape_type = 2;
1425
- new_kb.scrape_options = aiManager.setDefaultScrapeOptions();
1426
- } else {
1607
+ if (scrape_type === 0 || scrape_type === 4) {
1427
1608
  new_kb.scrape_type = scrape_type;
1428
1609
  new_kb.scrape_options = scrape_options;
1429
1610
  }
1611
+ else {
1612
+ new_kb.scrape_type = 2;
1613
+ new_kb.scrape_options = aiManager.setDefaultScrapeOptions();
1614
+ }
1430
1615
  }
1431
1616
 
1432
1617
  if (tags && Array.isArray(tags) && tags.every(tag => typeof tag === "string")) {
@@ -1451,6 +1636,8 @@ router.post('/', async (req, res) => {
1451
1636
  const embedding = normalizeEmbedding(namespace.embedding);
1452
1637
  embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
1453
1638
 
1639
+ const situated_context = normalizeSituatedContext();
1640
+
1454
1641
  const json = {
1455
1642
  id: saved_kb._id,
1456
1643
  type: saved_kb.type,
@@ -1461,6 +1648,7 @@ router.post('/', async (req, res) => {
1461
1648
  hybrid: namespace.hybrid,
1462
1649
  engine: namespace.engine || default_engine,
1463
1650
  embedding: embedding,
1651
+ ...(situated_context && { situated_context: situated_context }),
1464
1652
  ...(saved_kb.scrape_type && { scrape_type: saved_kb.scrape_type }),
1465
1653
  ...(saved_kb.scrape_options && { parameters_scrape_type_4: saved_kb.scrape_options }),
1466
1654
  ...(saved_kb.tags && { tags: saved_kb.tags }),
@@ -1617,10 +1805,18 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
1617
1805
  let embedding = normalizeEmbedding(namespace.embedding);
1618
1806
  embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
1619
1807
  let hybrid = namespace.hybrid;
1808
+ const situated_context = normalizeSituatedContext();
1620
1809
 
1621
1810
  let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
1622
1811
  resources = resources.map(({ _id, ...rest}) => {
1623
- return { id: _id, webhook: webhook, embedding: embedding, engine: engine, ...rest };
1812
+ return {
1813
+ id: _id,
1814
+ webhook: webhook,
1815
+ embedding: embedding,
1816
+ engine: engine,
1817
+ ...(situated_context && { situated_context: situated_context }),
1818
+ ...rest
1819
+ };
1624
1820
  })
1625
1821
  winston.verbose("resources to be sent to worker: ", resources);
1626
1822
 
@@ -1773,35 +1969,147 @@ router.post('/sitemap/import', async (req, res) => {
1773
1969
 
1774
1970
  router.put('/:kb_id', async (req, res) => {
1775
1971
 
1776
- let kb_id = req.params.kb_id;
1777
- winston.verbose("update kb_id " + kb_id);
1972
+ const id_project = req.projectid;
1973
+ const project = req.project;
1974
+ const kb_id = req.params.kb_id;
1975
+
1976
+ const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags } = req.body;
1977
+ const namespace_id = req.body.namespace;
1778
1978
 
1779
- let update = {};
1979
+ if (!namespace_id) {
1980
+ return res.status(400).send({ success: false, error: "Missing 'namespace' body parameter" })
1981
+ }
1982
+
1983
+ let namespace;
1984
+ try {
1985
+ namespace = await aiManager.checkNamespace(id_project, namespace_id);
1986
+ } catch (err) {
1987
+ let errorCode = err?.errorCode ?? 500;
1988
+ return res.status(errorCode).send({ success: false, error: err.error });
1989
+ }
1780
1990
 
1781
- if (req.body.name != undefined) {
1782
- update.name = req.body.name;
1991
+ let kb = await KB.findOne({ id_project, namespace: namespace_id, _id: kb_id }).lean().exec();
1992
+
1993
+ if (!kb) {
1994
+ return res.status(404).send({ success: false, error: "Content not found. Unable to update a non-existing content" })
1783
1995
  }
1784
1996
 
1785
- if (req.body.status != undefined) {
1786
- update.status = req.body.status;
1997
+ let data = {
1998
+ id: kb._id,
1999
+ namespace: namespace_id,
2000
+ engine: namespace.engine || default_engine
1787
2001
  }
1788
2002
 
1789
- winston.debug("kb update: ", update);
2003
+ // if (kb.type === 'sitemap') {
2004
+ // let new_sitemap = {
2005
+ // id_project,
2006
+ // name,
2007
+ // source,
2008
+ // type: 'sitemap',
2009
+ // content: "",
2010
+ // namespace: namespace_id,
2011
+ // status: -1,
2012
+ // scrape_type,
2013
+ // scrape_options,
2014
+ // refresh_rate,
2015
+ // ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {})
2016
+ // }
1790
2017
 
1791
- KB.findByIdAndUpdate(kb_id, update, { new: true }, (err, savedKb) => {
2018
+ // try {
2019
+ // let result = await aiManager.updateSitemap(id_project, namespace, kb, new_sitemap);
2020
+ // return res.status(200).send(result);
2021
+ // } catch (err) {
2022
+ // winston.error("Error updating sitemap: ", err);
2023
+ // return res.status(500).send({ success: false, error: err });
2024
+ // }
2025
+ // }
1792
2026
 
1793
- if (err) {
1794
- winston.error("KB findByIdAndUpdate error: ", err);
1795
- return res.status(500).send({ success: false, error: err });
2027
+ try {
2028
+ let delete_response = await aiService.deleteIndex(data);
2029
+
2030
+ if (delete_response.data.success === true) {
2031
+ await KB.findOneAndDelete({ id_project, namespace: namespace_id, source }).lean().exec();
2032
+ // continue the flow
2033
+ } else {
2034
+ winston.error("Unable to update content due to an error: ", delete_response.data.error);
2035
+ return res.status(500).send({ success: false, error: delete_response.data.error });
1796
2036
  }
1797
2037
 
1798
- if (!savedKb) {
1799
- winston.debug("Try to updating a non-existing kb");
1800
- return res.status(400).send({ success: false, message: "Content not found" })
2038
+ } catch (err) {
2039
+ winston.error("Error updating content: ", err);
2040
+ return res.status(500).send({ success: false, error: err });
2041
+ }
2042
+
2043
+ let new_content = {
2044
+ id_project,
2045
+ name,
2046
+ type,
2047
+ source,
2048
+ content,
2049
+ namespace: namespace_id,
2050
+ status: -1,
2051
+ }
2052
+
2053
+ if (new_content.type === 'url') {
2054
+ new_content.refresh_rate = refresh_rate || 'never';
2055
+ if (scrape_type === 0 || scrape_type === 4) {
2056
+ new_content.scrape_type = scrape_type;
2057
+ new_content.scrape_options = scrape_options;
2058
+ }
2059
+ else {
2060
+ new_content.scrape_type = 2;
2061
+ new_content.scrape_options = aiManager.setDefaultScrapeOptions();
1801
2062
  }
2063
+ }
1802
2064
 
1803
- res.status(200).send(savedKb)
1804
- })
2065
+ if (kb.sitemap_origin_id) {
2066
+ new_content.sitemap_origin_id = kb.sitemap_origin_id;
2067
+ new_content.sitemap_origin = kb.sitemap_origin;
2068
+ }
2069
+
2070
+ if (tags && Array.isArray(tags) && tags.every(tag => typeof tag === "string")) {
2071
+ new_content.tags = tags;
2072
+ }
2073
+
2074
+ winston.debug("Update content. New content: ", new_content);
2075
+
2076
+ let updated_content;
2077
+ try {
2078
+ updated_content = await KB.findOneAndUpdate({ id_project, namespace: namespace_id, source }, new_content, { upsert: true, new: true }).lean().exec();
2079
+ } catch (err) {
2080
+ winston.error("Error updating content: ", err);
2081
+ return res.status(500).send({ success: false, error: err });
2082
+ }
2083
+
2084
+ const embedding = normalizeEmbedding(namespace.embedding);
2085
+ embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
2086
+ let webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
2087
+ const situated_context = normalizeSituatedContext();
2088
+
2089
+ const json = {
2090
+ id: updated_content._id,
2091
+ type: updated_content.type,
2092
+ source: updated_content.source,
2093
+ content: updated_content.content || "",
2094
+ namespace: updated_content.namespace,
2095
+ webhook: webhook,
2096
+ hybrid: namespace.hybrid,
2097
+ engine: namespace.engine || default_engine,
2098
+ embedding: embedding,
2099
+ ...(situated_context && { situated_context: situated_context }),
2100
+ ...(updated_content.scrape_type && { scrape_type: updated_content.scrape_type }),
2101
+ ...(updated_content.scrape_options && { parameters_scrape_type_4: updated_content.scrape_options }),
2102
+ ...(updated_content.tags && { tags: updated_content.tags }),
2103
+ }
2104
+
2105
+ winston.debug("json: ", json);
2106
+
2107
+ if (process.env.NODE_ENV === 'test') {
2108
+ return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: updated_content, schedule_json: json });
2109
+ }
2110
+
2111
+ aiManager.scheduleScrape([json], namespace.hybrid);
2112
+ return res.status(200).send(updated_content);
1805
2113
 
1806
2114
  })
1807
2115
 
package/routes/quotes.js CHANGED
@@ -44,10 +44,17 @@ router.post('/incr/:type', async (req, res) => {
44
44
 
45
45
  let quoteManager = req.app.get('quote_manager');
46
46
 
47
- let multiplier = MODELS_MULTIPLIER[data.model];
47
+ let modelKey;
48
+ if (typeof data.model === 'string') {
49
+ modelKey = data.model;
50
+ } else if (data.model && typeof data.model.name === 'string') {
51
+ modelKey = data.model.name;
52
+ }
53
+
54
+ let multiplier = MODELS_MULTIPLIER[modelKey];
48
55
  if (!multiplier) {
49
56
  multiplier = 1;
50
- winston.info("No multiplier found for AI model (incr) " + data.model)
57
+ winston.info("No multiplier found for AI model (incr) " + modelKey)
51
58
  }
52
59
  data.multiplier = multiplier;
53
60
  data.createdAt = new Date();