@tiledesk/tiledesk-server 2.15.8 → 2.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/config/kb/prompt/rag/PromptManager.js +57 -0
- package/config/kb/prompt/rag/general.txt +9 -0
- package/config/kb/prompt/rag/gpt-3.5.txt +9 -0
- package/config/kb/prompt/rag/gpt-4.1.txt +9 -0
- package/config/kb/prompt/rag/gpt-4.txt +11 -0
- package/config/kb/prompt/rag/gpt-4o.txt +9 -0
- package/config/kb/prompt/rag/gpt-5.txt +32 -0
- package/config/kb/prompt/rag/gpt-5.x.txt +32 -0
- package/config/kb/situatedContext.js +6 -0
- package/event/messageEvent.js +24 -0
- package/middleware/file-type.js +109 -36
- package/models/request.js +1 -0
- package/package.json +1 -1
- package/pubmodules/queue/reconnect.js +16 -0
- package/pubmodules/routing-queue/listenerQueued.js +22 -3
- package/pubmodules/scheduler/tasks/closeAgentUnresponsiveRequestTask.js +2 -1
- package/pubmodules/scheduler/tasks/closeBotUnresponsiveRequestTask.js +2 -1
- package/routes/filesp.js +4 -3
- package/routes/kb.js +348 -40
- package/routes/quotes.js +9 -2
- package/routes/request.js +174 -92
- package/routes/webhook.js +5 -0
- package/services/aiManager.js +93 -1
- package/services/aiService.js +33 -8
- package/services/fileGridFsService.js +1 -2
package/routes/kb.js
CHANGED
|
@@ -80,6 +80,37 @@ let default_preview_settings = {
|
|
|
80
80
|
const default_engine = require('../config/kb/engine');
|
|
81
81
|
const default_engine_hybrid = require('../config/kb/engine.hybrid');
|
|
82
82
|
const default_embedding = require('../config/kb/embedding');
|
|
83
|
+
const PromptManager = require('../config/kb/prompt/rag/PromptManager');
|
|
84
|
+
const situatedContext = require('../config/kb/situatedContext');
|
|
85
|
+
|
|
86
|
+
const ragPromptManager = new PromptManager(path.join(__dirname, '../config/kb/prompt/rag'));
|
|
87
|
+
|
|
88
|
+
const RAG_CONTEXT_ENV_OVERRIDES = {
|
|
89
|
+
"gpt-3.5-turbo": process.env.GPT_3_5_CONTEXT,
|
|
90
|
+
"gpt-4": process.env.GPT_4_CONTEXT,
|
|
91
|
+
"gpt-4-turbo-preview": process.env.GPT_4T_CONTEXT,
|
|
92
|
+
"gpt-4o": process.env.GPT_4O_CONTEXT,
|
|
93
|
+
"gpt-4o-mini": process.env.GPT_4O_MINI_CONTEXT,
|
|
94
|
+
"gpt-4.1": process.env.GPT_4_1_CONTEXT,
|
|
95
|
+
"gpt-4.1-mini": process.env.GPT_4_1_MINI_CONTEXT,
|
|
96
|
+
"gpt-4.1-nano": process.env.GPT_4_1_NANO_CONTEXT,
|
|
97
|
+
"gpt-5": process.env.GPT_5_CONTEXT,
|
|
98
|
+
"gpt-5-mini": process.env.GPT_5_MINI_CONTEXT,
|
|
99
|
+
"gpt-5-nano": process.env.GPT_5_NANO_CONTEXT,
|
|
100
|
+
"general": process.env.GENERAL_CONTEXT
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
/** RAG system prompt per modello: file in config/kb/prompt/rag, sovrascrivibili via env (come prima). */
|
|
104
|
+
function getRagContextTemplate(modelName) {
|
|
105
|
+
const envOverride = RAG_CONTEXT_ENV_OVERRIDES[modelName];
|
|
106
|
+
if (envOverride) {
|
|
107
|
+
return envOverride;
|
|
108
|
+
}
|
|
109
|
+
if (!PromptManager.modelMap[modelName] && process.env.GENERAL_CONTEXT) {
|
|
110
|
+
return process.env.GENERAL_CONTEXT;
|
|
111
|
+
}
|
|
112
|
+
return ragPromptManager.getPrompt(modelName);
|
|
113
|
+
}
|
|
83
114
|
|
|
84
115
|
function normalizeEmbedding(embedding) {
|
|
85
116
|
const normalizedEmbedding = (embedding && typeof embedding.toObject === 'function')
|
|
@@ -88,19 +119,13 @@ function normalizeEmbedding(embedding) {
|
|
|
88
119
|
return { ...normalizedEmbedding };
|
|
89
120
|
}
|
|
90
121
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
"gpt-4.1-mini": process.env.GPT_4_1_MINI_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
|
|
99
|
-
"gpt-4.1-nano": process.env.GPT_4_1_NANO_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
|
|
100
|
-
"gpt-5": process.env.GPT_5_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
|
|
101
|
-
"gpt-5-mini": process.env.GPT_5_MINI_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
|
|
102
|
-
"gpt-5-nano": process.env.GPT_5_NANO_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end==",
|
|
103
|
-
"general": process.env.GENERAL_CONTEXT || "You are an helpful assistant for question-answering tasks. Follow these steps carefully:\n1. Answer in the same language of the user question, regardless of the retrieved context language\n2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.\n3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer\n\n==Retrieved context start==\n{context}\n==Retrieved context end=="
|
|
122
|
+
function normalizeSituatedContext() {
|
|
123
|
+
return situatedContext.enable
|
|
124
|
+
? {
|
|
125
|
+
...situatedContext,
|
|
126
|
+
api_key: process.env.SITUATED_CONTEXT_API_KEY || process.env.GPTKEY
|
|
127
|
+
}
|
|
128
|
+
: undefined;
|
|
104
129
|
}
|
|
105
130
|
|
|
106
131
|
/**
|
|
@@ -236,6 +261,11 @@ router.post('/scrape/single', async (req, res) => {
|
|
|
236
261
|
json.hybrid = true;
|
|
237
262
|
}
|
|
238
263
|
|
|
264
|
+
const situated_context = normalizeSituatedContext();
|
|
265
|
+
if (situated_context) {
|
|
266
|
+
json.situated_context = situated_context;
|
|
267
|
+
}
|
|
268
|
+
|
|
239
269
|
winston.verbose("/scrape/single json: ", json);
|
|
240
270
|
|
|
241
271
|
if (process.env.NODE_ENV === "test") {
|
|
@@ -361,7 +391,7 @@ router.post('/qa', async (req, res) => {
|
|
|
361
391
|
|
|
362
392
|
// Check if "Advanced Mode" is active. In such case the default_context must be not appended
|
|
363
393
|
if (!data.advancedPrompt) {
|
|
364
|
-
const contextTemplate =
|
|
394
|
+
const contextTemplate = getRagContextTemplate(data.model.name);
|
|
365
395
|
if (data.system_context) {
|
|
366
396
|
data.system_context = data.system_context + " \n" + contextTemplate;
|
|
367
397
|
} else {
|
|
@@ -393,7 +423,7 @@ router.post('/qa', async (req, res) => {
|
|
|
393
423
|
}
|
|
394
424
|
}
|
|
395
425
|
|
|
396
|
-
data.stream =
|
|
426
|
+
data.stream = data.stream === true;
|
|
397
427
|
data.debug = true;
|
|
398
428
|
delete data.advancedPrompt;
|
|
399
429
|
winston.verbose("ask data: ", data);
|
|
@@ -402,16 +432,163 @@ router.post('/qa', async (req, res) => {
|
|
|
402
432
|
return res.status(200).send({ success: true, message: "Question skipped in test environment", data: data });
|
|
403
433
|
}
|
|
404
434
|
|
|
435
|
+
if (data.stream === true) {
|
|
436
|
+
// Streaming SSE: use askStream and forward only content as JSON SSE events
|
|
437
|
+
res.status(200);
|
|
438
|
+
res.setHeader('Content-Type', 'text/event-stream');
|
|
439
|
+
res.setHeader('Cache-Control', 'no-cache');
|
|
440
|
+
res.setHeader('Connection', 'keep-alive');
|
|
441
|
+
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
442
|
+
|
|
443
|
+
const sendError = (message) => {
|
|
444
|
+
try {
|
|
445
|
+
res.write('data: ' + JSON.stringify({ error: message }) + '\n\n');
|
|
446
|
+
} catch (_) {}
|
|
447
|
+
res.end();
|
|
448
|
+
};
|
|
449
|
+
|
|
450
|
+
function extractContent(obj) {
|
|
451
|
+
if (obj.content != null) return obj.content;
|
|
452
|
+
if (obj.choices && obj.choices[0]) {
|
|
453
|
+
const c = obj.choices[0];
|
|
454
|
+
if (c.delta && c.delta.content != null) return c.delta.content;
|
|
455
|
+
if (c.message && c.message.content != null) return c.message.content;
|
|
456
|
+
}
|
|
457
|
+
return null;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
/** Same JSON shape as non-stream /qa: stream may wrap it in model_used */
|
|
461
|
+
function normalizeKbQaPayload(obj) {
|
|
462
|
+
if (obj && typeof obj === 'object' && obj.model_used != null && typeof obj.model_used === 'object') {
|
|
463
|
+
return obj.model_used;
|
|
464
|
+
}
|
|
465
|
+
return obj;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/** Flat final payload like non-stream /qa (answer, prompt_token_size, …) */
|
|
469
|
+
function isMetadataPayload(obj, streamedContent) {
|
|
470
|
+
if (obj == null || typeof obj !== 'object') return false;
|
|
471
|
+
if (streamedContent != null && streamedContent !== '') return false;
|
|
472
|
+
if (typeof obj.prompt_token_size === 'number') return true;
|
|
473
|
+
if (obj.answer != null) return true;
|
|
474
|
+
if (obj.sources != null) return true;
|
|
475
|
+
if (obj.chunks != null) return true;
|
|
476
|
+
if (obj.content_chunks != null) return true;
|
|
477
|
+
return false;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/** KB stream summary: full_response + model_used (same info as non-stream body, plus envelope) */
|
|
481
|
+
function isKbStreamCompletedSummary(obj) {
|
|
482
|
+
if (obj == null || typeof obj !== 'object') return false;
|
|
483
|
+
if (obj.status === 'completed') return true;
|
|
484
|
+
if (obj.full_response != null && obj.model_used != null && typeof obj.model_used === 'object') return true;
|
|
485
|
+
return false;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
function forwardSsePayload(payload) {
|
|
489
|
+
if (payload === '[DONE]') return;
|
|
490
|
+
let obj;
|
|
491
|
+
try {
|
|
492
|
+
obj = JSON.parse(payload);
|
|
493
|
+
} catch (_) {
|
|
494
|
+
return;
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
if (obj.status === 'started') {
|
|
498
|
+
return;
|
|
499
|
+
}
|
|
500
|
+
if (isKbStreamCompletedSummary(obj)) {
|
|
501
|
+
res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
if (obj.type === 'metadata' || obj.event === 'metadata') {
|
|
506
|
+
res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
const content = extractContent(obj);
|
|
510
|
+
if (content != null && content !== '') {
|
|
511
|
+
res.write('data: ' + JSON.stringify({ content }) + '\n\n');
|
|
512
|
+
return;
|
|
513
|
+
}
|
|
514
|
+
const normalized = normalizeKbQaPayload(obj);
|
|
515
|
+
if (isMetadataPayload(normalized, content)) {
|
|
516
|
+
res.write('data: ' + JSON.stringify(normalized) + '\n\n');
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
aiService.askStream(data).then((resp) => {
|
|
521
|
+
const stream = resp.data;
|
|
522
|
+
let buffer = '';
|
|
523
|
+
|
|
524
|
+
stream.on('data', (chunk) => {
|
|
525
|
+
buffer += chunk.toString();
|
|
526
|
+
const lines = buffer.split('\n');
|
|
527
|
+
buffer = lines.pop() || '';
|
|
528
|
+
|
|
529
|
+
for (const line of lines) {
|
|
530
|
+
const trimmed = line.trim();
|
|
531
|
+
if (!trimmed.startsWith('data: ')) continue;
|
|
532
|
+
const payload = trimmed.slice(6);
|
|
533
|
+
forwardSsePayload(payload);
|
|
534
|
+
}
|
|
535
|
+
});
|
|
536
|
+
|
|
537
|
+
stream.on('end', () => {
|
|
538
|
+
const tail = buffer.trim();
|
|
539
|
+
if (tail) {
|
|
540
|
+
for (const line of tail.split('\n')) {
|
|
541
|
+
const trimmed = line.trim();
|
|
542
|
+
if (!trimmed.startsWith('data: ')) continue;
|
|
543
|
+
forwardSsePayload(trimmed.slice(6));
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
res.write('data: [DONE]\n\n');
|
|
547
|
+
res.end();
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
stream.on('error', (err) => {
|
|
551
|
+
winston.error('qa stream err: ', err);
|
|
552
|
+
sendError(err.message || 'Stream error');
|
|
553
|
+
});
|
|
554
|
+
|
|
555
|
+
res.on('close', () => {
|
|
556
|
+
if (!res.writableEnded) {
|
|
557
|
+
stream.destroy();
|
|
558
|
+
}
|
|
559
|
+
});
|
|
560
|
+
}).catch((err) => {
|
|
561
|
+
winston.error('qa err: ', err);
|
|
562
|
+
winston.error('qa err.response: ', err.response);
|
|
563
|
+
const message = (err.response && err.response.data && typeof err.response.data.pipe !== 'function' && err.response.data.detail)
|
|
564
|
+
? err.response.data.detail
|
|
565
|
+
: (err.response && err.response.statusText) || err.message || String(err);
|
|
566
|
+
if (!res.headersSent) {
|
|
567
|
+
res.status(err.response && err.response.status ? err.response.status : 500);
|
|
568
|
+
}
|
|
569
|
+
sendError(message);
|
|
570
|
+
});
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
|
|
405
574
|
aiService.askNamespace(data).then((resp) => {
|
|
406
575
|
winston.debug("qa resp: ", resp.data);
|
|
407
576
|
let answer = resp.data;
|
|
408
577
|
|
|
409
578
|
if (publicKey === true) {
|
|
410
|
-
let
|
|
579
|
+
let modelKey;
|
|
580
|
+
if (typeof data.model === 'string') {
|
|
581
|
+
modelKey = data.model;
|
|
582
|
+
} else if (data.model && typeof data.model.name === 'string') {
|
|
583
|
+
modelKey = data.model.name;
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
let multiplier = MODELS_MULTIPLIER[modelKey];
|
|
411
587
|
if (!multiplier) {
|
|
412
588
|
multiplier = 1;
|
|
413
|
-
winston.info("No multiplier found for AI model (qa) " +
|
|
589
|
+
winston.info("No multiplier found for AI model (qa) " + modelKey);
|
|
414
590
|
}
|
|
591
|
+
|
|
415
592
|
obj.multiplier = multiplier;
|
|
416
593
|
obj.tokens = answer.prompt_token_size;
|
|
417
594
|
|
|
@@ -1039,6 +1216,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1039
1216
|
let embedding = normalizeEmbedding(ns.embedding);
|
|
1040
1217
|
embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
|
|
1041
1218
|
let hybrid = ns.hybrid;
|
|
1219
|
+
const situated_context = normalizeSituatedContext();
|
|
1042
1220
|
|
|
1043
1221
|
|
|
1044
1222
|
if (process.env.NODE_ENV !== "test") {
|
|
@@ -1076,7 +1254,13 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1076
1254
|
|
|
1077
1255
|
let resources = new_contents.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
|
|
1078
1256
|
resources = resources.map(({ _id, scrape_options, ...rest }) => {
|
|
1079
|
-
return {
|
|
1257
|
+
return {
|
|
1258
|
+
id: _id,
|
|
1259
|
+
parameters_scrape_type_4: scrape_options,
|
|
1260
|
+
embedding: embedding,
|
|
1261
|
+
engine: engine,
|
|
1262
|
+
...(situated_context && { situated_context: situated_context }),
|
|
1263
|
+
...rest}
|
|
1080
1264
|
});
|
|
1081
1265
|
|
|
1082
1266
|
winston.verbose("resources to be sent to worker: ", resources);
|
|
@@ -1420,13 +1604,14 @@ router.post('/', async (req, res) => {
|
|
|
1420
1604
|
}
|
|
1421
1605
|
if (type === 'url') {
|
|
1422
1606
|
new_kb.refresh_rate = refresh_rate || 'never';
|
|
1423
|
-
if (
|
|
1424
|
-
new_kb.scrape_type = 2;
|
|
1425
|
-
new_kb.scrape_options = aiManager.setDefaultScrapeOptions();
|
|
1426
|
-
} else {
|
|
1607
|
+
if (scrape_type === 0 || scrape_type === 4) {
|
|
1427
1608
|
new_kb.scrape_type = scrape_type;
|
|
1428
1609
|
new_kb.scrape_options = scrape_options;
|
|
1429
1610
|
}
|
|
1611
|
+
else {
|
|
1612
|
+
new_kb.scrape_type = 2;
|
|
1613
|
+
new_kb.scrape_options = aiManager.setDefaultScrapeOptions();
|
|
1614
|
+
}
|
|
1430
1615
|
}
|
|
1431
1616
|
|
|
1432
1617
|
if (tags && Array.isArray(tags) && tags.every(tag => typeof tag === "string")) {
|
|
@@ -1451,6 +1636,8 @@ router.post('/', async (req, res) => {
|
|
|
1451
1636
|
const embedding = normalizeEmbedding(namespace.embedding);
|
|
1452
1637
|
embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
|
|
1453
1638
|
|
|
1639
|
+
const situated_context = normalizeSituatedContext();
|
|
1640
|
+
|
|
1454
1641
|
const json = {
|
|
1455
1642
|
id: saved_kb._id,
|
|
1456
1643
|
type: saved_kb.type,
|
|
@@ -1461,6 +1648,7 @@ router.post('/', async (req, res) => {
|
|
|
1461
1648
|
hybrid: namespace.hybrid,
|
|
1462
1649
|
engine: namespace.engine || default_engine,
|
|
1463
1650
|
embedding: embedding,
|
|
1651
|
+
...(situated_context && { situated_context: situated_context }),
|
|
1464
1652
|
...(saved_kb.scrape_type && { scrape_type: saved_kb.scrape_type }),
|
|
1465
1653
|
...(saved_kb.scrape_options && { parameters_scrape_type_4: saved_kb.scrape_options }),
|
|
1466
1654
|
...(saved_kb.tags && { tags: saved_kb.tags }),
|
|
@@ -1617,10 +1805,18 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
|
|
|
1617
1805
|
let embedding = normalizeEmbedding(namespace.embedding);
|
|
1618
1806
|
embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
|
|
1619
1807
|
let hybrid = namespace.hybrid;
|
|
1808
|
+
const situated_context = normalizeSituatedContext();
|
|
1620
1809
|
|
|
1621
1810
|
let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
|
|
1622
1811
|
resources = resources.map(({ _id, ...rest}) => {
|
|
1623
|
-
return {
|
|
1812
|
+
return {
|
|
1813
|
+
id: _id,
|
|
1814
|
+
webhook: webhook,
|
|
1815
|
+
embedding: embedding,
|
|
1816
|
+
engine: engine,
|
|
1817
|
+
...(situated_context && { situated_context: situated_context }),
|
|
1818
|
+
...rest
|
|
1819
|
+
};
|
|
1624
1820
|
})
|
|
1625
1821
|
winston.verbose("resources to be sent to worker: ", resources);
|
|
1626
1822
|
|
|
@@ -1773,35 +1969,147 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
1773
1969
|
|
|
1774
1970
|
router.put('/:kb_id', async (req, res) => {
|
|
1775
1971
|
|
|
1776
|
-
|
|
1777
|
-
|
|
1972
|
+
const id_project = req.projectid;
|
|
1973
|
+
const project = req.project;
|
|
1974
|
+
const kb_id = req.params.kb_id;
|
|
1975
|
+
|
|
1976
|
+
const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags } = req.body;
|
|
1977
|
+
const namespace_id = req.body.namespace;
|
|
1778
1978
|
|
|
1779
|
-
|
|
1979
|
+
if (!namespace_id) {
|
|
1980
|
+
return res.status(400).send({ success: false, error: "Missing 'namespace' body parameter" })
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
let namespace;
|
|
1984
|
+
try {
|
|
1985
|
+
namespace = await aiManager.checkNamespace(id_project, namespace_id);
|
|
1986
|
+
} catch (err) {
|
|
1987
|
+
let errorCode = err?.errorCode ?? 500;
|
|
1988
|
+
return res.status(errorCode).send({ success: false, error: err.error });
|
|
1989
|
+
}
|
|
1780
1990
|
|
|
1781
|
-
|
|
1782
|
-
|
|
1991
|
+
let kb = await KB.findOne({ id_project, namespace: namespace_id, _id: kb_id }).lean().exec();
|
|
1992
|
+
|
|
1993
|
+
if (!kb) {
|
|
1994
|
+
return res.status(404).send({ success: false, error: "Content not found. Unable to update a non-existing content" })
|
|
1783
1995
|
}
|
|
1784
1996
|
|
|
1785
|
-
|
|
1786
|
-
|
|
1997
|
+
let data = {
|
|
1998
|
+
id: kb._id,
|
|
1999
|
+
namespace: namespace_id,
|
|
2000
|
+
engine: namespace.engine || default_engine
|
|
1787
2001
|
}
|
|
1788
2002
|
|
|
1789
|
-
|
|
2003
|
+
// if (kb.type === 'sitemap') {
|
|
2004
|
+
// let new_sitemap = {
|
|
2005
|
+
// id_project,
|
|
2006
|
+
// name,
|
|
2007
|
+
// source,
|
|
2008
|
+
// type: 'sitemap',
|
|
2009
|
+
// content: "",
|
|
2010
|
+
// namespace: namespace_id,
|
|
2011
|
+
// status: -1,
|
|
2012
|
+
// scrape_type,
|
|
2013
|
+
// scrape_options,
|
|
2014
|
+
// refresh_rate,
|
|
2015
|
+
// ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {})
|
|
2016
|
+
// }
|
|
1790
2017
|
|
|
1791
|
-
|
|
2018
|
+
// try {
|
|
2019
|
+
// let result = await aiManager.updateSitemap(id_project, namespace, kb, new_sitemap);
|
|
2020
|
+
// return res.status(200).send(result);
|
|
2021
|
+
// } catch (err) {
|
|
2022
|
+
// winston.error("Error updating sitemap: ", err);
|
|
2023
|
+
// return res.status(500).send({ success: false, error: err });
|
|
2024
|
+
// }
|
|
2025
|
+
// }
|
|
1792
2026
|
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
2027
|
+
try {
|
|
2028
|
+
let delete_response = await aiService.deleteIndex(data);
|
|
2029
|
+
|
|
2030
|
+
if (delete_response.data.success === true) {
|
|
2031
|
+
await KB.findOneAndDelete({ id_project, namespace: namespace_id, source }).lean().exec();
|
|
2032
|
+
// continue the flow
|
|
2033
|
+
} else {
|
|
2034
|
+
winston.error("Unable to update content due to an error: ", delete_response.data.error);
|
|
2035
|
+
return res.status(500).send({ success: false, error: delete_response.data.error });
|
|
1796
2036
|
}
|
|
1797
2037
|
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
2038
|
+
} catch (err) {
|
|
2039
|
+
winston.error("Error updating content: ", err);
|
|
2040
|
+
return res.status(500).send({ success: false, error: err });
|
|
2041
|
+
}
|
|
2042
|
+
|
|
2043
|
+
let new_content = {
|
|
2044
|
+
id_project,
|
|
2045
|
+
name,
|
|
2046
|
+
type,
|
|
2047
|
+
source,
|
|
2048
|
+
content,
|
|
2049
|
+
namespace: namespace_id,
|
|
2050
|
+
status: -1,
|
|
2051
|
+
}
|
|
2052
|
+
|
|
2053
|
+
if (new_content.type === 'url') {
|
|
2054
|
+
new_content.refresh_rate = refresh_rate || 'never';
|
|
2055
|
+
if (scrape_type === 0 || scrape_type === 4) {
|
|
2056
|
+
new_content.scrape_type = scrape_type;
|
|
2057
|
+
new_content.scrape_options = scrape_options;
|
|
2058
|
+
}
|
|
2059
|
+
else {
|
|
2060
|
+
new_content.scrape_type = 2;
|
|
2061
|
+
new_content.scrape_options = aiManager.setDefaultScrapeOptions();
|
|
1801
2062
|
}
|
|
2063
|
+
}
|
|
1802
2064
|
|
|
1803
|
-
|
|
1804
|
-
|
|
2065
|
+
if (kb.sitemap_origin_id) {
|
|
2066
|
+
new_content.sitemap_origin_id = kb.sitemap_origin_id;
|
|
2067
|
+
new_content.sitemap_origin = kb.sitemap_origin;
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
if (tags && Array.isArray(tags) && tags.every(tag => typeof tag === "string")) {
|
|
2071
|
+
new_content.tags = tags;
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
winston.debug("Update content. New content: ", new_content);
|
|
2075
|
+
|
|
2076
|
+
let updated_content;
|
|
2077
|
+
try {
|
|
2078
|
+
updated_content = await KB.findOneAndUpdate({ id_project, namespace: namespace_id, source }, new_content, { upsert: true, new: true }).lean().exec();
|
|
2079
|
+
} catch (err) {
|
|
2080
|
+
winston.error("Error updating content: ", err);
|
|
2081
|
+
return res.status(500).send({ success: false, error: err });
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
const embedding = normalizeEmbedding(namespace.embedding);
|
|
2085
|
+
embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
|
|
2086
|
+
let webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
|
|
2087
|
+
const situated_context = normalizeSituatedContext();
|
|
2088
|
+
|
|
2089
|
+
const json = {
|
|
2090
|
+
id: updated_content._id,
|
|
2091
|
+
type: updated_content.type,
|
|
2092
|
+
source: updated_content.source,
|
|
2093
|
+
content: updated_content.content || "",
|
|
2094
|
+
namespace: updated_content.namespace,
|
|
2095
|
+
webhook: webhook,
|
|
2096
|
+
hybrid: namespace.hybrid,
|
|
2097
|
+
engine: namespace.engine || default_engine,
|
|
2098
|
+
embedding: embedding,
|
|
2099
|
+
...(situated_context && { situated_context: situated_context }),
|
|
2100
|
+
...(updated_content.scrape_type && { scrape_type: updated_content.scrape_type }),
|
|
2101
|
+
...(updated_content.scrape_options && { parameters_scrape_type_4: updated_content.scrape_options }),
|
|
2102
|
+
...(updated_content.tags && { tags: updated_content.tags }),
|
|
2103
|
+
}
|
|
2104
|
+
|
|
2105
|
+
winston.debug("json: ", json);
|
|
2106
|
+
|
|
2107
|
+
if (process.env.NODE_ENV === 'test') {
|
|
2108
|
+
return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: updated_content, schedule_json: json });
|
|
2109
|
+
}
|
|
2110
|
+
|
|
2111
|
+
aiManager.scheduleScrape([json], namespace.hybrid);
|
|
2112
|
+
return res.status(200).send(updated_content);
|
|
1805
2113
|
|
|
1806
2114
|
})
|
|
1807
2115
|
|
package/routes/quotes.js
CHANGED
|
@@ -44,10 +44,17 @@ router.post('/incr/:type', async (req, res) => {
|
|
|
44
44
|
|
|
45
45
|
let quoteManager = req.app.get('quote_manager');
|
|
46
46
|
|
|
47
|
-
let
|
|
47
|
+
let modelKey;
|
|
48
|
+
if (typeof data.model === 'string') {
|
|
49
|
+
modelKey = data.model;
|
|
50
|
+
} else if (data.model && typeof data.model.name === 'string') {
|
|
51
|
+
modelKey = data.model.name;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
let multiplier = MODELS_MULTIPLIER[modelKey];
|
|
48
55
|
if (!multiplier) {
|
|
49
56
|
multiplier = 1;
|
|
50
|
-
winston.info("No multiplier found for AI model (incr) " +
|
|
57
|
+
winston.info("No multiplier found for AI model (incr) " + modelKey)
|
|
51
58
|
}
|
|
52
59
|
data.multiplier = multiplier;
|
|
53
60
|
data.createdAt = new Date();
|