@tiledesk/tiledesk-server 2.19.4 → 2.19.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3 -0
- package/package.json +1 -1
- package/routes/answered.js +243 -264
- package/routes/kb.js +150 -105
- package/routes/unanswered.js +277 -296
- package/services/aiManager.js +28 -19
- package/services/kbQuestionService.js +98 -0
package/routes/kb.js
CHANGED
|
@@ -35,12 +35,12 @@ let MAX_UPLOAD_FILE_SIZE = process.env.MAX_UPLOAD_FILE_SIZE;
|
|
|
35
35
|
let uploadlimits = undefined;
|
|
36
36
|
|
|
37
37
|
if (MAX_UPLOAD_FILE_SIZE) {
|
|
38
|
-
uploadlimits = {fileSize: parseInt(MAX_UPLOAD_FILE_SIZE)}
|
|
38
|
+
uploadlimits = { fileSize: parseInt(MAX_UPLOAD_FILE_SIZE) };
|
|
39
39
|
winston.debug("Max upload file size is : " + MAX_UPLOAD_FILE_SIZE);
|
|
40
40
|
} else {
|
|
41
41
|
winston.debug("Max upload file size is infinity");
|
|
42
42
|
}
|
|
43
|
-
var upload = multer({limits: uploadlimits});
|
|
43
|
+
var upload = multer({ limits: uploadlimits });
|
|
44
44
|
|
|
45
45
|
let jobManager = new JobManager(AMQP_MANAGER_URL, {
|
|
46
46
|
debug: false,
|
|
@@ -84,22 +84,23 @@ const default_engine_hybrid = require('../config/kb/engine.hybrid');
|
|
|
84
84
|
const default_embedding = require('../config/kb/embedding');
|
|
85
85
|
const PromptManager = require('../config/kb/prompt/rag/PromptManager');
|
|
86
86
|
const situatedContext = require('../config/kb/situatedContext');
|
|
87
|
+
const kbQuestionService = require('../services/kbQuestionService');
|
|
87
88
|
|
|
88
89
|
const ragPromptManager = new PromptManager(path.join(__dirname, '../config/kb/prompt/rag'));
|
|
89
90
|
|
|
90
91
|
const RAG_CONTEXT_ENV_OVERRIDES = {
|
|
91
|
-
"gpt-3.5-turbo":
|
|
92
|
-
"gpt-4":
|
|
92
|
+
"gpt-3.5-turbo": process.env.GPT_3_5_CONTEXT,
|
|
93
|
+
"gpt-4": process.env.GPT_4_CONTEXT,
|
|
93
94
|
"gpt-4-turbo-preview": process.env.GPT_4T_CONTEXT,
|
|
94
|
-
"gpt-4o":
|
|
95
|
-
"gpt-4o-mini":
|
|
96
|
-
"gpt-4.1":
|
|
97
|
-
"gpt-4.1-mini":
|
|
98
|
-
"gpt-4.1-nano":
|
|
99
|
-
"gpt-5":
|
|
100
|
-
"gpt-5-mini":
|
|
101
|
-
"gpt-5-nano":
|
|
102
|
-
"general":
|
|
95
|
+
"gpt-4o": process.env.GPT_4O_CONTEXT,
|
|
96
|
+
"gpt-4o-mini": process.env.GPT_4O_MINI_CONTEXT,
|
|
97
|
+
"gpt-4.1": process.env.GPT_4_1_CONTEXT,
|
|
98
|
+
"gpt-4.1-mini": process.env.GPT_4_1_MINI_CONTEXT,
|
|
99
|
+
"gpt-4.1-nano": process.env.GPT_4_1_NANO_CONTEXT,
|
|
100
|
+
"gpt-5": process.env.GPT_5_CONTEXT,
|
|
101
|
+
"gpt-5-mini": process.env.GPT_5_MINI_CONTEXT,
|
|
102
|
+
"gpt-5-nano": process.env.GPT_5_NANO_CONTEXT,
|
|
103
|
+
"general": process.env.GENERAL_CONTEXT
|
|
103
104
|
};
|
|
104
105
|
|
|
105
106
|
/** RAG system prompt per modello: file in config/kb/prompt/rag, sovrascrivibili via env (come prima). */
|
|
@@ -142,15 +143,15 @@ router.post('/scrape/single', async (req, res) => {
|
|
|
142
143
|
|
|
143
144
|
let data = req.body;
|
|
144
145
|
winston.debug("/scrape/single data: ", data)
|
|
145
|
-
|
|
146
|
+
|
|
146
147
|
let namespace;
|
|
147
148
|
try {
|
|
148
149
|
namespace = await aiManager.checkNamespace(project_id, data.namespace);
|
|
149
150
|
} catch (err) {
|
|
150
151
|
let errorCode = err?.errorCode ?? 500;
|
|
151
152
|
return res.status(errorCode).send({ success: false, error: err.error });
|
|
152
|
-
}
|
|
153
|
-
|
|
153
|
+
}
|
|
154
|
+
|
|
154
155
|
if (data.type === "sitemap") {
|
|
155
156
|
|
|
156
157
|
const urls = await aiManager.fetchSitemap(data.source).catch((err) => {
|
|
@@ -166,18 +167,18 @@ router.post('/scrape/single', async (req, res) => {
|
|
|
166
167
|
try {
|
|
167
168
|
sitemapKb = await KB.findById(data.id);
|
|
168
169
|
} catch (err) {
|
|
169
|
-
winston.error("Error finding sitemap content with id " +
|
|
170
|
+
winston.error("Error finding sitemap content with id " + data.id);
|
|
170
171
|
return res.status(500).send({ success: false, error: "Error finding sitemap content with id " + data.id });
|
|
171
172
|
}
|
|
172
173
|
|
|
173
174
|
if (!sitemapKb) {
|
|
174
175
|
return res.status(404).send({ success: false, error: "Content not found with id " + data.id });
|
|
175
176
|
}
|
|
176
|
-
|
|
177
|
+
|
|
177
178
|
let existingKbs;
|
|
178
179
|
try {
|
|
179
180
|
existingKbs = await KB.find({ id_project: project_id, namespace: data.namespace, sitemap_origin_id: data.id }).lean().exec();
|
|
180
|
-
} catch(err) {
|
|
181
|
+
} catch (err) {
|
|
181
182
|
winston.error("Error finding existing contents: ", err);
|
|
182
183
|
return res.status(500).send({ success: false, error: "Error finding existing sitemap contents" });
|
|
183
184
|
}
|
|
@@ -396,7 +397,7 @@ router.post('/qa', async (req, res) => {
|
|
|
396
397
|
if (publicKey === true) {
|
|
397
398
|
let isAvailable = await quoteManager.checkQuote(req.project, obj, 'tokens');
|
|
398
399
|
if (isAvailable === false) {
|
|
399
|
-
return res.status(403).send({ success: false, message: "Tokens quota exceeded", error_code: 13001})
|
|
400
|
+
return res.status(403).send({ success: false, message: "Tokens quota exceeded", error_code: 13001 })
|
|
400
401
|
}
|
|
401
402
|
}
|
|
402
403
|
|
|
@@ -416,10 +417,10 @@ router.post('/qa', async (req, res) => {
|
|
|
416
417
|
|
|
417
418
|
if (namespace.hybrid === true) {
|
|
418
419
|
data.search_type = 'hybrid';
|
|
419
|
-
|
|
420
|
+
|
|
420
421
|
if (data.reranking === true) {
|
|
421
422
|
data.reranker_model = process.env.RERANKING_MODEL || "cross-encoder/ms-marco-MiniLM-L-6-v2";
|
|
422
|
-
|
|
423
|
+
|
|
423
424
|
if (!data.reranking_multiplier) {
|
|
424
425
|
data.reranking_multiplier = 3;
|
|
425
426
|
}
|
|
@@ -432,10 +433,10 @@ router.post('/qa', async (req, res) => {
|
|
|
432
433
|
data.reranking_multiplier = calculatedRerankingMultiplier;
|
|
433
434
|
}
|
|
434
435
|
}
|
|
435
|
-
}
|
|
436
|
+
}
|
|
436
437
|
|
|
437
438
|
if (!namespace.hybrid && data.reranking === true && PINECONE_RERANKING) {
|
|
438
|
-
|
|
439
|
+
|
|
439
440
|
data.reranking = {
|
|
440
441
|
"provider": "pinecone",
|
|
441
442
|
"api_key": process.env.PINECONE_API_KEY,
|
|
@@ -476,7 +477,7 @@ router.post('/qa', async (req, res) => {
|
|
|
476
477
|
const sendError = (message) => {
|
|
477
478
|
try {
|
|
478
479
|
res.write('data: ' + JSON.stringify({ error: message }) + '\n\n');
|
|
479
|
-
} catch (_) {}
|
|
480
|
+
} catch (_) { }
|
|
480
481
|
res.end();
|
|
481
482
|
};
|
|
482
483
|
|
|
@@ -518,6 +519,32 @@ router.post('/qa', async (req, res) => {
|
|
|
518
519
|
return false;
|
|
519
520
|
}
|
|
520
521
|
|
|
522
|
+
let streamQaResult = null;
|
|
523
|
+
const qaStartedAt = Date.now();
|
|
524
|
+
let streamDurationSeconds = null;
|
|
525
|
+
|
|
526
|
+
function captureStreamDurationFromFirstChunk(obj) {
|
|
527
|
+
if (streamDurationSeconds != null) return;
|
|
528
|
+
const content = extractContent(obj);
|
|
529
|
+
if (content != null && content !== '') {
|
|
530
|
+
streamDurationSeconds = (Date.now() - qaStartedAt) / 1000;
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
function withMeasuredDuration(payload) {
|
|
535
|
+
if (streamDurationSeconds != null && payload && typeof payload === 'object') {
|
|
536
|
+
payload.duration = streamDurationSeconds;
|
|
537
|
+
}
|
|
538
|
+
return payload;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
function captureStreamQaResult(obj) {
|
|
542
|
+
const normalized = normalizeKbQaPayload(obj);
|
|
543
|
+
if (normalized && typeof normalized === 'object' && typeof normalized.success === 'boolean') {
|
|
544
|
+
streamQaResult = normalized;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
521
548
|
function forwardSsePayload(payload) {
|
|
522
549
|
if (payload === '[DONE]') return;
|
|
523
550
|
let obj;
|
|
@@ -527,16 +554,19 @@ router.post('/qa', async (req, res) => {
|
|
|
527
554
|
return;
|
|
528
555
|
}
|
|
529
556
|
|
|
557
|
+
captureStreamQaResult(obj);
|
|
558
|
+
captureStreamDurationFromFirstChunk(obj);
|
|
559
|
+
|
|
530
560
|
if (obj.status === 'started') {
|
|
531
561
|
return;
|
|
532
562
|
}
|
|
533
563
|
if (isKbStreamCompletedSummary(obj)) {
|
|
534
|
-
res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
|
|
564
|
+
res.write('data: ' + JSON.stringify(withMeasuredDuration(normalizeKbQaPayload(obj))) + '\n\n');
|
|
535
565
|
return;
|
|
536
566
|
}
|
|
537
567
|
|
|
538
568
|
if (obj.type === 'metadata' || obj.event === 'metadata') {
|
|
539
|
-
res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
|
|
569
|
+
res.write('data: ' + JSON.stringify(withMeasuredDuration(normalizeKbQaPayload(obj))) + '\n\n');
|
|
540
570
|
return;
|
|
541
571
|
}
|
|
542
572
|
const content = extractContent(obj);
|
|
@@ -546,7 +576,7 @@ router.post('/qa', async (req, res) => {
|
|
|
546
576
|
}
|
|
547
577
|
const normalized = normalizeKbQaPayload(obj);
|
|
548
578
|
if (isMetadataPayload(normalized, content)) {
|
|
549
|
-
res.write('data: ' + JSON.stringify(normalized) + '\n\n');
|
|
579
|
+
res.write('data: ' + JSON.stringify(withMeasuredDuration(normalized)) + '\n\n');
|
|
550
580
|
}
|
|
551
581
|
}
|
|
552
582
|
|
|
@@ -567,7 +597,7 @@ router.post('/qa', async (req, res) => {
|
|
|
567
597
|
}
|
|
568
598
|
});
|
|
569
599
|
|
|
570
|
-
stream.on('end', () => {
|
|
600
|
+
stream.on('end', async () => {
|
|
571
601
|
const tail = buffer.trim();
|
|
572
602
|
if (tail) {
|
|
573
603
|
for (const line of tail.split('\n')) {
|
|
@@ -576,6 +606,26 @@ router.post('/qa', async (req, res) => {
|
|
|
576
606
|
forwardSsePayload(trimmed.slice(6));
|
|
577
607
|
}
|
|
578
608
|
}
|
|
609
|
+
if (streamQaResult) {
|
|
610
|
+
withMeasuredDuration(streamQaResult);
|
|
611
|
+
const tokens = await aiManager.getTrackingTokens({
|
|
612
|
+
qaResult: streamQaResult,
|
|
613
|
+
publicKey,
|
|
614
|
+
model: data.model,
|
|
615
|
+
obj,
|
|
616
|
+
quoteManager,
|
|
617
|
+
project: req.project,
|
|
618
|
+
});
|
|
619
|
+
kbQuestionService.trackKbQaResult(
|
|
620
|
+
id_project,
|
|
621
|
+
{ namespace: data.namespace, question: data.question },
|
|
622
|
+
streamQaResult,
|
|
623
|
+
tokens
|
|
624
|
+
);
|
|
625
|
+
if (publicKey === true && typeof streamQaResult.legacy_prompt_token_size === 'number') {
|
|
626
|
+
res.write('data: ' + JSON.stringify(streamQaResult) + '\n\n');
|
|
627
|
+
}
|
|
628
|
+
}
|
|
579
629
|
res.write('data: [DONE]\n\n');
|
|
580
630
|
res.end();
|
|
581
631
|
});
|
|
@@ -604,35 +654,29 @@ router.post('/qa', async (req, res) => {
|
|
|
604
654
|
return;
|
|
605
655
|
}
|
|
606
656
|
|
|
607
|
-
|
|
657
|
+
const qaStartedAt = Date.now();
|
|
608
658
|
|
|
609
|
-
aiService.askNamespace(data).then((resp) => {
|
|
659
|
+
aiService.askNamespace(data).then(async (resp) => {
|
|
610
660
|
winston.debug("qa resp: ", resp.data);
|
|
611
661
|
let answer = resp.data;
|
|
612
662
|
|
|
613
|
-
answer.duration = (Date.now() -
|
|
614
|
-
|
|
615
|
-
if (publicKey === true) {
|
|
616
|
-
let modelKey;
|
|
617
|
-
if (typeof data.model === 'string') {
|
|
618
|
-
modelKey = data.model;
|
|
619
|
-
} else if (data.model && typeof data.model.name === 'string') {
|
|
620
|
-
modelKey = data.model.name;
|
|
621
|
-
}
|
|
663
|
+
answer.duration = (Date.now() - qaStartedAt) / 1000;
|
|
622
664
|
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
665
|
+
const tokens = await aiManager.getTrackingTokens({
|
|
666
|
+
qaResult: answer,
|
|
667
|
+
publicKey,
|
|
668
|
+
model: data.model,
|
|
669
|
+
obj,
|
|
670
|
+
quoteManager,
|
|
671
|
+
project: req.project,
|
|
672
|
+
});
|
|
673
|
+
kbQuestionService.trackKbQaResult(
|
|
674
|
+
id_project,
|
|
675
|
+
{ namespace: data.namespace, question: data.question },
|
|
676
|
+
answer,
|
|
677
|
+
tokens
|
|
678
|
+
);
|
|
631
679
|
|
|
632
|
-
let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
|
|
633
|
-
winston.verbose("incremented_key: ", incremented_key);
|
|
634
|
-
}
|
|
635
|
-
|
|
636
680
|
return res.status(200).send(answer);
|
|
637
681
|
|
|
638
682
|
}).catch((err) => {
|
|
@@ -738,7 +782,7 @@ router.post('/qa', async (req, res) => {
|
|
|
738
782
|
|
|
739
783
|
// if (namespace.hybrid === true) {
|
|
740
784
|
// data.search_type = 'hybrid';
|
|
741
|
-
|
|
785
|
+
|
|
742
786
|
// if (data.reranking === true) {
|
|
743
787
|
// data.reranking_multiplier = 3;
|
|
744
788
|
// data.reranker_model = "cross-encoder/ms-marco-MiniLM-L-6-v2";
|
|
@@ -771,7 +815,7 @@ router.post('/qa', async (req, res) => {
|
|
|
771
815
|
|
|
772
816
|
// delete data.advancedPrompt;
|
|
773
817
|
// winston.verbose("ask data: ", data);
|
|
774
|
-
|
|
818
|
+
|
|
775
819
|
// if (process.env.NODE_ENV === 'test') {
|
|
776
820
|
// return res.status(200).send({ success: true, message: "Question skipped in test environment", data: data });
|
|
777
821
|
// }
|
|
@@ -794,7 +838,7 @@ router.post('/qa', async (req, res) => {
|
|
|
794
838
|
// let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
|
|
795
839
|
// winston.verbose("incremented_key: ", incremented_key);
|
|
796
840
|
// }
|
|
797
|
-
|
|
841
|
+
|
|
798
842
|
// return res.status(200).send(answer);
|
|
799
843
|
|
|
800
844
|
// }).catch((err) => {
|
|
@@ -886,7 +930,7 @@ router.get('/namespace/all', async (req, res) => {
|
|
|
886
930
|
|
|
887
931
|
let project_id = req.projectid;
|
|
888
932
|
|
|
889
|
-
Namespace.find({ id_project: project_id }).lean().exec(
|
|
933
|
+
Namespace.find({ id_project: project_id }).lean().exec(async (err, namespaces) => {
|
|
890
934
|
|
|
891
935
|
if (err) {
|
|
892
936
|
winston.error("find namespaces error: ", err);
|
|
@@ -935,7 +979,7 @@ router.get('/namespace/all', async (req, res) => {
|
|
|
935
979
|
} else {
|
|
936
980
|
namespaceObjArray = namespaces.map(({ _id, __v, ...keepAttrs }) => keepAttrs)
|
|
937
981
|
}
|
|
938
|
-
|
|
982
|
+
|
|
939
983
|
winston.debug("namespaceObjArray: ", namespaceObjArray);
|
|
940
984
|
return res.status(200).send(namespaceObjArray);
|
|
941
985
|
}
|
|
@@ -961,14 +1005,14 @@ router.get('/namespace/:id/chunks/:content_id', async (req, res) => {
|
|
|
961
1005
|
return res.status(403).send({ success: false, error: err })
|
|
962
1006
|
})
|
|
963
1007
|
|
|
964
|
-
if(!content) {
|
|
1008
|
+
if (!content) {
|
|
965
1009
|
return res.status(403).send({ success: false, error: "Not allowed. The content does not belong to the current namespace." })
|
|
966
1010
|
}
|
|
967
1011
|
|
|
968
1012
|
let engine = namespace.engine || default_engine;
|
|
969
1013
|
|
|
970
1014
|
if (process.env.NODE_ENV === 'test') {
|
|
971
|
-
return res.status(200).send({ success: true, message: "Get chunks skipped in test environment"});
|
|
1015
|
+
return res.status(200).send({ success: true, message: "Get chunks skipped in test environment" });
|
|
972
1016
|
}
|
|
973
1017
|
|
|
974
1018
|
aiService.getContentChunks(namespace_id, content_id, engine).then((resp) => {
|
|
@@ -989,7 +1033,7 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
|
|
|
989
1033
|
|
|
990
1034
|
let project_id = req.projectid;
|
|
991
1035
|
let namespace_id = req.params.id;
|
|
992
|
-
|
|
1036
|
+
|
|
993
1037
|
let chatbotsArray = [];
|
|
994
1038
|
|
|
995
1039
|
let namespace;
|
|
@@ -1031,12 +1075,12 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
|
|
|
1031
1075
|
await Promise.all(chatbotPromises);
|
|
1032
1076
|
|
|
1033
1077
|
winston.debug("chatbotsArray: ", chatbotsArray);
|
|
1034
|
-
|
|
1078
|
+
|
|
1035
1079
|
res.status(200).send(chatbotsArray);
|
|
1036
1080
|
})
|
|
1037
1081
|
|
|
1038
1082
|
router.get('/namespace/export/:id', async (req, res) => {
|
|
1039
|
-
|
|
1083
|
+
|
|
1040
1084
|
let id_project = req.projectid;
|
|
1041
1085
|
let namespace_id = req.params.id;
|
|
1042
1086
|
|
|
@@ -1048,7 +1092,7 @@ router.get('/namespace/export/:id', async (req, res) => {
|
|
|
1048
1092
|
query.status = parseInt(req.query.status)
|
|
1049
1093
|
}
|
|
1050
1094
|
|
|
1051
|
-
query.type = { $in: [
|
|
1095
|
+
query.type = { $in: [kbTypes.URL, kbTypes.TEXT, kbTypes.FAQ] };
|
|
1052
1096
|
|
|
1053
1097
|
let namespace;
|
|
1054
1098
|
try {
|
|
@@ -1078,11 +1122,11 @@ router.get('/namespace/export/:id', async (req, res) => {
|
|
|
1078
1122
|
let json_string = JSON.stringify(json);
|
|
1079
1123
|
res.set({ "Content-Disposition": `attachment; filename="${filename}.json"` });
|
|
1080
1124
|
return res.send(json_string);
|
|
1081
|
-
} catch(err) {
|
|
1125
|
+
} catch (err) {
|
|
1082
1126
|
winston.error("Error genereting json ", err);
|
|
1083
1127
|
return res.status(500).send({ success: false, error: "Error genereting json file" })
|
|
1084
1128
|
}
|
|
1085
|
-
|
|
1129
|
+
|
|
1086
1130
|
})
|
|
1087
1131
|
|
|
1088
1132
|
router.post('/namespace', async (req, res) => {
|
|
@@ -1212,7 +1256,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1212
1256
|
}
|
|
1213
1257
|
|
|
1214
1258
|
let addingContents = [];
|
|
1215
|
-
contents.forEach(
|
|
1259
|
+
contents.forEach(async (e) => {
|
|
1216
1260
|
let content = {
|
|
1217
1261
|
id_project: id_project,
|
|
1218
1262
|
name: e.name,
|
|
@@ -1295,15 +1339,16 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1295
1339
|
|
|
1296
1340
|
let resources = new_contents.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
|
|
1297
1341
|
resources = resources.map(({ _id, scrape_options, ...rest }) => {
|
|
1298
|
-
return {
|
|
1299
|
-
id: _id,
|
|
1300
|
-
parameters_scrape_type_4: scrape_options,
|
|
1301
|
-
embedding: embedding,
|
|
1302
|
-
engine: engine,
|
|
1303
|
-
...(situated_context && { situated_context: situated_context }),
|
|
1304
|
-
...rest
|
|
1342
|
+
return {
|
|
1343
|
+
id: _id,
|
|
1344
|
+
parameters_scrape_type_4: scrape_options,
|
|
1345
|
+
embedding: embedding,
|
|
1346
|
+
engine: engine,
|
|
1347
|
+
...(situated_context && { situated_context: situated_context }),
|
|
1348
|
+
...rest
|
|
1349
|
+
}
|
|
1305
1350
|
});
|
|
1306
|
-
|
|
1351
|
+
|
|
1307
1352
|
winston.verbose("resources to be sent to worker: ", resources);
|
|
1308
1353
|
|
|
1309
1354
|
if (process.env.NODE_ENV !== "test") {
|
|
@@ -1314,7 +1359,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1314
1359
|
|
|
1315
1360
|
|
|
1316
1361
|
// saveBulk(operations, addingContents, id_project).then((result) => {
|
|
1317
|
-
|
|
1362
|
+
|
|
1318
1363
|
// let ns = namespaces.find(n => n.id === namespace_id);
|
|
1319
1364
|
// let engine = ns.engine || default_engine;
|
|
1320
1365
|
|
|
@@ -1328,7 +1373,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1328
1373
|
// if (process.env.NODE_ENV !== 'test') {
|
|
1329
1374
|
// scheduleScrape(resources);
|
|
1330
1375
|
// }
|
|
1331
|
-
|
|
1376
|
+
|
|
1332
1377
|
// //res.status(200).send(result);
|
|
1333
1378
|
// res.status(200).send({ success: true, message: "Contents imported successfully"});
|
|
1334
1379
|
|
|
@@ -1336,7 +1381,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1336
1381
|
// winston.error("Unable to save kbs in bulk ", err)
|
|
1337
1382
|
// res.status(500).send(err);
|
|
1338
1383
|
// })
|
|
1339
|
-
|
|
1384
|
+
|
|
1340
1385
|
})
|
|
1341
1386
|
|
|
1342
1387
|
router.put('/namespace/:id', async (req, res) => {
|
|
@@ -1614,7 +1659,7 @@ router.post('/', async (req, res) => {
|
|
|
1614
1659
|
|
|
1615
1660
|
const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags, situated_context, chunk_regex } = req.body;
|
|
1616
1661
|
const namespace_id = req.body?.namespace;
|
|
1617
|
-
|
|
1662
|
+
|
|
1618
1663
|
if (!namespace_id) {
|
|
1619
1664
|
return res.status(400).send({ success: false, error: "parameter 'namespace' is not defined" });
|
|
1620
1665
|
}
|
|
@@ -1626,11 +1671,11 @@ router.post('/', async (req, res) => {
|
|
|
1626
1671
|
let errorCode = err?.errorCode ?? 500;
|
|
1627
1672
|
return res.status(errorCode).send({ success: false, error: err.error });
|
|
1628
1673
|
}
|
|
1629
|
-
|
|
1674
|
+
|
|
1630
1675
|
const quoteManager = req.app.get('quote_manager');
|
|
1631
1676
|
try {
|
|
1632
1677
|
await aiManager.checkQuotaAvailability(quoteManager, project, 1)
|
|
1633
|
-
} catch(err) {
|
|
1678
|
+
} catch (err) {
|
|
1634
1679
|
let errorCode = err?.errorCode ?? 500;
|
|
1635
1680
|
return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
|
|
1636
1681
|
}
|
|
@@ -1683,7 +1728,7 @@ router.post('/', async (req, res) => {
|
|
|
1683
1728
|
delete raw_content.ok;
|
|
1684
1729
|
delete raw_content.$clusterTime;
|
|
1685
1730
|
delete raw_content.operationTime;
|
|
1686
|
-
|
|
1731
|
+
|
|
1687
1732
|
const saved_kb = raw_content.value;
|
|
1688
1733
|
const webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
|
|
1689
1734
|
const embedding = normalizeEmbedding(namespace.embedding);
|
|
@@ -1713,7 +1758,7 @@ router.post('/', async (req, res) => {
|
|
|
1713
1758
|
if (process.env.NODE_ENV === 'test') {
|
|
1714
1759
|
return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: raw_content, schedule_json: json });
|
|
1715
1760
|
}
|
|
1716
|
-
|
|
1761
|
+
|
|
1717
1762
|
aiManager.scheduleScrape([json], namespace.hybrid);
|
|
1718
1763
|
return res.status(200).send(raw_content);
|
|
1719
1764
|
|
|
@@ -1763,7 +1808,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
|
|
|
1763
1808
|
const quoteManager = req.app.get('quote_manager');
|
|
1764
1809
|
try {
|
|
1765
1810
|
await aiManager.checkQuotaAvailability(quoteManager, project, list.length)
|
|
1766
|
-
} catch(err) {
|
|
1811
|
+
} catch (err) {
|
|
1767
1812
|
let errorCode = err?.errorCode ?? 500;
|
|
1768
1813
|
return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
|
|
1769
1814
|
}
|
|
@@ -1844,7 +1889,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
|
|
|
1844
1889
|
const quoteManager = req.app.get('quote_manager');
|
|
1845
1890
|
try {
|
|
1846
1891
|
await aiManager.checkQuotaAvailability(quoteManager, req.project, kbs.length)
|
|
1847
|
-
} catch(err) {
|
|
1892
|
+
} catch (err) {
|
|
1848
1893
|
let errorCode = err?.errorCode ?? 500;
|
|
1849
1894
|
return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
|
|
1850
1895
|
}
|
|
@@ -1872,15 +1917,15 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
|
|
|
1872
1917
|
situated_context_obj = normalizeSituatedContext(situated_context);
|
|
1873
1918
|
}
|
|
1874
1919
|
|
|
1875
|
-
let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context,
|
|
1876
|
-
resources = resources.map(({ _id, ...rest}) => {
|
|
1877
|
-
return {
|
|
1878
|
-
id: _id,
|
|
1879
|
-
webhook: webhook,
|
|
1880
|
-
embedding: embedding,
|
|
1881
|
-
engine: engine,
|
|
1882
|
-
...(situated_context_obj && { situated_context: situated_context_obj }),
|
|
1883
|
-
...rest
|
|
1920
|
+
let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context, ...keepAttrs }) => keepAttrs)
|
|
1921
|
+
resources = resources.map(({ _id, ...rest }) => {
|
|
1922
|
+
return {
|
|
1923
|
+
id: _id,
|
|
1924
|
+
webhook: webhook,
|
|
1925
|
+
embedding: embedding,
|
|
1926
|
+
engine: engine,
|
|
1927
|
+
...(situated_context_obj && { situated_context: situated_context_obj }),
|
|
1928
|
+
...rest
|
|
1884
1929
|
};
|
|
1885
1930
|
})
|
|
1886
1931
|
winston.verbose("resources to be sent to worker: ", resources);
|
|
@@ -1888,7 +1933,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
|
|
|
1888
1933
|
if (process.env.NODE_ENV === 'test') {
|
|
1889
1934
|
return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: result, schedule_json: resources });
|
|
1890
1935
|
}
|
|
1891
|
-
|
|
1936
|
+
|
|
1892
1937
|
aiManager.scheduleScrape(resources, hybrid);
|
|
1893
1938
|
return res.status(200).send(result);
|
|
1894
1939
|
|
|
@@ -1937,13 +1982,13 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
1937
1982
|
}
|
|
1938
1983
|
|
|
1939
1984
|
if (type !== "sitemap") {
|
|
1940
|
-
return res.status(403).send({success: false, error: "Endpoint available for sitemap type only." });
|
|
1985
|
+
return res.status(403).send({ success: false, error: "Endpoint available for sitemap type only." });
|
|
1941
1986
|
}
|
|
1942
1987
|
|
|
1943
1988
|
if (!namespace_id) {
|
|
1944
1989
|
return res.status(400).send({ success: false, error: "queryParam 'namespace' is not defined" })
|
|
1945
1990
|
}
|
|
1946
|
-
|
|
1991
|
+
|
|
1947
1992
|
let namespace;
|
|
1948
1993
|
try {
|
|
1949
1994
|
namespace = await aiManager.checkNamespace(id_project, namespace_id);
|
|
@@ -1974,7 +2019,7 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
1974
2019
|
|
|
1975
2020
|
if (data.errors && data.errors.length > 0) {
|
|
1976
2021
|
winston.error("An error occurred during sitemap fetch: ", data.errors[0])
|
|
1977
|
-
return res.status(500).send({ success: false, error: "Unable to fecth sitemap due to an error: " + data.errors[0].message})
|
|
2022
|
+
return res.status(500).send({ success: false, error: "Unable to fecth sitemap due to an error: " + data.errors[0].message })
|
|
1978
2023
|
}
|
|
1979
2024
|
|
|
1980
2025
|
const urls = Array.isArray(data.sites) ? data.sites : [];
|
|
@@ -2018,7 +2063,7 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
2018
2063
|
...(Array.isArray(tags) && tags.length > 0 ? { tags } : {}),
|
|
2019
2064
|
...(situated_context && situated_context === true && scrape_type === 0 ? { situated_context: true } : {}),
|
|
2020
2065
|
}
|
|
2021
|
-
|
|
2066
|
+
|
|
2022
2067
|
try {
|
|
2023
2068
|
await aiManager.scheduleSitemap(namespace, saved_content, options);
|
|
2024
2069
|
let result = await aiManager.addMultipleUrls(namespace, urls, options);
|
|
@@ -2030,8 +2075,8 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
2030
2075
|
return res.status(200).send(result);
|
|
2031
2076
|
} catch (err) {
|
|
2032
2077
|
return res.status(500).send({ success: false, error: "Unable to add multiple urls from sitemap due to an error." });
|
|
2033
|
-
}
|
|
2034
|
-
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2035
2080
|
})
|
|
2036
2081
|
|
|
2037
2082
|
router.put('/:kb_id', async (req, res) => {
|
|
@@ -2039,7 +2084,7 @@ router.put('/:kb_id', async (req, res) => {
|
|
|
2039
2084
|
const id_project = req.projectid;
|
|
2040
2085
|
const project = req.project;
|
|
2041
2086
|
const kb_id = req.params.kb_id;
|
|
2042
|
-
|
|
2087
|
+
|
|
2043
2088
|
const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags } = req.body;
|
|
2044
2089
|
const namespace_id = req.body.namespace;
|
|
2045
2090
|
let situated_context = req.body.situated_context;
|
|
@@ -2183,7 +2228,7 @@ router.put('/:kb_id', async (req, res) => {
|
|
|
2183
2228
|
if (process.env.NODE_ENV === 'test') {
|
|
2184
2229
|
return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: updated_content, schedule_json: json });
|
|
2185
2230
|
}
|
|
2186
|
-
|
|
2231
|
+
|
|
2187
2232
|
aiManager.scheduleScrape([json], namespace.hybrid);
|
|
2188
2233
|
return res.status(200).send(updated_content);
|
|
2189
2234
|
|
|
@@ -2195,11 +2240,11 @@ router.delete('/:kb_id', async (req, res) => {
|
|
|
2195
2240
|
let kb_id = req.params.kb_id;
|
|
2196
2241
|
winston.verbose("delete kb_id " + kb_id);
|
|
2197
2242
|
|
|
2198
|
-
let kb = await KB.findOne({ id_project: project_id, _id: kb_id}).catch((err) => {
|
|
2243
|
+
let kb = await KB.findOne({ id_project: project_id, _id: kb_id }).catch((err) => {
|
|
2199
2244
|
winston.warn("Unable to find kb. Error: ", err);
|
|
2200
2245
|
return res.status(500).send({ success: false, error: err })
|
|
2201
2246
|
})
|
|
2202
|
-
|
|
2247
|
+
|
|
2203
2248
|
if (!kb) {
|
|
2204
2249
|
winston.error("Unable to delete kb. Kb not found...")
|
|
2205
2250
|
return res.status(404).send({ success: false, error: "Content not found" })
|
|
@@ -2234,7 +2279,7 @@ router.delete('/:kb_id', async (req, res) => {
|
|
|
2234
2279
|
if (!data.namespace) {
|
|
2235
2280
|
data.namespace = project_id;
|
|
2236
2281
|
}
|
|
2237
|
-
|
|
2282
|
+
|
|
2238
2283
|
data.engine = namespace.engine || default_engine;
|
|
2239
2284
|
winston.verbose("/:delete_id data: ", data);
|
|
2240
2285
|
|