@tiledesk/tiledesk-server 2.19.3 → 2.19.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/models/kb_setting.js +101 -1
- package/package.json +1 -1
- package/routes/answered.js +243 -264
- package/routes/kb.js +154 -102
- package/routes/unanswered.js +277 -296
- package/routes/webhook.js +1 -1
- package/services/aiManager.js +29 -20
- package/services/kbQuestionService.js +98 -0
package/routes/kb.js
CHANGED
|
@@ -35,12 +35,12 @@ let MAX_UPLOAD_FILE_SIZE = process.env.MAX_UPLOAD_FILE_SIZE;
|
|
|
35
35
|
let uploadlimits = undefined;
|
|
36
36
|
|
|
37
37
|
if (MAX_UPLOAD_FILE_SIZE) {
|
|
38
|
-
uploadlimits = {fileSize: parseInt(MAX_UPLOAD_FILE_SIZE)}
|
|
38
|
+
uploadlimits = { fileSize: parseInt(MAX_UPLOAD_FILE_SIZE) };
|
|
39
39
|
winston.debug("Max upload file size is : " + MAX_UPLOAD_FILE_SIZE);
|
|
40
40
|
} else {
|
|
41
41
|
winston.debug("Max upload file size is infinity");
|
|
42
42
|
}
|
|
43
|
-
var upload = multer({limits: uploadlimits});
|
|
43
|
+
var upload = multer({ limits: uploadlimits });
|
|
44
44
|
|
|
45
45
|
let jobManager = new JobManager(AMQP_MANAGER_URL, {
|
|
46
46
|
debug: false,
|
|
@@ -84,22 +84,23 @@ const default_engine_hybrid = require('../config/kb/engine.hybrid');
|
|
|
84
84
|
const default_embedding = require('../config/kb/embedding');
|
|
85
85
|
const PromptManager = require('../config/kb/prompt/rag/PromptManager');
|
|
86
86
|
const situatedContext = require('../config/kb/situatedContext');
|
|
87
|
+
const kbQuestionService = require('../services/kbQuestionService');
|
|
87
88
|
|
|
88
89
|
const ragPromptManager = new PromptManager(path.join(__dirname, '../config/kb/prompt/rag'));
|
|
89
90
|
|
|
90
91
|
const RAG_CONTEXT_ENV_OVERRIDES = {
|
|
91
|
-
"gpt-3.5-turbo":
|
|
92
|
-
"gpt-4":
|
|
92
|
+
"gpt-3.5-turbo": process.env.GPT_3_5_CONTEXT,
|
|
93
|
+
"gpt-4": process.env.GPT_4_CONTEXT,
|
|
93
94
|
"gpt-4-turbo-preview": process.env.GPT_4T_CONTEXT,
|
|
94
|
-
"gpt-4o":
|
|
95
|
-
"gpt-4o-mini":
|
|
96
|
-
"gpt-4.1":
|
|
97
|
-
"gpt-4.1-mini":
|
|
98
|
-
"gpt-4.1-nano":
|
|
99
|
-
"gpt-5":
|
|
100
|
-
"gpt-5-mini":
|
|
101
|
-
"gpt-5-nano":
|
|
102
|
-
"general":
|
|
95
|
+
"gpt-4o": process.env.GPT_4O_CONTEXT,
|
|
96
|
+
"gpt-4o-mini": process.env.GPT_4O_MINI_CONTEXT,
|
|
97
|
+
"gpt-4.1": process.env.GPT_4_1_CONTEXT,
|
|
98
|
+
"gpt-4.1-mini": process.env.GPT_4_1_MINI_CONTEXT,
|
|
99
|
+
"gpt-4.1-nano": process.env.GPT_4_1_NANO_CONTEXT,
|
|
100
|
+
"gpt-5": process.env.GPT_5_CONTEXT,
|
|
101
|
+
"gpt-5-mini": process.env.GPT_5_MINI_CONTEXT,
|
|
102
|
+
"gpt-5-nano": process.env.GPT_5_NANO_CONTEXT,
|
|
103
|
+
"general": process.env.GENERAL_CONTEXT
|
|
103
104
|
};
|
|
104
105
|
|
|
105
106
|
/** RAG system prompt per modello: file in config/kb/prompt/rag, sovrascrivibili via env (come prima). */
|
|
@@ -142,15 +143,15 @@ router.post('/scrape/single', async (req, res) => {
|
|
|
142
143
|
|
|
143
144
|
let data = req.body;
|
|
144
145
|
winston.debug("/scrape/single data: ", data)
|
|
145
|
-
|
|
146
|
+
|
|
146
147
|
let namespace;
|
|
147
148
|
try {
|
|
148
149
|
namespace = await aiManager.checkNamespace(project_id, data.namespace);
|
|
149
150
|
} catch (err) {
|
|
150
151
|
let errorCode = err?.errorCode ?? 500;
|
|
151
152
|
return res.status(errorCode).send({ success: false, error: err.error });
|
|
152
|
-
}
|
|
153
|
-
|
|
153
|
+
}
|
|
154
|
+
|
|
154
155
|
if (data.type === "sitemap") {
|
|
155
156
|
|
|
156
157
|
const urls = await aiManager.fetchSitemap(data.source).catch((err) => {
|
|
@@ -166,18 +167,18 @@ router.post('/scrape/single', async (req, res) => {
|
|
|
166
167
|
try {
|
|
167
168
|
sitemapKb = await KB.findById(data.id);
|
|
168
169
|
} catch (err) {
|
|
169
|
-
winston.error("Error finding sitemap content with id " +
|
|
170
|
+
winston.error("Error finding sitemap content with id " + data.id);
|
|
170
171
|
return res.status(500).send({ success: false, error: "Error finding sitemap content with id " + data.id });
|
|
171
172
|
}
|
|
172
173
|
|
|
173
174
|
if (!sitemapKb) {
|
|
174
175
|
return res.status(404).send({ success: false, error: "Content not found with id " + data.id });
|
|
175
176
|
}
|
|
176
|
-
|
|
177
|
+
|
|
177
178
|
let existingKbs;
|
|
178
179
|
try {
|
|
179
180
|
existingKbs = await KB.find({ id_project: project_id, namespace: data.namespace, sitemap_origin_id: data.id }).lean().exec();
|
|
180
|
-
} catch(err) {
|
|
181
|
+
} catch (err) {
|
|
181
182
|
winston.error("Error finding existing contents: ", err);
|
|
182
183
|
return res.status(500).send({ success: false, error: "Error finding existing sitemap contents" });
|
|
183
184
|
}
|
|
@@ -276,6 +277,8 @@ router.post('/scrape/single', async (req, res) => {
|
|
|
276
277
|
|
|
277
278
|
winston.verbose("/scrape/single json: ", json);
|
|
278
279
|
|
|
280
|
+
json.id_project = project_id;
|
|
281
|
+
|
|
279
282
|
if (process.env.NODE_ENV === "test") {
|
|
280
283
|
res.status(200).send({ success: true, message: "Skip indexing in test environment", data: json })
|
|
281
284
|
}
|
|
@@ -362,6 +365,7 @@ router.post('/qa', async (req, res) => {
|
|
|
362
365
|
let id_project = req.projectid;
|
|
363
366
|
let publicKey = false;
|
|
364
367
|
let data = req.body;
|
|
368
|
+
data.id_project = id_project;
|
|
365
369
|
|
|
366
370
|
let namespace;
|
|
367
371
|
try {
|
|
@@ -393,7 +397,7 @@ router.post('/qa', async (req, res) => {
|
|
|
393
397
|
if (publicKey === true) {
|
|
394
398
|
let isAvailable = await quoteManager.checkQuote(req.project, obj, 'tokens');
|
|
395
399
|
if (isAvailable === false) {
|
|
396
|
-
return res.status(403).send({ success: false, message: "Tokens quota exceeded", error_code: 13001})
|
|
400
|
+
return res.status(403).send({ success: false, message: "Tokens quota exceeded", error_code: 13001 })
|
|
397
401
|
}
|
|
398
402
|
}
|
|
399
403
|
|
|
@@ -413,10 +417,10 @@ router.post('/qa', async (req, res) => {
|
|
|
413
417
|
|
|
414
418
|
if (namespace.hybrid === true) {
|
|
415
419
|
data.search_type = 'hybrid';
|
|
416
|
-
|
|
420
|
+
|
|
417
421
|
if (data.reranking === true) {
|
|
418
422
|
data.reranker_model = process.env.RERANKING_MODEL || "cross-encoder/ms-marco-MiniLM-L-6-v2";
|
|
419
|
-
|
|
423
|
+
|
|
420
424
|
if (!data.reranking_multiplier) {
|
|
421
425
|
data.reranking_multiplier = 3;
|
|
422
426
|
}
|
|
@@ -429,10 +433,10 @@ router.post('/qa', async (req, res) => {
|
|
|
429
433
|
data.reranking_multiplier = calculatedRerankingMultiplier;
|
|
430
434
|
}
|
|
431
435
|
}
|
|
432
|
-
}
|
|
436
|
+
}
|
|
433
437
|
|
|
434
438
|
if (!namespace.hybrid && data.reranking === true && PINECONE_RERANKING) {
|
|
435
|
-
|
|
439
|
+
|
|
436
440
|
data.reranking = {
|
|
437
441
|
"provider": "pinecone",
|
|
438
442
|
"api_key": process.env.PINECONE_API_KEY,
|
|
@@ -473,7 +477,7 @@ router.post('/qa', async (req, res) => {
|
|
|
473
477
|
const sendError = (message) => {
|
|
474
478
|
try {
|
|
475
479
|
res.write('data: ' + JSON.stringify({ error: message }) + '\n\n');
|
|
476
|
-
} catch (_) {}
|
|
480
|
+
} catch (_) { }
|
|
477
481
|
res.end();
|
|
478
482
|
};
|
|
479
483
|
|
|
@@ -515,6 +519,32 @@ router.post('/qa', async (req, res) => {
|
|
|
515
519
|
return false;
|
|
516
520
|
}
|
|
517
521
|
|
|
522
|
+
let streamQaResult = null;
|
|
523
|
+
const qaStartedAt = Date.now();
|
|
524
|
+
let streamDurationSeconds = null;
|
|
525
|
+
|
|
526
|
+
function captureStreamDurationFromFirstChunk(obj) {
|
|
527
|
+
if (streamDurationSeconds != null) return;
|
|
528
|
+
const content = extractContent(obj);
|
|
529
|
+
if (content != null && content !== '') {
|
|
530
|
+
streamDurationSeconds = (Date.now() - qaStartedAt) / 1000;
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
function withMeasuredDuration(payload) {
|
|
535
|
+
if (streamDurationSeconds != null && payload && typeof payload === 'object') {
|
|
536
|
+
payload.duration = streamDurationSeconds;
|
|
537
|
+
}
|
|
538
|
+
return payload;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
function captureStreamQaResult(obj) {
|
|
542
|
+
const normalized = normalizeKbQaPayload(obj);
|
|
543
|
+
if (normalized && typeof normalized === 'object' && typeof normalized.success === 'boolean') {
|
|
544
|
+
streamQaResult = normalized;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
518
548
|
function forwardSsePayload(payload) {
|
|
519
549
|
if (payload === '[DONE]') return;
|
|
520
550
|
let obj;
|
|
@@ -524,16 +554,19 @@ router.post('/qa', async (req, res) => {
|
|
|
524
554
|
return;
|
|
525
555
|
}
|
|
526
556
|
|
|
557
|
+
captureStreamQaResult(obj);
|
|
558
|
+
captureStreamDurationFromFirstChunk(obj);
|
|
559
|
+
|
|
527
560
|
if (obj.status === 'started') {
|
|
528
561
|
return;
|
|
529
562
|
}
|
|
530
563
|
if (isKbStreamCompletedSummary(obj)) {
|
|
531
|
-
res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
|
|
564
|
+
res.write('data: ' + JSON.stringify(withMeasuredDuration(normalizeKbQaPayload(obj))) + '\n\n');
|
|
532
565
|
return;
|
|
533
566
|
}
|
|
534
567
|
|
|
535
568
|
if (obj.type === 'metadata' || obj.event === 'metadata') {
|
|
536
|
-
res.write('data: ' + JSON.stringify(normalizeKbQaPayload(obj)) + '\n\n');
|
|
569
|
+
res.write('data: ' + JSON.stringify(withMeasuredDuration(normalizeKbQaPayload(obj))) + '\n\n');
|
|
537
570
|
return;
|
|
538
571
|
}
|
|
539
572
|
const content = extractContent(obj);
|
|
@@ -543,7 +576,7 @@ router.post('/qa', async (req, res) => {
|
|
|
543
576
|
}
|
|
544
577
|
const normalized = normalizeKbQaPayload(obj);
|
|
545
578
|
if (isMetadataPayload(normalized, content)) {
|
|
546
|
-
res.write('data: ' + JSON.stringify(normalized) + '\n\n');
|
|
579
|
+
res.write('data: ' + JSON.stringify(withMeasuredDuration(normalized)) + '\n\n');
|
|
547
580
|
}
|
|
548
581
|
}
|
|
549
582
|
|
|
@@ -564,7 +597,7 @@ router.post('/qa', async (req, res) => {
|
|
|
564
597
|
}
|
|
565
598
|
});
|
|
566
599
|
|
|
567
|
-
stream.on('end', () => {
|
|
600
|
+
stream.on('end', async () => {
|
|
568
601
|
const tail = buffer.trim();
|
|
569
602
|
if (tail) {
|
|
570
603
|
for (const line of tail.split('\n')) {
|
|
@@ -573,6 +606,26 @@ router.post('/qa', async (req, res) => {
|
|
|
573
606
|
forwardSsePayload(trimmed.slice(6));
|
|
574
607
|
}
|
|
575
608
|
}
|
|
609
|
+
if (streamQaResult) {
|
|
610
|
+
withMeasuredDuration(streamQaResult);
|
|
611
|
+
const tokens = await aiManager.getTrackingTokens({
|
|
612
|
+
qaResult: streamQaResult,
|
|
613
|
+
publicKey,
|
|
614
|
+
model: data.model,
|
|
615
|
+
obj,
|
|
616
|
+
quoteManager,
|
|
617
|
+
project: req.project,
|
|
618
|
+
});
|
|
619
|
+
kbQuestionService.trackKbQaResult(
|
|
620
|
+
id_project,
|
|
621
|
+
{ namespace: data.namespace, question: data.question },
|
|
622
|
+
streamQaResult,
|
|
623
|
+
tokens
|
|
624
|
+
);
|
|
625
|
+
if (publicKey === true && typeof streamQaResult.legacy_prompt_token_size === 'number') {
|
|
626
|
+
res.write('data: ' + JSON.stringify(streamQaResult) + '\n\n');
|
|
627
|
+
}
|
|
628
|
+
}
|
|
576
629
|
res.write('data: [DONE]\n\n');
|
|
577
630
|
res.end();
|
|
578
631
|
});
|
|
@@ -601,31 +654,29 @@ router.post('/qa', async (req, res) => {
|
|
|
601
654
|
return;
|
|
602
655
|
}
|
|
603
656
|
|
|
604
|
-
|
|
657
|
+
const qaStartedAt = Date.now();
|
|
658
|
+
|
|
659
|
+
aiService.askNamespace(data).then(async (resp) => {
|
|
605
660
|
winston.debug("qa resp: ", resp.data);
|
|
606
661
|
let answer = resp.data;
|
|
607
662
|
|
|
608
|
-
|
|
609
|
-
let modelKey;
|
|
610
|
-
if (typeof data.model === 'string') {
|
|
611
|
-
modelKey = data.model;
|
|
612
|
-
} else if (data.model && typeof data.model.name === 'string') {
|
|
613
|
-
modelKey = data.model.name;
|
|
614
|
-
}
|
|
663
|
+
answer.duration = (Date.now() - qaStartedAt) / 1000;
|
|
615
664
|
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
665
|
+
const tokens = await aiManager.getTrackingTokens({
|
|
666
|
+
qaResult: answer,
|
|
667
|
+
publicKey,
|
|
668
|
+
model: data.model,
|
|
669
|
+
obj,
|
|
670
|
+
quoteManager,
|
|
671
|
+
project: req.project,
|
|
672
|
+
});
|
|
673
|
+
kbQuestionService.trackKbQaResult(
|
|
674
|
+
id_project,
|
|
675
|
+
{ namespace: data.namespace, question: data.question },
|
|
676
|
+
answer,
|
|
677
|
+
tokens
|
|
678
|
+
);
|
|
624
679
|
|
|
625
|
-
let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
|
|
626
|
-
winston.verbose("incremented_key: ", incremented_key);
|
|
627
|
-
}
|
|
628
|
-
|
|
629
680
|
return res.status(200).send(answer);
|
|
630
681
|
|
|
631
682
|
}).catch((err) => {
|
|
@@ -731,7 +782,7 @@ router.post('/qa', async (req, res) => {
|
|
|
731
782
|
|
|
732
783
|
// if (namespace.hybrid === true) {
|
|
733
784
|
// data.search_type = 'hybrid';
|
|
734
|
-
|
|
785
|
+
|
|
735
786
|
// if (data.reranking === true) {
|
|
736
787
|
// data.reranking_multiplier = 3;
|
|
737
788
|
// data.reranker_model = "cross-encoder/ms-marco-MiniLM-L-6-v2";
|
|
@@ -764,7 +815,7 @@ router.post('/qa', async (req, res) => {
|
|
|
764
815
|
|
|
765
816
|
// delete data.advancedPrompt;
|
|
766
817
|
// winston.verbose("ask data: ", data);
|
|
767
|
-
|
|
818
|
+
|
|
768
819
|
// if (process.env.NODE_ENV === 'test') {
|
|
769
820
|
// return res.status(200).send({ success: true, message: "Question skipped in test environment", data: data });
|
|
770
821
|
// }
|
|
@@ -787,7 +838,7 @@ router.post('/qa', async (req, res) => {
|
|
|
787
838
|
// let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
|
|
788
839
|
// winston.verbose("incremented_key: ", incremented_key);
|
|
789
840
|
// }
|
|
790
|
-
|
|
841
|
+
|
|
791
842
|
// return res.status(200).send(answer);
|
|
792
843
|
|
|
793
844
|
// }).catch((err) => {
|
|
@@ -879,7 +930,7 @@ router.get('/namespace/all', async (req, res) => {
|
|
|
879
930
|
|
|
880
931
|
let project_id = req.projectid;
|
|
881
932
|
|
|
882
|
-
Namespace.find({ id_project: project_id }).lean().exec(
|
|
933
|
+
Namespace.find({ id_project: project_id }).lean().exec(async (err, namespaces) => {
|
|
883
934
|
|
|
884
935
|
if (err) {
|
|
885
936
|
winston.error("find namespaces error: ", err);
|
|
@@ -928,7 +979,7 @@ router.get('/namespace/all', async (req, res) => {
|
|
|
928
979
|
} else {
|
|
929
980
|
namespaceObjArray = namespaces.map(({ _id, __v, ...keepAttrs }) => keepAttrs)
|
|
930
981
|
}
|
|
931
|
-
|
|
982
|
+
|
|
932
983
|
winston.debug("namespaceObjArray: ", namespaceObjArray);
|
|
933
984
|
return res.status(200).send(namespaceObjArray);
|
|
934
985
|
}
|
|
@@ -954,14 +1005,14 @@ router.get('/namespace/:id/chunks/:content_id', async (req, res) => {
|
|
|
954
1005
|
return res.status(403).send({ success: false, error: err })
|
|
955
1006
|
})
|
|
956
1007
|
|
|
957
|
-
if(!content) {
|
|
1008
|
+
if (!content) {
|
|
958
1009
|
return res.status(403).send({ success: false, error: "Not allowed. The content does not belong to the current namespace." })
|
|
959
1010
|
}
|
|
960
1011
|
|
|
961
1012
|
let engine = namespace.engine || default_engine;
|
|
962
1013
|
|
|
963
1014
|
if (process.env.NODE_ENV === 'test') {
|
|
964
|
-
return res.status(200).send({ success: true, message: "Get chunks skipped in test environment"});
|
|
1015
|
+
return res.status(200).send({ success: true, message: "Get chunks skipped in test environment" });
|
|
965
1016
|
}
|
|
966
1017
|
|
|
967
1018
|
aiService.getContentChunks(namespace_id, content_id, engine).then((resp) => {
|
|
@@ -982,7 +1033,7 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
|
|
|
982
1033
|
|
|
983
1034
|
let project_id = req.projectid;
|
|
984
1035
|
let namespace_id = req.params.id;
|
|
985
|
-
|
|
1036
|
+
|
|
986
1037
|
let chatbotsArray = [];
|
|
987
1038
|
|
|
988
1039
|
let namespace;
|
|
@@ -1024,12 +1075,12 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
|
|
|
1024
1075
|
await Promise.all(chatbotPromises);
|
|
1025
1076
|
|
|
1026
1077
|
winston.debug("chatbotsArray: ", chatbotsArray);
|
|
1027
|
-
|
|
1078
|
+
|
|
1028
1079
|
res.status(200).send(chatbotsArray);
|
|
1029
1080
|
})
|
|
1030
1081
|
|
|
1031
1082
|
router.get('/namespace/export/:id', async (req, res) => {
|
|
1032
|
-
|
|
1083
|
+
|
|
1033
1084
|
let id_project = req.projectid;
|
|
1034
1085
|
let namespace_id = req.params.id;
|
|
1035
1086
|
|
|
@@ -1041,7 +1092,7 @@ router.get('/namespace/export/:id', async (req, res) => {
|
|
|
1041
1092
|
query.status = parseInt(req.query.status)
|
|
1042
1093
|
}
|
|
1043
1094
|
|
|
1044
|
-
query.type = { $in: [
|
|
1095
|
+
query.type = { $in: [kbTypes.URL, kbTypes.TEXT, kbTypes.FAQ] };
|
|
1045
1096
|
|
|
1046
1097
|
let namespace;
|
|
1047
1098
|
try {
|
|
@@ -1071,11 +1122,11 @@ router.get('/namespace/export/:id', async (req, res) => {
|
|
|
1071
1122
|
let json_string = JSON.stringify(json);
|
|
1072
1123
|
res.set({ "Content-Disposition": `attachment; filename="${filename}.json"` });
|
|
1073
1124
|
return res.send(json_string);
|
|
1074
|
-
} catch(err) {
|
|
1125
|
+
} catch (err) {
|
|
1075
1126
|
winston.error("Error genereting json ", err);
|
|
1076
1127
|
return res.status(500).send({ success: false, error: "Error genereting json file" })
|
|
1077
1128
|
}
|
|
1078
|
-
|
|
1129
|
+
|
|
1079
1130
|
})
|
|
1080
1131
|
|
|
1081
1132
|
router.post('/namespace', async (req, res) => {
|
|
@@ -1205,7 +1256,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1205
1256
|
}
|
|
1206
1257
|
|
|
1207
1258
|
let addingContents = [];
|
|
1208
|
-
contents.forEach(
|
|
1259
|
+
contents.forEach(async (e) => {
|
|
1209
1260
|
let content = {
|
|
1210
1261
|
id_project: id_project,
|
|
1211
1262
|
name: e.name,
|
|
@@ -1288,15 +1339,16 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1288
1339
|
|
|
1289
1340
|
let resources = new_contents.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
|
|
1290
1341
|
resources = resources.map(({ _id, scrape_options, ...rest }) => {
|
|
1291
|
-
return {
|
|
1292
|
-
id: _id,
|
|
1293
|
-
parameters_scrape_type_4: scrape_options,
|
|
1294
|
-
embedding: embedding,
|
|
1295
|
-
engine: engine,
|
|
1296
|
-
...(situated_context && { situated_context: situated_context }),
|
|
1297
|
-
...rest
|
|
1342
|
+
return {
|
|
1343
|
+
id: _id,
|
|
1344
|
+
parameters_scrape_type_4: scrape_options,
|
|
1345
|
+
embedding: embedding,
|
|
1346
|
+
engine: engine,
|
|
1347
|
+
...(situated_context && { situated_context: situated_context }),
|
|
1348
|
+
...rest
|
|
1349
|
+
}
|
|
1298
1350
|
});
|
|
1299
|
-
|
|
1351
|
+
|
|
1300
1352
|
winston.verbose("resources to be sent to worker: ", resources);
|
|
1301
1353
|
|
|
1302
1354
|
if (process.env.NODE_ENV !== "test") {
|
|
@@ -1307,7 +1359,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1307
1359
|
|
|
1308
1360
|
|
|
1309
1361
|
// saveBulk(operations, addingContents, id_project).then((result) => {
|
|
1310
|
-
|
|
1362
|
+
|
|
1311
1363
|
// let ns = namespaces.find(n => n.id === namespace_id);
|
|
1312
1364
|
// let engine = ns.engine || default_engine;
|
|
1313
1365
|
|
|
@@ -1321,7 +1373,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1321
1373
|
// if (process.env.NODE_ENV !== 'test') {
|
|
1322
1374
|
// scheduleScrape(resources);
|
|
1323
1375
|
// }
|
|
1324
|
-
|
|
1376
|
+
|
|
1325
1377
|
// //res.status(200).send(result);
|
|
1326
1378
|
// res.status(200).send({ success: true, message: "Contents imported successfully"});
|
|
1327
1379
|
|
|
@@ -1329,7 +1381,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1329
1381
|
// winston.error("Unable to save kbs in bulk ", err)
|
|
1330
1382
|
// res.status(500).send(err);
|
|
1331
1383
|
// })
|
|
1332
|
-
|
|
1384
|
+
|
|
1333
1385
|
})
|
|
1334
1386
|
|
|
1335
1387
|
router.put('/namespace/:id', async (req, res) => {
|
|
@@ -1607,7 +1659,7 @@ router.post('/', async (req, res) => {
|
|
|
1607
1659
|
|
|
1608
1660
|
const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags, situated_context, chunk_regex } = req.body;
|
|
1609
1661
|
const namespace_id = req.body?.namespace;
|
|
1610
|
-
|
|
1662
|
+
|
|
1611
1663
|
if (!namespace_id) {
|
|
1612
1664
|
return res.status(400).send({ success: false, error: "parameter 'namespace' is not defined" });
|
|
1613
1665
|
}
|
|
@@ -1619,11 +1671,11 @@ router.post('/', async (req, res) => {
|
|
|
1619
1671
|
let errorCode = err?.errorCode ?? 500;
|
|
1620
1672
|
return res.status(errorCode).send({ success: false, error: err.error });
|
|
1621
1673
|
}
|
|
1622
|
-
|
|
1674
|
+
|
|
1623
1675
|
const quoteManager = req.app.get('quote_manager');
|
|
1624
1676
|
try {
|
|
1625
1677
|
await aiManager.checkQuotaAvailability(quoteManager, project, 1)
|
|
1626
|
-
} catch(err) {
|
|
1678
|
+
} catch (err) {
|
|
1627
1679
|
let errorCode = err?.errorCode ?? 500;
|
|
1628
1680
|
return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
|
|
1629
1681
|
}
|
|
@@ -1676,7 +1728,7 @@ router.post('/', async (req, res) => {
|
|
|
1676
1728
|
delete raw_content.ok;
|
|
1677
1729
|
delete raw_content.$clusterTime;
|
|
1678
1730
|
delete raw_content.operationTime;
|
|
1679
|
-
|
|
1731
|
+
|
|
1680
1732
|
const saved_kb = raw_content.value;
|
|
1681
1733
|
const webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
|
|
1682
1734
|
const embedding = normalizeEmbedding(namespace.embedding);
|
|
@@ -1706,7 +1758,7 @@ router.post('/', async (req, res) => {
|
|
|
1706
1758
|
if (process.env.NODE_ENV === 'test') {
|
|
1707
1759
|
return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: raw_content, schedule_json: json });
|
|
1708
1760
|
}
|
|
1709
|
-
|
|
1761
|
+
|
|
1710
1762
|
aiManager.scheduleScrape([json], namespace.hybrid);
|
|
1711
1763
|
return res.status(200).send(raw_content);
|
|
1712
1764
|
|
|
@@ -1756,7 +1808,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
|
|
|
1756
1808
|
const quoteManager = req.app.get('quote_manager');
|
|
1757
1809
|
try {
|
|
1758
1810
|
await aiManager.checkQuotaAvailability(quoteManager, project, list.length)
|
|
1759
|
-
} catch(err) {
|
|
1811
|
+
} catch (err) {
|
|
1760
1812
|
let errorCode = err?.errorCode ?? 500;
|
|
1761
1813
|
return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
|
|
1762
1814
|
}
|
|
@@ -1837,7 +1889,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
|
|
|
1837
1889
|
const quoteManager = req.app.get('quote_manager');
|
|
1838
1890
|
try {
|
|
1839
1891
|
await aiManager.checkQuotaAvailability(quoteManager, req.project, kbs.length)
|
|
1840
|
-
} catch(err) {
|
|
1892
|
+
} catch (err) {
|
|
1841
1893
|
let errorCode = err?.errorCode ?? 500;
|
|
1842
1894
|
return res.status(errorCode).send({ success: false, error: err.error, plan_limit: err.plan_limit })
|
|
1843
1895
|
}
|
|
@@ -1865,15 +1917,15 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
|
|
|
1865
1917
|
situated_context_obj = normalizeSituatedContext(situated_context);
|
|
1866
1918
|
}
|
|
1867
1919
|
|
|
1868
|
-
let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context,
|
|
1869
|
-
resources = resources.map(({ _id, ...rest}) => {
|
|
1870
|
-
return {
|
|
1871
|
-
id: _id,
|
|
1872
|
-
webhook: webhook,
|
|
1873
|
-
embedding: embedding,
|
|
1874
|
-
engine: engine,
|
|
1875
|
-
...(situated_context_obj && { situated_context: situated_context_obj }),
|
|
1876
|
-
...rest
|
|
1920
|
+
let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context, ...keepAttrs }) => keepAttrs)
|
|
1921
|
+
resources = resources.map(({ _id, ...rest }) => {
|
|
1922
|
+
return {
|
|
1923
|
+
id: _id,
|
|
1924
|
+
webhook: webhook,
|
|
1925
|
+
embedding: embedding,
|
|
1926
|
+
engine: engine,
|
|
1927
|
+
...(situated_context_obj && { situated_context: situated_context_obj }),
|
|
1928
|
+
...rest
|
|
1877
1929
|
};
|
|
1878
1930
|
})
|
|
1879
1931
|
winston.verbose("resources to be sent to worker: ", resources);
|
|
@@ -1881,7 +1933,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
|
|
|
1881
1933
|
if (process.env.NODE_ENV === 'test') {
|
|
1882
1934
|
return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: result, schedule_json: resources });
|
|
1883
1935
|
}
|
|
1884
|
-
|
|
1936
|
+
|
|
1885
1937
|
aiManager.scheduleScrape(resources, hybrid);
|
|
1886
1938
|
return res.status(200).send(result);
|
|
1887
1939
|
|
|
@@ -1930,13 +1982,13 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
1930
1982
|
}
|
|
1931
1983
|
|
|
1932
1984
|
if (type !== "sitemap") {
|
|
1933
|
-
return res.status(403).send({success: false, error: "Endpoint available for sitemap type only." });
|
|
1985
|
+
return res.status(403).send({ success: false, error: "Endpoint available for sitemap type only." });
|
|
1934
1986
|
}
|
|
1935
1987
|
|
|
1936
1988
|
if (!namespace_id) {
|
|
1937
1989
|
return res.status(400).send({ success: false, error: "queryParam 'namespace' is not defined" })
|
|
1938
1990
|
}
|
|
1939
|
-
|
|
1991
|
+
|
|
1940
1992
|
let namespace;
|
|
1941
1993
|
try {
|
|
1942
1994
|
namespace = await aiManager.checkNamespace(id_project, namespace_id);
|
|
@@ -1967,7 +2019,7 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
1967
2019
|
|
|
1968
2020
|
if (data.errors && data.errors.length > 0) {
|
|
1969
2021
|
winston.error("An error occurred during sitemap fetch: ", data.errors[0])
|
|
1970
|
-
return res.status(500).send({ success: false, error: "Unable to fecth sitemap due to an error: " + data.errors[0].message})
|
|
2022
|
+
return res.status(500).send({ success: false, error: "Unable to fecth sitemap due to an error: " + data.errors[0].message })
|
|
1971
2023
|
}
|
|
1972
2024
|
|
|
1973
2025
|
const urls = Array.isArray(data.sites) ? data.sites : [];
|
|
@@ -2011,7 +2063,7 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
2011
2063
|
...(Array.isArray(tags) && tags.length > 0 ? { tags } : {}),
|
|
2012
2064
|
...(situated_context && situated_context === true && scrape_type === 0 ? { situated_context: true } : {}),
|
|
2013
2065
|
}
|
|
2014
|
-
|
|
2066
|
+
|
|
2015
2067
|
try {
|
|
2016
2068
|
await aiManager.scheduleSitemap(namespace, saved_content, options);
|
|
2017
2069
|
let result = await aiManager.addMultipleUrls(namespace, urls, options);
|
|
@@ -2023,8 +2075,8 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
2023
2075
|
return res.status(200).send(result);
|
|
2024
2076
|
} catch (err) {
|
|
2025
2077
|
return res.status(500).send({ success: false, error: "Unable to add multiple urls from sitemap due to an error." });
|
|
2026
|
-
}
|
|
2027
|
-
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2028
2080
|
})
|
|
2029
2081
|
|
|
2030
2082
|
router.put('/:kb_id', async (req, res) => {
|
|
@@ -2032,7 +2084,7 @@ router.put('/:kb_id', async (req, res) => {
|
|
|
2032
2084
|
const id_project = req.projectid;
|
|
2033
2085
|
const project = req.project;
|
|
2034
2086
|
const kb_id = req.params.kb_id;
|
|
2035
|
-
|
|
2087
|
+
|
|
2036
2088
|
const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags } = req.body;
|
|
2037
2089
|
const namespace_id = req.body.namespace;
|
|
2038
2090
|
let situated_context = req.body.situated_context;
|
|
@@ -2176,7 +2228,7 @@ router.put('/:kb_id', async (req, res) => {
|
|
|
2176
2228
|
if (process.env.NODE_ENV === 'test') {
|
|
2177
2229
|
return res.status(200).send({ success: true, message: "Schedule scrape skipped in test environment", data: updated_content, schedule_json: json });
|
|
2178
2230
|
}
|
|
2179
|
-
|
|
2231
|
+
|
|
2180
2232
|
aiManager.scheduleScrape([json], namespace.hybrid);
|
|
2181
2233
|
return res.status(200).send(updated_content);
|
|
2182
2234
|
|
|
@@ -2188,11 +2240,11 @@ router.delete('/:kb_id', async (req, res) => {
|
|
|
2188
2240
|
let kb_id = req.params.kb_id;
|
|
2189
2241
|
winston.verbose("delete kb_id " + kb_id);
|
|
2190
2242
|
|
|
2191
|
-
let kb = await KB.findOne({ id_project: project_id, _id: kb_id}).catch((err) => {
|
|
2243
|
+
let kb = await KB.findOne({ id_project: project_id, _id: kb_id }).catch((err) => {
|
|
2192
2244
|
winston.warn("Unable to find kb. Error: ", err);
|
|
2193
2245
|
return res.status(500).send({ success: false, error: err })
|
|
2194
2246
|
})
|
|
2195
|
-
|
|
2247
|
+
|
|
2196
2248
|
if (!kb) {
|
|
2197
2249
|
winston.error("Unable to delete kb. Kb not found...")
|
|
2198
2250
|
return res.status(404).send({ success: false, error: "Content not found" })
|
|
@@ -2227,7 +2279,7 @@ router.delete('/:kb_id', async (req, res) => {
|
|
|
2227
2279
|
if (!data.namespace) {
|
|
2228
2280
|
data.namespace = project_id;
|
|
2229
2281
|
}
|
|
2230
|
-
|
|
2282
|
+
|
|
2231
2283
|
data.engine = namespace.engine || default_engine;
|
|
2232
2284
|
winston.verbose("/:delete_id data: ", data);
|
|
2233
2285
|
|