@tiledesk/tiledesk-server 2.20.2 → 2.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -0
- package/config/winston.js +14 -3
- package/package.json +2 -2
- package/routes/kb.js +64 -42
- package/routes/llm.js +4 -2
- package/utils/logRedactor.js +86 -0
- package/utils/maskIntegrationSecrets.js +1 -1
package/CHANGELOG.md
CHANGED
package/config/winston.js
CHANGED
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
var appRoot = require('app-root-path');
|
|
2
2
|
var winston = require('winston');
|
|
3
3
|
var config = require('./database');
|
|
4
|
+
var { redactInfo } = require('../utils/logRedactor');
|
|
4
5
|
|
|
5
6
|
var level = process.env.LOG_LEVEL || 'info'
|
|
6
7
|
// console.log("level",level);
|
|
7
8
|
|
|
9
|
+
// Scrub credentials (provider API keys, Pinecone key, bearer tokens) from every
|
|
10
|
+
// logged value BEFORE any transport writes it. AxiosError objects carry the full
|
|
11
|
+
// request body (with keys) in config.data; this strips them at the transport
|
|
12
|
+
// layer so no call site can leak a key into stdout/app.log/MongoDB. See
|
|
13
|
+
// utils/logRedactor.js.
|
|
14
|
+
var redactFormat = winston.format((info) => redactInfo(info))();
|
|
15
|
+
|
|
8
16
|
var options = {
|
|
9
17
|
file: {
|
|
10
18
|
level:level ,
|
|
@@ -26,7 +34,7 @@ var options = {
|
|
|
26
34
|
json: true,
|
|
27
35
|
colorize: true,
|
|
28
36
|
// timestamp: true,
|
|
29
|
-
format: winston.format.simple()
|
|
37
|
+
format: winston.format.simple()
|
|
30
38
|
// format: winston.format.combine(
|
|
31
39
|
// winston.format.splat(),
|
|
32
40
|
// winston.format.simple()
|
|
@@ -45,11 +53,14 @@ var options = {
|
|
|
45
53
|
|
|
46
54
|
|
|
47
55
|
|
|
48
|
-
let logger = winston.createLogger({
|
|
56
|
+
let logger = winston.createLogger({
|
|
57
|
+
// Logger-level format runs before every transport format, so redaction here
|
|
58
|
+
// covers Console, File AND the MongoDB transports added below.
|
|
59
|
+
format: redactFormat,
|
|
49
60
|
transports: [
|
|
50
61
|
new (winston.transports.Console)(options.console),
|
|
51
62
|
new (winston.transports.File)(options.file),
|
|
52
|
-
//new (winston.transports.MongoDB)( {db: logsDb, collection: "logs"})
|
|
63
|
+
//new (winston.transports.MongoDB)( {db: logsDb, collection: "logs"})
|
|
53
64
|
],
|
|
54
65
|
exitOnError: false, // do not exit on handled exceptions
|
|
55
66
|
});
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tiledesk/tiledesk-server",
|
|
3
3
|
"description": "The Tiledesk server module",
|
|
4
|
-
"version": "2.
|
|
4
|
+
"version": "2.22.0",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"start": "node ./bin/www",
|
|
7
7
|
"pretest": "mongodb-runner start",
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
"@tiledesk/tiledesk-rasa-connector": "^1.0.10",
|
|
50
50
|
"@tiledesk/tiledesk-sms-connector": "^0.1.13",
|
|
51
51
|
"@tiledesk/tiledesk-telegram-connector": "^0.1.14",
|
|
52
|
-
"@tiledesk/tiledesk-tybot-connector": "^2.1.
|
|
52
|
+
"@tiledesk/tiledesk-tybot-connector": "^2.1.11",
|
|
53
53
|
"@tiledesk/tiledesk-voice-twilio-connector": "^0.3.2",
|
|
54
54
|
"@tiledesk/tiledesk-vxml-connector": "^0.1.91",
|
|
55
55
|
"@tiledesk/tiledesk-whatsapp-connector": "1.0.26",
|
package/routes/kb.js
CHANGED
|
@@ -22,6 +22,28 @@ const aiService = require('../services/aiService');
|
|
|
22
22
|
const aiManager = require('../services/aiManager');
|
|
23
23
|
const integrationService = require('../services/integrationService');
|
|
24
24
|
const kbEvent = require('../event/kbEvent');
|
|
25
|
+
// Never return a raw error to the client: an AxiosError from the LLM
|
|
26
|
+
// microservice serializes config.data (the request body, which carries the
|
|
27
|
+
// provider/gptkey + Pinecone key) straight into the response. sanitizeAxiosError
|
|
28
|
+
// reduces it to a safe { message }.
|
|
29
|
+
const { sanitizeAxiosError } = require('../utils/sanitizeAxiosError');
|
|
30
|
+
|
|
31
|
+
// Strip the vector-store credential from a namespace (or array of namespaces)
|
|
32
|
+
// before returning it to a client. engine.apikey is seeded from
|
|
33
|
+
// VECTOR_STORE_APIKEY and must never reach the dashboard/browser. Storage is
|
|
34
|
+
// left untouched (the key is still used server-side when building LLM calls);
|
|
35
|
+
// only the outbound representation is scrubbed.
|
|
36
|
+
function stripNamespaceSecrets(ns) {
|
|
37
|
+
if (Array.isArray(ns)) {
|
|
38
|
+
ns.forEach(stripNamespaceSecrets);
|
|
39
|
+
return ns;
|
|
40
|
+
}
|
|
41
|
+
if (ns && ns.engine && typeof ns.engine === 'object') {
|
|
42
|
+
delete ns.engine.apikey;
|
|
43
|
+
delete ns.engine.api_key;
|
|
44
|
+
}
|
|
45
|
+
return ns;
|
|
46
|
+
}
|
|
25
47
|
|
|
26
48
|
const AMQP_MANAGER_URL = process.env.AMQP_MANAGER_URL;
|
|
27
49
|
const JOB_TOPIC_EXCHANGE = process.env.JOB_TOPIC_EXCHANGE_TRAIN || 'tiledesk-trainer';
|
|
@@ -149,7 +171,7 @@ router.post('/scrape/single', async (req, res) => {
|
|
|
149
171
|
|
|
150
172
|
const urls = await aiManager.fetchSitemap(data.source).catch((err) => {
|
|
151
173
|
winston.error("Error fetching sitemap: ", err);
|
|
152
|
-
return res.status(500).send({ success: false, error: err });
|
|
174
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
153
175
|
})
|
|
154
176
|
|
|
155
177
|
if (urls.length === 0) {
|
|
@@ -220,7 +242,7 @@ router.post('/scrape/single', async (req, res) => {
|
|
|
220
242
|
KB.findById(data.id, (err, kb) => {
|
|
221
243
|
if (err) {
|
|
222
244
|
winston.error("findById error: ", err);
|
|
223
|
-
return res.status(500).send({ success: false, error: err });
|
|
245
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
224
246
|
}
|
|
225
247
|
|
|
226
248
|
else if (!kb) {
|
|
@@ -345,7 +367,7 @@ router.post('/scrape/status', async (req, res) => {
|
|
|
345
367
|
}).catch((err) => {
|
|
346
368
|
winston.error("scrapeStatus err: ", err);
|
|
347
369
|
let status = err.response.status;
|
|
348
|
-
res.status(status).send({ statusText: err.response.statusText, error: err
|
|
370
|
+
res.status(status).send({ statusText: err.response.statusText, error: sanitizeAxiosError(err) });
|
|
349
371
|
})
|
|
350
372
|
})
|
|
351
373
|
|
|
@@ -630,9 +652,9 @@ router.post('/qa', async (req, res) => {
|
|
|
630
652
|
}).catch((err) => {
|
|
631
653
|
winston.error('qa err: ', err);
|
|
632
654
|
winston.error('qa err.response: ', err.response);
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
655
|
+
// sanitizeAxiosError never exposes the request body / detail[].input (which
|
|
656
|
+
// carries the provider key); it yields only a safe message string.
|
|
657
|
+
const message = sanitizeAxiosError(err).message;
|
|
636
658
|
if (!res.headersSent) {
|
|
637
659
|
res.status(err.response && err.response.status ? err.response.status : 500);
|
|
638
660
|
}
|
|
@@ -671,10 +693,10 @@ router.post('/qa', async (req, res) => {
|
|
|
671
693
|
winston.error("qa err.response: ", err.response);
|
|
672
694
|
if (err.response && err.response.status) {
|
|
673
695
|
let status = err.response.status;
|
|
674
|
-
res.status(status).send({ success: false, statusText: err.response.statusText, error: err
|
|
696
|
+
res.status(status).send({ success: false, statusText: err.response.statusText, error: sanitizeAxiosError(err) });
|
|
675
697
|
}
|
|
676
698
|
else {
|
|
677
|
-
res.status(500).send({ success: false, error: err });
|
|
699
|
+
res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
678
700
|
}
|
|
679
701
|
|
|
680
702
|
})
|
|
@@ -864,7 +886,7 @@ router.delete('/delete', async (req, res) => {
|
|
|
864
886
|
}).catch((err) => {
|
|
865
887
|
winston.error("delete err: ", err);
|
|
866
888
|
let status = err.response.status;
|
|
867
|
-
res.status(status).send({ statusText: err.response.statusText, error: err
|
|
889
|
+
res.status(status).send({ statusText: err.response.statusText, error: sanitizeAxiosError(err) });
|
|
868
890
|
})
|
|
869
891
|
|
|
870
892
|
})
|
|
@@ -901,7 +923,7 @@ router.delete('/deleteall', async (req, res) => {
|
|
|
901
923
|
}).catch((err) => {
|
|
902
924
|
winston.error("delete namespace err: ", err);
|
|
903
925
|
let status = err.response.status;
|
|
904
|
-
res.status(status).send({ statusText: err.response.statusText, error: err
|
|
926
|
+
res.status(status).send({ statusText: err.response.statusText, error: sanitizeAxiosError(err) });
|
|
905
927
|
})
|
|
906
928
|
})
|
|
907
929
|
/**
|
|
@@ -927,7 +949,7 @@ router.get('/namespace/all', async (req, res) => {
|
|
|
927
949
|
|
|
928
950
|
if (err) {
|
|
929
951
|
winston.error("find namespaces error: ", err);
|
|
930
|
-
return res.status(500).send({ success: false, error: err });
|
|
952
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
931
953
|
}
|
|
932
954
|
else if (namespaces.length == 0) {
|
|
933
955
|
|
|
@@ -945,7 +967,7 @@ router.get('/namespace/all', async (req, res) => {
|
|
|
945
967
|
new_namespace.save((err, savedNamespace) => {
|
|
946
968
|
if (err) {
|
|
947
969
|
winston.error("create default namespace error: ", err);
|
|
948
|
-
return res.status(500).send({ success: false, error: err });
|
|
970
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
949
971
|
}
|
|
950
972
|
|
|
951
973
|
let namespaceObj = savedNamespace.toObject();
|
|
@@ -955,7 +977,7 @@ router.get('/namespace/all', async (req, res) => {
|
|
|
955
977
|
let namespaces = [];
|
|
956
978
|
namespaces.push(namespaceObj);
|
|
957
979
|
|
|
958
|
-
return res.status(200).send(namespaces);
|
|
980
|
+
return res.status(200).send(stripNamespaceSecrets(namespaces));
|
|
959
981
|
|
|
960
982
|
})
|
|
961
983
|
|
|
@@ -974,7 +996,7 @@ router.get('/namespace/all', async (req, res) => {
|
|
|
974
996
|
}
|
|
975
997
|
|
|
976
998
|
winston.debug("namespaceObjArray: ", namespaceObjArray);
|
|
977
|
-
return res.status(200).send(namespaceObjArray);
|
|
999
|
+
return res.status(200).send(stripNamespaceSecrets(namespaceObjArray));
|
|
978
1000
|
}
|
|
979
1001
|
})
|
|
980
1002
|
})
|
|
@@ -995,7 +1017,7 @@ router.get('/namespace/:id/chunks/:content_id', async (req, res) => {
|
|
|
995
1017
|
|
|
996
1018
|
let content = await KB.find({ id_project: project_id, namespace: namespace_id, _id: content_id }).catch((err) => {
|
|
997
1019
|
winston.error("find content error: ", err);
|
|
998
|
-
return res.status(403).send({ success: false, error: err })
|
|
1020
|
+
return res.status(403).send({ success: false, error: sanitizeAxiosError(err) })
|
|
999
1021
|
})
|
|
1000
1022
|
|
|
1001
1023
|
if (!content) {
|
|
@@ -1017,7 +1039,7 @@ router.get('/namespace/:id/chunks/:content_id', async (req, res) => {
|
|
|
1017
1039
|
}).catch((err) => {
|
|
1018
1040
|
winston.error("error getting content chunks err.response: ", err.response)
|
|
1019
1041
|
winston.error("error getting content chunks err.data: ", err.data)
|
|
1020
|
-
return res.status(500).send({ success: false, error: err });
|
|
1042
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1021
1043
|
})
|
|
1022
1044
|
|
|
1023
1045
|
})
|
|
@@ -1039,7 +1061,7 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
|
|
|
1039
1061
|
|
|
1040
1062
|
let intents = await faq.find({ id_project: project_id, 'actions.namespace': namespace_id }).catch((err) => {
|
|
1041
1063
|
winston.error("Error find intents: ", err);
|
|
1042
|
-
return res.status(500).send({ success: false, message: 'Unable to retrieve intents using the selected namespace', error: err });
|
|
1064
|
+
return res.status(500).send({ success: false, message: 'Unable to retrieve intents using the selected namespace', error: sanitizeAxiosError(err) });
|
|
1043
1065
|
})
|
|
1044
1066
|
|
|
1045
1067
|
if (!intents || intents.length == 0) {
|
|
@@ -1176,7 +1198,7 @@ router.post('/namespace', async (req, res) => {
|
|
|
1176
1198
|
new_namespace.save((err, savedNamespace) => {
|
|
1177
1199
|
if (err) {
|
|
1178
1200
|
winston.error("create namespace error: ", err);
|
|
1179
|
-
return res.status(500).send({ success: false, error: err });
|
|
1201
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1180
1202
|
}
|
|
1181
1203
|
|
|
1182
1204
|
kbEvent.emit('kb.namespace.create', { req, savedNamespace, body, namespace_id, project_id });
|
|
@@ -1184,7 +1206,7 @@ router.post('/namespace', async (req, res) => {
|
|
|
1184
1206
|
let namespaceObj = savedNamespace.toObject();
|
|
1185
1207
|
delete namespaceObj._id;
|
|
1186
1208
|
delete namespaceObj.__v;
|
|
1187
|
-
return res.status(200).send(namespaceObj);
|
|
1209
|
+
return res.status(200).send(stripNamespaceSecrets(namespaceObj));
|
|
1188
1210
|
})
|
|
1189
1211
|
})
|
|
1190
1212
|
|
|
@@ -1216,7 +1238,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1216
1238
|
|
|
1217
1239
|
let namespaces = await Namespace.find({ id_project: id_project }).catch((err) => {
|
|
1218
1240
|
winston.error("find namespaces error: ", err)
|
|
1219
|
-
res.status(500).send({ success: false, error: err })
|
|
1241
|
+
res.status(500).send({ success: false, error: sanitizeAxiosError(err) })
|
|
1220
1242
|
})
|
|
1221
1243
|
|
|
1222
1244
|
if (!namespaces || namespaces.length == 0) {
|
|
@@ -1310,7 +1332,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
|
|
|
1310
1332
|
|
|
1311
1333
|
let deleteResponse = await KB.deleteMany({ id_project: id_project, namespace: namespace_id, type: { $in: ['url', 'text', 'faq'] } }).catch((err) => {
|
|
1312
1334
|
winston.error("deleteMany error: ", err);
|
|
1313
|
-
return res.status(500).send({ success: false, error: err });
|
|
1335
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1314
1336
|
})
|
|
1315
1337
|
|
|
1316
1338
|
winston.verbose("Content deletetion response: ", deleteResponse);
|
|
@@ -1401,7 +1423,7 @@ router.put('/namespace/:id', async (req, res) => {
|
|
|
1401
1423
|
Namespace.findOneAndUpdate({ id: namespace_id }, update, { new: true, upsert: true }, (err, updatedNamespace) => {
|
|
1402
1424
|
if (err) {
|
|
1403
1425
|
winston.error("update namespace error: ", err);
|
|
1404
|
-
return res.status(500).send({ success: false, error: err });
|
|
1426
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1405
1427
|
}
|
|
1406
1428
|
|
|
1407
1429
|
kbEvent.emit('kb.namespace.update', { updatedNamespace, namespace_id });
|
|
@@ -1409,7 +1431,7 @@ router.put('/namespace/:id', async (req, res) => {
|
|
|
1409
1431
|
let namespaceObj = updatedNamespace.toObject();
|
|
1410
1432
|
delete namespaceObj._id;
|
|
1411
1433
|
delete namespaceObj.__v;
|
|
1412
|
-
res.status(200).send(namespaceObj);
|
|
1434
|
+
res.status(200).send(stripNamespaceSecrets(namespaceObj));
|
|
1413
1435
|
})
|
|
1414
1436
|
})
|
|
1415
1437
|
|
|
@@ -1420,7 +1442,7 @@ router.delete('/namespace/:id', async (req, res) => {
|
|
|
1420
1442
|
|
|
1421
1443
|
let namespace = await Namespace.findOne({ id_project: project_id, id: namespace_id }).catch((err) => {
|
|
1422
1444
|
winston.error("find namespace error: ", err);
|
|
1423
|
-
res.status(500).send({ success: false, error: err });
|
|
1445
|
+
res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1424
1446
|
})
|
|
1425
1447
|
|
|
1426
1448
|
if (!namespace) {
|
|
@@ -1441,7 +1463,7 @@ router.delete('/namespace/:id', async (req, res) => {
|
|
|
1441
1463
|
|
|
1442
1464
|
let deleteResponse = await KB.deleteMany({ id_project: project_id, namespace: namespace_id }).catch((err) => {
|
|
1443
1465
|
winston.error("deleteMany error: ", err);
|
|
1444
|
-
return res.status(500).send({ success: false, error: err });
|
|
1466
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1445
1467
|
})
|
|
1446
1468
|
winston.debug("delete all contents response: ", deleteResponse);
|
|
1447
1469
|
|
|
@@ -1462,9 +1484,9 @@ router.delete('/namespace/:id', async (req, res) => {
|
|
|
1462
1484
|
winston.error("deleteNamespace err.response: ", err.response);
|
|
1463
1485
|
if (err.response && err.response.status) {
|
|
1464
1486
|
let status = err.response.status;
|
|
1465
|
-
res.status(status).send({ success: false, statusText: err.response.statusText, error: err
|
|
1487
|
+
res.status(status).send({ success: false, statusText: err.response.statusText, error: sanitizeAxiosError(err) });
|
|
1466
1488
|
} else {
|
|
1467
|
-
res.status(500).send({ success: false, message: "Unable to delete namespace", error: err })
|
|
1489
|
+
res.status(500).send({ success: false, message: "Unable to delete namespace", error: sanitizeAxiosError(err) })
|
|
1468
1490
|
}
|
|
1469
1491
|
})
|
|
1470
1492
|
|
|
@@ -1472,7 +1494,7 @@ router.delete('/namespace/:id', async (req, res) => {
|
|
|
1472
1494
|
|
|
1473
1495
|
let namespace = await Namespace.findOne({ id: namespace_id }).catch((err) => {
|
|
1474
1496
|
winston.error("findOne namespace error: ", err);
|
|
1475
|
-
return res.status(500).send({ success: false, error: err });
|
|
1497
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1476
1498
|
})
|
|
1477
1499
|
if (namespace.default === true) {
|
|
1478
1500
|
winston.error("Default namespace cannot be deleted");
|
|
@@ -1484,13 +1506,13 @@ router.delete('/namespace/:id', async (req, res) => {
|
|
|
1484
1506
|
|
|
1485
1507
|
let deleteResponse = await KB.deleteMany({ id_project: project_id, namespace: namespace_id }).catch((err) => {
|
|
1486
1508
|
winston.error("deleteMany error: ", err);
|
|
1487
|
-
return res.status(500).send({ success: false, error: err });
|
|
1509
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1488
1510
|
})
|
|
1489
1511
|
winston.debug("delete all contents response: ", deleteResponse);
|
|
1490
1512
|
|
|
1491
1513
|
let deleteNamespaceResponse = await Namespace.findOneAndDelete({ id: namespace_id }).catch((err) => {
|
|
1492
1514
|
winston.error("deleteOne namespace error: ", err);
|
|
1493
|
-
return res.status(500).send({ success: false, error: err });
|
|
1515
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1494
1516
|
})
|
|
1495
1517
|
winston.debug("delete namespace response: ", deleteNamespaceResponse);
|
|
1496
1518
|
|
|
@@ -1609,7 +1631,7 @@ router.get('/', async (req, res) => {
|
|
|
1609
1631
|
.exec((err, kbs) => {
|
|
1610
1632
|
if (err) {
|
|
1611
1633
|
winston.error("Find all kbs error: ", err);
|
|
1612
|
-
return res.status(500).send({ success: false, error: err });
|
|
1634
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1613
1635
|
}
|
|
1614
1636
|
|
|
1615
1637
|
winston.debug("KBs found: ", kbs);
|
|
@@ -1735,7 +1757,7 @@ router.post('/', async (req, res) => {
|
|
|
1735
1757
|
KB.findOneAndUpdate({ id_project, type, source }, new_kb, { upsert: true, new: true, rawResult: true }, async (err, raw_content) => {
|
|
1736
1758
|
if (err) {
|
|
1737
1759
|
winston.error("findOneAndUpdate with upsert error: ", err);
|
|
1738
|
-
res.status(500).send({ success: false, error: err });
|
|
1760
|
+
res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1739
1761
|
}
|
|
1740
1762
|
else {
|
|
1741
1763
|
|
|
@@ -1988,7 +2010,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
|
|
|
1988
2010
|
})
|
|
1989
2011
|
.on("error", (err) => {
|
|
1990
2012
|
winston.error("CSV parsing error: ", err);
|
|
1991
|
-
res.status(400).send({ success: false, error: err });
|
|
2013
|
+
res.status(400).send({ success: false, error: sanitizeAxiosError(err) });
|
|
1992
2014
|
})
|
|
1993
2015
|
|
|
1994
2016
|
})
|
|
@@ -2009,7 +2031,7 @@ router.post('/sitemap', async (req, res) => {
|
|
|
2009
2031
|
res.status(200).send(data);
|
|
2010
2032
|
}).catch((err) => {
|
|
2011
2033
|
winston.error("Error fetching sitemap: ", err)
|
|
2012
|
-
res.status(500).send({ success: false, error: err });
|
|
2034
|
+
res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
2013
2035
|
})
|
|
2014
2036
|
|
|
2015
2037
|
})
|
|
@@ -2057,7 +2079,7 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
2057
2079
|
|
|
2058
2080
|
const data = await sitemap.fetch().catch((err) => {
|
|
2059
2081
|
winston.error("Error fetching sitemap: ", err);
|
|
2060
|
-
return res.status(500).send({ success: false, error: err });
|
|
2082
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
2061
2083
|
})
|
|
2062
2084
|
|
|
2063
2085
|
if (data.errors && data.errors.length > 0) {
|
|
@@ -2094,7 +2116,7 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
2094
2116
|
saved_content = await KB.findOneAndUpdate({ id_project, type: 'sitemap', source, namespace: namespace_id }, sitemap_content, { upsert: true, new: true }).lean().exec();
|
|
2095
2117
|
} catch (err) {
|
|
2096
2118
|
winston.error("Error saving content: ", err);
|
|
2097
|
-
return res.status(500).send({ success: false, error: err });
|
|
2119
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
2098
2120
|
}
|
|
2099
2121
|
|
|
2100
2122
|
const options = {
|
|
@@ -2205,7 +2227,7 @@ router.put('/:kb_id', async (req, res) => {
|
|
|
2205
2227
|
|
|
2206
2228
|
} catch (err) {
|
|
2207
2229
|
winston.error("Error updating content: ", err);
|
|
2208
|
-
return res.status(500).send({ success: false, error: err });
|
|
2230
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
2209
2231
|
}
|
|
2210
2232
|
|
|
2211
2233
|
if (situated_context && situated_context === true && scrape_type === 0) {
|
|
@@ -2254,7 +2276,7 @@ router.put('/:kb_id', async (req, res) => {
|
|
|
2254
2276
|
updated_content = await KB.findOneAndUpdate({ id_project, namespace: namespace_id, source }, new_content, { upsert: true, new: true }).lean().exec();
|
|
2255
2277
|
} catch (err) {
|
|
2256
2278
|
winston.error("Error updating content: ", err);
|
|
2257
|
-
return res.status(500).send({ success: false, error: err });
|
|
2279
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
2258
2280
|
}
|
|
2259
2281
|
|
|
2260
2282
|
const embedding = aiManager.normalizeEmbedding(namespace.embedding);
|
|
@@ -2297,7 +2319,7 @@ router.delete('/:kb_id', async (req, res) => {
|
|
|
2297
2319
|
|
|
2298
2320
|
let kb = await KB.findOne({ id_project: project_id, _id: kb_id }).catch((err) => {
|
|
2299
2321
|
winston.warn("Unable to find kb. Error: ", err);
|
|
2300
|
-
return res.status(500).send({ success: false, error: err })
|
|
2322
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) })
|
|
2301
2323
|
})
|
|
2302
2324
|
|
|
2303
2325
|
if (!kb) {
|
|
@@ -2322,7 +2344,7 @@ router.delete('/:kb_id', async (req, res) => {
|
|
|
2322
2344
|
//return res.status(200).send({ success: true, message: "Scheduled jobs for deleting sitemap" });
|
|
2323
2345
|
} catch (err) {
|
|
2324
2346
|
winston.error("Error deleting sitemap: ", err);
|
|
2325
|
-
return res.status(500).send({ success: false, error: err });
|
|
2347
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
2326
2348
|
}
|
|
2327
2349
|
}
|
|
2328
2350
|
|
|
@@ -2368,7 +2390,7 @@ router.delete('/:kb_id', async (req, res) => {
|
|
|
2368
2390
|
|
|
2369
2391
|
if (err) {
|
|
2370
2392
|
winston.error("Delete kb error: ", err);
|
|
2371
|
-
return res.status(500).send({ success: false, error: err });
|
|
2393
|
+
return res.status(500).send({ success: false, error: sanitizeAxiosError(err) });
|
|
2372
2394
|
}
|
|
2373
2395
|
emitKbContentDelete(deletedKb);
|
|
2374
2396
|
res.status(200).send(deletedKb);
|
|
@@ -2394,7 +2416,7 @@ router.delete('/:kb_id', async (req, res) => {
|
|
|
2394
2416
|
|
|
2395
2417
|
}).catch((err) => {
|
|
2396
2418
|
let status = err.response?.status || 500;
|
|
2397
|
-
res.status(status).send({ success: false, statusText: err.response.statusText, error: err
|
|
2419
|
+
res.status(status).send({ success: false, statusText: err.response.statusText, error: sanitizeAxiosError(err) });
|
|
2398
2420
|
})
|
|
2399
2421
|
|
|
2400
2422
|
})
|
package/routes/llm.js
CHANGED
|
@@ -130,9 +130,11 @@ router.post('/preview', async (req, res) => {
|
|
|
130
130
|
res.status(200).send(response.data)
|
|
131
131
|
}).catch((err) => {
|
|
132
132
|
if (err.response?.data?.detail?.[0]) {
|
|
133
|
-
|
|
133
|
+
// Only the message — never `detail` itself: detail[].input echoes the
|
|
134
|
+
// request body, which carries the provider key.
|
|
135
|
+
res.status(400).send({ success: false, error: err.response.data.detail[0]?.msg });
|
|
134
136
|
} else if (err.response?.data?.detail?.answer) {
|
|
135
|
-
res.status(400).send({ success: false, error: err.response.data.detail.answer
|
|
137
|
+
res.status(400).send({ success: false, error: err.response.data.detail.answer });
|
|
136
138
|
} else if (err.response?.data) {
|
|
137
139
|
res.status(axiosErrorStatus(err)).send({ success: false, error: sanitizeAxiosError(err) });
|
|
138
140
|
} else {
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Log redaction helpers.
|
|
3
|
+
*
|
|
4
|
+
* Winston serializes whatever is handed to it. Because credentials (OpenAI /
|
|
5
|
+
* provider keys, the Pinecone key, bearer tokens) travel inside request bodies
|
|
6
|
+
* to the LLM microservice, an AxiosError carries the full body in
|
|
7
|
+
* `err.config.data` and the bearer in `err.config.headers.Authorization`, and a
|
|
8
|
+
* `winston.error("...", err)` or `winston.debug("data:", data)` leaks them to
|
|
9
|
+
* stdout / app.log / the MongoDB log transport.
|
|
10
|
+
*
|
|
11
|
+
* `redactFormat` is a winston format that walks every logged value (message,
|
|
12
|
+
* merged metadata and splat args) and scrubs sensitive keys and key-shaped
|
|
13
|
+
* strings BEFORE any transport writes them. This closes the logging leak at the
|
|
14
|
+
* transport layer instead of at ~40 individual call sites.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
// Object keys whose value is a secret and must never be logged.
|
|
18
|
+
const SENSITIVE_KEY_RE = /^(api_?key|gptkey|llm_key|apikey|embedding_key|token|password|passwd|secret|client_secret|authorization|x-auth-token|kb_webhook_token)$/i;
|
|
19
|
+
|
|
20
|
+
const REDACTED = '[REDACTED]';
|
|
21
|
+
const MAX_DEPTH = 8;
|
|
22
|
+
|
|
23
|
+
// Scrub secrets that appear inside a raw string (e.g. the JSON string held in
|
|
24
|
+
// axios `config.data`, or a bearer header rendered into a message).
|
|
25
|
+
function redactString(str) {
|
|
26
|
+
if (typeof str !== 'string' || str.length === 0) return str;
|
|
27
|
+
return str
|
|
28
|
+
// provider keys: OpenAI (sk-..., sk-proj-...), Anthropic (sk-ant-...), Pinecone (pcsk_...), Google (AIza...)
|
|
29
|
+
.replace(/sk-(?:proj-|ant-)?[A-Za-z0-9_-]{12,}/g, 'sk-[REDACTED]')
|
|
30
|
+
.replace(/pcsk_[A-Za-z0-9_-]{12,}/g, 'pcsk_[REDACTED]')
|
|
31
|
+
.replace(/AIza[A-Za-z0-9_-]{20,}/g, 'AIza[REDACTED]')
|
|
32
|
+
// JSON-encoded secret fields inside a serialized body: "api_key":"..."
|
|
33
|
+
.replace(/("(?:api_?key|gptkey|llm_key|apikey|embedding_key|token|password|secret|client_secret)"\s*:\s*)"[^"]*"/gi, '$1"[REDACTED]"')
|
|
34
|
+
// bearer tokens
|
|
35
|
+
.replace(/Bearer\s+[A-Za-z0-9._-]+/gi, 'Bearer [REDACTED]');
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Recursively redact a value (object, array, or scalar). Returns a redacted copy
|
|
39
|
+
// for objects/arrays; scalars are returned scrubbed. Guards against cycles and
|
|
40
|
+
// runaway depth.
|
|
41
|
+
function redactValue(value, seen, depth) {
|
|
42
|
+
if (depth > MAX_DEPTH) return '[Truncated]';
|
|
43
|
+
if (value === null || value === undefined) return value;
|
|
44
|
+
if (typeof value === 'string') return redactString(value);
|
|
45
|
+
if (typeof value !== 'object') return value;
|
|
46
|
+
if (seen.has(value)) return '[Circular]';
|
|
47
|
+
seen.add(value);
|
|
48
|
+
|
|
49
|
+
if (Array.isArray(value)) {
|
|
50
|
+
return value.map((v) => redactValue(v, seen, depth + 1));
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Preserve Error identity/message while scrubbing any attached props (axios
|
|
54
|
+
// errors attach `config`, `response`, etc.).
|
|
55
|
+
const out = {};
|
|
56
|
+
for (const key of Object.keys(value)) {
|
|
57
|
+
if (SENSITIVE_KEY_RE.test(key)) {
|
|
58
|
+
out[key] = REDACTED;
|
|
59
|
+
} else {
|
|
60
|
+
out[key] = redactValue(value[key], seen, depth + 1);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Redact the winston `info` object in place, including the Symbol(splat) args
|
|
67
|
+
// that hold the extra arguments of calls like `winston.error("msg:", err)`.
|
|
68
|
+
function redactInfo(info) {
|
|
69
|
+
const seen = new WeakSet();
|
|
70
|
+
for (const key of Object.keys(info)) {
|
|
71
|
+
if (key === 'level') continue;
|
|
72
|
+
if (SENSITIVE_KEY_RE.test(key)) {
|
|
73
|
+
info[key] = REDACTED;
|
|
74
|
+
} else {
|
|
75
|
+
info[key] = redactValue(info[key], seen, 0);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const splatSym = Symbol.for('splat');
|
|
79
|
+
const splat = info[splatSym];
|
|
80
|
+
if (Array.isArray(splat)) {
|
|
81
|
+
info[splatSym] = splat.map((arg) => redactValue(arg, new WeakSet(), 0));
|
|
82
|
+
}
|
|
83
|
+
return info;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
module.exports = { redactString, redactValue, redactInfo, SENSITIVE_KEY_RE };
|