@tiledesk/tiledesk-server 3.2.2 โ 3.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/controllers/chatbot.controller.js +2 -2
- package/middleware/freezePineconeWrites.js +27 -0
- package/models/faq_kb.js +5 -1
- package/package.json +3 -3
- package/routes/auth.js +57 -21
- package/routes/faq_kb.js +4 -2
- package/routes/kb.js +87 -12
- package/routes/users.js +6 -8
- package/routes/webhook.js +2 -1
- package/services/aiService.js +16 -0
- package/services/chatbotService.js +3 -1
- package/services/faqService.js +7 -4
- package/test/authenticationJwt.js +158 -0
- package/test/freezePineconeWrites.js +62 -0
- package/test/kbRoute.js +136 -0
- package/test/passwordPolicy.test.js +56 -0
- package/test/publishFailLoud.test.js +14 -2
- package/utils/passwordPolicy.js +37 -0
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,23 @@
|
|
|
5
5
|
๐ IN PRODUCTION ๐
|
|
6
6
|
(https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77)
|
|
7
7
|
|
|
8
|
+
# 3.3.4
|
|
9
|
+
- Updated AskGPTV2 Pinecone reranking (standard namespaces, with `PINECONE_RERANKING` on) sends the reranker model as `name`. The payload used `model`, so the model (`PINECONE_RERANKING_MODEL`, otherwise `RERANKING_MODEL`, otherwise `bge-reranker-v2-m3`) was ignored. Hybrid reranking is unchanged
|
|
10
|
+
- Bumped tybot-connector version to 3.0.8
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# 3.3.2
|
|
14
|
+
- Publish creates the forked copy already with `trashed: true`. A copy left behind when the following update never lands no longer shows up as a second chatbot. `trashedAt` is still not set, so the published snapshot is not removed by the TTL
|
|
15
|
+
|
|
16
|
+
# 3.3.1
|
|
17
|
+
- Added possibility to block indexing APIs
|
|
18
|
+
|
|
19
|
+
# 3.3.0
|
|
20
|
+
- New `POST /:projectid/kb/crawl/discover` (`{ source, max_pages?, max_depth?, max_seconds?, crawl_file_types? }`): lists the pages of a site through the llm crawler without indexing anything. Answers 200 with the llm body (`{ pages, truncated, fetched, skipped }`); 400 for an invalid `source`/parameter or a start URL the llm refuses; 501 `crawl_discover_unavailable` when the llm has no such endpoint; 504 `discover_timeout`; 502 for any other llm failure. Only the fields the caller gave are forwarded, capped at 300 pages, depth 5 and `KB_CRAWL_DISCOVER_MAX_SECONDS` (default 300, never above the llm's 1800); `ignore_robots` and `browser_headers` are never forwarded. The call times out after `max_seconds + 120` seconds
|
|
21
|
+
|
|
22
|
+
# 3.2.2
|
|
23
|
+
- Fixed: `GET /:projectid/users/availables` no longer returns trashed project users
|
|
24
|
+
|
|
8
25
|
# 3.2.1
|
|
9
26
|
- Bump version `tiledesk-chatbot-template` to 0.1.7
|
|
10
27
|
- Bump verion `tiledesk-vxml-connector` to 0.2.7
|
|
@@ -546,7 +546,7 @@ class ChatbotController {
|
|
|
546
546
|
}
|
|
547
547
|
}
|
|
548
548
|
|
|
549
|
-
async import(json, id_project, chatbot_id, create, replace, overwrite, user_id, skipSubagents) {
|
|
549
|
+
async import(json, id_project, chatbot_id, create, replace, overwrite, user_id, skipSubagents, options) {
|
|
550
550
|
let chatbot;
|
|
551
551
|
const sourceParentId = this._sourceId(json);
|
|
552
552
|
this._stripImportedIdentity(json);
|
|
@@ -556,7 +556,7 @@ class ChatbotController {
|
|
|
556
556
|
if (create) {
|
|
557
557
|
json.template = "empty";
|
|
558
558
|
try {
|
|
559
|
-
chatbot = await faqService.create(id_project, user_id, json);
|
|
559
|
+
chatbot = await faqService.create(id_project, user_id, json, options);
|
|
560
560
|
} catch (err) {
|
|
561
561
|
winston.error("(ChatbotController) import create error: ", err);
|
|
562
562
|
throw new AppError(400, "Error creating new chatbot");
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
var winston = require('../config/winston');
|
|
2
|
+
|
|
3
|
+
// Temporary freeze while vector-store indexes are copied.
|
|
4
|
+
// Blocks only when PINECONE_WRITE_DISABLED=true. Absent or false leaves writes enabled.
|
|
5
|
+
function vectorStoreWritesFrozen() {
|
|
6
|
+
return process.env.PINECONE_WRITE_DISABLED === 'true';
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
function freezePineconeWrites(req, res, next) {
|
|
10
|
+
if (!vectorStoreWritesFrozen()) {
|
|
11
|
+
return next();
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
winston.warn("Vector store write rejected during index migration: " + req.method + " " + req.originalUrl);
|
|
15
|
+
res.set('Retry-After', '3600');
|
|
16
|
+
return res.status(503).send({
|
|
17
|
+
success: false,
|
|
18
|
+
error: "Knowledge base indexing is temporarily unavailable",
|
|
19
|
+
code: "vector_store_writes_disabled"
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if (vectorStoreWritesFrozen()) {
|
|
24
|
+
winston.warn("Vector store writes are disabled for the index migration (PINECONE_WRITE_DISABLED=true).");
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
module.exports = { vectorStoreWritesFrozen, freezePineconeWrites };
|
package/models/faq_kb.js
CHANGED
|
@@ -264,7 +264,11 @@ Faq_kbSchema.pre('findOneAndUpdate', async function (next) {
|
|
|
264
264
|
});
|
|
265
265
|
|
|
266
266
|
Faq_kbSchema.post('findOneAndUpdate', async function (doc) {
|
|
267
|
-
|
|
267
|
+
// Only the update that sets trashed. A publish copy is already trashed, and the
|
|
268
|
+
// writes that follow (its attributes, the tilebot url) must not soft-delete its blocks.
|
|
269
|
+
const update = this.getUpdate() || {};
|
|
270
|
+
const setsTrashed = update.trashed === true || (update.$set && update.$set.trashed === true);
|
|
271
|
+
if (doc && doc.trashed === true && setsTrashed) {
|
|
268
272
|
botEvent.emit('faqbot.update.virtual.delete', doc)
|
|
269
273
|
}
|
|
270
274
|
})
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tiledesk/tiledesk-server",
|
|
3
3
|
"description": "The Tiledesk server module",
|
|
4
|
-
"version": "3.
|
|
4
|
+
"version": "3.3.4",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"start": "node ./bin/www",
|
|
7
7
|
"quota-restore": "node ./bin/quota-restore",
|
|
@@ -51,10 +51,10 @@
|
|
|
51
51
|
"@tiledesk/tiledesk-rasa-connector": "^1.0.10",
|
|
52
52
|
"@tiledesk/tiledesk-sms-connector": "^0.1.13",
|
|
53
53
|
"@tiledesk/tiledesk-telegram-connector": "^0.1.14",
|
|
54
|
-
"@tiledesk/tiledesk-tybot-connector": "^3.0.
|
|
54
|
+
"@tiledesk/tiledesk-tybot-connector": "^3.0.8",
|
|
55
55
|
"@tiledesk/tiledesk-voice-twilio-connector": "^0.3.2",
|
|
56
56
|
"@tiledesk/tiledesk-vxml-connector": "^0.2.7",
|
|
57
|
-
"@tiledesk/tiledesk-whatsapp-connector": "1.0.
|
|
57
|
+
"@tiledesk/tiledesk-whatsapp-connector": "1.0.33",
|
|
58
58
|
"@tiledesk/tiledesk-whatsapp-jobworker": "^0.0.14",
|
|
59
59
|
"amqplib": "^0.5.5",
|
|
60
60
|
"app-root-path": "^3.0.0",
|
package/routes/auth.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
var config = require('../config/database');
|
|
2
2
|
var express = require('express');
|
|
3
3
|
var jwt = require('jsonwebtoken');
|
|
4
|
+
var mongoose = require('mongoose');
|
|
4
5
|
var router = express.Router();
|
|
5
6
|
var User = require("../models/user");
|
|
6
7
|
var Subscription = require("../models/subscription");
|
|
8
|
+
var Project = require("../models/project");
|
|
7
9
|
var Project_user = require("../models/project_user");
|
|
8
10
|
var RoleConstants = require("../models/roleConstants");
|
|
9
11
|
var uniqid = require('uniqid');
|
|
@@ -24,6 +26,7 @@ var validtoken = require('../middleware/valid-token');
|
|
|
24
26
|
var PendingInvitation = require("../models/pending-invitation");
|
|
25
27
|
const { check, validationResult } = require('express-validator');
|
|
26
28
|
var UserUtil = require('../utils/userUtil');
|
|
29
|
+
const { validatePassword } = require('../utils/passwordPolicy');
|
|
27
30
|
|
|
28
31
|
let configSecret = process.env.GLOBAL_SECRET || config.secret;
|
|
29
32
|
var pKey = process.env.GLOBAL_SECRET_OR_PRIVATE_KEY;
|
|
@@ -77,12 +80,10 @@ router.post('/signup',
|
|
|
77
80
|
return res.json({ success: false, msg: 'Please pass email and password.' });
|
|
78
81
|
} else {
|
|
79
82
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
// return res.status(403).send({ success: false, message: "The password does not meet the minimum vulnerability requirements"})
|
|
85
|
-
// }
|
|
83
|
+
const passwordCheck = validatePassword(req.body.password);
|
|
84
|
+
if (!passwordCheck.valid) {
|
|
85
|
+
return res.status(403).send({ success: false, message: passwordCheck.message });
|
|
86
|
+
}
|
|
86
87
|
|
|
87
88
|
return userService.signup(req.body.email, req.body.password, req.body.firstname, req.body.lastname, false, req.body.phone)
|
|
88
89
|
.then( async function (savedUser) {
|
|
@@ -281,9 +282,29 @@ router.post('/signinWithCustomToken', [
|
|
|
281
282
|
return res.status(400).send({ success: false, msg: 'JWT Aud.AudienceId field is required for AudienceType subscriptions' });
|
|
282
283
|
}
|
|
283
284
|
|
|
284
|
-
var subscription = await Subscription.findById(AudienceId).exec();
|
|
285
|
+
var subscription = await Subscription.findById(AudienceId).select('+global').exec();
|
|
285
286
|
winston.debug("signinWithCustomToken subscription: ", subscription );
|
|
287
|
+
if (!subscription) {
|
|
288
|
+
return res.status(401).send({ success: false, msg: 'Subscription not found' });
|
|
289
|
+
}
|
|
286
290
|
id_project = subscription.id_project;
|
|
291
|
+
// Try on WhatsApp opens the conversation in the bot's project. Only this
|
|
292
|
+
// dedicated global subscription may choose that project from the body.
|
|
293
|
+
if (subscription.event === 'whatsapp.bottest.signin' && (subscription.global === true || String(subscription.id_project) === '*')) {
|
|
294
|
+
const requestedProjectId = req.body && req.body.id_project;
|
|
295
|
+
if (!requestedProjectId || String(requestedProjectId) === '*') {
|
|
296
|
+
return res.status(400).send({ success: false, msg: 'id_project is required' });
|
|
297
|
+
}
|
|
298
|
+
if (!mongoose.Types.ObjectId.isValid(String(requestedProjectId))) {
|
|
299
|
+
return res.status(400).send({ success: false, msg: 'id_project is invalid' });
|
|
300
|
+
}
|
|
301
|
+
const project = await Project.findOne({ _id: requestedProjectId, status: 100 }).select('_id').exec();
|
|
302
|
+
if (!project) {
|
|
303
|
+
return res.status(404).send({ success: false, msg: 'Project not found' });
|
|
304
|
+
}
|
|
305
|
+
id_project = String(project._id);
|
|
306
|
+
winston.verbose("signinWithCustomToken whatsapp bottest project: " + id_project);
|
|
307
|
+
}
|
|
287
308
|
winston.debug("signinWithCustomToken subscription req.user._id: "+ req.user._id );
|
|
288
309
|
winston.debug("signinWithCustomToken subscription.id_project:"+ id_project );
|
|
289
310
|
|
|
@@ -1008,9 +1029,8 @@ router.put('/requestresetpsw', function (req, res) {
|
|
|
1008
1029
|
*/
|
|
1009
1030
|
router.put('/resetpsw/:resetpswrequestid', function (req, res) {
|
|
1010
1031
|
winston.debug("--> RESET PSW - REQUEST ID", req.params.resetpswrequestid);
|
|
1011
|
-
winston.debug("--> RESET PSW - NEW PSW ", req.body.password);
|
|
1012
1032
|
|
|
1013
|
-
User.findOne({ resetpswrequestid: req.params.resetpswrequestid }
|
|
1033
|
+
User.findOne({ resetpswrequestid: req.params.resetpswrequestid }).select('+password').exec(function (err, user) {
|
|
1014
1034
|
|
|
1015
1035
|
if (err) {
|
|
1016
1036
|
winston.error('--> RESET PSW - Error getting user ', err)
|
|
@@ -1023,28 +1043,44 @@ router.put('/resetpsw/:resetpswrequestid', function (req, res) {
|
|
|
1023
1043
|
}
|
|
1024
1044
|
|
|
1025
1045
|
if (user && req.body.password) {
|
|
1026
|
-
winston.debug('--> RESET PSW - User Found ', user);
|
|
1027
1046
|
winston.debug('--> RESET PSW - User ID Found ', user._id);
|
|
1028
1047
|
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1048
|
+
const passwordCheck = validatePassword(req.body.password);
|
|
1049
|
+
if (!passwordCheck.valid) {
|
|
1050
|
+
return res.status(403).send({ success: false, message: passwordCheck.message });
|
|
1051
|
+
}
|
|
1033
1052
|
|
|
1034
|
-
|
|
1035
|
-
|
|
1053
|
+
user.comparePassword(req.body.password, function (compareErr, isMatch) {
|
|
1054
|
+
if (compareErr) {
|
|
1055
|
+
winston.error('--> RESET PSW - compare error ', compareErr);
|
|
1036
1056
|
return res.status(500).send({ success: false, msg: 'Error saving object.' });
|
|
1037
1057
|
}
|
|
1038
|
-
|
|
1058
|
+
if (isMatch) {
|
|
1059
|
+
return res.status(403).send({ success: false, message: "The new password must be different from the previous one." });
|
|
1060
|
+
}
|
|
1039
1061
|
|
|
1040
|
-
|
|
1062
|
+
user.password = req.body.password;
|
|
1063
|
+
user.resetpswrequestid = '';
|
|
1041
1064
|
|
|
1042
|
-
|
|
1043
|
-
authEvent.emit('user.resetpassword', {saveUser:saveUser, req:req});
|
|
1065
|
+
user.save(function (err, saveUser) {
|
|
1044
1066
|
|
|
1067
|
+
if (err) {
|
|
1068
|
+
winston.error('--- > USER SAVE -ERROR ', err)
|
|
1069
|
+
return res.status(500).send({ success: false, msg: 'Error saving object.' });
|
|
1070
|
+
}
|
|
1071
|
+
winston.debug('--- > USER SAVED ', saveUser)
|
|
1072
|
+
|
|
1073
|
+
emailService.sendYourPswHasBeenChangedEmail(saveUser.email, saveUser.firstname, saveUser.lastname);
|
|
1074
|
+
|
|
1075
|
+
// TODO emit user.update?
|
|
1076
|
+
authEvent.emit('user.resetpassword', {saveUser:saveUser, req:req});
|
|
1077
|
+
|
|
1078
|
+
const userJson = saveUser.toJSON();
|
|
1079
|
+
delete userJson.password;
|
|
1045
1080
|
|
|
1046
|
-
|
|
1081
|
+
res.status(200).json({ message: 'Password change successful', user: userJson });
|
|
1047
1082
|
|
|
1083
|
+
});
|
|
1048
1084
|
});
|
|
1049
1085
|
}
|
|
1050
1086
|
});
|
package/routes/faq_kb.js
CHANGED
|
@@ -975,6 +975,7 @@ router.post('/fork/:id_faq_kb', roleChecker.hasRole('admin'), asyncHandler(async
|
|
|
975
975
|
// Publish passes skipSubagents so actions keep the development subagent id.
|
|
976
976
|
// A plain fork still clones subagents into the copy.
|
|
977
977
|
const skipSubagents = req.query.skipSubagents === 'true';
|
|
978
|
+
const forPublish = req.query.for_publish === 'true';
|
|
978
979
|
|
|
979
980
|
const savedChatbot = await chatbotController.import(
|
|
980
981
|
chatbot,
|
|
@@ -984,10 +985,11 @@ router.post('/fork/:id_faq_kb', roleChecker.hasRole('admin'), asyncHandler(async
|
|
|
984
985
|
false,
|
|
985
986
|
false,
|
|
986
987
|
user_id,
|
|
987
|
-
skipSubagents
|
|
988
|
+
skipSubagents,
|
|
989
|
+
forPublish ? { trashed: true } : undefined
|
|
988
990
|
);
|
|
989
991
|
|
|
990
|
-
if (
|
|
992
|
+
if (!forPublish) {
|
|
991
993
|
botEvent.emit('faqbot.created', { req, chatbot: savedChatbot, id_project: landing_project_id });
|
|
992
994
|
}
|
|
993
995
|
|
package/routes/kb.js
CHANGED
|
@@ -27,6 +27,7 @@ const kbEvent = require('../event/kbEvent');
|
|
|
27
27
|
// provider/gptkey + Pinecone key) straight into the response. sanitizeAxiosError
|
|
28
28
|
// reduces it to a safe { message }.
|
|
29
29
|
const { sanitizeAxiosError } = require('../utils/sanitizeAxiosError');
|
|
30
|
+
const { freezePineconeWrites } = require('../middleware/freezePineconeWrites');
|
|
30
31
|
|
|
31
32
|
// Strip the vector-store credential from a namespace (or array of namespaces)
|
|
32
33
|
// before returning it to a client. engine.apikey is seeded from
|
|
@@ -152,7 +153,81 @@ function normalizeSituatedContext(enable = false) {
|
|
|
152
153
|
* Proxy Section - Start
|
|
153
154
|
* ****************************************
|
|
154
155
|
*/
|
|
155
|
-
|
|
156
|
+
// Caps applied to what the caller asks; the llm has its own (larger) limits.
|
|
157
|
+
const CRAWL_DISCOVER_MAX_PAGES = 300;
|
|
158
|
+
const CRAWL_DISCOVER_MAX_DEPTH = 5;
|
|
159
|
+
const CRAWL_DISCOVER_DEFAULT_SECONDS = 300;
|
|
160
|
+
// The llm refuses more than 1800 s (its own maximum).
|
|
161
|
+
const CRAWL_DISCOVER_LLM_MAX_SECONDS = 1800;
|
|
162
|
+
// KB_CRAWL_DISCOVER_MAX_SECONDS (default 300), never above the llm's maximum. Read per request.
|
|
163
|
+
const crawlDiscoverMaxSeconds = () => {
|
|
164
|
+
const configured = parseInt(process.env.KB_CRAWL_DISCOVER_MAX_SECONDS);
|
|
165
|
+
return Math.min(configured > 0 ? configured : CRAWL_DISCOVER_DEFAULT_SECONDS, CRAWL_DISCOVER_LLM_MAX_SECONDS);
|
|
166
|
+
};
|
|
167
|
+
// The http call must outlive the crawl's own budget, so the llm answers (with truncated) before axios gives up.
|
|
168
|
+
const CRAWL_DISCOVER_TIMEOUT_MARGIN_SECONDS = 120;
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Lists the pages of a site (no indexing). Only the fields the caller gave are
|
|
172
|
+
* forwarded to the llm (capped); ignore_robots and browser_headers are never forwarded.
|
|
173
|
+
*/
|
|
174
|
+
router.post('/crawl/discover', async (req, res) => {
|
|
175
|
+
|
|
176
|
+
const body = req.body || {};
|
|
177
|
+
|
|
178
|
+
let source;
|
|
179
|
+
try {
|
|
180
|
+
const url = new URL(body.source);
|
|
181
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
182
|
+
throw new Error('unsupported protocol');
|
|
183
|
+
}
|
|
184
|
+
source = body.source;
|
|
185
|
+
} catch (err) {
|
|
186
|
+
return res.status(400).send({ success: false, error: "source must be an http(s) url" });
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const payload = { source };
|
|
190
|
+
for (const [field, cap] of [
|
|
191
|
+
['max_pages', CRAWL_DISCOVER_MAX_PAGES],
|
|
192
|
+
['max_depth', CRAWL_DISCOVER_MAX_DEPTH],
|
|
193
|
+
['max_seconds', crawlDiscoverMaxSeconds()]
|
|
194
|
+
]) {
|
|
195
|
+
if (body[field] === undefined || body[field] === null) continue;
|
|
196
|
+
if (typeof body[field] !== 'number' || !Number.isFinite(body[field])) {
|
|
197
|
+
return res.status(400).send({ success: false, error: field + " must be a number" });
|
|
198
|
+
}
|
|
199
|
+
payload[field] = Math.min(Math.floor(body[field]), cap);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if (body.crawl_file_types !== undefined && body.crawl_file_types !== null) {
|
|
203
|
+
if (!Array.isArray(body.crawl_file_types) || !body.crawl_file_types.every((t) => typeof t === 'string')) {
|
|
204
|
+
return res.status(400).send({ success: false, error: "crawl_file_types must be an array of strings" });
|
|
205
|
+
}
|
|
206
|
+
payload.crawl_file_types = body.crawl_file_types;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const timeoutMs = ((payload.max_seconds ?? CRAWL_DISCOVER_DEFAULT_SECONDS) + CRAWL_DISCOVER_TIMEOUT_MARGIN_SECONDS) * 1000;
|
|
210
|
+
|
|
211
|
+
try {
|
|
212
|
+
const resbody = await aiService.crawlDiscover(payload, timeoutMs);
|
|
213
|
+
return res.status(200).send(resbody.data);
|
|
214
|
+
} catch (err) {
|
|
215
|
+
winston.error("/crawl/discover error: ", sanitizeAxiosError(err));
|
|
216
|
+
if (err.code === 'ECONNABORTED' || err.code === 'ETIMEDOUT') {
|
|
217
|
+
return res.status(504).send({ success: false, error: "discover_timeout" });
|
|
218
|
+
}
|
|
219
|
+
const status = err.response?.status;
|
|
220
|
+
if (status === 404) {
|
|
221
|
+
return res.status(501).send({ success: false, error: "crawl_discover_unavailable" });
|
|
222
|
+
}
|
|
223
|
+
if (status === 400 || status === 422) {
|
|
224
|
+
return res.status(400).send({ success: false, error: sanitizeAxiosError(err).message });
|
|
225
|
+
}
|
|
226
|
+
return res.status(502).send({ success: false, error: sanitizeAxiosError(err).message });
|
|
227
|
+
}
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
router.post('/scrape/single', freezePineconeWrites, async (req, res) => {
|
|
156
231
|
|
|
157
232
|
let project_id = req.projectid;
|
|
158
233
|
|
|
@@ -453,7 +528,7 @@ router.post('/qa', async (req, res) => {
|
|
|
453
528
|
data.reranking = {
|
|
454
529
|
"provider": "pinecone",
|
|
455
530
|
"api_key": process.env.PINECONE_API_KEY,
|
|
456
|
-
"
|
|
531
|
+
"name": process.env.PINECONE_RERANKING_MODEL || process.env.RERANKING_MODEL || "bge-reranker-v2-m3"
|
|
457
532
|
}
|
|
458
533
|
|
|
459
534
|
if (!data.reranking_multiplier) {
|
|
@@ -868,7 +943,7 @@ router.post('/qa', async (req, res) => {
|
|
|
868
943
|
// })
|
|
869
944
|
// })
|
|
870
945
|
|
|
871
|
-
router.delete('/delete', async (req, res) => {
|
|
946
|
+
router.delete('/delete', freezePineconeWrites, async (req, res) => {
|
|
872
947
|
|
|
873
948
|
let project_id = req.projectid;
|
|
874
949
|
let data = req.body;
|
|
@@ -895,7 +970,7 @@ router.delete('/delete', async (req, res) => {
|
|
|
895
970
|
|
|
896
971
|
})
|
|
897
972
|
|
|
898
|
-
router.delete('/deleteall', async (req, res) => {
|
|
973
|
+
router.delete('/deleteall', freezePineconeWrites, async (req, res) => {
|
|
899
974
|
|
|
900
975
|
let project_id = req.projectid;
|
|
901
976
|
let data = req.body;
|
|
@@ -1214,7 +1289,7 @@ router.post('/namespace', async (req, res) => {
|
|
|
1214
1289
|
})
|
|
1215
1290
|
})
|
|
1216
1291
|
|
|
1217
|
-
router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, res) => {
|
|
1292
|
+
router.post('/namespace/import/:id', freezePineconeWrites, upload.single('uploadFile'), async (req, res) => {
|
|
1218
1293
|
|
|
1219
1294
|
let id_project = req.projectid;
|
|
1220
1295
|
let namespace_id = req.params.id;
|
|
@@ -1439,7 +1514,7 @@ router.put('/namespace/:id', async (req, res) => {
|
|
|
1439
1514
|
})
|
|
1440
1515
|
})
|
|
1441
1516
|
|
|
1442
|
-
router.delete('/namespace/:id', async (req, res) => {
|
|
1517
|
+
router.delete('/namespace/:id', freezePineconeWrites, async (req, res) => {
|
|
1443
1518
|
|
|
1444
1519
|
let project_id = req.projectid;
|
|
1445
1520
|
let namespace_id = req.params.id;
|
|
@@ -1692,7 +1767,7 @@ router.get('/:kb_id', async (req, res) => {
|
|
|
1692
1767
|
|
|
1693
1768
|
})
|
|
1694
1769
|
|
|
1695
|
-
router.post('/', async (req, res) => {
|
|
1770
|
+
router.post('/', freezePineconeWrites, async (req, res) => {
|
|
1696
1771
|
|
|
1697
1772
|
const id_project = req.projectid;
|
|
1698
1773
|
const project = req.project
|
|
@@ -1819,7 +1894,7 @@ router.post('/', async (req, res) => {
|
|
|
1819
1894
|
|
|
1820
1895
|
})
|
|
1821
1896
|
|
|
1822
|
-
router.post('/multi', upload.single('uploadFile'), async (req, res) => {
|
|
1897
|
+
router.post('/multi', freezePineconeWrites, upload.single('uploadFile'), async (req, res) => {
|
|
1823
1898
|
|
|
1824
1899
|
let list;
|
|
1825
1900
|
if (req.file) {
|
|
@@ -1896,7 +1971,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
|
|
|
1896
1971
|
|
|
1897
1972
|
})
|
|
1898
1973
|
|
|
1899
|
-
router.post('/csv', upload.single('uploadFile'), async (req, res) => {
|
|
1974
|
+
router.post('/csv', freezePineconeWrites, upload.single('uploadFile'), async (req, res) => {
|
|
1900
1975
|
|
|
1901
1976
|
let project_id = req.projectid;
|
|
1902
1977
|
let namespace_id = req.query.namespace;
|
|
@@ -2040,7 +2115,7 @@ router.post('/sitemap', async (req, res) => {
|
|
|
2040
2115
|
|
|
2041
2116
|
})
|
|
2042
2117
|
|
|
2043
|
-
router.post('/sitemap/import', async (req, res) => {
|
|
2118
|
+
router.post('/sitemap/import', freezePineconeWrites, async (req, res) => {
|
|
2044
2119
|
|
|
2045
2120
|
const id_project = req.projectid;
|
|
2046
2121
|
const namespace_id = req.query.namespace;
|
|
@@ -2160,7 +2235,7 @@ router.post('/sitemap/import', async (req, res) => {
|
|
|
2160
2235
|
|
|
2161
2236
|
})
|
|
2162
2237
|
|
|
2163
|
-
router.put('/:kb_id', async (req, res) => {
|
|
2238
|
+
router.put('/:kb_id', freezePineconeWrites, async (req, res) => {
|
|
2164
2239
|
|
|
2165
2240
|
const id_project = req.projectid;
|
|
2166
2241
|
const project = req.project;
|
|
@@ -2315,7 +2390,7 @@ router.put('/:kb_id', async (req, res) => {
|
|
|
2315
2390
|
|
|
2316
2391
|
})
|
|
2317
2392
|
|
|
2318
|
-
router.delete('/:kb_id', async (req, res) => {
|
|
2393
|
+
router.delete('/:kb_id', freezePineconeWrites, async (req, res) => {
|
|
2319
2394
|
|
|
2320
2395
|
let project_id = req.projectid;
|
|
2321
2396
|
let kb_id = req.params.kb_id;
|
package/routes/users.js
CHANGED
|
@@ -7,6 +7,7 @@ var winston = require('../config/winston');
|
|
|
7
7
|
const authEvent = require('../event/authEvent');
|
|
8
8
|
const uuidv4 = require('uuid/v4');
|
|
9
9
|
var uniqid = require('uniqid');
|
|
10
|
+
const { validatePassword } = require('../utils/passwordPolicy');
|
|
10
11
|
|
|
11
12
|
router.put('/', function (req, res) {
|
|
12
13
|
|
|
@@ -127,24 +128,21 @@ router.put('/changepsw', function (req, res) {
|
|
|
127
128
|
// check if password matches
|
|
128
129
|
|
|
129
130
|
if (req.body.oldpsw) {
|
|
130
|
-
winston.debug('CHANGE PSW - OLD PSW: ', req.body.oldpsw);
|
|
131
|
-
|
|
132
131
|
user.comparePassword(req.body.oldpsw, function (err, isMatch) {
|
|
133
132
|
if (isMatch && !err) {
|
|
134
133
|
// if user is found and old password is right
|
|
135
134
|
winston.debug('* THE PSW MATCH CURRENT PSW * PROCEED WITH THE UPDATE')
|
|
136
|
-
winston.debug('CHANGE PSW - NEW PSW: ', req.body.newpsw);
|
|
137
135
|
|
|
138
136
|
if (req.body.newpsw === req.body.oldpsw) {
|
|
139
137
|
winston.warn("New password can't match the old one");
|
|
140
138
|
return res.status(403).send({ success: false, message: "The new password must be different from the previous one."})
|
|
141
139
|
}
|
|
142
140
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
141
|
+
const passwordCheck = validatePassword(req.body.newpsw);
|
|
142
|
+
if (!passwordCheck.valid) {
|
|
143
|
+
return res.status(403).send({ success: false, message: passwordCheck.message });
|
|
144
|
+
}
|
|
145
|
+
|
|
148
146
|
user.password = req.body.newpsw
|
|
149
147
|
|
|
150
148
|
user.save(function (err, saveUser) {
|
package/routes/webhook.js
CHANGED
|
@@ -10,6 +10,7 @@ const webhookService = require('../services/webhookService');
|
|
|
10
10
|
const webhookEvent = require('../event/webhookEvent');
|
|
11
11
|
const errorCodes = require('../errorCodes');
|
|
12
12
|
const aiManager = require('../services/aiManager');
|
|
13
|
+
const { freezePineconeWrites } = require('../middleware/freezePineconeWrites');
|
|
13
14
|
var ObjectId = require('mongoose').Types.ObjectId;
|
|
14
15
|
const default_embedding = require('../config/kb/embedding');
|
|
15
16
|
const webhookSettingsValidator = require('../services/webhookSettingsValidator');
|
|
@@ -104,7 +105,7 @@ let default_engine_hybrid = {
|
|
|
104
105
|
index_name: process.env.PINECONE_INDEX_HYBRID
|
|
105
106
|
}
|
|
106
107
|
|
|
107
|
-
router.post('/kb/reindex', async (req, res) => {
|
|
108
|
+
router.post('/kb/reindex', freezePineconeWrites, async (req, res) => {
|
|
108
109
|
|
|
109
110
|
winston.verbose("/kb/reindex webhook called")
|
|
110
111
|
winston.debug("(webhook) req.body: ", req.body);
|
package/services/aiService.js
CHANGED
|
@@ -177,6 +177,22 @@ class AiService {
|
|
|
177
177
|
})
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
+
/**
|
|
181
|
+
* Lists the pages of a site through the llm crawler (no indexing).
|
|
182
|
+
* timeoutMs is passed by the caller: a crawl can legitimately run for minutes.
|
|
183
|
+
*/
|
|
184
|
+
crawlDiscover(data, timeoutMs) {
|
|
185
|
+
return axios({
|
|
186
|
+
url: kb_endpoint_train + "/crawl/discover",
|
|
187
|
+
headers: {
|
|
188
|
+
'Content-Type': 'application/json'
|
|
189
|
+
},
|
|
190
|
+
data: data,
|
|
191
|
+
method: 'POST',
|
|
192
|
+
timeout: timeoutMs
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
|
|
180
196
|
scrapeStatus(data) {
|
|
181
197
|
let base_url = kb_endpoint_train;
|
|
182
198
|
winston.debug("[OPENAI SERVICE] kb endpoint: " + base_url);
|
|
@@ -210,7 +210,9 @@ class ChatbotService {
|
|
|
210
210
|
await this.assertPublishable(sourceId, project_id);
|
|
211
211
|
|
|
212
212
|
// Keep callsubagent.botId on the development subagent. Its url points at the published snapshot.
|
|
213
|
-
//
|
|
213
|
+
// The fork already saves the copy with trashed: true (no trashedAt, so the TTL does not
|
|
214
|
+
// apply). for_publish skips FAQ_KB_CREATE; FAQ_KB_PUBLISH is emitted below.
|
|
215
|
+
// The update still sets root_id, the release and publishedAt, and unsets the slug.
|
|
214
216
|
let forked;
|
|
215
217
|
try {
|
|
216
218
|
forked = await this.fork(sourceId, api_url, token, project_id, { skipSubagents: true, forPublish: true });
|
package/services/faqService.js
CHANGED
|
@@ -74,7 +74,7 @@ class FaqService {
|
|
|
74
74
|
// });
|
|
75
75
|
// }
|
|
76
76
|
|
|
77
|
-
async create(id_project, id_user, data) {
|
|
77
|
+
async create(id_project, id_user, data, createOptions) {
|
|
78
78
|
|
|
79
79
|
if (!this.validateSubagent(data)) {
|
|
80
80
|
throw new Error("parent_id is required for subtype 'subagent'");
|
|
@@ -92,7 +92,7 @@ class FaqService {
|
|
|
92
92
|
}
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
-
let chatbot = this.createChatbotObject(id_project, id_user, data);
|
|
95
|
+
let chatbot = this.createChatbotObject(id_project, id_user, data, createOptions);
|
|
96
96
|
|
|
97
97
|
let template;
|
|
98
98
|
if (data.type === "internal" || data.type === "tilebot") {
|
|
@@ -146,7 +146,7 @@ class FaqService {
|
|
|
146
146
|
};
|
|
147
147
|
}
|
|
148
148
|
|
|
149
|
-
createChatbotObject(id_project, id_user, data) {
|
|
149
|
+
createChatbotObject(id_project, id_user, data, createOptions) {
|
|
150
150
|
return new Faq_kb({
|
|
151
151
|
name: data.name,
|
|
152
152
|
slug: data.slug,
|
|
@@ -162,7 +162,10 @@ class FaqService {
|
|
|
162
162
|
certified: false,
|
|
163
163
|
mainCategory: data.mainCategory,
|
|
164
164
|
intentsEngine: data.intentsEngine,
|
|
165
|
-
trashed
|
|
165
|
+
// A publish fork is born trashed, so a copy left behind when the following
|
|
166
|
+
// update never lands stays out of the chatbot list. No trashedAt: that field
|
|
167
|
+
// is the TTL, and a published snapshot must not expire.
|
|
168
|
+
trashed: !!(createOptions && createOptions.trashed === true),
|
|
166
169
|
createdBy: id_user,
|
|
167
170
|
updatedBy: id_user,
|
|
168
171
|
attributes: data.attributes,
|
|
@@ -21,6 +21,7 @@ var jwt = require('jsonwebtoken');
|
|
|
21
21
|
var config = require('../config/database'); // get db config file
|
|
22
22
|
var faqService = require('../services/faqService');
|
|
23
23
|
var Subscription = require('../models/subscription');
|
|
24
|
+
var Project_user = require('../models/project_user');
|
|
24
25
|
|
|
25
26
|
let log = false;
|
|
26
27
|
|
|
@@ -557,6 +558,163 @@ describe('AuthenticationJWT', () => {
|
|
|
557
558
|
});
|
|
558
559
|
});
|
|
559
560
|
|
|
561
|
+
it('signinWithCustomToken-whatsapp-bottest-uses-body-project', (done) => {
|
|
562
|
+
|
|
563
|
+
var email = "test-signup-" + Date.now() + "@email.com";
|
|
564
|
+
var pwd = "pwd";
|
|
565
|
+
|
|
566
|
+
userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
|
|
567
|
+
projectService.create("test-bottest-official", savedUser._id).then(function (officialProject) {
|
|
568
|
+
projectService.create("test-bottest-customer", savedUser._id).then(function (customerProject) {
|
|
569
|
+
var subscription = new Subscription({
|
|
570
|
+
event: "whatsapp.bottest.signin",
|
|
571
|
+
target: "https://whatsapp.example.com/tiledesk/test",
|
|
572
|
+
id_project: "*",
|
|
573
|
+
global: true,
|
|
574
|
+
createdBy: savedUser._id
|
|
575
|
+
});
|
|
576
|
+
subscription.save(function (err, savedSub) {
|
|
577
|
+
if (err) { console.error("err: ", err); return done(err); }
|
|
578
|
+
|
|
579
|
+
Subscription.findById(savedSub._id).select('+secret').exec(function (err, withSecret) {
|
|
580
|
+
if (err) { return done(err); }
|
|
581
|
+
|
|
582
|
+
var externalId = "wab-39333" + Date.now();
|
|
583
|
+
var payload = {
|
|
584
|
+
_id: externalId,
|
|
585
|
+
first_name: "Whatsapp",
|
|
586
|
+
last_name: "Tester",
|
|
587
|
+
sub: "userexternal",
|
|
588
|
+
aud: "https://tiledesk.com/subscriptions/" + withSecret._id
|
|
589
|
+
};
|
|
590
|
+
var jwtToken = jwt.sign(payload, withSecret.secret);
|
|
591
|
+
|
|
592
|
+
chai.request(server)
|
|
593
|
+
.post('/auth/signinWithCustomToken')
|
|
594
|
+
.set('Authorization', 'JWT ' + jwtToken)
|
|
595
|
+
.send({ id_project: customerProject._id.toString() })
|
|
596
|
+
.end((err, res) => {
|
|
597
|
+
if (err) { console.error("err: ", err); }
|
|
598
|
+
if (log) { console.log("res.body", res.body); }
|
|
599
|
+
|
|
600
|
+
res.should.have.status(200);
|
|
601
|
+
Project_user.findOne({ id_project: customerProject._id, uuid_user: externalId }).exec(function (findErr, projectUser) {
|
|
602
|
+
if (findErr) { return done(findErr); }
|
|
603
|
+
expect(projectUser).to.not.equal(null);
|
|
604
|
+
Project_user.findOne({ id_project: officialProject._id, uuid_user: externalId }).exec(function (findErr2, officialUser) {
|
|
605
|
+
if (findErr2) { return done(findErr2); }
|
|
606
|
+
expect(officialUser).to.equal(null);
|
|
607
|
+
done();
|
|
608
|
+
});
|
|
609
|
+
});
|
|
610
|
+
});
|
|
611
|
+
});
|
|
612
|
+
});
|
|
613
|
+
});
|
|
614
|
+
});
|
|
615
|
+
});
|
|
616
|
+
}).timeout(20000);
|
|
617
|
+
|
|
618
|
+
it('signinWithCustomToken-whatsapp-bottest-requires-project', (done) => {
|
|
619
|
+
|
|
620
|
+
var email = "test-signup-" + Date.now() + "@email.com";
|
|
621
|
+
var pwd = "pwd";
|
|
622
|
+
|
|
623
|
+
userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
|
|
624
|
+
var subscription = new Subscription({
|
|
625
|
+
event: "whatsapp.bottest.signin",
|
|
626
|
+
target: "https://whatsapp.example.com/tiledesk/test",
|
|
627
|
+
id_project: "*",
|
|
628
|
+
global: true,
|
|
629
|
+
createdBy: savedUser._id
|
|
630
|
+
});
|
|
631
|
+
subscription.save(function (err, savedSub) {
|
|
632
|
+
if (err) { console.error("err: ", err); return done(err); }
|
|
633
|
+
|
|
634
|
+
Subscription.findById(savedSub._id).select('+secret').exec(function (err, withSecret) {
|
|
635
|
+
if (err) { return done(err); }
|
|
636
|
+
|
|
637
|
+
var payload = {
|
|
638
|
+
_id: "wab-39333" + Date.now(),
|
|
639
|
+
first_name: "Whatsapp",
|
|
640
|
+
last_name: "Tester",
|
|
641
|
+
sub: "userexternal",
|
|
642
|
+
aud: "https://tiledesk.com/subscriptions/" + withSecret._id
|
|
643
|
+
};
|
|
644
|
+
var jwtToken = jwt.sign(payload, withSecret.secret);
|
|
645
|
+
|
|
646
|
+
chai.request(server)
|
|
647
|
+
.post('/auth/signinWithCustomToken')
|
|
648
|
+
.set('Authorization', 'JWT ' + jwtToken)
|
|
649
|
+
.send({})
|
|
650
|
+
.end((err, res) => {
|
|
651
|
+
if (err) { console.error("err: ", err); }
|
|
652
|
+
if (log) { console.log("res.body", res.body); }
|
|
653
|
+
|
|
654
|
+
res.should.have.status(400);
|
|
655
|
+
done();
|
|
656
|
+
});
|
|
657
|
+
});
|
|
658
|
+
});
|
|
659
|
+
});
|
|
660
|
+
}).timeout(20000);
|
|
661
|
+
|
|
662
|
+
it('signinWithCustomToken-regular-subscription-ignores-body-project', (done) => {
|
|
663
|
+
|
|
664
|
+
var email = "test-signup-" + Date.now() + "@email.com";
|
|
665
|
+
var pwd = "pwd";
|
|
666
|
+
|
|
667
|
+
userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
|
|
668
|
+
projectService.create("test-sub-home", savedUser._id).then(function (projectA) {
|
|
669
|
+
projectService.create("test-sub-other", savedUser._id).then(function (projectB) {
|
|
670
|
+
var subscription = new Subscription({
|
|
671
|
+
event: "message.create.request.channel.whatsapp." + Date.now(),
|
|
672
|
+
target: "https://hooks.example.com/hook",
|
|
673
|
+
id_project: projectA._id,
|
|
674
|
+
createdBy: savedUser._id
|
|
675
|
+
});
|
|
676
|
+
subscription.save(function (err, savedSub) {
|
|
677
|
+
if (err) { console.error("err: ", err); return done(err); }
|
|
678
|
+
|
|
679
|
+
Subscription.findById(savedSub._id).select('+secret').exec(function (err, withSecret) {
|
|
680
|
+
if (err) { return done(err); }
|
|
681
|
+
|
|
682
|
+
var externalId = "wab-39334" + Date.now();
|
|
683
|
+
var payload = {
|
|
684
|
+
_id: externalId,
|
|
685
|
+
first_name: "Whatsapp",
|
|
686
|
+
last_name: "Tester",
|
|
687
|
+
sub: "userexternal",
|
|
688
|
+
aud: "https://tiledesk.com/subscriptions/" + withSecret._id
|
|
689
|
+
};
|
|
690
|
+
var jwtToken = jwt.sign(payload, withSecret.secret);
|
|
691
|
+
|
|
692
|
+
chai.request(server)
|
|
693
|
+
.post('/auth/signinWithCustomToken')
|
|
694
|
+
.set('Authorization', 'JWT ' + jwtToken)
|
|
695
|
+
.send({ id_project: projectB._id.toString() })
|
|
696
|
+
.end((err, res) => {
|
|
697
|
+
if (err) { console.error("err: ", err); }
|
|
698
|
+
if (log) { console.log("res.body", res.body); }
|
|
699
|
+
|
|
700
|
+
res.should.have.status(200);
|
|
701
|
+
Project_user.findOne({ id_project: projectA._id, uuid_user: externalId }).exec(function (findErr, homeUser) {
|
|
702
|
+
if (findErr) { return done(findErr); }
|
|
703
|
+
expect(homeUser).to.not.equal(null);
|
|
704
|
+
Project_user.findOne({ id_project: projectB._id, uuid_user: externalId }).exec(function (findErr2, otherUser) {
|
|
705
|
+
if (findErr2) { return done(findErr2); }
|
|
706
|
+
expect(otherUser).to.equal(null);
|
|
707
|
+
done();
|
|
708
|
+
});
|
|
709
|
+
});
|
|
710
|
+
});
|
|
711
|
+
});
|
|
712
|
+
});
|
|
713
|
+
});
|
|
714
|
+
});
|
|
715
|
+
});
|
|
716
|
+
}).timeout(20000);
|
|
717
|
+
|
|
560
718
|
});
|
|
561
719
|
|
|
562
720
|
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
process.env.NODE_ENV = 'test';
|
|
2
|
+
|
|
3
|
+
const { expect } = require('chai');
|
|
4
|
+
const { vectorStoreWritesFrozen, freezePineconeWrites } = require('../middleware/freezePineconeWrites');
|
|
5
|
+
|
|
6
|
+
describe('freezePineconeWrites', () => {
|
|
7
|
+
const previous = process.env.PINECONE_WRITE_DISABLED;
|
|
8
|
+
|
|
9
|
+
afterEach(() => {
|
|
10
|
+
if (previous === undefined) {
|
|
11
|
+
delete process.env.PINECONE_WRITE_DISABLED;
|
|
12
|
+
} else {
|
|
13
|
+
process.env.PINECONE_WRITE_DISABLED = previous;
|
|
14
|
+
}
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it('stays open when the variable is absent or false', () => {
|
|
18
|
+
delete process.env.PINECONE_WRITE_DISABLED;
|
|
19
|
+
expect(vectorStoreWritesFrozen()).to.equal(false);
|
|
20
|
+
|
|
21
|
+
process.env.PINECONE_WRITE_DISABLED = 'false';
|
|
22
|
+
expect(vectorStoreWritesFrozen()).to.equal(false);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('answers 503 and does not call next when PINECONE_WRITE_DISABLED=true', () => {
|
|
26
|
+
process.env.PINECONE_WRITE_DISABLED = 'true';
|
|
27
|
+
let nextCalled = false;
|
|
28
|
+
const headers = {};
|
|
29
|
+
const res = {
|
|
30
|
+
set(name, value) {
|
|
31
|
+
headers[name] = value;
|
|
32
|
+
return this;
|
|
33
|
+
},
|
|
34
|
+
status(code) {
|
|
35
|
+
this.statusCode = code;
|
|
36
|
+
return this;
|
|
37
|
+
},
|
|
38
|
+
send(body) {
|
|
39
|
+
this.body = body;
|
|
40
|
+
return this;
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
freezePineconeWrites({ method: 'POST', originalUrl: '/kb' }, res, () => {
|
|
45
|
+
nextCalled = true;
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
expect(nextCalled).to.equal(false);
|
|
49
|
+
expect(res.statusCode).to.equal(503);
|
|
50
|
+
expect(headers['Retry-After']).to.equal('3600');
|
|
51
|
+
expect(res.body.code).to.equal('vector_store_writes_disabled');
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it('lets the request through when writes are enabled', () => {
|
|
55
|
+
delete process.env.PINECONE_WRITE_DISABLED;
|
|
56
|
+
let nextCalled = false;
|
|
57
|
+
freezePineconeWrites({ method: 'POST', originalUrl: '/kb' }, {}, () => {
|
|
58
|
+
nextCalled = true;
|
|
59
|
+
});
|
|
60
|
+
expect(nextCalled).to.equal(true);
|
|
61
|
+
});
|
|
62
|
+
});
|
package/test/kbRoute.js
CHANGED
|
@@ -2131,6 +2131,142 @@ describe('KbRoute', () => {
|
|
|
2131
2131
|
|
|
2132
2132
|
|
|
2133
2133
|
|
|
2134
|
+
describe('/crawl/discover', () => {
|
|
2135
|
+
const aiService = require('../services/aiService');
|
|
2136
|
+
let email, pwd, projectId;
|
|
2137
|
+
const llmUrl = () => process.env.KB_ENDPOINT_TRAIN;
|
|
2138
|
+
|
|
2139
|
+
before(async () => {
|
|
2140
|
+
email = "test-signup-" + Date.now() + "@email.com";
|
|
2141
|
+
pwd = "pwd";
|
|
2142
|
+
const savedUser = await userService.signup(email, pwd, "Test Firstname", "Test lastname");
|
|
2143
|
+
const savedProject = await projectService.create("test-crawl-discover", savedUser._id);
|
|
2144
|
+
projectId = savedProject._id;
|
|
2145
|
+
});
|
|
2146
|
+
|
|
2147
|
+
afterEach(() => { nock.cleanAll(); });
|
|
2148
|
+
|
|
2149
|
+
const discover = (body) => chai.request(server)
|
|
2150
|
+
.post('/' + projectId + '/kb/crawl/discover')
|
|
2151
|
+
.auth(email, pwd)
|
|
2152
|
+
.send(body);
|
|
2153
|
+
|
|
2154
|
+
it('forwards only the whitelisted fields, capped', async () => {
|
|
2155
|
+
const llmBody = { pages: [{ url: "https://ex.org/a", title: "A", kind: "html", file_type: null, depth: 1 }], truncated: false, fetched: 1, skipped: 0 };
|
|
2156
|
+
let received;
|
|
2157
|
+
const scope = nock(llmUrl())
|
|
2158
|
+
.post('/crawl/discover', (b) => { received = b; return true; })
|
|
2159
|
+
.reply(200, llmBody);
|
|
2160
|
+
|
|
2161
|
+
const res = await discover({ source: "https://ex.org", max_pages: 900, max_depth: 9, max_seconds: 999, ignore_robots: true, browser_headers: { a: "b" } });
|
|
2162
|
+
|
|
2163
|
+
res.should.have.status(200);
|
|
2164
|
+
expect(res.body).to.deep.equal(llmBody);
|
|
2165
|
+
expect(received).to.deep.equal({ source: "https://ex.org", max_pages: 300, max_depth: 5, max_seconds: 300 });
|
|
2166
|
+
scope.done();
|
|
2167
|
+
});
|
|
2168
|
+
|
|
2169
|
+
it('omits fields the caller did not give and forwards crawl_file_types', async () => {
|
|
2170
|
+
let received;
|
|
2171
|
+
nock(llmUrl()).post('/crawl/discover', (b) => { received = b; return true; })
|
|
2172
|
+
.reply(200, { pages: [], truncated: false, fetched: 0, skipped: 0 });
|
|
2173
|
+
|
|
2174
|
+
const res = await discover({ source: "https://ex.org", crawl_file_types: ["pdf", "docx"] });
|
|
2175
|
+
|
|
2176
|
+
res.should.have.status(200);
|
|
2177
|
+
expect(received).to.deep.equal({ source: "https://ex.org", crawl_file_types: ["pdf", "docx"] });
|
|
2178
|
+
});
|
|
2179
|
+
|
|
2180
|
+
it('axios timeout is (max_seconds ?? 300) + 120 seconds', async () => {
|
|
2181
|
+
const original = aiService.crawlDiscover;
|
|
2182
|
+
const timeouts = [];
|
|
2183
|
+
aiService.crawlDiscover = (data, timeoutMs) => { timeouts.push(timeoutMs); return Promise.resolve({ data: { pages: [], truncated: false, fetched: 0, skipped: 0 } }); };
|
|
2184
|
+
try {
|
|
2185
|
+
await discover({ source: "https://ex.org" });
|
|
2186
|
+
await discover({ source: "https://ex.org", max_seconds: 60 });
|
|
2187
|
+
await discover({ source: "https://ex.org", max_seconds: 5000 });
|
|
2188
|
+
} finally {
|
|
2189
|
+
aiService.crawlDiscover = original;
|
|
2190
|
+
}
|
|
2191
|
+
expect(timeouts).to.deep.equal([420000, 180000, 420000]);
|
|
2192
|
+
});
|
|
2193
|
+
|
|
2194
|
+
it('caps max_seconds at KB_CRAWL_DISCOVER_MAX_SECONDS, never above the llm maximum of 1800', async () => {
|
|
2195
|
+
const original = process.env.KB_CRAWL_DISCOVER_MAX_SECONDS;
|
|
2196
|
+
const received = [];
|
|
2197
|
+
nock(llmUrl()).post('/crawl/discover', (b) => { received.push(b.max_seconds); return true; }).times(2)
|
|
2198
|
+
.reply(200, { pages: [], truncated: false, fetched: 0, skipped: 0 });
|
|
2199
|
+
try {
|
|
2200
|
+
process.env.KB_CRAWL_DISCOVER_MAX_SECONDS = "600";
|
|
2201
|
+
(await discover({ source: "https://ex.org", max_seconds: 9999 })).should.have.status(200);
|
|
2202
|
+
process.env.KB_CRAWL_DISCOVER_MAX_SECONDS = "5000";
|
|
2203
|
+
(await discover({ source: "https://ex.org", max_seconds: 9999 })).should.have.status(200);
|
|
2204
|
+
} finally {
|
|
2205
|
+
if (original === undefined) delete process.env.KB_CRAWL_DISCOVER_MAX_SECONDS;
|
|
2206
|
+
else process.env.KB_CRAWL_DISCOVER_MAX_SECONDS = original;
|
|
2207
|
+
}
|
|
2208
|
+
expect(received).to.deep.equal([600, 1800]);
|
|
2209
|
+
});
|
|
2210
|
+
|
|
2211
|
+
it('refuses a non-http source', async () => {
|
|
2212
|
+
const scope = nock(llmUrl()).post('/crawl/discover').reply(200, {});
|
|
2213
|
+
const res = await discover({ source: "ftp://x" });
|
|
2214
|
+
res.should.have.status(400);
|
|
2215
|
+
expect(res.body.success).to.equal(false);
|
|
2216
|
+
expect(scope.isDone()).to.equal(false);
|
|
2217
|
+
});
|
|
2218
|
+
|
|
2219
|
+
it('refuses a missing source, a bad number and bad crawl_file_types', async () => {
|
|
2220
|
+
const scope = nock(llmUrl()).post('/crawl/discover').reply(200, {});
|
|
2221
|
+
(await discover({})).should.have.status(400);
|
|
2222
|
+
(await discover({ source: "https://ex.org", max_pages: "lots" })).should.have.status(400);
|
|
2223
|
+
(await discover({ source: "https://ex.org", crawl_file_types: "pdf" })).should.have.status(400);
|
|
2224
|
+
(await discover({ source: "https://ex.org", crawl_file_types: [1] })).should.have.status(400);
|
|
2225
|
+
expect(scope.isDone()).to.equal(false);
|
|
2226
|
+
});
|
|
2227
|
+
|
|
2228
|
+
it('maps llm 404 to 501 crawl_discover_unavailable', async () => {
|
|
2229
|
+
nock(llmUrl()).post('/crawl/discover').reply(404, { detail: "Not Found" });
|
|
2230
|
+
const res = await discover({ source: "https://ex.org" });
|
|
2231
|
+
res.should.have.status(501);
|
|
2232
|
+
expect(res.body).to.deep.equal({ success: false, error: "crawl_discover_unavailable" });
|
|
2233
|
+
});
|
|
2234
|
+
|
|
2235
|
+
it('maps llm 400 detail to 400', async () => {
|
|
2236
|
+
nock(llmUrl()).post('/crawl/discover').reply(400, { detail: "start url refused" });
|
|
2237
|
+
const res = await discover({ source: "https://ex.org" });
|
|
2238
|
+
res.should.have.status(400);
|
|
2239
|
+
expect(res.body).to.deep.equal({ success: false, error: "start url refused" });
|
|
2240
|
+
});
|
|
2241
|
+
|
|
2242
|
+
it('maps llm 422 to 400', async () => {
|
|
2243
|
+
nock(llmUrl()).post('/crawl/discover').reply(422, { detail: [{ msg: "max_seconds too small" }] });
|
|
2244
|
+
const res = await discover({ source: "https://ex.org", max_seconds: 1 });
|
|
2245
|
+
res.should.have.status(400);
|
|
2246
|
+
expect(res.body.success).to.equal(false);
|
|
2247
|
+
});
|
|
2248
|
+
|
|
2249
|
+
it('maps llm 500 to 502', async () => {
|
|
2250
|
+
nock(llmUrl()).post('/crawl/discover').reply(500, { detail: "boom" });
|
|
2251
|
+
const res = await discover({ source: "https://ex.org" });
|
|
2252
|
+
res.should.have.status(502);
|
|
2253
|
+
expect(res.body.success).to.equal(false);
|
|
2254
|
+
});
|
|
2255
|
+
|
|
2256
|
+
it('maps an unreachable llm to 502', async () => {
|
|
2257
|
+
nock(llmUrl()).post('/crawl/discover').replyWithError({ message: "connect ECONNREFUSED", code: "ECONNREFUSED" });
|
|
2258
|
+
const res = await discover({ source: "https://ex.org" });
|
|
2259
|
+
res.should.have.status(502);
|
|
2260
|
+
});
|
|
2261
|
+
|
|
2262
|
+
it('maps a timeout to 504', async () => {
|
|
2263
|
+
nock(llmUrl()).post('/crawl/discover').replyWithError({ message: "timeout of 420000ms exceeded", code: "ECONNABORTED" });
|
|
2264
|
+
const res = await discover({ source: "https://ex.org" });
|
|
2265
|
+
res.should.have.status(504);
|
|
2266
|
+
expect(res.body).to.deep.equal({ success: false, error: "discover_timeout" });
|
|
2267
|
+
});
|
|
2268
|
+
});
|
|
2269
|
+
|
|
2134
2270
|
describe('Unanswered Questions', () => {
|
|
2135
2271
|
|
|
2136
2272
|
it('add-unanswered-question', (done) => {
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
const { expect } = require('chai');
|
|
2
|
+
const { validatePassword } = require('../utils/passwordPolicy');
|
|
3
|
+
|
|
4
|
+
describe('passwordPolicy', () => {
|
|
5
|
+
|
|
6
|
+
it('rejects a one-character password', () => {
|
|
7
|
+
expect(validatePassword('a').valid).to.equal(false);
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
it('rejects passwords shorter than 8 characters', () => {
|
|
11
|
+
expect(validatePassword('Ab1!xyz').valid).to.equal(false);
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
it('rejects a password without a lowercase letter', () => {
|
|
15
|
+
expect(validatePassword('ABCDEF1!').valid).to.equal(false);
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it('rejects a password without an uppercase letter', () => {
|
|
19
|
+
expect(validatePassword('abcdef1!').valid).to.equal(false);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it('rejects a password without a number', () => {
|
|
23
|
+
expect(validatePassword('Abcdefg!').valid).to.equal(false);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it('rejects a password without a required special character', () => {
|
|
27
|
+
expect(validatePassword('Abcdefg1').valid).to.equal(false);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('rejects a non-string password', () => {
|
|
31
|
+
expect(validatePassword(1).valid).to.equal(false);
|
|
32
|
+
expect(validatePassword(null).valid).to.equal(false);
|
|
33
|
+
expect(validatePassword(undefined).valid).to.equal(false);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('accepts a password that meets every rule', () => {
|
|
37
|
+
expect(validatePassword('Abcdef1!').valid).to.equal(true);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it('accepts + as a special character, matching the dashboard strength meter', () => {
|
|
41
|
+
expect(validatePassword('Abcdef1+').valid).to.equal(true);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('rejects a password longer than 512 characters', () => {
|
|
45
|
+
expect(validatePassword('Abcdef1+' + 'a'.repeat(505)).valid).to.equal(false);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('accepts characters outside the required classes', () => {
|
|
49
|
+
expect(validatePassword('Abcdef1! รจร -_').valid).to.equal(true);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it('treats case as significant', () => {
|
|
53
|
+
expect(validatePassword('ABCDEF1!').valid).to.equal(false);
|
|
54
|
+
expect(validatePassword('Abcdef1!').valid).to.equal(true);
|
|
55
|
+
});
|
|
56
|
+
});
|
|
@@ -245,7 +245,13 @@ describe('Publish fails loud', () => {
|
|
|
245
245
|
const res = await publish(ctx, bot._id);
|
|
246
246
|
res.should.have.status(200);
|
|
247
247
|
expect(res.body.message).to.equal("Chatbot published successfully");
|
|
248
|
-
|
|
248
|
+
const copyBlocks = await blocksOf(res.body.bot_id);
|
|
249
|
+
expect(copyBlocks).to.have.length(draftBlocks.length);
|
|
250
|
+
expect(copyBlocks.every((block) => block.trashed !== true)).to.equal(true);
|
|
251
|
+
const copy = await Faq_kb.findById(res.body.bot_id).lean().exec();
|
|
252
|
+
expect(copy.trashed).to.equal(true);
|
|
253
|
+
expect(copy.trashedAt).to.equal(undefined);
|
|
254
|
+
expect(copy.root_id).to.equal(String(bot._id));
|
|
249
255
|
const draft = await Faq_kb.findById(bot._id).lean().exec();
|
|
250
256
|
expect(draft.url).to.match(new RegExp(res.body.bot_id + '$'));
|
|
251
257
|
}).timeout(30000);
|
|
@@ -259,7 +265,13 @@ describe('Publish fails loud', () => {
|
|
|
259
265
|
const res = await publish(ctx, bot._id);
|
|
260
266
|
res.should.have.status(200);
|
|
261
267
|
expect(res.body.message).to.equal("Chatbot published successfully");
|
|
262
|
-
|
|
268
|
+
const copyBlocks = await blocksOf(res.body.bot_id);
|
|
269
|
+
expect(copyBlocks).to.have.length(draftBlocks.length);
|
|
270
|
+
expect(copyBlocks.every((block) => block.trashed !== true)).to.equal(true);
|
|
271
|
+
const copy = await Faq_kb.findById(res.body.bot_id).lean().exec();
|
|
272
|
+
expect(copy.trashed).to.equal(true);
|
|
273
|
+
expect(copy.trashedAt).to.equal(undefined);
|
|
274
|
+
expect(copy.root_id).to.equal(String(bot._id));
|
|
263
275
|
expect(await publishedCopies(ctx, bot._id)).to.have.length(1);
|
|
264
276
|
}).timeout(30000);
|
|
265
277
|
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Same rules as tiledesk-dashboard password-strength:
|
|
4
|
+
// lowercase, uppercase, digit, and the symbol class used by the strength meter.
|
|
5
|
+
// Signup, change password and reset also cap the length at 512.
|
|
6
|
+
// Any other character is allowed; the symbol class is not a whitelist.
|
|
7
|
+
const MIN_LENGTH = 8;
|
|
8
|
+
const MAX_LENGTH = 512;
|
|
9
|
+
const SPECIAL_CHARACTERS = /[$-/:-?{-~!"^@#`\[\]]/;
|
|
10
|
+
|
|
11
|
+
const PASSWORD_POLICY_MESSAGE = 'Password must be 8 to 512 characters and include a lowercase letter, an uppercase letter, a number, and a special character.';
|
|
12
|
+
|
|
13
|
+
function validatePassword(password) {
|
|
14
|
+
if (typeof password !== 'string') {
|
|
15
|
+
return { valid: false, message: PASSWORD_POLICY_MESSAGE };
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const valid = password.length >= MIN_LENGTH
|
|
19
|
+
&& password.length <= MAX_LENGTH
|
|
20
|
+
&& /[a-z]/.test(password)
|
|
21
|
+
&& /[A-Z]/.test(password)
|
|
22
|
+
&& /[0-9]/.test(password)
|
|
23
|
+
&& SPECIAL_CHARACTERS.test(password);
|
|
24
|
+
|
|
25
|
+
if (!valid) {
|
|
26
|
+
return { valid: false, message: PASSWORD_POLICY_MESSAGE };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
return { valid: true };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
module.exports = {
|
|
33
|
+
validatePassword,
|
|
34
|
+
PASSWORD_POLICY_MESSAGE,
|
|
35
|
+
MIN_LENGTH,
|
|
36
|
+
MAX_LENGTH
|
|
37
|
+
};
|