@tiledesk/tiledesk-server 2.10.51 → 2.10.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/app.js +2 -7
- package/jobs.js +0 -5
- package/models/kb_setting.js +8 -0
- package/package.json +2 -3
- package/routes/faq.js +31 -20
- package/routes/kb.js +14 -16
- package/routes/kbsettings.js +4 -4
- package/routes/openai.js +2 -2
- package/routes/request.js +1 -1
- package/routes/webhook.js +134 -1
- package/services/QuoteManager.js +37 -10
- package/services/aiReindexService.js +151 -0
- package/services/{openaiService.js → aiService.js} +3 -3
- package/services/requestService.js +449 -435
- package/test/faqRoute.js +865 -746
- package/test/kbRoute.js +74 -2
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,13 @@
|
|
|
5
5
|
🚀 IN PRODUCTION 🚀
|
|
6
6
|
(https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77)
|
|
7
7
|
|
|
8
|
+
# 2.10.53
|
|
9
|
+
- added voice quota duration
|
|
10
|
+
|
|
11
|
+
# 2.10.52
|
|
12
|
+
- updated tybot-connector to 0.2.148
|
|
13
|
+
- updated faqRoute /get endpoint with restricted mode
|
|
14
|
+
|
|
8
15
|
# 2.10.51
|
|
9
16
|
- updated /replace endpoint adding trashed: false inside query
|
|
10
17
|
|
package/app.js
CHANGED
|
@@ -199,11 +199,6 @@ let whatsappQueue = require('@tiledesk/tiledesk-whatsapp-jobworker');
|
|
|
199
199
|
winston.info("whatsappQueue");
|
|
200
200
|
jobsManager.listenWhatsappQueue(whatsappQueue);
|
|
201
201
|
|
|
202
|
-
// let trainingQueue = require('@tiledesk/tiledesk-train-jobworker');
|
|
203
|
-
// winston.info("trainingQueue");
|
|
204
|
-
// jobsManager.listenTrainingQueue(trainingQueue);
|
|
205
|
-
|
|
206
|
-
|
|
207
202
|
var channelManager = require('./channels/channelManager');
|
|
208
203
|
channelManager.listen();
|
|
209
204
|
|
|
@@ -557,8 +552,8 @@ app.use('/:projectid/tags', [passport.authenticate(['basic', 'jwt'], { session:
|
|
|
557
552
|
app.use('/:projectid/subscriptions', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRole('admin')], resthook);
|
|
558
553
|
|
|
559
554
|
//deprecated
|
|
560
|
-
app.use('/:projectid/faq', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken
|
|
561
|
-
app.use('/:projectid/intents', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken
|
|
555
|
+
app.use('/:projectid/faq', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken], faq);
|
|
556
|
+
app.use('/:projectid/intents', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken], faq);
|
|
562
557
|
|
|
563
558
|
//Deprecated??
|
|
564
559
|
app.use('/:projectid/faqpub', faqpub);
|
package/jobs.js
CHANGED
|
@@ -104,11 +104,6 @@ async function main()
|
|
|
104
104
|
winston.info("whatsappQueue");
|
|
105
105
|
jobsManager.listenWhatsappQueue(whatsappQueue);
|
|
106
106
|
|
|
107
|
-
// let trainingQueue = require('@tiledesk/tiledesk-train-jobworker');
|
|
108
|
-
// winston.info("trainingQueue");
|
|
109
|
-
// jobsManager.listenTrainingQueue(trainingQueue);
|
|
110
|
-
|
|
111
|
-
|
|
112
107
|
let scheduler = require('./pubmodules/scheduler');
|
|
113
108
|
jobsManager.listenScheduler(scheduler);
|
|
114
109
|
|
package/models/kb_setting.js
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tiledesk/tiledesk-server",
|
|
3
3
|
"description": "The Tiledesk server module",
|
|
4
|
-
"version": "2.10.
|
|
4
|
+
"version": "2.10.53",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"start": "node ./bin/www",
|
|
7
7
|
"pretest": "mongodb-runner start",
|
|
@@ -47,8 +47,7 @@
|
|
|
47
47
|
"@tiledesk/tiledesk-messenger-connector": "^0.1.23",
|
|
48
48
|
"@tiledesk/tiledesk-rasa-connector": "^1.0.10",
|
|
49
49
|
"@tiledesk/tiledesk-telegram-connector": "^0.1.14",
|
|
50
|
-
"@tiledesk/tiledesk-
|
|
51
|
-
"@tiledesk/tiledesk-tybot-connector": "^0.2.147",
|
|
50
|
+
"@tiledesk/tiledesk-tybot-connector": "^0.2.148",
|
|
52
51
|
"@tiledesk/tiledesk-whatsapp-connector": "^0.1.76",
|
|
53
52
|
"@tiledesk/tiledesk-whatsapp-jobworker": "^0.0.11",
|
|
54
53
|
"@tiledesk/tiledesk-sms-connector": "^0.1.11",
|
package/routes/faq.js
CHANGED
|
@@ -16,11 +16,12 @@ csv.separator = ';';
|
|
|
16
16
|
const axios = require("axios").default;
|
|
17
17
|
var configGlobal = require('../config/global');
|
|
18
18
|
const roleConstants = require('../models/roleConstants');
|
|
19
|
+
const roleChecker = require('../middleware/has-role');
|
|
19
20
|
|
|
20
21
|
const apiUrl = process.env.API_URL || configGlobal.apiUrl;
|
|
21
22
|
|
|
22
23
|
// POST CSV FILE UPLOAD FROM CLIENT
|
|
23
|
-
router.post('/uploadcsv', upload.single('uploadFile'), function (req, res, next) {
|
|
24
|
+
router.post('/uploadcsv', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), upload.single('uploadFile'), function (req, res, next) {
|
|
24
25
|
winston.debug(' -> -> REQ BODY ', req.body);
|
|
25
26
|
winston.debug(' -> ID FAQ-KB ', req.body.id_faq_kb);
|
|
26
27
|
winston.debug(' -> DELIMITER ', req.body.delimiter);
|
|
@@ -138,7 +139,7 @@ router.post('/uploadcsv', upload.single('uploadFile'), function (req, res, next)
|
|
|
138
139
|
});
|
|
139
140
|
|
|
140
141
|
|
|
141
|
-
router.post('/', function (req, res) {
|
|
142
|
+
router.post('/', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
|
142
143
|
|
|
143
144
|
winston.debug(req.body);
|
|
144
145
|
|
|
@@ -205,7 +206,7 @@ router.post('/', function (req, res) {
|
|
|
205
206
|
});
|
|
206
207
|
});
|
|
207
208
|
|
|
208
|
-
router.post('/ops_update', async (req, res) => {
|
|
209
|
+
router.post('/ops_update', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), async (req, res) => {
|
|
209
210
|
|
|
210
211
|
let id_faq_kb = req.body.id_faq_kb;
|
|
211
212
|
let operations = req.body.operations;
|
|
@@ -321,7 +322,7 @@ router.post('/ops_update', async (req, res) => {
|
|
|
321
322
|
|
|
322
323
|
})
|
|
323
324
|
|
|
324
|
-
router.patch('/:faqid/attributes', function (req, res) {
|
|
325
|
+
router.patch('/:faqid/attributes', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
|
325
326
|
let data = req.body;
|
|
326
327
|
winston.debug("data: ", data);
|
|
327
328
|
|
|
@@ -374,7 +375,7 @@ router.patch('/:faqid/attributes', function (req, res) {
|
|
|
374
375
|
})
|
|
375
376
|
})
|
|
376
377
|
|
|
377
|
-
router.put('/:faqid', function (req, res) {
|
|
378
|
+
router.put('/:faqid', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
|
378
379
|
|
|
379
380
|
winston.debug('UPDATE FAQ ', req.body);
|
|
380
381
|
let faqid = req.params.faqid;
|
|
@@ -470,7 +471,7 @@ router.put('/:faqid', function (req, res) {
|
|
|
470
471
|
|
|
471
472
|
|
|
472
473
|
// DELETE REMOTE AND LOCAL FAQ
|
|
473
|
-
router.delete('/:faqid', function (req, res) {
|
|
474
|
+
router.delete('/:faqid', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
|
474
475
|
|
|
475
476
|
winston.debug('DELETE FAQ - FAQ ID ', req.params.faqid);
|
|
476
477
|
|
|
@@ -521,7 +522,7 @@ router.delete('/:faqid', function (req, res) {
|
|
|
521
522
|
});
|
|
522
523
|
|
|
523
524
|
// EXPORT FAQ TO CSV
|
|
524
|
-
router.get('/csv', function (req, res) {
|
|
525
|
+
router.get('/csv', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
|
525
526
|
var query = {};
|
|
526
527
|
|
|
527
528
|
winston.debug('req.query', req.query);
|
|
@@ -554,7 +555,7 @@ router.get('/csv', function (req, res) {
|
|
|
554
555
|
});
|
|
555
556
|
|
|
556
557
|
|
|
557
|
-
router.get('/:faqid', function (req, res) {
|
|
558
|
+
router.get('/:faqid', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
|
558
559
|
|
|
559
560
|
winston.debug(req.body);
|
|
560
561
|
|
|
@@ -570,12 +571,18 @@ router.get('/:faqid', function (req, res) {
|
|
|
570
571
|
});
|
|
571
572
|
|
|
572
573
|
|
|
573
|
-
|
|
574
|
-
router.get('/', function (req, res, next) {
|
|
574
|
+
router.get('/', roleChecker.hasRoleOrTypes('agent', ['bot', 'subscription']), function (req, res, next) {
|
|
575
575
|
var query = {};
|
|
576
576
|
|
|
577
577
|
winston.debug("GET ALL FAQ OF THE BOT ID (req.query): ", req.query);
|
|
578
578
|
|
|
579
|
+
let restricted_mode;
|
|
580
|
+
|
|
581
|
+
let project_user = req.projectuser;
|
|
582
|
+
if (project_user && project_user.role === roleConstants.AGENT) {
|
|
583
|
+
restricted_mode = true;
|
|
584
|
+
}
|
|
585
|
+
|
|
579
586
|
if (req.query.id_faq_kb) {
|
|
580
587
|
query.id_faq_kb = req.query.id_faq_kb;
|
|
581
588
|
}
|
|
@@ -610,8 +617,7 @@ router.get('/', function (req, res, next) {
|
|
|
610
617
|
query.intent_display_name = req.query.intent_display_name
|
|
611
618
|
}
|
|
612
619
|
|
|
613
|
-
|
|
614
|
-
if (project_user && project_user.role === roleConstants.AGENT) {
|
|
620
|
+
if (restricted_mode) {
|
|
615
621
|
query.agents_available = {
|
|
616
622
|
$in: [ null, true ]
|
|
617
623
|
}
|
|
@@ -631,21 +637,26 @@ router.get('/', function (req, res, next) {
|
|
|
631
637
|
// console.log("result: ", result);
|
|
632
638
|
// })
|
|
633
639
|
|
|
634
|
-
return Faq.find(query)
|
|
635
|
-
skip(skip).limit(limit)
|
|
636
|
-
populate({ path: 'faq_kb' })
|
|
637
|
-
.
|
|
638
|
-
|
|
640
|
+
return Faq.find(query)
|
|
641
|
+
.skip(skip).limit(limit)
|
|
642
|
+
.populate({ path: 'faq_kb' })
|
|
643
|
+
.lean()
|
|
644
|
+
.exec(function (err, faqs) {
|
|
645
|
+
|
|
646
|
+
winston.debug("GET FAQ ", faqs);
|
|
639
647
|
|
|
640
648
|
if (err) {
|
|
641
649
|
winston.debug('GET FAQ err ', err)
|
|
642
650
|
return next(err)
|
|
643
651
|
};
|
|
644
|
-
winston.debug('GET FAQ ', faq)
|
|
645
|
-
res.json(faq);
|
|
646
652
|
|
|
647
|
-
|
|
653
|
+
if (restricted_mode === true) {
|
|
654
|
+
faqs = faqs.map(({ webhook_enabled, faq_kb, actions, attributes, createdBy, createdAt, updatedAt, __v, ...keepAttrs }) => keepAttrs)
|
|
655
|
+
faqs = faqs.filter(f => (f.intent_display_name !== "start" && f.intent_display_name !== 'defaultFallback'));
|
|
656
|
+
}
|
|
648
657
|
|
|
658
|
+
res.status(200).send(faqs);
|
|
659
|
+
});
|
|
649
660
|
|
|
650
661
|
});
|
|
651
662
|
|
package/routes/kb.js
CHANGED
|
@@ -6,7 +6,7 @@ var router = express.Router();
|
|
|
6
6
|
var winston = require('../config/winston');
|
|
7
7
|
var multer = require('multer')
|
|
8
8
|
var upload = multer()
|
|
9
|
-
const
|
|
9
|
+
const aiService = require('../services/aiService');
|
|
10
10
|
const JobManager = require('../utils/jobs-worker-queue-manager/JobManagerV2');
|
|
11
11
|
const { Scheduler } = require('../services/Scheduler');
|
|
12
12
|
var configGlobal = require('../config/global');
|
|
@@ -18,7 +18,6 @@ let Integration = require('../models/integrations');
|
|
|
18
18
|
var parsecsv = require("fast-csv");
|
|
19
19
|
|
|
20
20
|
const { MODELS_MULTIPLIER } = require('../utils/aiUtils');
|
|
21
|
-
const { body } = require('express-validator');
|
|
22
21
|
|
|
23
22
|
const AMQP_MANAGER_URL = process.env.AMQP_MANAGER_URL;
|
|
24
23
|
const JOB_TOPIC_EXCHANGE = process.env.JOB_TOPIC_EXCHANGE_TRAIN || 'tiledesk-trainer';
|
|
@@ -190,7 +189,7 @@ router.post('/scrape/status', async (req, res) => {
|
|
|
190
189
|
let ns = namespaces.find(n => n.id === data.namespace);
|
|
191
190
|
data.engine = ns.engine || default_engine;
|
|
192
191
|
|
|
193
|
-
|
|
192
|
+
aiService.scrapeStatus(data).then(async (response) => {
|
|
194
193
|
|
|
195
194
|
winston.debug("scrapeStatus response.data: ", response.data);
|
|
196
195
|
|
|
@@ -294,7 +293,7 @@ router.post('/qa', async (req, res) => {
|
|
|
294
293
|
return res.status(200).send({ success: true, message: "Question skipped in test environment"});
|
|
295
294
|
}
|
|
296
295
|
|
|
297
|
-
|
|
296
|
+
aiService.askNamespace(data).then((resp) => {
|
|
298
297
|
winston.debug("qa resp: ", resp.data);
|
|
299
298
|
let answer = resp.data;
|
|
300
299
|
|
|
@@ -353,7 +352,7 @@ router.delete('/delete', async (req, res) => {
|
|
|
353
352
|
let ns = namespaces.find(n => n.id === data.namespace);
|
|
354
353
|
data.engine = ns.engine || default_engine;
|
|
355
354
|
|
|
356
|
-
|
|
355
|
+
aiService.deleteIndex(data).then((resp) => {
|
|
357
356
|
winston.debug("delete resp: ", resp.data);
|
|
358
357
|
res.status(200).send(resp.data);
|
|
359
358
|
}).catch((err) => {
|
|
@@ -392,7 +391,7 @@ router.delete('/deleteall', async (req, res) => {
|
|
|
392
391
|
|
|
393
392
|
winston.verbose("/deleteall data: ", data);
|
|
394
393
|
|
|
395
|
-
|
|
394
|
+
aiService.deleteNamespace(data).then((resp) => {
|
|
396
395
|
winston.debug("delete namespace resp: ", resp.data);
|
|
397
396
|
res.status(200).send(resp.data);
|
|
398
397
|
}).catch((err) => {
|
|
@@ -497,7 +496,7 @@ router.get('/namespace/:id/chunks/:content_id', async (req, res) => {
|
|
|
497
496
|
return res.status(200).send({ success: true, message: "Get chunks skipped in test environment"});
|
|
498
497
|
}
|
|
499
498
|
|
|
500
|
-
|
|
499
|
+
aiService.getContentChunks(namespace_id, content_id, engine).then((resp) => {
|
|
501
500
|
let chunks = resp.data;
|
|
502
501
|
winston.debug("chunks for content " + content_id);
|
|
503
502
|
winston.debug("chunks found ", chunks);
|
|
@@ -659,7 +658,7 @@ router.delete('/namespace/:id', async (req, res) => {
|
|
|
659
658
|
|
|
660
659
|
if (req.query.contents_only && (req.query.contents_only === true || req.query.contents_only === 'true')) {
|
|
661
660
|
|
|
662
|
-
|
|
661
|
+
aiService.deleteNamespace(data).then(async (resp) => {
|
|
663
662
|
winston.debug("delete namespace resp: ", resp.data);
|
|
664
663
|
|
|
665
664
|
let deleteResponse = await KB.deleteMany({ id_project: project_id, namespace: namespace_id }).catch((err) => {
|
|
@@ -693,7 +692,7 @@ router.delete('/namespace/:id', async (req, res) => {
|
|
|
693
692
|
return res.status(403).send({ success: false, error: "Default namespace cannot be deleted" });
|
|
694
693
|
}
|
|
695
694
|
|
|
696
|
-
|
|
695
|
+
aiService.deleteNamespace(data).then(async (resp) => {
|
|
697
696
|
winston.debug("delete namespace resp: ", resp.data);
|
|
698
697
|
|
|
699
698
|
let deleteResponse = await KB.deleteMany({ id_project: project_id, namespace: namespace_id }).catch((err) => {
|
|
@@ -926,6 +925,7 @@ router.post('/', async (req, res) => {
|
|
|
926
925
|
new_kb.scrape_type = 1;
|
|
927
926
|
}
|
|
928
927
|
if (new_kb.type === 'url') {
|
|
928
|
+
new_kb.refresh = body.refresh;
|
|
929
929
|
if (!body.scrape_type || body.scrape_type === 2) {
|
|
930
930
|
new_kb.scrape_type = 2;
|
|
931
931
|
new_kb.scrape_options = await setDefaultScrapeOptions();
|
|
@@ -999,6 +999,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
|
|
|
999
999
|
let project_id = req.projectid;
|
|
1000
1000
|
let scrape_type = req.body.scrape_type;
|
|
1001
1001
|
let scrape_options = req.body.scrape_options;
|
|
1002
|
+
let refresh_rate = req.body.refresh_rate;
|
|
1002
1003
|
|
|
1003
1004
|
let namespace_id = req.query.namespace;
|
|
1004
1005
|
if (!namespace_id) {
|
|
@@ -1056,7 +1057,8 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
|
|
|
1056
1057
|
content: "",
|
|
1057
1058
|
namespace: namespace_id,
|
|
1058
1059
|
status: -1,
|
|
1059
|
-
scrape_type: scrape_type
|
|
1060
|
+
scrape_type: scrape_type,
|
|
1061
|
+
refresh_rate: refresh_rate
|
|
1060
1062
|
}
|
|
1061
1063
|
|
|
1062
1064
|
if (!kb.scrape_type) {
|
|
@@ -1316,7 +1318,7 @@ router.delete('/:kb_id', async (req, res) => {
|
|
|
1316
1318
|
|
|
1317
1319
|
winston.verbose("/:delete_id data: ", data);
|
|
1318
1320
|
|
|
1319
|
-
|
|
1321
|
+
aiService.deleteIndex(data).then((resp) => {
|
|
1320
1322
|
winston.debug("delete resp: ", resp.data);
|
|
1321
1323
|
if (resp.data.success === true) {
|
|
1322
1324
|
KB.findByIdAndDelete(kb_id, (err, deletedKb) => {
|
|
@@ -1432,9 +1434,6 @@ async function updateStatus(id, status) {
|
|
|
1432
1434
|
|
|
1433
1435
|
async function scheduleScrape(resources) {
|
|
1434
1436
|
|
|
1435
|
-
// let data = {
|
|
1436
|
-
// resources: resources
|
|
1437
|
-
// }
|
|
1438
1437
|
let scheduler = new Scheduler({ jobManager: jobManager });
|
|
1439
1438
|
|
|
1440
1439
|
resources.forEach(r => {
|
|
@@ -1451,7 +1450,6 @@ async function scheduleScrape(resources) {
|
|
|
1451
1450
|
});
|
|
1452
1451
|
})
|
|
1453
1452
|
|
|
1454
|
-
|
|
1455
1453
|
return true;
|
|
1456
1454
|
}
|
|
1457
1455
|
|
|
@@ -1470,7 +1468,7 @@ async function startScrape(data) {
|
|
|
1470
1468
|
winston.verbose("status of kb " + data.id + " updated: " + status_updated);
|
|
1471
1469
|
|
|
1472
1470
|
return new Promise((resolve, reject) => {
|
|
1473
|
-
|
|
1471
|
+
aiService.singleScrape(data).then(async (resp) => {
|
|
1474
1472
|
winston.debug("singleScrape resp: ", resp.data);
|
|
1475
1473
|
let status_updated = await updateStatus(data.id, 300);
|
|
1476
1474
|
winston.verbose("status of kb " + data.id + " updated: " + status_updated);
|
package/routes/kbsettings.js
CHANGED
|
@@ -4,7 +4,7 @@ var { KBSettings } = require('../models/kb_setting');
|
|
|
4
4
|
// var KB = require('../models/kb_setting')
|
|
5
5
|
var router = express.Router();
|
|
6
6
|
var winston = require('../config/winston');
|
|
7
|
-
const
|
|
7
|
+
const aiService = require('../services/aiService');
|
|
8
8
|
|
|
9
9
|
router.get('/', async (req, res) => {
|
|
10
10
|
let project_id = req.projectid;
|
|
@@ -105,7 +105,7 @@ router.post('/qa', async (req, res) => {
|
|
|
105
105
|
let data = req.body;
|
|
106
106
|
winston.debug("/qa data: ", data);
|
|
107
107
|
|
|
108
|
-
|
|
108
|
+
aiService.ask(data).then((resp) => {
|
|
109
109
|
winston.debug("qa resp: ", resp.data);
|
|
110
110
|
res.status(200).send(resp.data);
|
|
111
111
|
}).catch((err) => {
|
|
@@ -121,7 +121,7 @@ router.post('/startscrape', async (req, res) => {
|
|
|
121
121
|
let data = req.body;
|
|
122
122
|
winston.debug("/startscrape data: ", data);
|
|
123
123
|
|
|
124
|
-
|
|
124
|
+
aiService.startScrape(data).then((resp) => {
|
|
125
125
|
winston.debug("startScrape resp: ", resp.data);
|
|
126
126
|
res.status(200).send(resp.data);
|
|
127
127
|
}).catch((err) => {
|
|
@@ -143,7 +143,7 @@ router.post('/checkstatus', async (req, res) => {
|
|
|
143
143
|
url_list: [ req.body.full_url ]
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
-
|
|
146
|
+
aiService.checkStatus(data).then((resp) => {
|
|
147
147
|
winston.debug("checkStatus resp: ", resp);
|
|
148
148
|
winston.debug("checkStatus resp: ", resp.data);
|
|
149
149
|
winston.debug("checkStatus resp: ", resp.data[full_url]);
|
package/routes/openai.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
var express = require('express');
|
|
2
2
|
var router = express.Router();
|
|
3
3
|
var { KBSettings } = require('../models/kb_setting');
|
|
4
|
-
var
|
|
4
|
+
var aiService = require('../services/aiService');
|
|
5
5
|
var winston = require('../config/winston');
|
|
6
6
|
const { QuoteManager } = require('../services/QuoteManager');
|
|
7
7
|
const { MODELS_MULTIPLIER } = require('../utils/aiUtils');
|
|
@@ -70,7 +70,7 @@ router.post('/', async (req, res) => {
|
|
|
70
70
|
winston.info("No multiplier found for AI model")
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
-
|
|
73
|
+
aiService.completions(json, gptkey).then(async (response) => {
|
|
74
74
|
let data = { createdAt: new Date(), tokens: response.data.usage.total_tokens, multiplier: multiplier }
|
|
75
75
|
if (usePublicKey === true) {
|
|
76
76
|
let incremented_key = await quoteManager.incrementTokenCount(req.project, data);
|
package/routes/request.js
CHANGED
package/routes/webhook.js
CHANGED
|
@@ -1,9 +1,124 @@
|
|
|
1
1
|
var express = require('express');
|
|
2
2
|
var router = express.Router();
|
|
3
|
-
var { KB } = require('../models/kb_setting');
|
|
3
|
+
var { KB, Namespace } = require('../models/kb_setting');
|
|
4
4
|
var winston = require('../config/winston');
|
|
5
|
+
const JobManager = require('../utils/jobs-worker-queue-manager/JobManagerV2');
|
|
6
|
+
const { Scheduler } = require('../services/Scheduler');
|
|
7
|
+
const { AiReindexService } = require('../services/aiReindexService');
|
|
5
8
|
|
|
6
9
|
const KB_WEBHOOK_TOKEN = process.env.KB_WEBHOOK_TOKEN || 'kbcustomtoken';
|
|
10
|
+
const AMQP_MANAGER_URL = process.env.AMQP_MANAGER_URL;
|
|
11
|
+
const JOB_TOPIC_EXCHANGE = process.env.JOB_TOPIC_EXCHANGE_TRAIN || 'tiledesk-trainer';
|
|
12
|
+
|
|
13
|
+
let jobManager = new JobManager(AMQP_MANAGER_URL, {
|
|
14
|
+
debug: false,
|
|
15
|
+
topic: JOB_TOPIC_EXCHANGE,
|
|
16
|
+
exchange: JOB_TOPIC_EXCHANGE
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
jobManager.connectAndStartPublisher((status, error) => {
|
|
20
|
+
if (error) {
|
|
21
|
+
winston.error("connectAndStartPublisher error: ", error);
|
|
22
|
+
} else {
|
|
23
|
+
winston.info("KbRoute - ConnectPublisher done with status: ", status);
|
|
24
|
+
}
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
let default_engine = {
|
|
28
|
+
name: "pinecone",
|
|
29
|
+
type: process.env.PINECONE_TYPE,
|
|
30
|
+
apikey: "",
|
|
31
|
+
vector_size: 1536,
|
|
32
|
+
index_name: process.env.PINECONE_INDEX
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
router.post('/kb/reindex', async (req, res) => {
|
|
36
|
+
|
|
37
|
+
winston.verbose("/kb/reindex webhook called")
|
|
38
|
+
winston.debug("(webhook) req.body: ", req.body);
|
|
39
|
+
|
|
40
|
+
if (!req.headers['x-auth-token']) {
|
|
41
|
+
winston.error("(webhook) Unauthorized: A x-auth-token must be provided")
|
|
42
|
+
return res.status(401).send({ success: false, error: "Unauthorized", message: "A x-auth-token must be provided" })
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
if (req.headers['x-auth-token'] != KB_WEBHOOK_TOKEN) {
|
|
46
|
+
winston.error("(webhook) Unauthorized: You don't have the authorization to accomplish this operation")
|
|
47
|
+
return res.status(401).send({ success: false, error: "Unauthorized", message: "You don't have the authorization to accomplish this operation" });
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
let content_id = req.body.content_id;
|
|
51
|
+
|
|
52
|
+
let kb = await KB.findById(content_id).catch( async (err) => {
|
|
53
|
+
winston.error("(webhook) Error getting kb content: ", err);
|
|
54
|
+
return res.status(500).send({ success: false, error: "Error getting content with id " + content_id });
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
if (!kb) {
|
|
58
|
+
winston.warn("(webhook) Kb content not found with id " + content_id + ". Deleting scheduler...");
|
|
59
|
+
|
|
60
|
+
// Assuming the content has been deleted. The scheduler should be stopped and deleted.
|
|
61
|
+
res.status(200).send({ success: true, message: "Content no longer exists. Deleting scheduler..." })
|
|
62
|
+
|
|
63
|
+
setTimeout( async () => {
|
|
64
|
+
let aiReindexService = new AiReindexService();
|
|
65
|
+
let deleteResponse = await aiReindexService.delete(content_id).catch((err) => {
|
|
66
|
+
winston.error("(webhook) Error deleting scheduler ", err);
|
|
67
|
+
winston.error("(webhook) Error deleting scheduler " + err);
|
|
68
|
+
return;
|
|
69
|
+
});
|
|
70
|
+
winston.verbose("(webhook) delete response: ", deleteResponse);
|
|
71
|
+
return;
|
|
72
|
+
}, 10000);
|
|
73
|
+
|
|
74
|
+
} else {
|
|
75
|
+
|
|
76
|
+
let json = {
|
|
77
|
+
id: kb._id,
|
|
78
|
+
type: kb.type,
|
|
79
|
+
source: kb.source,
|
|
80
|
+
content: "",
|
|
81
|
+
namespace: kb.namespace,
|
|
82
|
+
refresh_rate: kb.refresh_rate,
|
|
83
|
+
last_refresh: kb.last_refresh
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
if (kb.scrape_type) {
|
|
87
|
+
json.scrape_type = kb.scrape_type
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (kb.scrape_options) {
|
|
91
|
+
json.parameters_scrape_type_4 = {
|
|
92
|
+
tags_to_extract: kb.scrape_options.tags_to_extract,
|
|
93
|
+
unwanted_tags: kb.scrape_options.unwanted_tags,
|
|
94
|
+
unwanted_classnames: kb.scrape_options.unwanted_classnames
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
let namespace = await Namespace.findOne({ id: kb.namespace }).catch((err) => {
|
|
99
|
+
winston.error("(webhook) Error getting namespace ", err)
|
|
100
|
+
return res.status(500).send({ success: false, error: err })
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
if (!namespace) {
|
|
104
|
+
winston.warn("(webhook) Namespace not found with id " + kb.namespace);
|
|
105
|
+
return res.status(500).send({ success: false, error: err })
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
json.engine = namespace.engine || default_engine;
|
|
109
|
+
|
|
110
|
+
let resources = [];
|
|
111
|
+
resources.push(json);
|
|
112
|
+
|
|
113
|
+
if (process.env.NODE_ENV !== 'test') {
|
|
114
|
+
scheduleScrape(resources);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
res.status(200).send({ success: true, message: "Content queued for reindexing" });
|
|
118
|
+
|
|
119
|
+
}
|
|
120
|
+
})
|
|
121
|
+
|
|
7
122
|
|
|
8
123
|
router.post('/kb/status', async (req, res) => {
|
|
9
124
|
|
|
@@ -52,4 +167,22 @@ router.post('/kb/status', async (req, res) => {
|
|
|
52
167
|
|
|
53
168
|
})
|
|
54
169
|
|
|
170
|
+
async function scheduleScrape(resources) {
|
|
171
|
+
|
|
172
|
+
let scheduler = new Scheduler({ jobManager: jobManager });
|
|
173
|
+
|
|
174
|
+
resources.forEach(r => {
|
|
175
|
+
winston.debug("(Webhook) Schedule job with following data: ", r);
|
|
176
|
+
scheduler.trainSchedule(r, async (err, result) => {
|
|
177
|
+
if (err) {
|
|
178
|
+
winston.error("Scheduling error: ", err);
|
|
179
|
+
} else {
|
|
180
|
+
winston.info("Scheduling result: ", result);
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
})
|
|
184
|
+
|
|
185
|
+
return true;
|
|
186
|
+
}
|
|
187
|
+
|
|
55
188
|
module.exports = router;
|
package/services/QuoteManager.js
CHANGED
|
@@ -16,15 +16,15 @@ const emailEvent = require('../event/emailEvent');
|
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
const PLANS_LIST = {
|
|
19
|
-
FREE_TRIAL: { requests: 200, messages: 0, tokens: 100000,
|
|
20
|
-
SANDBOX: { requests: 200, messages: 0, tokens: 100000,
|
|
21
|
-
BASIC: { requests: 800, messages: 0, tokens: 2000000,
|
|
22
|
-
PREMIUM: { requests: 3000, messages: 0, tokens: 5000000,
|
|
23
|
-
TEAM: { requests: 5000, messages: 0, tokens: 10000000,
|
|
24
|
-
CUSTOM: { requests: 5000, messages: 0, tokens: 10000000,
|
|
19
|
+
FREE_TRIAL: { requests: 200, messages: 0, tokens: 100000, voice_duration: 0, email: 200, chatbots: 20, namespace: 3, kbs: 50 }, // same as PREMIUM
|
|
20
|
+
SANDBOX: { requests: 200, messages: 0, tokens: 100000, voice_duration: 0, email: 200, chatbots: 2, namespace: 1, kbs: 50 },
|
|
21
|
+
BASIC: { requests: 800, messages: 0, tokens: 2000000, voice_duration: 0, email: 200, chatbots: 5, namespace: 1, kbs: 150 },
|
|
22
|
+
PREMIUM: { requests: 3000, messages: 0, tokens: 5000000, voice_duration: 0, email: 200, chatbots: 20, namespace: 3, kbs: 300 },
|
|
23
|
+
TEAM: { requests: 5000, messages: 0, tokens: 10000000, voice_duration: 0, email: 200, chatbots: 50, namespace: 10, kbs: 1000 },
|
|
24
|
+
CUSTOM: { requests: 5000, messages: 0, tokens: 10000000, voice_duration: 120000, email: 200, chatbots: 50, namespace: 10, kbs: 1000 },
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
const typesList = ['requests', 'messages', 'email', 'tokens', 'chatbots', 'kbs']
|
|
27
|
+
const typesList = ['requests', 'messages', 'email', 'tokens', 'voice_duration', 'chatbots', 'kbs']
|
|
28
28
|
|
|
29
29
|
let quotes_enabled = true;
|
|
30
30
|
|
|
@@ -95,6 +95,25 @@ class QuoteManager {
|
|
|
95
95
|
this.sendEmailIfQuotaExceeded(project, data, 'tokens', key);
|
|
96
96
|
return key;
|
|
97
97
|
}
|
|
98
|
+
|
|
99
|
+
async incrementVoiceDurationCount(project, request) {
|
|
100
|
+
|
|
101
|
+
this.project = project;
|
|
102
|
+
let key = await this.generateKey(request, 'voice_duration');
|
|
103
|
+
winston.verbose("[QuoteManager] incrementVoiceDurationCount key: " + key);
|
|
104
|
+
|
|
105
|
+
if (quotes_enabled === false) {
|
|
106
|
+
winston.debug("QUOTES DISABLED - incrementVoiceDurationCount")
|
|
107
|
+
return key;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (request?.duration) {
|
|
111
|
+
let duration = Math.round(request.duration / 1000); // from ms to s
|
|
112
|
+
await this.tdCache.incrby(key, duration);
|
|
113
|
+
|
|
114
|
+
this.sendEmailIfQuotaExceeded(project, request, 'voice_duration', key);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
98
117
|
// INCREMENT KEY SECTION - END
|
|
99
118
|
|
|
100
119
|
|
|
@@ -228,7 +247,7 @@ class QuoteManager {
|
|
|
228
247
|
winston.verbose("limits for current plan: ", limits)
|
|
229
248
|
|
|
230
249
|
let quote = await this.getCurrentQuote(project, object, type);
|
|
231
|
-
winston.verbose("getCurrentQuote resp: "
|
|
250
|
+
winston.verbose("getCurrentQuote resp: " + quote)
|
|
232
251
|
|
|
233
252
|
if (quote == null) {
|
|
234
253
|
return true;
|
|
@@ -494,13 +513,21 @@ class QuoteManager {
|
|
|
494
513
|
if (quotes_enabled === true) {
|
|
495
514
|
winston.verbose("request.create.quote event catched");
|
|
496
515
|
let result = await this.incrementRequestsCount(payload.project, payload.request);
|
|
497
|
-
|
|
498
|
-
|
|
499
516
|
return result;
|
|
500
517
|
} else {
|
|
501
518
|
winston.verbose("QUOTES DISABLED - request.create.quote event")
|
|
502
519
|
}
|
|
503
520
|
})
|
|
521
|
+
|
|
522
|
+
requestEvent.on('request.close.quote', async (payload) => {
|
|
523
|
+
if (quotes_enabled === true) {
|
|
524
|
+
winston.verbose("request.close.quote event catched");
|
|
525
|
+
let result = await this.incrementVoiceDurationCount(payload.project, payload.request);
|
|
526
|
+
return result;
|
|
527
|
+
} else {
|
|
528
|
+
winston.verbose("QUOTES DISABLED - request.close.quote event")
|
|
529
|
+
}
|
|
530
|
+
})
|
|
504
531
|
// REQUESTS EVENTS - END
|
|
505
532
|
|
|
506
533
|
|