@tiledesk/tiledesk-server 2.10.51 → 2.10.53
Sign up to get free protection for your applications and to get access to all the features.
- package/CHANGELOG.md +7 -0
- package/app.js +2 -7
- package/jobs.js +0 -5
- package/models/kb_setting.js +8 -0
- package/package.json +2 -3
- package/routes/faq.js +31 -20
- package/routes/kb.js +14 -16
- package/routes/kbsettings.js +4 -4
- package/routes/openai.js +2 -2
- package/routes/request.js +1 -1
- package/routes/webhook.js +134 -1
- package/services/QuoteManager.js +37 -10
- package/services/aiReindexService.js +151 -0
- package/services/{openaiService.js → aiService.js} +3 -3
- package/services/requestService.js +449 -435
- package/test/faqRoute.js +865 -746
- package/test/kbRoute.js +74 -2
package/CHANGELOG.md
CHANGED
@@ -5,6 +5,13 @@
|
|
5
5
|
🚀 IN PRODUCTION 🚀
|
6
6
|
(https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77)
|
7
7
|
|
8
|
+
# 2.10.53
|
9
|
+
- added voice quota duration
|
10
|
+
|
11
|
+
# 2.10.52
|
12
|
+
- updated tybot-connector to 0.2.148
|
13
|
+
- updated faqRoute /get endpoint with restricted mode
|
14
|
+
|
8
15
|
# 2.10.51
|
9
16
|
- updated /replace endpoint adding trashed: false inside query
|
10
17
|
|
package/app.js
CHANGED
@@ -199,11 +199,6 @@ let whatsappQueue = require('@tiledesk/tiledesk-whatsapp-jobworker');
|
|
199
199
|
winston.info("whatsappQueue");
|
200
200
|
jobsManager.listenWhatsappQueue(whatsappQueue);
|
201
201
|
|
202
|
-
// let trainingQueue = require('@tiledesk/tiledesk-train-jobworker');
|
203
|
-
// winston.info("trainingQueue");
|
204
|
-
// jobsManager.listenTrainingQueue(trainingQueue);
|
205
|
-
|
206
|
-
|
207
202
|
var channelManager = require('./channels/channelManager');
|
208
203
|
channelManager.listen();
|
209
204
|
|
@@ -557,8 +552,8 @@ app.use('/:projectid/tags', [passport.authenticate(['basic', 'jwt'], { session:
|
|
557
552
|
app.use('/:projectid/subscriptions', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRole('admin')], resthook);
|
558
553
|
|
559
554
|
//deprecated
|
560
|
-
app.use('/:projectid/faq', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken
|
561
|
-
app.use('/:projectid/intents', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken
|
555
|
+
app.use('/:projectid/faq', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken], faq);
|
556
|
+
app.use('/:projectid/intents', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken], faq);
|
562
557
|
|
563
558
|
//Deprecated??
|
564
559
|
app.use('/:projectid/faqpub', faqpub);
|
package/jobs.js
CHANGED
@@ -104,11 +104,6 @@ async function main()
|
|
104
104
|
winston.info("whatsappQueue");
|
105
105
|
jobsManager.listenWhatsappQueue(whatsappQueue);
|
106
106
|
|
107
|
-
// let trainingQueue = require('@tiledesk/tiledesk-train-jobworker');
|
108
|
-
// winston.info("trainingQueue");
|
109
|
-
// jobsManager.listenTrainingQueue(trainingQueue);
|
110
|
-
|
111
|
-
|
112
107
|
let scheduler = require('./pubmodules/scheduler');
|
113
108
|
jobsManager.listenScheduler(scheduler);
|
114
109
|
|
package/models/kb_setting.js
CHANGED
package/package.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
{
|
2
2
|
"name": "@tiledesk/tiledesk-server",
|
3
3
|
"description": "The Tiledesk server module",
|
4
|
-
"version": "2.10.
|
4
|
+
"version": "2.10.53",
|
5
5
|
"scripts": {
|
6
6
|
"start": "node ./bin/www",
|
7
7
|
"pretest": "mongodb-runner start",
|
@@ -47,8 +47,7 @@
|
|
47
47
|
"@tiledesk/tiledesk-messenger-connector": "^0.1.23",
|
48
48
|
"@tiledesk/tiledesk-rasa-connector": "^1.0.10",
|
49
49
|
"@tiledesk/tiledesk-telegram-connector": "^0.1.14",
|
50
|
-
"@tiledesk/tiledesk-
|
51
|
-
"@tiledesk/tiledesk-tybot-connector": "^0.2.147",
|
50
|
+
"@tiledesk/tiledesk-tybot-connector": "^0.2.148",
|
52
51
|
"@tiledesk/tiledesk-whatsapp-connector": "^0.1.76",
|
53
52
|
"@tiledesk/tiledesk-whatsapp-jobworker": "^0.0.11",
|
54
53
|
"@tiledesk/tiledesk-sms-connector": "^0.1.11",
|
package/routes/faq.js
CHANGED
@@ -16,11 +16,12 @@ csv.separator = ';';
|
|
16
16
|
const axios = require("axios").default;
|
17
17
|
var configGlobal = require('../config/global');
|
18
18
|
const roleConstants = require('../models/roleConstants');
|
19
|
+
const roleChecker = require('../middleware/has-role');
|
19
20
|
|
20
21
|
const apiUrl = process.env.API_URL || configGlobal.apiUrl;
|
21
22
|
|
22
23
|
// POST CSV FILE UPLOAD FROM CLIENT
|
23
|
-
router.post('/uploadcsv', upload.single('uploadFile'), function (req, res, next) {
|
24
|
+
router.post('/uploadcsv', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), upload.single('uploadFile'), function (req, res, next) {
|
24
25
|
winston.debug(' -> -> REQ BODY ', req.body);
|
25
26
|
winston.debug(' -> ID FAQ-KB ', req.body.id_faq_kb);
|
26
27
|
winston.debug(' -> DELIMITER ', req.body.delimiter);
|
@@ -138,7 +139,7 @@ router.post('/uploadcsv', upload.single('uploadFile'), function (req, res, next)
|
|
138
139
|
});
|
139
140
|
|
140
141
|
|
141
|
-
router.post('/', function (req, res) {
|
142
|
+
router.post('/', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
142
143
|
|
143
144
|
winston.debug(req.body);
|
144
145
|
|
@@ -205,7 +206,7 @@ router.post('/', function (req, res) {
|
|
205
206
|
});
|
206
207
|
});
|
207
208
|
|
208
|
-
router.post('/ops_update', async (req, res) => {
|
209
|
+
router.post('/ops_update', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), async (req, res) => {
|
209
210
|
|
210
211
|
let id_faq_kb = req.body.id_faq_kb;
|
211
212
|
let operations = req.body.operations;
|
@@ -321,7 +322,7 @@ router.post('/ops_update', async (req, res) => {
|
|
321
322
|
|
322
323
|
})
|
323
324
|
|
324
|
-
router.patch('/:faqid/attributes', function (req, res) {
|
325
|
+
router.patch('/:faqid/attributes', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
325
326
|
let data = req.body;
|
326
327
|
winston.debug("data: ", data);
|
327
328
|
|
@@ -374,7 +375,7 @@ router.patch('/:faqid/attributes', function (req, res) {
|
|
374
375
|
})
|
375
376
|
})
|
376
377
|
|
377
|
-
router.put('/:faqid', function (req, res) {
|
378
|
+
router.put('/:faqid', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
378
379
|
|
379
380
|
winston.debug('UPDATE FAQ ', req.body);
|
380
381
|
let faqid = req.params.faqid;
|
@@ -470,7 +471,7 @@ router.put('/:faqid', function (req, res) {
|
|
470
471
|
|
471
472
|
|
472
473
|
// DELETE REMOTE AND LOCAL FAQ
|
473
|
-
router.delete('/:faqid', function (req, res) {
|
474
|
+
router.delete('/:faqid', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
474
475
|
|
475
476
|
winston.debug('DELETE FAQ - FAQ ID ', req.params.faqid);
|
476
477
|
|
@@ -521,7 +522,7 @@ router.delete('/:faqid', function (req, res) {
|
|
521
522
|
});
|
522
523
|
|
523
524
|
// EXPORT FAQ TO CSV
|
524
|
-
router.get('/csv', function (req, res) {
|
525
|
+
router.get('/csv', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
525
526
|
var query = {};
|
526
527
|
|
527
528
|
winston.debug('req.query', req.query);
|
@@ -554,7 +555,7 @@ router.get('/csv', function (req, res) {
|
|
554
555
|
});
|
555
556
|
|
556
557
|
|
557
|
-
router.get('/:faqid', function (req, res) {
|
558
|
+
router.get('/:faqid', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscription']), function (req, res) {
|
558
559
|
|
559
560
|
winston.debug(req.body);
|
560
561
|
|
@@ -570,12 +571,18 @@ router.get('/:faqid', function (req, res) {
|
|
570
571
|
});
|
571
572
|
|
572
573
|
|
573
|
-
|
574
|
-
router.get('/', function (req, res, next) {
|
574
|
+
router.get('/', roleChecker.hasRoleOrTypes('agent', ['bot', 'subscription']), function (req, res, next) {
|
575
575
|
var query = {};
|
576
576
|
|
577
577
|
winston.debug("GET ALL FAQ OF THE BOT ID (req.query): ", req.query);
|
578
578
|
|
579
|
+
let restricted_mode;
|
580
|
+
|
581
|
+
let project_user = req.projectuser;
|
582
|
+
if (project_user && project_user.role === roleConstants.AGENT) {
|
583
|
+
restricted_mode = true;
|
584
|
+
}
|
585
|
+
|
579
586
|
if (req.query.id_faq_kb) {
|
580
587
|
query.id_faq_kb = req.query.id_faq_kb;
|
581
588
|
}
|
@@ -610,8 +617,7 @@ router.get('/', function (req, res, next) {
|
|
610
617
|
query.intent_display_name = req.query.intent_display_name
|
611
618
|
}
|
612
619
|
|
613
|
-
|
614
|
-
if (project_user && project_user.role === roleConstants.AGENT) {
|
620
|
+
if (restricted_mode) {
|
615
621
|
query.agents_available = {
|
616
622
|
$in: [ null, true ]
|
617
623
|
}
|
@@ -631,21 +637,26 @@ router.get('/', function (req, res, next) {
|
|
631
637
|
// console.log("result: ", result);
|
632
638
|
// })
|
633
639
|
|
634
|
-
return Faq.find(query)
|
635
|
-
skip(skip).limit(limit)
|
636
|
-
populate({ path: 'faq_kb' })
|
637
|
-
.
|
638
|
-
|
640
|
+
return Faq.find(query)
|
641
|
+
.skip(skip).limit(limit)
|
642
|
+
.populate({ path: 'faq_kb' })
|
643
|
+
.lean()
|
644
|
+
.exec(function (err, faqs) {
|
645
|
+
|
646
|
+
winston.debug("GET FAQ ", faqs);
|
639
647
|
|
640
648
|
if (err) {
|
641
649
|
winston.debug('GET FAQ err ', err)
|
642
650
|
return next(err)
|
643
651
|
};
|
644
|
-
winston.debug('GET FAQ ', faq)
|
645
|
-
res.json(faq);
|
646
652
|
|
647
|
-
|
653
|
+
if (restricted_mode === true) {
|
654
|
+
faqs = faqs.map(({ webhook_enabled, faq_kb, actions, attributes, createdBy, createdAt, updatedAt, __v, ...keepAttrs }) => keepAttrs)
|
655
|
+
faqs = faqs.filter(f => (f.intent_display_name !== "start" && f.intent_display_name !== 'defaultFallback'));
|
656
|
+
}
|
648
657
|
|
658
|
+
res.status(200).send(faqs);
|
659
|
+
});
|
649
660
|
|
650
661
|
});
|
651
662
|
|
package/routes/kb.js
CHANGED
@@ -6,7 +6,7 @@ var router = express.Router();
|
|
6
6
|
var winston = require('../config/winston');
|
7
7
|
var multer = require('multer')
|
8
8
|
var upload = multer()
|
9
|
-
const
|
9
|
+
const aiService = require('../services/aiService');
|
10
10
|
const JobManager = require('../utils/jobs-worker-queue-manager/JobManagerV2');
|
11
11
|
const { Scheduler } = require('../services/Scheduler');
|
12
12
|
var configGlobal = require('../config/global');
|
@@ -18,7 +18,6 @@ let Integration = require('../models/integrations');
|
|
18
18
|
var parsecsv = require("fast-csv");
|
19
19
|
|
20
20
|
const { MODELS_MULTIPLIER } = require('../utils/aiUtils');
|
21
|
-
const { body } = require('express-validator');
|
22
21
|
|
23
22
|
const AMQP_MANAGER_URL = process.env.AMQP_MANAGER_URL;
|
24
23
|
const JOB_TOPIC_EXCHANGE = process.env.JOB_TOPIC_EXCHANGE_TRAIN || 'tiledesk-trainer';
|
@@ -190,7 +189,7 @@ router.post('/scrape/status', async (req, res) => {
|
|
190
189
|
let ns = namespaces.find(n => n.id === data.namespace);
|
191
190
|
data.engine = ns.engine || default_engine;
|
192
191
|
|
193
|
-
|
192
|
+
aiService.scrapeStatus(data).then(async (response) => {
|
194
193
|
|
195
194
|
winston.debug("scrapeStatus response.data: ", response.data);
|
196
195
|
|
@@ -294,7 +293,7 @@ router.post('/qa', async (req, res) => {
|
|
294
293
|
return res.status(200).send({ success: true, message: "Question skipped in test environment"});
|
295
294
|
}
|
296
295
|
|
297
|
-
|
296
|
+
aiService.askNamespace(data).then((resp) => {
|
298
297
|
winston.debug("qa resp: ", resp.data);
|
299
298
|
let answer = resp.data;
|
300
299
|
|
@@ -353,7 +352,7 @@ router.delete('/delete', async (req, res) => {
|
|
353
352
|
let ns = namespaces.find(n => n.id === data.namespace);
|
354
353
|
data.engine = ns.engine || default_engine;
|
355
354
|
|
356
|
-
|
355
|
+
aiService.deleteIndex(data).then((resp) => {
|
357
356
|
winston.debug("delete resp: ", resp.data);
|
358
357
|
res.status(200).send(resp.data);
|
359
358
|
}).catch((err) => {
|
@@ -392,7 +391,7 @@ router.delete('/deleteall', async (req, res) => {
|
|
392
391
|
|
393
392
|
winston.verbose("/deleteall data: ", data);
|
394
393
|
|
395
|
-
|
394
|
+
aiService.deleteNamespace(data).then((resp) => {
|
396
395
|
winston.debug("delete namespace resp: ", resp.data);
|
397
396
|
res.status(200).send(resp.data);
|
398
397
|
}).catch((err) => {
|
@@ -497,7 +496,7 @@ router.get('/namespace/:id/chunks/:content_id', async (req, res) => {
|
|
497
496
|
return res.status(200).send({ success: true, message: "Get chunks skipped in test environment"});
|
498
497
|
}
|
499
498
|
|
500
|
-
|
499
|
+
aiService.getContentChunks(namespace_id, content_id, engine).then((resp) => {
|
501
500
|
let chunks = resp.data;
|
502
501
|
winston.debug("chunks for content " + content_id);
|
503
502
|
winston.debug("chunks found ", chunks);
|
@@ -659,7 +658,7 @@ router.delete('/namespace/:id', async (req, res) => {
|
|
659
658
|
|
660
659
|
if (req.query.contents_only && (req.query.contents_only === true || req.query.contents_only === 'true')) {
|
661
660
|
|
662
|
-
|
661
|
+
aiService.deleteNamespace(data).then(async (resp) => {
|
663
662
|
winston.debug("delete namespace resp: ", resp.data);
|
664
663
|
|
665
664
|
let deleteResponse = await KB.deleteMany({ id_project: project_id, namespace: namespace_id }).catch((err) => {
|
@@ -693,7 +692,7 @@ router.delete('/namespace/:id', async (req, res) => {
|
|
693
692
|
return res.status(403).send({ success: false, error: "Default namespace cannot be deleted" });
|
694
693
|
}
|
695
694
|
|
696
|
-
|
695
|
+
aiService.deleteNamespace(data).then(async (resp) => {
|
697
696
|
winston.debug("delete namespace resp: ", resp.data);
|
698
697
|
|
699
698
|
let deleteResponse = await KB.deleteMany({ id_project: project_id, namespace: namespace_id }).catch((err) => {
|
@@ -926,6 +925,7 @@ router.post('/', async (req, res) => {
|
|
926
925
|
new_kb.scrape_type = 1;
|
927
926
|
}
|
928
927
|
if (new_kb.type === 'url') {
|
928
|
+
new_kb.refresh = body.refresh;
|
929
929
|
if (!body.scrape_type || body.scrape_type === 2) {
|
930
930
|
new_kb.scrape_type = 2;
|
931
931
|
new_kb.scrape_options = await setDefaultScrapeOptions();
|
@@ -999,6 +999,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
|
|
999
999
|
let project_id = req.projectid;
|
1000
1000
|
let scrape_type = req.body.scrape_type;
|
1001
1001
|
let scrape_options = req.body.scrape_options;
|
1002
|
+
let refresh_rate = req.body.refresh_rate;
|
1002
1003
|
|
1003
1004
|
let namespace_id = req.query.namespace;
|
1004
1005
|
if (!namespace_id) {
|
@@ -1056,7 +1057,8 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
|
|
1056
1057
|
content: "",
|
1057
1058
|
namespace: namespace_id,
|
1058
1059
|
status: -1,
|
1059
|
-
scrape_type: scrape_type
|
1060
|
+
scrape_type: scrape_type,
|
1061
|
+
refresh_rate: refresh_rate
|
1060
1062
|
}
|
1061
1063
|
|
1062
1064
|
if (!kb.scrape_type) {
|
@@ -1316,7 +1318,7 @@ router.delete('/:kb_id', async (req, res) => {
|
|
1316
1318
|
|
1317
1319
|
winston.verbose("/:delete_id data: ", data);
|
1318
1320
|
|
1319
|
-
|
1321
|
+
aiService.deleteIndex(data).then((resp) => {
|
1320
1322
|
winston.debug("delete resp: ", resp.data);
|
1321
1323
|
if (resp.data.success === true) {
|
1322
1324
|
KB.findByIdAndDelete(kb_id, (err, deletedKb) => {
|
@@ -1432,9 +1434,6 @@ async function updateStatus(id, status) {
|
|
1432
1434
|
|
1433
1435
|
async function scheduleScrape(resources) {
|
1434
1436
|
|
1435
|
-
// let data = {
|
1436
|
-
// resources: resources
|
1437
|
-
// }
|
1438
1437
|
let scheduler = new Scheduler({ jobManager: jobManager });
|
1439
1438
|
|
1440
1439
|
resources.forEach(r => {
|
@@ -1451,7 +1450,6 @@ async function scheduleScrape(resources) {
|
|
1451
1450
|
});
|
1452
1451
|
})
|
1453
1452
|
|
1454
|
-
|
1455
1453
|
return true;
|
1456
1454
|
}
|
1457
1455
|
|
@@ -1470,7 +1468,7 @@ async function startScrape(data) {
|
|
1470
1468
|
winston.verbose("status of kb " + data.id + " updated: " + status_updated);
|
1471
1469
|
|
1472
1470
|
return new Promise((resolve, reject) => {
|
1473
|
-
|
1471
|
+
aiService.singleScrape(data).then(async (resp) => {
|
1474
1472
|
winston.debug("singleScrape resp: ", resp.data);
|
1475
1473
|
let status_updated = await updateStatus(data.id, 300);
|
1476
1474
|
winston.verbose("status of kb " + data.id + " updated: " + status_updated);
|
package/routes/kbsettings.js
CHANGED
@@ -4,7 +4,7 @@ var { KBSettings } = require('../models/kb_setting');
|
|
4
4
|
// var KB = require('../models/kb_setting')
|
5
5
|
var router = express.Router();
|
6
6
|
var winston = require('../config/winston');
|
7
|
-
const
|
7
|
+
const aiService = require('../services/aiService');
|
8
8
|
|
9
9
|
router.get('/', async (req, res) => {
|
10
10
|
let project_id = req.projectid;
|
@@ -105,7 +105,7 @@ router.post('/qa', async (req, res) => {
|
|
105
105
|
let data = req.body;
|
106
106
|
winston.debug("/qa data: ", data);
|
107
107
|
|
108
|
-
|
108
|
+
aiService.ask(data).then((resp) => {
|
109
109
|
winston.debug("qa resp: ", resp.data);
|
110
110
|
res.status(200).send(resp.data);
|
111
111
|
}).catch((err) => {
|
@@ -121,7 +121,7 @@ router.post('/startscrape', async (req, res) => {
|
|
121
121
|
let data = req.body;
|
122
122
|
winston.debug("/startscrape data: ", data);
|
123
123
|
|
124
|
-
|
124
|
+
aiService.startScrape(data).then((resp) => {
|
125
125
|
winston.debug("startScrape resp: ", resp.data);
|
126
126
|
res.status(200).send(resp.data);
|
127
127
|
}).catch((err) => {
|
@@ -143,7 +143,7 @@ router.post('/checkstatus', async (req, res) => {
|
|
143
143
|
url_list: [ req.body.full_url ]
|
144
144
|
}
|
145
145
|
|
146
|
-
|
146
|
+
aiService.checkStatus(data).then((resp) => {
|
147
147
|
winston.debug("checkStatus resp: ", resp);
|
148
148
|
winston.debug("checkStatus resp: ", resp.data);
|
149
149
|
winston.debug("checkStatus resp: ", resp.data[full_url]);
|
package/routes/openai.js
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
var express = require('express');
|
2
2
|
var router = express.Router();
|
3
3
|
var { KBSettings } = require('../models/kb_setting');
|
4
|
-
var
|
4
|
+
var aiService = require('../services/aiService');
|
5
5
|
var winston = require('../config/winston');
|
6
6
|
const { QuoteManager } = require('../services/QuoteManager');
|
7
7
|
const { MODELS_MULTIPLIER } = require('../utils/aiUtils');
|
@@ -70,7 +70,7 @@ router.post('/', async (req, res) => {
|
|
70
70
|
winston.info("No multiplier found for AI model")
|
71
71
|
}
|
72
72
|
|
73
|
-
|
73
|
+
aiService.completions(json, gptkey).then(async (response) => {
|
74
74
|
let data = { createdAt: new Date(), tokens: response.data.usage.total_tokens, multiplier: multiplier }
|
75
75
|
if (usePublicKey === true) {
|
76
76
|
let incremented_key = await quoteManager.incrementTokenCount(req.project, data);
|
package/routes/request.js
CHANGED
package/routes/webhook.js
CHANGED
@@ -1,9 +1,124 @@
|
|
1
1
|
var express = require('express');
|
2
2
|
var router = express.Router();
|
3
|
-
var { KB } = require('../models/kb_setting');
|
3
|
+
var { KB, Namespace } = require('../models/kb_setting');
|
4
4
|
var winston = require('../config/winston');
|
5
|
+
const JobManager = require('../utils/jobs-worker-queue-manager/JobManagerV2');
|
6
|
+
const { Scheduler } = require('../services/Scheduler');
|
7
|
+
const { AiReindexService } = require('../services/aiReindexService');
|
5
8
|
|
6
9
|
const KB_WEBHOOK_TOKEN = process.env.KB_WEBHOOK_TOKEN || 'kbcustomtoken';
|
10
|
+
const AMQP_MANAGER_URL = process.env.AMQP_MANAGER_URL;
|
11
|
+
const JOB_TOPIC_EXCHANGE = process.env.JOB_TOPIC_EXCHANGE_TRAIN || 'tiledesk-trainer';
|
12
|
+
|
13
|
+
let jobManager = new JobManager(AMQP_MANAGER_URL, {
|
14
|
+
debug: false,
|
15
|
+
topic: JOB_TOPIC_EXCHANGE,
|
16
|
+
exchange: JOB_TOPIC_EXCHANGE
|
17
|
+
})
|
18
|
+
|
19
|
+
jobManager.connectAndStartPublisher((status, error) => {
|
20
|
+
if (error) {
|
21
|
+
winston.error("connectAndStartPublisher error: ", error);
|
22
|
+
} else {
|
23
|
+
winston.info("KbRoute - ConnectPublisher done with status: ", status);
|
24
|
+
}
|
25
|
+
})
|
26
|
+
|
27
|
+
let default_engine = {
|
28
|
+
name: "pinecone",
|
29
|
+
type: process.env.PINECONE_TYPE,
|
30
|
+
apikey: "",
|
31
|
+
vector_size: 1536,
|
32
|
+
index_name: process.env.PINECONE_INDEX
|
33
|
+
}
|
34
|
+
|
35
|
+
router.post('/kb/reindex', async (req, res) => {
|
36
|
+
|
37
|
+
winston.verbose("/kb/reindex webhook called")
|
38
|
+
winston.debug("(webhook) req.body: ", req.body);
|
39
|
+
|
40
|
+
if (!req.headers['x-auth-token']) {
|
41
|
+
winston.error("(webhook) Unauthorized: A x-auth-token must be provided")
|
42
|
+
return res.status(401).send({ success: false, error: "Unauthorized", message: "A x-auth-token must be provided" })
|
43
|
+
}
|
44
|
+
|
45
|
+
if (req.headers['x-auth-token'] != KB_WEBHOOK_TOKEN) {
|
46
|
+
winston.error("(webhook) Unauthorized: You don't have the authorization to accomplish this operation")
|
47
|
+
return res.status(401).send({ success: false, error: "Unauthorized", message: "You don't have the authorization to accomplish this operation" });
|
48
|
+
}
|
49
|
+
|
50
|
+
let content_id = req.body.content_id;
|
51
|
+
|
52
|
+
let kb = await KB.findById(content_id).catch( async (err) => {
|
53
|
+
winston.error("(webhook) Error getting kb content: ", err);
|
54
|
+
return res.status(500).send({ success: false, error: "Error getting content with id " + content_id });
|
55
|
+
})
|
56
|
+
|
57
|
+
if (!kb) {
|
58
|
+
winston.warn("(webhook) Kb content not found with id " + content_id + ". Deleting scheduler...");
|
59
|
+
|
60
|
+
// Assuming the content has been deleted. The scheduler should be stopped and deleted.
|
61
|
+
res.status(200).send({ success: true, message: "Content no longer exists. Deleting scheduler..." })
|
62
|
+
|
63
|
+
setTimeout( async () => {
|
64
|
+
let aiReindexService = new AiReindexService();
|
65
|
+
let deleteResponse = await aiReindexService.delete(content_id).catch((err) => {
|
66
|
+
winston.error("(webhook) Error deleting scheduler ", err);
|
67
|
+
winston.error("(webhook) Error deleting scheduler " + err);
|
68
|
+
return;
|
69
|
+
});
|
70
|
+
winston.verbose("(webhook) delete response: ", deleteResponse);
|
71
|
+
return;
|
72
|
+
}, 10000);
|
73
|
+
|
74
|
+
} else {
|
75
|
+
|
76
|
+
let json = {
|
77
|
+
id: kb._id,
|
78
|
+
type: kb.type,
|
79
|
+
source: kb.source,
|
80
|
+
content: "",
|
81
|
+
namespace: kb.namespace,
|
82
|
+
refresh_rate: kb.refresh_rate,
|
83
|
+
last_refresh: kb.last_refresh
|
84
|
+
}
|
85
|
+
|
86
|
+
if (kb.scrape_type) {
|
87
|
+
json.scrape_type = kb.scrape_type
|
88
|
+
}
|
89
|
+
|
90
|
+
if (kb.scrape_options) {
|
91
|
+
json.parameters_scrape_type_4 = {
|
92
|
+
tags_to_extract: kb.scrape_options.tags_to_extract,
|
93
|
+
unwanted_tags: kb.scrape_options.unwanted_tags,
|
94
|
+
unwanted_classnames: kb.scrape_options.unwanted_classnames
|
95
|
+
}
|
96
|
+
}
|
97
|
+
|
98
|
+
let namespace = await Namespace.findOne({ id: kb.namespace }).catch((err) => {
|
99
|
+
winston.error("(webhook) Error getting namespace ", err)
|
100
|
+
return res.status(500).send({ success: false, error: err })
|
101
|
+
})
|
102
|
+
|
103
|
+
if (!namespace) {
|
104
|
+
winston.warn("(webhook) Namespace not found with id " + kb.namespace);
|
105
|
+
return res.status(500).send({ success: false, error: err })
|
106
|
+
}
|
107
|
+
|
108
|
+
json.engine = namespace.engine || default_engine;
|
109
|
+
|
110
|
+
let resources = [];
|
111
|
+
resources.push(json);
|
112
|
+
|
113
|
+
if (process.env.NODE_ENV !== 'test') {
|
114
|
+
scheduleScrape(resources);
|
115
|
+
}
|
116
|
+
|
117
|
+
res.status(200).send({ success: true, message: "Content queued for reindexing" });
|
118
|
+
|
119
|
+
}
|
120
|
+
})
|
121
|
+
|
7
122
|
|
8
123
|
router.post('/kb/status', async (req, res) => {
|
9
124
|
|
@@ -52,4 +167,22 @@ router.post('/kb/status', async (req, res) => {
|
|
52
167
|
|
53
168
|
})
|
54
169
|
|
170
|
+
async function scheduleScrape(resources) {
|
171
|
+
|
172
|
+
let scheduler = new Scheduler({ jobManager: jobManager });
|
173
|
+
|
174
|
+
resources.forEach(r => {
|
175
|
+
winston.debug("(Webhook) Schedule job with following data: ", r);
|
176
|
+
scheduler.trainSchedule(r, async (err, result) => {
|
177
|
+
if (err) {
|
178
|
+
winston.error("Scheduling error: ", err);
|
179
|
+
} else {
|
180
|
+
winston.info("Scheduling result: ", result);
|
181
|
+
}
|
182
|
+
});
|
183
|
+
})
|
184
|
+
|
185
|
+
return true;
|
186
|
+
}
|
187
|
+
|
55
188
|
module.exports = router;
|
package/services/QuoteManager.js
CHANGED
@@ -16,15 +16,15 @@ const emailEvent = require('../event/emailEvent');
|
|
16
16
|
|
17
17
|
|
18
18
|
const PLANS_LIST = {
|
19
|
-
FREE_TRIAL: { requests: 200, messages: 0, tokens: 100000,
|
20
|
-
SANDBOX: { requests: 200, messages: 0, tokens: 100000,
|
21
|
-
BASIC: { requests: 800, messages: 0, tokens: 2000000,
|
22
|
-
PREMIUM: { requests: 3000, messages: 0, tokens: 5000000,
|
23
|
-
TEAM: { requests: 5000, messages: 0, tokens: 10000000,
|
24
|
-
CUSTOM: { requests: 5000, messages: 0, tokens: 10000000,
|
19
|
+
FREE_TRIAL: { requests: 200, messages: 0, tokens: 100000, voice_duration: 0, email: 200, chatbots: 20, namespace: 3, kbs: 50 }, // same as PREMIUM
|
20
|
+
SANDBOX: { requests: 200, messages: 0, tokens: 100000, voice_duration: 0, email: 200, chatbots: 2, namespace: 1, kbs: 50 },
|
21
|
+
BASIC: { requests: 800, messages: 0, tokens: 2000000, voice_duration: 0, email: 200, chatbots: 5, namespace: 1, kbs: 150 },
|
22
|
+
PREMIUM: { requests: 3000, messages: 0, tokens: 5000000, voice_duration: 0, email: 200, chatbots: 20, namespace: 3, kbs: 300 },
|
23
|
+
TEAM: { requests: 5000, messages: 0, tokens: 10000000, voice_duration: 0, email: 200, chatbots: 50, namespace: 10, kbs: 1000 },
|
24
|
+
CUSTOM: { requests: 5000, messages: 0, tokens: 10000000, voice_duration: 120000, email: 200, chatbots: 50, namespace: 10, kbs: 1000 },
|
25
25
|
}
|
26
26
|
|
27
|
-
const typesList = ['requests', 'messages', 'email', 'tokens', 'chatbots', 'kbs']
|
27
|
+
const typesList = ['requests', 'messages', 'email', 'tokens', 'voice_duration', 'chatbots', 'kbs']
|
28
28
|
|
29
29
|
let quotes_enabled = true;
|
30
30
|
|
@@ -95,6 +95,25 @@ class QuoteManager {
|
|
95
95
|
this.sendEmailIfQuotaExceeded(project, data, 'tokens', key);
|
96
96
|
return key;
|
97
97
|
}
|
98
|
+
|
99
|
+
async incrementVoiceDurationCount(project, request) {
|
100
|
+
|
101
|
+
this.project = project;
|
102
|
+
let key = await this.generateKey(request, 'voice_duration');
|
103
|
+
winston.verbose("[QuoteManager] incrementVoiceDurationCount key: " + key);
|
104
|
+
|
105
|
+
if (quotes_enabled === false) {
|
106
|
+
winston.debug("QUOTES DISABLED - incrementVoiceDurationCount")
|
107
|
+
return key;
|
108
|
+
}
|
109
|
+
|
110
|
+
if (request?.duration) {
|
111
|
+
let duration = Math.round(request.duration / 1000); // from ms to s
|
112
|
+
await this.tdCache.incrby(key, duration);
|
113
|
+
|
114
|
+
this.sendEmailIfQuotaExceeded(project, request, 'voice_duration', key);
|
115
|
+
}
|
116
|
+
}
|
98
117
|
// INCREMENT KEY SECTION - END
|
99
118
|
|
100
119
|
|
@@ -228,7 +247,7 @@ class QuoteManager {
|
|
228
247
|
winston.verbose("limits for current plan: ", limits)
|
229
248
|
|
230
249
|
let quote = await this.getCurrentQuote(project, object, type);
|
231
|
-
winston.verbose("getCurrentQuote resp: "
|
250
|
+
winston.verbose("getCurrentQuote resp: " + quote)
|
232
251
|
|
233
252
|
if (quote == null) {
|
234
253
|
return true;
|
@@ -494,13 +513,21 @@ class QuoteManager {
|
|
494
513
|
if (quotes_enabled === true) {
|
495
514
|
winston.verbose("request.create.quote event catched");
|
496
515
|
let result = await this.incrementRequestsCount(payload.project, payload.request);
|
497
|
-
|
498
|
-
|
499
516
|
return result;
|
500
517
|
} else {
|
501
518
|
winston.verbose("QUOTES DISABLED - request.create.quote event")
|
502
519
|
}
|
503
520
|
})
|
521
|
+
|
522
|
+
requestEvent.on('request.close.quote', async (payload) => {
|
523
|
+
if (quotes_enabled === true) {
|
524
|
+
winston.verbose("request.close.quote event catched");
|
525
|
+
let result = await this.incrementVoiceDurationCount(payload.project, payload.request);
|
526
|
+
return result;
|
527
|
+
} else {
|
528
|
+
winston.verbose("QUOTES DISABLED - request.close.quote event")
|
529
|
+
}
|
530
|
+
})
|
504
531
|
// REQUESTS EVENTS - END
|
505
532
|
|
506
533
|
|