@tiledesk/tiledesk-server 2.5.3 → 2.7.1

package/routes/kb.js

@@ -2,8 +2,26 @@ var express = require('express');
 var { KB } = require('../models/kb_setting');
 var router = express.Router();
 var winston = require('../config/winston');
+var multer = require('multer')
+var upload = multer()
 const openaiService = require('../services/openaiService');
+const JobManager = require('../utils/jobs-worker-queue-manager/JobManagerV2');
+const { Scheduler } = require('../services/Scheduler');
 
+const Sitemapper = require('sitemapper');
+
+const AMQP_MANAGER_URL = process.env.AMQP_MANAGER_URL;
+const JOB_TOPIC_EXCHANGE = process.env.JOB_TOPIC_EXCHANGE_TRAIN || 'tiledesk-trainer';
+
+let jobManager = new JobManager(AMQP_MANAGER_URL, {
+    debug: false,
+    topic: JOB_TOPIC_EXCHANGE,
+    exchange: JOB_TOPIC_EXCHANGE
+})
+
+jobManager.connectAndStartPublisher(() => {
+    winston.info("ConnectPublisher done");
+})
 
 router.get('/', async (req, res) => {
 
@@ -27,7 +45,7 @@ router.get('/', async (req, res) => {
         limit = parseInt(req.query.limit);
         winston.debug("Get kb limit: " + limit)
     }
-    
+
     if (req.query.page) {
         page = parseInt(req.query.page);
         winston.debug("Get kb page: " + page)
@@ -55,54 +73,54 @@ router.get('/', async (req, res) => {
     let sortQuery = {};
     sortQuery[sortField] = direction;
     winston.debug("Get kb sortQuery: " + sortQuery);
-     
+
     KB.countDocuments(query, (err, kbs_count) => {
-            if (err) {
-                winston.error("Find all kbs error: ", err);
-            }
-            winston.debug("KBs count: ", kbs_count);
-            
-            KB.find(query)
-                .skip(skip)
-                .limit(limit)
-                .sort(sortQuery)
-                .exec((err, kbs) => {
-                    if (err) {
-                        winston.error("Find all kbs error: ", err);
-                        return res.status(500).send({ success: false, error: err });
-                    }
-
-                    winston.debug("KBs found: ", kbs);
-
-                    let response = {
-                        count: kbs_count,
-                        query: {},
-                        kbs: kbs
-                    }
-                    if (status) {
-                        response.query.status = status;
-                    }
-                    if (limit) {
-                        response.query.limit = limit;
-                    }
-                    if (status) {
-                        response.query.page = page;
-                    }
-                    if (sortField) {
-                        response.query.sortField = sortField;
-                    }
-                    if (direction) {
-                        response.query.direction = direction;
-                    }
-                    if (text) {
-                        response.query.search = text;
-                    }
-
-
-                    return res.status(200).send(response);
-                })
+        if (err) {
+            winston.error("Find all kbs error: ", err);
+        }
+        winston.debug("KBs count: ", kbs_count);
 
-        })
+        KB.find(query)
+            .skip(skip)
+            .limit(limit)
+            .sort(sortQuery)
+            .exec((err, kbs) => {
+                if (err) {
+                    winston.error("Find all kbs error: ", err);
+                    return res.status(500).send({ success: false, error: err });
+                }
+
+                winston.debug("KBs found: ", kbs);
+
+                let response = {
+                    count: kbs_count,
+                    query: {},
+                    kbs: kbs
+                }
+                if (status) {
+                    response.query.status = status;
+                }
+                if (limit) {
+                    response.query.limit = limit;
+                }
+                if (status) {
+                    response.query.page = page;
+                }
+                if (sortField) {
+                    response.query.sortField = sortField;
+                }
+                if (direction) {
+                    response.query.direction = direction;
+                }
+                if (text) {
+                    response.query.search = text;
+                }
+
+
+                return res.status(200).send(response);
+            })
+
+    })
 
 
 })
@@ -127,6 +145,31 @@ router.post('/', async (req, res) => {
     let project_id = req.projectid;
     let body = req.body;
 
+    // add or override namespace value if it is passed for security reason
+    body.namespace = project_id;
+
+    // if (req.body.namespace) {
+    //     if (req.body.namespace != req.projectid) {
+    //         return res.status(403).send({ success: false, error: "Not allowed. The namespace does not belong to the current project."})
+    //     }
+    // }
+
+    let quoteManager = req.app.get('quote_manager');
+    let limits = await quoteManager.getPlanLimits(req.project);
+    let kbs_limit = limits.kbs;
+    winston.verbose("Limit of kbs for current plan: " + kbs_limit);
+
+    let kbs_count = await KB.countDocuments({ id_project: project_id }).exec();
+    winston.verbose("Kbs count: " + kbs_count);
+
+    if (kbs_count >= kbs_limit) {
+        return res.status(403).send({ success: false, error: "Maximum number of resources reached for the current plan", plan_limit: kbs_limit })
+    }
+
+    let total_count = kbs_count + 1;
+    if (total_count > kbs_limit) {
+        return res.status(403).send({ success: false, error: "Cannot exceed the number of resources in the current plan", plan_limit: kbs_limit })
+    }
     let new_kb = {
         id_project: project_id,
         name: body.name,
@@ -141,7 +184,6 @@ router.post('/', async (req, res) => {
     }
     winston.debug("adding kb: ", new_kb);
 
-
     KB.findOneAndUpdate({ id_project: project_id, type: 'url', source: new_kb.source }, new_kb, { upsert: true, new: true, rawResult: true }, async (err, raw) => {
         if (err) {
             winston.error("findOneAndUpdate with upsert error: ", err);
@@ -163,17 +205,118 @@ router.post('/', async (req, res) => {
                 json.content = raw.value.content;
             }
 
-            startScrape(json).then((response) => {
-                winston.verbose("startScrape response: ", response);
-            }).catch((err) => {
-                winston.error("startScrape err: ", err);
-            })
+            let resources = [];
+
+            resources.push(json);
+            scheduleScrape(resources);
+
+            // startScrape(json).then((response) => {
+            //     winston.verbose("startScrape response: ", response);
+            // }).catch((err) => {
+            //     winston.error("startScrape err: ", err);
+            // })
+
+        }
+    })
+
+})
+
+
+router.post('/multi', upload.single('uploadFile'), async (req, res) => {
+
+    let list;
+    if (req.file) {
+        file_string = req.file.buffer.toString('utf-8');
+        list = file_string.trim().split('\n');
+    } else {
+        list = req.body.list;
+    }
+
+    let project_id = req.projectid;
+
+    let quoteManager = req.app.get('quote_manager');
+    let limits = await quoteManager.getPlanLimits(req.project);
+    let kbs_limit = limits.kbs;
+    winston.verbose("Limit of kbs for current plan: " + kbs_limit);
+
+    let kbs_count = await KB.countDocuments({ id_project: project_id }).exec();
+    winston.verbose("Kbs count: " + kbs_count);
+
+    if (kbs_count >= kbs_limit) {
+        return res.status(403).send({ success: false, error: "Maximum number of resources reached for the current plan", plan_limit: kbs_limit })
+    }
+
+    let total_count = kbs_count + list.length;
+    if (total_count > kbs_limit) {
+        return res.status(403).send({ success: false, error: "Cannot exceed the number of resources in the current plan", plan_limit: kbs_limit })
+    }
+
+    if (list.length > 300) {
+        winston.error("Too many urls. Can't index more than 300 urls at a time.");
+        return res.status(403).send({ success: false, error: "Too many urls. Can't index more than 300 urls at a time."})
+    }
 
+    let kbs = [];
+    list.forEach(url => {
+        kbs.push({
+            id_project: project_id,
+            name: url,
+            source: url,
+            type: 'url',
+            content: "",
+            namespace: project_id,
+            status: -1
+        })
+    })
+
+    let operations = kbs.map(doc => {
+        return {
+            updateOne: {
+                filter: { id_project: doc.id_project, type: 'url', source: doc.source },
+                update: doc,
+                upsert: true,
+                returnOriginal: false
+            }
         }
     })
 
+    saveBulk(operations, kbs, project_id).then((result) => {
+
+        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
+        resources = resources.map(({ _id, ...rest }) => {
+            return { id: _id, ...rest };
+        });
+        winston.verbose("resources to be sent to worker: ", resources)        
+        scheduleScrape(resources);
+        res.status(200).send(result);
+
+    }).catch((err) => {
+        winston.error("Unable to save kbs in bulk ", err)
+        res.status(500).send(err);
+    })
+
+})
+
+router.post('/sitemap', async (req, res) => {
+
+    let sitemap_url = req.body.sitemap;
+    
+    const sitemap = new Sitemapper({
+        url: sitemap_url,
+        timeout: 15000
+    });
+
+    sitemap.fetch().then((data) => {
+        winston.debug("data: ", data);
+        res.status(200).send(data);
+    }).catch((err) => {
+        console.error("err ", err)
+        res.status(500).send({ success: false, error: err });
+    })
+
 })
 
+
 router.put('/:kb_id', async (req, res) => {
 
     let kb_id = req.params.kb_id;
@@ -230,10 +373,14 @@ router.post('/scrape/single', async (req, res) => {
                 id: kb._id,
                 type: kb.type,
                 source: kb.source,
-                content: kb.content,
+                content: "",
                 namespace: kb.namespace
             }
 
+            if (kb.content) {
+                json.content = kb.content;
+            }
+
             startScrape(json).then((response) => {
                 winston.verbose("startScrape response: ", response);
                 res.status(200).send(response);
@@ -256,24 +403,27 @@ router.post('/scrape/status', async (req, res) => {
 
     if (req.query &&
         req.query.returnObject &&
-        (req.query.returnObject === true || req.query.returnObject === true)) {
+        (req.query.returnObject === true || req.query.returnObject === 'true')) {
         returnObject = true;
     }
 
-    openaiService.scrapeStatus(data).then((response) => {
+    openaiService.scrapeStatus(data).then( async (response) => {
 
         winston.debug("scrapeStatus response.data: ", response.data);
 
         let update = {};
 
         if (response.data.status_code) {
-            update.status = response.data.status_code;
+            // update.status = response.data.status_code;
+            update.status = await statusConverter(response.data.status_code)
+                
         }
 
         KB.findByIdAndUpdate(data.id, update, { new: true }, (err, savedKb) => {
 
             if (err) {
                 winston.verbose("Status was successfully recovered, but the update on the db failed");
+                winston.error("find kb by id and updated error: ", err);
 
                 if (returnObject) {
                     return res.status(206).send({ warning: "Unable to udpate content on db", message: "The original reply was forwarded", data: response.data });
@@ -298,8 +448,14 @@ router.post('/scrape/status', async (req, res) => {
 
 router.post('/qa', async (req, res) => {
     let data = req.body;
+    // add or override namespace value if it is passed for security reason
+    data.namespace = req.projectid;
     winston.debug("/qa data: ", data);
 
+    // if (req.body.namespace != req.projectid) {
+    //     return res.status(403).send({ success: false, error: "Not allowed. The namespace does not belong to the current project."})
+    // }
+
     if (!data.gptkey) {
         let gptkey = process.env.GPTKEY;
         if (!gptkey) {
@@ -310,11 +466,38 @@ router.post('/qa', async (req, res) => {
 
     openaiService.askNamespace(data).then((resp) => {
         winston.debug("qa resp: ", resp.data);
-        res.status(200).send(resp.data);
+        let answer = resp.data;
+
+        let id = answer.id;
+        let index = id.indexOf("#");
+        if (index != -1) {
+            id = id.substring(index + 1);
+        }
+
+        KB.findById(id, (err, resource) => {
+
+            if (err) {
+                winston.error("Unable to find resource with id " + id + " in namespace " + answer.namespace + ". The standard answer is returned.")
+                return res.status(200).send(resp.data);
+            }
+
+            answer.source = resource.name;
+            return res.status(200).send(answer);
+        })
+    
+
     }).catch((err) => {
         winston.error("qa err: ", err);
-        let status = err.response.status;
-        res.status(status).send({ success: false,  statusText: err.response.statusText, error: err.response.data.detail });
+        console.log(err.response)
+        if (err.response 
+            && err.response.status) {
+                let status = err.response.status;
+                res.status(status).send({ success: false, statusText: err.response.statusText, error: err.response.data.detail });
+        }
+        else {
+            res.status(500).send({ success: false, error: err });
+        }
+        
     })
 })
 
@@ -375,7 +558,7 @@ router.delete('/:kb_id', async (req, res) => {
             })
 
         } else {
-            winston.info("resp.data: ", resp.data);
+            winston.verbose("resp.data: ", resp.data);
 
             KB.findOneAndDelete({ _id: kb_id, status: { $in: [-1, 3, 4] } }, (err, deletedKb) => {
                 if (err) {
@@ -398,6 +581,51 @@ router.delete('/:kb_id', async (req, res) => {
 
 })
 
+async function saveBulk(operations, kbs, project_id) {
+
+    return new Promise((resolve, reject) => {
+        KB.bulkWrite(operations, { ordered: false }).then((result) => {
+            winston.verbose("bulkWrite operations result: ", result);
+
+            KB.find({ id_project: project_id, source: { $in: kbs.map(kb => kb.source) } }).lean().then((documents) => {
+                winston.debug("documents: ", documents);
+                resolve(documents)
+            }).catch((err) => {
+                winston.error("Error finding documents ", err)
+                reject(err);
+            })
+
+        }).catch((err) => {
+            reject(err);
+        })
+    })
+
+}
+
+async function statusConverter(status) {
+    return new Promise((resolve) => {
+
+        let td_status;
+        switch(status) {
+            case 0:
+                td_status = -1;
+                break;
+            case 2:
+                td_status = 200;
+                break;    
+            case 3:
+                td_status = 300;
+                break;
+            case 4:
+                td_status = 400;
+                break;
+            default:
+                td_status = -1
+        }
+        resolve(td_status);
+    })
+}
+
 async function updateStatus(id, status) {
     return new Promise((resolve) => {
 
@@ -412,6 +640,23 @@ async function updateStatus(id, status) {
     })
 }
 
+async function scheduleScrape(resources) {
+
+    let data = {
+        resources: resources
+    }
+    winston.info("Schedule job with following data: ", data);
+    let scheduler = new Scheduler({ jobManager: jobManager });
+    scheduler.trainSchedule(data, (err, result) => {
+        if (err) {
+            winston.error("Scheduling error: ", err);
+        }
+        winston.info("Scheduling result: ", result);
+    });
+
+    return true;
+}
+
 async function startScrape(data) {
 
     if (!data.gptkey) {
@@ -425,7 +670,7 @@ async function startScrape(data) {
     return new Promise((resolve, reject) => {
         openaiService.singleScrape(data).then(async (resp) => {
             winston.debug("singleScrape resp: ", resp.data);
-            let status_updated = await updateStatus(data.id, 0);
+            let status_updated = await updateStatus(data.id, 100);
             winston.verbose("status of kb " + data.id + " updated: " + status_updated);
             resolve(resp.data);
         }).catch((err) => {

package/services/QuoteManager.js

@@ -5,11 +5,11 @@ const messageEvent = require('../event/messageEvent');
 const emailEvent = require('../event/emailEvent');
 
 const PLANS_LIST = {
-    FREE_TRIAL: { requests: 3000,   messages: 0,    tokens: 250000,     email: 200 }, // same as PREMIUM
-    SANDBOX:    { requests: 200,    messages: 0,    tokens: 10000,      email: 200 },
-    BASIC:      { requests: 800,    messages: 0,    tokens: 50000,      email: 200 },
-    PREMIUM:    { requests: 3000,   messages: 0,    tokens: 250000,     email: 200 },
-    CUSTOM:     { requests: 3000,   messages: 0,    tokens: 250000,     email: 200 }
+    FREE_TRIAL: { requests: 3000,   messages: 0,    tokens: 250000,     email: 200,     kbs: 50 }, // same as PREMIUM
+    SANDBOX:    { requests: 200,    messages: 0,    tokens: 10000,      email: 200,     kbs: 50 },
+    BASIC:      { requests: 800,    messages: 0,    tokens: 50000,      email: 200,     kbs: 150},
+    PREMIUM:    { requests: 3000,   messages: 0,    tokens: 250000,     email: 200,     kbs: 300},
+    CUSTOM:     { requests: 3000,   messages: 0,    tokens: 250000,     email: 200,     kbs: 300}
 }
 
 const typesList = ['requests', 'messages', 'email', 'tokens']
@@ -58,7 +58,7 @@ class QuoteManager {
 
         this.project = project;
         let key = await this.generateKey(email, 'email');
-        winston.info("[QuoteManager] incrementEmailCount key: " + key);
+        winston.verbose("[QuoteManager] incrementEmailCount key: " + key);
 
         await this.tdCache.incr(key)
         return key;
@@ -68,7 +68,7 @@ class QuoteManager {
 
         this.project = project;
         let key = await this.generateKey(data, 'tokens');
-        winston.info("[QuoteManager] incrementTokenCount key: " + key);
+        winston.verbose("[QuoteManager] incrementTokenCount key: " + key);
 
         if (quotes_enabled === false) {
             winston.debug("QUOTES DISABLED - incrementTokenCount")
@@ -83,8 +83,8 @@ class QuoteManager {
 
     async generateKey(object, type) {
 
-        winston.info("generateKey object ", object)
-        winston.info("generateKey type " + type)
+        winston.debug("generateKey object ", object)
+        winston.debug("generateKey type " + type)
         let subscriptionDate;
         if (this.project.profile.subStart) {
             subscriptionDate = this.project.profile.subStart;
@@ -92,7 +92,7 @@ class QuoteManager {
             subscriptionDate = this.project.createdAt;
         }
         let objectDate = object.createdAt;
-        winston.info("objectDate " + objectDate);
+        winston.debug("objectDate " + objectDate);
 
         // converts date in timestamps and transform from ms to s
         const objectDateTimestamp = ceil(objectDate.getTime() / 1000);
@@ -114,7 +114,7 @@ class QuoteManager {
 
         this.project = project;
         let key = await this.generateKey(object, type);
-        winston.info("[QuoteManager] getCurrentQuote key: " + key);
+        winston.verbose("[QuoteManager] getCurrentQuote key: " + key);
 
         let quote = await this.tdCache.get(key);
         return Number(quote);
@@ -147,17 +147,17 @@ class QuoteManager {
      */
     async checkQuote(project, object, type) {
 
-        winston.info("checkQuote type " + type);
+        winston.verbose("checkQuote type " + type);
         if (quotes_enabled === false) {
-            winston.info("QUOTES DISABLED - checkQuote for type " + type);
+            winston.verbose("QUOTES DISABLED - checkQuote for type " + type);
             return true;
         }
 
         this.project = project;
         let limits = await this.getPlanLimits();
-        winston.info("limits for current plan: ", limits)
+        winston.verbose("limits for current plan: ", limits)
         let quote = await this.getCurrentQuote(project, object, type);
-        winston.info("getCurrentQuote resp: ", quote)
+        winston.verbose("getCurrentQuote resp: ", quote)
 
         if (quote == null) {
             return true;
@@ -171,7 +171,11 @@ class QuoteManager {
     }
 
 
-    async getPlanLimits() {
+    async getPlanLimits(project) {
+
+        if (project) {
+            this.project = project
+        };
 
         let limits;
         if (this.project.profile.type === 'payment') {
@@ -222,6 +226,8 @@ class QuoteManager {
             }
         }
 
+        winston.info("QUOTES ENABLED ? ", quotes_enabled);
+
         // TODO - Try to generalize to avoid repetition
         let incrementEventHandler = (object) => { }
         let checkEventHandler = (object) => { }
@@ -244,7 +250,7 @@ class QuoteManager {
                 let result = await this.incrementRequestsCount(payload.project, payload.request);
                 return result;
             } else {
-                winston.info("QUOTES DISABLED - request.create.quote event")
+                winston.verbose("QUOTES DISABLED - request.create.quote event")
             }
         })
         // REQUESTS EVENTS - END
@@ -267,7 +273,7 @@ class QuoteManager {
                 let result = await this.incrementMessagesCount(payload.project, payload.message);
                 return result;
             } else {
-                winston.info("QUOTES DISABLED - message.create.quote event")
+                winston.verbose("QUOTES DISABLED - message.create.quote event")
             }
         })
         // MESSAGES EVENTS - END
@@ -286,11 +292,11 @@ class QuoteManager {
 
         emailEvent.on('email.send.quote', async (payload) => {
             if (quotes_enabled === true) {
-                winston.info("email.send event catched");
+                winston.verbose("email.send event catched");
                 let result = await this.incrementEmailCount(payload.project, payload.email);
                 return result;
             } else {
-                winston.info("QUOTES DISABLED - email.send event")
+                winston.verbose("QUOTES DISABLED - email.send event")
             }
         })
         // EMAIL EVENTS - END

package/services/Scheduler.js

@@ -0,0 +1,33 @@
+let JobManager = require("jobs-worker-queued");
+let winston = require('../config/winston');
+
+let jobManager;
+
+class Scheduler {
+
+    constructor(config) {
+
+        if (!config) {
+            throw new Error('(Scheduler) config is mandatory');
+        }
+
+        if (!config.jobManager) {
+            throw new Error('(Scheduler) config.jobManager is mandatory');
+        }
+
+        this.jobManager = config.jobManager;
+    }
+
+    trainSchedule(data, callback) {
+
+        winston.debug("(trainScheduler) data: ", data);
+        this.jobManager.publish(data, (err, ok) => {
+            let response_data = { success: true, message: "Scheduled" };
+            if (callback) {
+                callback(err, response_data);
+            }
+        });
+    }
+}
+
+module.exports = { Scheduler };