@tiledesk/tiledesk-server 2.4.99 → 2.4.100

package/CHANGELOG.md

@@ -5,6 +5,10 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.4.100
+- Updated tybot-connector to 0.2.50
+- Added new route for knowledge base
+
 # 2.4.99
 - Updated whatsapp-connector to 0.1.62
 - Updated messenger-connector to 0.1.16

package/app.js

@@ -114,6 +114,7 @@ var widgets = require('./routes/widget');
 var widgetsLoader = require('./routes/widgetLoader');
 var openai = require('./routes/openai');
 var kbsettings = require('./routes/kbsettings');
+var kb = require('./routes/kb');
 
 // var admin = require('./routes/admin');
 var faqpub = require('./routes/faqpub');
@@ -561,6 +562,7 @@ app.use('/:projectid/segments',[passport.authenticate(['basic', 'jwt'], { sessio
 
 app.use('/:projectid/openai', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('agent')], openai);
 app.use('/:projectid/kbsettings', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('agent', ['bot','subscription'])], kbsettings);
+app.use('/:projectid/kb', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('admin', ['bot','subscription'])], kb);
 
 app.use('/:projectid/logs', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRole('admin')], logs);
 

package/models/kb_setting.js

@@ -3,18 +3,42 @@ var Schema = mongoose.Schema;
 var winston = require('../config/winston');
 
 var KBSchema = new Schema({
+  id_project: {
+    type: String,
+    required: true,
+  },
   name: {
     type: String,
     required: true
   },
   url: {
     type: String,
-    required: true
+    required: false
+  },
+  source: {
+    type: String,
+    required: false
+  },
+  type: {
+    type: String,
+    required: false
+  },
+  content: {
+    type: String,
+    required: false
+  },
+  namespace: {
+    type: String,
+    required: false
   },
   createdAt: {
     type: Date,
     default: Date.now,
   },
+  updatedAt: {
+    type: Date,
+    default: Date.now
+  },
   status: {
     type: Number,
     required: false,
@@ -44,5 +68,12 @@ var KBSettingSchema = new Schema({
 });
 
 
-module.exports = mongoose.model('KBSettings', KBSettingSchema);
-// module.exports = mongoose.model('KB', KBSchema)
+//module.exports = mongoose.model('KBSettings', KBSettingSchema);
+const KBSettings = mongoose.model('KBSettings', KBSettingSchema);
+const KB = mongoose.model('KB', KBSchema)
+
+module.exports = {
+  KBSettings: KBSettings,
+  KB: KB
+}
+

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.4.99",
+  "version": "2.4.100",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",
@@ -45,7 +45,7 @@
     "@tiledesk/tiledesk-messenger-connector": "^0.1.15",
     "@tiledesk/tiledesk-rasa-connector": "^1.0.10",
     "@tiledesk/tiledesk-telegram-connector": "^0.1.10",
-    "@tiledesk/tiledesk-tybot-connector": "^0.2.49",
+    "@tiledesk/tiledesk-tybot-connector": "^0.2.50",
     "@tiledesk/tiledesk-whatsapp-connector": "^0.1.61",
     "@tiledesk/tiledesk-whatsapp-jobworker": "^0.0.7",
     "@tiledesk/tiledesk-chatbot-templates": "^0.1.2",

package/routes/kb.js

@@ -0,0 +1,353 @@
+var express = require('express');
+var { KB } = require('../models/kb_setting');
+var router = express.Router();
+var winston = require('../config/winston');
+const openaiService = require('../services/openaiService');
+
+
+router.get('/', async (req, res) => {
+
+    let project_id = req.projectid;
+
+    KB.find({ id_project: project_id }, (err, kbs) => {
+        if (err) {
+            winston.error("Find all kbs error: ", err);
+            return res.status(500).send({ success: false, error: err });
+        }
+
+        winston.debug("KBs found: ", kbs);
+        return res.status(200).send(kbs);
+    })
+})
+
+router.get('/:kb_id', async (req, res) => {
+
+    let kb_id = req.params.kb_id;
+
+    KB.findById(kb_id, (err, kb) => {
+        if (err) {
+            winston.error("Find kb by id error: ", err);
+            return res.status(500).send({ success: false, error: err });
+        }
+
+        return res.status(200).send(kb);
+    })
+})
+
+router.post('/', async (req, res) => {
+
+
+    let project_id = req.projectid;
+    let body = req.body;
+    console.log("create new kb project_id " + project_id);
+    console.log("create new kb body ", body);
+
+    let new_kb = {
+        id_project: project_id,
+        name: body.name,
+        type: body.type,
+        source: body.source,
+        content: body.content,
+        namespace: body.namespace
+    }
+    if (!new_kb.namespace) {
+        new_kb.namespace = project_id;
+    }
+    winston.debug("adding kb: ", new_kb);
+
+
+    KB.findOneAndUpdate({ id_project: project_id, type: 'url', source: new_kb.source }, new_kb, { upsert: true, new: true, rawResult: true }, async (err, raw) => {
+        if (err) {
+            winston.error("findOneAndUpdate with upsert error: ", err);
+            res.status(500).send({ success: false, error: err });
+        }
+        else {
+
+            res.status(200).send(raw);
+
+            let json = {
+                id: raw.value._id,
+                type: raw.value.type,
+                source: raw.value.source,
+                content: "",
+                namespace: raw.value.namespace
+            }
+
+            if (raw.value.content) {
+                json.content = raw.value.content;
+            }
+
+            startScrape(json).then((response) => {
+                winston.verbose("startScrape response: ", response);
+            }).catch((err) => {
+                winston.error("startScrape err: ", err);
+            })
+
+        }
+    })
+
+})
+
+router.put('/:kb_id', async (req, res) => {
+
+    let kb_id = req.params.kb_id;
+    winston.verbose("update kb_id " + kb_id);
+
+    let update = {};
+
+    if (req.body.name != undefined) {
+        update.name = req.body.name;
+    }
+
+    if (req.body.status != undefined) {
+        update.status = req.body.status;
+    }
+
+    update.updatedAt = new Date();
+    winston.debug("kb update: ", update);
+
+    KB.findByIdAndUpdate(kb_id, update, { new: true }, (err, savedKb) => {
+
+        if (err) {
+            winston.error("KB findByIdAndUpdate error: ", err);
+            return res.status(500).send({ success: false, error: err });
+        }
+
+        if (!savedKb) {
+            winston.debug("Try to updating a non-existing kb");
+            return res.status(400).send({ success: false, message: "Content not found" })
+        }
+
+        res.status(200).send(savedKb)
+    })
+
+})
+
+
+// PROXY PUGLIA AI V2 - START
+router.post('/scrape/single', async (req, res) => {
+
+    let data = req.body;
+    winston.debug("/scrape/single data: ", data);
+
+    KB.findById(data.id, (err, kb) => {
+        if (err) {
+            winston.error("findById error: ", err);
+            return res.status(500).send({ success: false, error: err });
+        }
+
+        else if (!kb) {
+            return res.status(404).send({ success: false, error: "Unable to find the kb requested" })
+        }
+        else {
+
+            let json = {
+                id: kb._id,
+                type: kb.type,
+                source: kb.source,
+                content: kb.content,
+                namespace: kb.namespace
+            }
+
+            startScrape(json).then((response) => {
+                winston.verbose("startScrape response: ", response);
+                res.status(200).send(response);
+            }).catch((err) => {
+                winston.error("startScrape err: ", err);
+                res.status(500).send({ success: false, error: err });
+            })
+
+        }
+    })
+
+})
+
+router.post('/scrape/status', async (req, res) => {
+
+    let data = req.body;
+    winston.debug("/scrapeStatus req.body: ", req.body);
+
+    let returnObject = false;
+
+    if (req.query &&
+        req.query.returnObject &&
+        (req.query.returnObject === true || req.query.returnObject === true)) {
+        returnObject = true;
+    }
+
+    openaiService.scrapeStatus(data).then((response) => {
+
+        winston.debug("scrapeStatus response.data: ", response.data);
+
+        let update = {};
+
+        if (response.data.status_code) {
+            update.status = response.data.status_code;
+        }
+
+        KB.findByIdAndUpdate(data.id, update, { new: true }, (err, savedKb) => {
+
+            if (err) {
+                winston.verbose("Status was successfully recovered, but the update on the db failed");
+
+                if (returnObject) {
+                    return res.status(206).send({ warning: "Unable to udpate content on db", message: "The original reply was forwarded", data: response.data });
+                } else {
+                    return res.status(200).send(response.data);
+                }
+            }
+
+            if (returnObject) {
+                return res.status(200).send(savedKb);
+            } else {
+                return res.status(200).send(response.data);
+            }
+        })
+
+    }).catch((err) => {
+        winston.error("scrapeStatus err: ", err);
+        let status = err.response.status;
+        res.status(status).send({ statusText: err.response.statusText, error: err.response.data.detail });
+    })
+})
+
+router.post('/qa', async (req, res) => {
+    let data = req.body;
+    winston.debug("/qa data: ", data);
+
+    if (!data.gptkey) {
+        let gptkey = process.env.GPTKEY;
+        if (!gptkey) {
+            return res.status(403).send({ success: false, error: "GPT apikey undefined" })
+        }
+        data.gptkey = gptkey;
+    }
+
+    openaiService.askNamespace(data).then((resp) => {
+        winston.debug("qa resp: ", resp.data);
+        res.status(200).send(resp.data);
+    }).catch((err) => {
+        winston.error("qa err: ", err);
+        let status = err.response.status;
+        res.status(status).send({ success: false,  statusText: err.response.statusText, error: err.response.data.detail });
+    })
+})
+
+router.delete('/delete', async (req, res) => {
+
+    let data = req.body;
+    winston.debug("/delete data: ", data);
+
+    openaiService.deleteIndex(data).then((resp) => {
+        winston.debug("delete resp: ", resp.data);
+        res.status(200).send(resp.data);
+    }).catch((err) => {
+        winston.error("delete err: ", err);
+        let status = err.response.status;
+        res.status(status).send({ statusText: err.response.statusText, error: err.response.data.detail });
+    })
+
+})
+
+router.delete('/deleteall', async (req, res) => {
+
+    let data = req.body;
+    winston.debug('/delete all data: ', data);
+
+    openaiService.deleteNamespace(data).then((resp) => {
+        winston.debug("delete namespace resp: ", resp.data);
+        res.status(200).send(resp.data);
+    }).catch((err) => {
+        winston.error("delete namespace err: ", err);
+        let status = err.response.status;
+        res.status(status).send({ statusText: err.response.statusText, error: err.response.data.detail });
+    })
+})
+// PROXY PUGLIA AI V2 - END
+
+router.delete('/:kb_id', async (req, res) => {
+
+    let project_id = req.projectid;
+    let kb_id = req.params.kb_id;
+    winston.verbose("delete kb_id " + kb_id);
+
+    let data = {
+        id: kb_id,
+        namespace: project_id
+    }
+
+    openaiService.deleteIndex(data).then((resp) => {
+        winston.debug("delete resp: ", resp.data);
+
+        if (resp.data.success === true) {
+            KB.findByIdAndDelete(kb_id, (err, deletedKb) => {
+
+                if (err) {
+                    winston.error("Delete kb error: ", err);
+                    return res.status(500).send({ success: false, error: err });
+                }
+                res.status(200).send(deletedKb);
+            })
+
+        } else {
+            winston.info("resp.data: ", resp.data);
+
+            KB.findOneAndDelete({ _id: kb_id, status: { $in: [-1, 3, 4] } }, (err, deletedKb) => {
+                if (err) {
+                    winston.error("findOneAndDelete err: ", err);
+                    return res.status(500).send({ success: false, error: "Unable to delete the content due to an error" })
+                }
+                else if (!deletedKb) {
+                    winston.verbose("Unable to delete the content in indexing status")
+                    return res.status(500).send({ success: false, error: "Unable to delete the content in indexing status" })
+                } else {
+                    res.status(200).send(deletedKb);
+                }
+            })
+        }
+
+    }).catch((err) => {
+        let status = err.response.status;
+        res.status(status).send({ success: false, statusText: err.response.statusText, error: err.response.data.detail });
+    })
+
+})
+
+async function updateStatus(id, status) {
+    return new Promise((resolve) => {
+
+        KB.findByIdAndUpdate(id, { status: status }, (err, updatedKb) => {
+            if (err) {
+                resolve(false)
+            } else {
+                winston.debug("updatedKb: ", updatedKb)
+                resolve(true);
+            }
+        })
+    })
+}
+
+async function startScrape(data) {
+
+    if (!data.gptkey) {
+        let gptkey = process.env.GPTKEY;
+        if (!gptkey) {
+            return { error: "GPT apikey undefined" }
+        }
+        data.gptkey = gptkey;
+    }
+
+    return new Promise((resolve, reject) => {
+        openaiService.singleScrape(data).then(async (resp) => {
+            winston.debug("singleScrape resp: ", resp.data);
+            let status_updated = await updateStatus(data.id, 0);
+            winston.verbose("status of kb " + data.id + " updated: " + status_updated);
+            resolve(resp.data);
+        }).catch((err) => {
+            winston.error("singleScrape err: ", err);
+            reject(err);
+        })
+    })
+}
+
+module.exports = router;

package/routes/kbsettings.js

@@ -1,5 +1,6 @@
 var express = require('express');
-var KBSettings = require('../models/kb_setting');
+var { KBSettings } = require('../models/kb_setting');
+var { KB } = require('../models/kb_setting');
 // var KB = require('../models/kb_setting')
 var router = express.Router();
 var winston = require('../config/winston');
@@ -98,11 +99,75 @@ router.delete('/:settings_id/:kb_id', async (req, res) => {
 
 })
 
+// PROXY PUGLIA AI V2 - START
+router.post('/scrape/single', async (req, res) => {
+
+    let data = req.body;
+    winston.debug("/scrape/single data: ", data);
+
+    let gptkey = process.env.GPTKEY;
+    if (!gptkey) {
+        return res.status(403).send({ success: false, error: "GPT apikey undefined"})
+    }
+
+    data.gptkey = gptkey;
+    
+    openaiService.singleScrape(data).then((resp) => {
+        winston.debug("singleScrape resp: ", resp.data);
+        return res.status(200).send(resp.data);
+    }).catch((err) => {
+        winston.error("singleScrape err: ", err);
+        let status = err.response.status;
+        return res.status(status).send({ statusText: err.response.statusText, detail: err.response.data.detail });
+    })
+})
+
+router.post('/scrape/status', async (req, res) => {
+
+    let data = req.body;
+    winston.debug("/scrapeStatus req.body: ", req.body);
+
+    openaiService.scrapeStatus(data).then((response) => {
+
+        winston.debug("scrapeStatus response.data: ", response.data);
+        res.status(200).send(response.data);
+    }).catch((err) => {
+        winston.error("scrapeStatus err: ", err);
+        let status = err.response.status;
+        res.status(status).send({ statusText: err.response.statusText, detail: err.response.data.detail });
+    })
+})
+
+router.post('/ask', async (req, res) => {
+    let data = req.body;
+    winston.debug("/qa data: ", data);
+
+    if (!data.gptkey) {
+        let gptkey = process.env.GPTKEY;
+        if (!gptkey) {
+            return res.status(403).send({ success: false, error: "GPT apikey undefined"})
+        }
+        data.gptkey = gptkey;
+    }
+
+    openaiService.askNamespace(data).then((resp) => {
+        winston.debug("qa resp: ", resp.data);
+        res.status(200).send(resp.data);
+    }).catch((err) => {
+        winston.error("qa err: ", err);
+        let status = err.response.status;
+        res.status(status).send({ statusText: err.response.statusText, detail: err.response.data.detail });
+    })
+})
+
+// PROXY PUGLIA AI V2 - END
+
 
 // PROXY PUGLIA AI - START
 router.post('/qa', async (req, res) => {
     let data = req.body;
     winston.debug("/qa data: ", data);
+    winston.info("/qa data: ", data);
 
     openaiService.ask(data).then((resp) => {
         winston.debug("qa resp: ", resp.data);
@@ -110,6 +175,7 @@ router.post('/qa', async (req, res) => {
     }).catch((err) => {
         winston.error("qa err: ", err);
         let status = err.response.status;
+        winston.info("status on error: ", status)
         res.status(status).send({ statusText: err.response.statusText, detail: err.response.data.detail });
     })
 })
@@ -132,7 +198,7 @@ router.post('/startscrape', async (req, res) => {
 
 router.post('/checkstatus', async (req, res) => {
 
-    // let data = req.body;
+    //let data = req.body;
     winston.debug("/checkstatus req.body: ", req.body);
 
     let full_url = req.body.full_url;
@@ -141,7 +207,7 @@ router.post('/checkstatus', async (req, res) => {
     }
 
     openaiService.checkStatus(data).then((resp) => {
-        winston.debug("checkStatus resp: ", resp.data);
+        winston.debug("checkStatus resp: ", resp);
         winston.debug("checkStatus resp: ", resp.data);
         winston.debug("checkStatus resp: ", resp.data[full_url]);
 
@@ -168,7 +234,7 @@ router.post('/checkstatus', async (req, res) => {
         }
 
         
-        res.status(200).send(return_data);
+        res.status(200).send(resp);
     }).catch((err) => {
         winston.error("checkstatus err: ", err);
         let status = err.response.status;
@@ -188,10 +254,14 @@ router.post('/:settings_id', async (req, res) => {
             return res.status(500).send({ success: false, error: err});
         } else {
 
-            let new_kb = {
+            let new_kb = new KB({
                 name: body.name,
-                url: body.url
-            }
+                url: body.url,
+                source: body.source,
+                type: body.type,
+                content: body.content,
+                namespace: body.namespace
+            })
             settings.kbs.push(new_kb);
 
             KBSettings.findByIdAndUpdate( settings_id, settings, { new: true }, (err, savedSettings) => {
@@ -199,7 +269,7 @@ router.post('/:settings_id', async (req, res) => {
                     winston.err("findByIdAndUpdate error: ", err);
                     res.status(500).send({ success: false, error: err });
                 } else {
-                    res.status(200).send(savedSettings);
+                    res.status(200).send(new_kb);
                 }
             })
         }

package/routes/openai.js

@@ -1,6 +1,6 @@
 var express = require('express');
 var router = express.Router();
-var KBSettings = require('../models/kb_setting');
+var { KBSettings } = require('../models/kb_setting');
 var openaiService = require('../services/openaiService');
 var winston = require('../config/winston');
 

package/services/openaiService.js

@@ -7,97 +7,203 @@ let openai_endpoint = process.env.OPENAI_ENDPOINT;
 let kb_endpoint = process.env.KB_ENDPOINT;
 
 class OpenaiService {
-    
-    // OPEN AI
-    completions(data, gptkey) {
-        
-        winston.debug("[OPENAI SERVICE] openai endpoint: ", openai_endpoint);
-
-        return new Promise((resolve, reject) => {
-
-            axios({
-                url: openai_endpoint + "/chat/completions",
-                headers: {
-                    'Content-Type': 'application/json',
-                    'Authorization': "Bearer " + gptkey
-                },
-                data: data,
-                method: 'POST'
-            }).then((resbody) => {
-                resolve(resbody);
-            }).catch((err) => {
-                reject(err);
-            })
-
-        })
-
-    }
-    
-
-    // PUGLIA AI
-    checkStatus(data) {
-        winston.debug("[OPENAI SERVICE] kb endpoint: ", kb_endpoint);
-
-        return new Promise((resolve, reject) => {
-
-            axios({
-                url: kb_endpoint + "/scrape/status",
-                headers: {
-                    'Content-Type': 'application/json'
-                },
-                data: data,
-                method: 'POST'
-            }).then((resbody) => {
-                resolve(resbody);
-            }).catch((err) => {
-                reject(err);
-            })
-
-        })
-    }
-
-    startScrape(data) {
-        winston.debug("[OPENAI SERVICE] kb endpoint: ", kb_endpoint);
-
-        return new Promise((resolve, reject) => {
-
-            axios({
-                url: kb_endpoint + "/scrape",
-                headers: {
-                    'Content-Type': 'application/json'
-                },
-                data: data,
-                method: 'POST'
-            }).then((resbody) => {
-                resolve(resbody);
-            }).catch((err) => {
-                reject(err);
-            })
-
-        })
-    }
-
-    ask(data) {
-        winston.debug("[OPENAI SERVICE] kb endpoint: ", kb_endpoint);
-
-        return new Promise((resolve, reject) => {
-
-            axios({
-                url: kb_endpoint + "/qa",
-                headers: {
-                    'Content-Type': 'application/json'
-                },
-                data: data,
-                method: 'POST'
-            }).then((resbody) => {
-                resolve(resbody);
-            }).catch((err) => {
-                console.log("err: ", err);
-                reject(err);
-            })
-
-        })
-    }
+
+  // OPEN AI
+  completions(data, gptkey) {
+
+    winston.debug("[OPENAI SERVICE] openai endpoint: ", openai_endpoint);
+
+    return new Promise((resolve, reject) => {
+
+      axios({
+        url: openai_endpoint + "/chat/completions",
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': "Bearer " + gptkey
+        },
+        data: data,
+        method: 'POST'
+      }).then((resbody) => {
+        resolve(resbody);
+      }).catch((err) => {
+        reject(err);
+      })
+
+    })
+
+  }
+
+
+  // PUGLIA AI
+  checkStatus(data) {
+    winston.debug("[OPENAI SERVICE] kb endpoint: ", kb_endpoint);
+
+    return new Promise((resolve, reject) => {
+
+      axios({
+        url: kb_endpoint + "/scrape/status",
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        data: data,
+        method: 'POST'
+      }).then((resbody) => {
+        resolve(resbody);
+      }).catch((err) => {
+        reject(err);
+      })
+
+    })
+  }
+
+  startScrape(data) {
+    winston.debug("[OPENAI SERVICE] kb endpoint: ", kb_endpoint);
+
+    return new Promise((resolve, reject) => {
+
+      axios({
+        url: kb_endpoint + "/scrape",
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        data: data,
+        method: 'POST'
+      }).then((resbody) => {
+        resolve(resbody);
+      }).catch((err) => {
+        reject(err);
+      })
+
+    })
+  }
+
+  ask(data) {
+    winston.debug("[OPENAI SERVICE] kb endpoint: ", kb_endpoint);
+
+    return new Promise((resolve, reject) => {
+
+      axios({
+        url: kb_endpoint + "/qa",
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        data: data,
+        method: 'POST'
+      }).then((resbody) => {
+        resolve(resbody);
+      }).catch((err) => {
+        console.log("err: ", err);
+        reject(err);
+      })
+
+    })
+  }
+
+  // PUGLIA AI V2
+  singleScrape(data) {
+    winston.debug("[OPENAI SERVICE] kb endpoint: ", kb_endpoint);
+
+    return new Promise((resolve, reject) => {
+
+      axios({
+        url: kb_endpoint + "/scrape/single",
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        data: data,
+        method: 'POST'
+      }).then((resbody) => {
+        resolve(resbody);
+      }).catch((err) => {
+        reject(err);
+      })
+
+    })
+  }
+
+  scrapeStatus(data) {
+    winston.debug("[OPENAI SERVICE] kb endpoint: ", kb_endpoint);
+
+    return new Promise((resolve, reject) => {
+
+      axios({
+        url: kb_endpoint + "/scrape/status",
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        data: data,
+        method: 'POST'
+      }).then((resbody) => {
+        resolve(resbody);
+      }).catch((err) => {
+        reject(err);
+      })
+    })
+  }
+
+  askNamespace(data) {
+    winston.debug("[OPENAI SERVICE] kb endpoint: ", kb_endpoint);
+
+    return new Promise((resolve, reject) => {
+
+      axios({
+        url: kb_endpoint + "/qa",
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        data: data,
+        method: 'POST'
+      }).then((resbody) => {
+        resolve(resbody);
+      }).catch((err) => {
+        reject(err);
+      })
+
+    })
+  }
+
+
+  deleteIndex(data) {
+    winston.debug("[OPENAI SERVICE] kb endpoint: ", kb_endpoint);
+
+    return new Promise((resolve, reject) => {
+
+      axios({
+        url: kb_endpoint + "/delete/id",
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        data: data,
+        method: 'POST'
+      }).then((resbody) => {
+        resolve(resbody);
+      }).catch((err) => {
+        reject(err);
+      })
+    })
+  }
+
+  deleteNamespace(data) {
+    winston.debug("[OPENAI SERVICE] kb endpoint: ", kb_endpoint);
+
+    return new Promise((resolve, reject) => {
+
+      axios({
+        url: kb_endpoint + "/delete/namespace",
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        data: data,
+        method: 'POST'
+      }).then((resbody) => {
+        resolve(resbody);
+      }).catch((err) => {
+        reject(err);
+      })
+    })
+  }
+
+
 }
 
 var openaiService = new OpenaiService();

package/test/kbRoute.js

@@ -0,0 +1,134 @@
+//During the test the env variable is set to test
+process.env.NODE_ENV = 'test';
+
+var userService = require('../services/userService');
+var projectService = require('../services/projectService');
+
+let log = true;
+
+//Require the dev-dependencies
+let chai = require('chai');
+let chaiHttp = require('chai-http');
+let server = require('../app');
+let should = chai.should();
+var fs = require('fs');
+const path = require('path');
+
+// chai.config.includeStack = true;
+
+var expect = chai.expect;
+var assert = chai.assert;
+
+chai.use(chaiHttp);
+
+describe('KbRoute', () => {
+
+    describe('/create', () => {
+
+        it('createNewKb', (done) => {
+
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+                projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+                    let kb = {
+                        name: "example_name5",
+                        type: "url",
+                        source: "https://www.exampleurl5.com",
+                        content: ""
+                    }
+
+                    chai.request(server)
+                        .post('/' + savedProject._id + '/kb')
+                        .auth(email, pwd)
+                        .send(kb) // can be empty
+                        .end((err, res) => {
+                            if (log) { console.log("create kb res.body: ", res.body); }
+                            res.should.have.status(200);
+                            // res.body.should.be.a('object');
+                            // expect(res.body.id_project).to.equal(savedProject._id.toString());
+
+                            done();
+                            // chai.request(server)
+                            //     .get('/' + savedProject._id + "/kbsettings")
+                            //     .auth(email, pwd)
+                            //     .end((err, res) => {
+                            //         if (log) { console.log("get kbsettings res.body: ", res.body); }
+                            //         res.should.have.status(200);
+                            //         res.body.should.be.a('object');
+                            //         expect(res.body.id_project).to.equal(savedProject._id.toString())
+                            //         expect(res.body.maxKbsNumber).to.equal(3);
+                            //         expect(res.body.maxPagesNumber).to.equal(1000);
+                            //         expect(res.body.kbs).is.an('array').that.is.empty;
+
+
+                            //     })
+
+                        })
+
+                });
+            });
+
+        });
+
+        it('scrapeSingle', (done) => {
+
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+                projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+                    let kb = {
+                        name: "example_name6",
+                        type: "url",
+                        source: "https://www.exampleurl6.com",
+                        content: ""
+                    }
+
+                    chai.request(server)
+                        .post('/' + savedProject._id + '/kb')
+                        .auth(email, pwd)
+                        .send(kb) // can be empty
+                        .end((err, res) => {
+                            if (log) { console.log("create kb res.body: ", res.body); }
+                            res.should.have.status(200);
+
+                            let kbid = res.body.value._id;
+                            console.log("kbid: ", kbid)
+                            chai.request(server)
+                                .post('/' + savedProject._id + "/kb/scrape/single")
+                                .auth(email, pwd)
+                                .send({ id: kbid })
+                                .end((err, res) => {
+                                    if (log) { console.log("single scrape res.body: ", res.body); }
+                                    //res.should.have.status(200);
+                                    // res.body.should.be.a('object');
+                                    // expect(res.body.id_project).to.equal(savedProject._id.toString())
+                                    // expect(res.body.maxKbsNumber).to.equal(3);
+                                    // expect(res.body.maxPagesNumber).to.equal(1000);
+                                    // expect(res.body.kbs).is.an('array').that.is.empty;
+                                    done();
+
+                                })
+
+
+                            // res.body.should.be.a('object');
+                            // expect(res.body.id_project).to.equal(savedProject._id.toString());
+
+
+
+
+                        })
+
+                });
+            });
+
+        });
+
+    })
+});
+
+

package/test/kbsettingsRoute.js

@@ -25,7 +25,7 @@ describe('KbSettingsRoute', () => {
 
     describe('/create', () => {
 
-        it('create kb settings', (done) => {
+        it('createKbSettings', (done) => {
 
             var email = "test-signup-" + Date.now() + "@email.com";
             var pwd = "pwd";
@@ -65,7 +65,7 @@ describe('KbSettingsRoute', () => {
 
         });
 
-        it('create kb settings if not exists', (done) => {
+        it('createKbSettingsIfNotExists', (done) => {
 
             var email = "test-signup-" + Date.now() + "@email.com";
             var pwd = "pwd";
@@ -93,7 +93,7 @@ describe('KbSettingsRoute', () => {
         });
 
 
-        it('add kb to kb settings', (done) => {
+        it('addKbToKbSettings', (done) => {
 
             var email = "test-signup-" + Date.now() + "@email.com";
             var pwd = "pwd";
@@ -111,25 +111,27 @@ describe('KbSettingsRoute', () => {
                             res.body.should.be.a('object');
                             expect(res.body.id_project).to.equal(savedProject._id.toString());
 
+                            let settings_id = res.body._id;
+
                             chai.request(server)
-                                .post('/' + savedProject._id + "/kbsettings/" + res.body._id)
+                                .post('/' + savedProject._id + "/kbsettings/" + settings_id)
                                 .auth(email, pwd)
                                 .send({ name: "exampleurl.com/kb/", url: "https://exampleurl.com/kb/" })
                                 .end((err, res) => {
                                     if (log) { console.log("add kb to kb settings res.body: ", res.body); }
                                     res.should.have.status(200);
                                     res.body.should.be.a('object');
-                                    expect(res.body.kbs).to.have.length(1)
+                                    //expect(res.body.kbs).to.have.length(1)
 
                                     chai.request(server)
-                                        .post('/' + savedProject._id + "/kbsettings/" + res.body._id)
+                                        .post('/' + savedProject._id + "/kbsettings/" + settings_id)
                                         .auth(email, pwd)
                                         .send({ name: "secondurl.com/support/", url: "https://secondurl.com/support/" })
                                         .end((err, res) => {
                                             if (log) { console.log("add kb to kb settings res.body: ", res.body); }
                                             res.should.have.status(200);
                                             res.body.should.be.a('object');
-                                            expect(res.body.kbs).to.have.length(2)
+                                            //expect(res.body.kbs).to.have.length(2)
 
                                             done();
                                         })
@@ -142,7 +144,7 @@ describe('KbSettingsRoute', () => {
 
         });
 
-        it('update kb settings', (done) => {
+        it('updateKbSettings', (done) => {
 
             var email = "test-signup-" + Date.now() + "@email.com";
             var pwd = "pwd";
@@ -178,7 +180,7 @@ describe('KbSettingsRoute', () => {
 
         });
 
-        it('delete kb from list', (done) => {
+        it('deleteKbFromList', (done) => {
 
             var email = "test-signup-" + Date.now() + "@email.com";
             var pwd = "pwd";
@@ -206,9 +208,9 @@ describe('KbSettingsRoute', () => {
                                     if (log) { console.log("add kb to kb settings res.body: ", res.body); }
                                     res.should.have.status(200);
                                     res.body.should.be.a('object');
-                                    expect(res.body.kbs).to.have.length(1)
+                                    //expect(res.body.kbs).to.have.length(1)
 
-                                    let kb_to_delete_id = res.body.kbs[0]._id;
+                                    let kb_to_delete_id = res.body._id;
 
                                     chai.request(server)
                                         .post('/' + savedProject._id + "/kbsettings/" + settings_id)
@@ -218,7 +220,7 @@ describe('KbSettingsRoute', () => {
                                             if (log) { console.log("add kb to kb settings res.body: ", res.body); }
                                             res.should.have.status(200);
                                             res.body.should.be.a('object');
-                                            expect(res.body.kbs).to.have.length(2)
+                                            //expect(res.body.kbs).to.have.length(2)
 
                                             chai.request(server)
                                                 .delete('/' + savedProject._id + "/kbsettings/" + settings_id + "/" + kb_to_delete_id)