@tiledesk/tiledesk-server 2.9.32 → 2.10.0

package/CHANGELOG.md

@@ -5,6 +5,14 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.10.0
+- Added support for engine to namespace
+- Added support for advanced context to kb
+- Added support for scrape types to kb
+- Updated tybot-connector to 0.2.112
+- Updated vxml-connector to 0.1.54
+- Updated widget.json translation file
+
 # 2.9.32
 - Added voice filters in get requests
 - Added endpoint to get all projects

package/config/labels/widget.json

@@ -61,6 +61,7 @@
             "CLOSE_CHAT": "Close chat",
             "MINIMIZE":"Minimize",
             "MAXIMIZE":"Maximize",
+            "CONFIRM_CLOSE_CHAT":"Are you sure you wanna close this chat?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT":"you",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB":"you have been added ",
@@ -160,6 +161,7 @@
             "CLOSE_CHAT": "Chiudi chat",
             "MINIMIZE":"Minimizza",
             "MAXIMIZE":"Massimizza",
+            "CONFIRM_CLOSE_CHAT":"Vuoi davvero chiudere questa chat?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT":"tu",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB":"sei stato aggiunto ",
@@ -259,6 +261,7 @@
             "CLOSE_CHAT": "Fermer le chat",
             "MINIMIZE":"Minimiser",
             "MAXIMIZE":"Maximiser",
+            "CONFIRM_CLOSE_CHAT":"Etes-vous sûr de vouloir fermer cette discussion?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT":"toi",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB":"ont été ajoutés à ",
@@ -358,6 +361,7 @@
             "CLOSE_CHAT": "Cerrar chat",
             "MINIMIZE":"Minimizar",
             "MAXIMIZE":"Maximizar",
+            "CONFIRM_CLOSE_CHAT":"¿Estás segura de que quieres cerrar este chat?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT":"tú",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB":"han sido agregados a ",
@@ -457,6 +461,7 @@
             "CLOSE_CHAT": "Chat schließen",
             "MINIMIZE":"Minimieren",
             "MAXIMIZE":"Maximieren",
+            "CONFIRM_CLOSE_CHAT":"Möchten Sie diesen Chat wirklich schließen?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT": "Sie",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB": "Sie wurden hinzugefügt ",
@@ -556,6 +561,7 @@
             "CLOSE_CHAT": "Fechar bate-papo",
             "MINIMIZE":"Minimizar",
             "MAXIMIZE":"Maximizar",
+            "CONFIRM_CLOSE_CHAT":"Tem certeza de que deseja fechar este bate-papo?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT": "você",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB": "você foi adicionado ",
@@ -655,6 +661,7 @@
             "CLOSE_CHAT": "Закрыть чат",
             "MINIMIZE":"Свести к минимуму",
             "MAXIMIZE":"Максимизировать",
+            "CONFIRM_CLOSE_CHAT":"Вы уверены, что хотите закрыть этот чат?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT": "вы",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB": "вы добавлены ",
@@ -754,6 +761,7 @@
             "CLOSE_CHAT": "Sohbeti kapat",
             "MINIMIZE":"Küçült",
             "MAXIMIZE":"Büyüt",
+            "CONFIRM_CLOSE_CHAT":"Bu sohbeti kapatmak istediğinizden emin misiniz?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT": "siz",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB": "eklendiniz ",
@@ -855,6 +863,7 @@
             "CLOSE_CHAT": "Затвори ћаскање",
             "MINIMIZE":"Минимизирајте",
             "MAXIMIZE":"Максимизирајте",
+            "CONFIRM_CLOSE_CHAT":"Да ли сте сигурни да желите да затворите ово ћаскање?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT":"ти",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB":"додани сте",
@@ -956,6 +965,7 @@
             "CLOSE_CHAT": "إغلاق الدردشة",
             "MINIMIZE":"تصغير",
             "MAXIMIZE":"تحقيق أقصى قدر",
+            "CONFIRM_CLOSE_CHAT":"هل أنت متأكد أنك تريد إغلاق هذه الدردشة؟",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT": "أنت",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB": "لقد تم اضافتك ",
@@ -1057,6 +1067,7 @@
             "CLOSE_CHAT": "Закрити чат",
             "MINIMIZE":"Згорнути",
             "MAXIMIZE":"Максимізувати",
+            "CONFIRM_CLOSE_CHAT":"Ви впевнені, що бажаєте закрити цей чат?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT": "ти",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB": "вас додали ",
@@ -1157,6 +1168,7 @@
             "CLOSE_CHAT":"Stäng chatten",
             "MINIMIZE":"Minimera",
             "MAXIMIZE":"Maximera",
+            "CONFIRM_CLOSE_CHAT":"Är du säker på att du vill stänga den här chatten?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT": "du",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB": "du har lagts till ",
@@ -1257,6 +1269,7 @@
             "CLOSE_CHAT":"Чатты жабу",
             "MINIMIZE":"Кішірейту",
             "MAXIMIZE":"Үлкейту",
+            "CONFIRM_CLOSE_CHAT":"Бұл чатты шынымен жапқыңыз келе ме?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT": "сен",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB": "сіз қосылдыңыз ",
@@ -1357,6 +1370,7 @@
             "CLOSE_CHAT":"Chatni yopish",
             "MINIMIZE":"Minimallashtirish",
             "MAXIMIZE":"Maksimallashtirish",
+            "CONFIRM_CLOSE_CHAT":"Bu chatni yopmoqchi ekanligingizga ishonchingiz komilmi?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT": "siz",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB": "qo'shildingiz ",
@@ -1457,6 +1471,7 @@
             "CLOSE_CHAT": "Söhbəti bağlayın",
             "MINIMIZE":"Minimallaşdırın",
             "MAXIMIZE":"Maksimallaşdırın",
+            "CONFIRM_CLOSE_CHAT":"Bu söhbəti bağlamaq istədiyinizə əminsiniz?",
 
             "INFO_SUPPORT_USER_ADDED_SUBJECT": "Sən",
             "INFO_SUPPORT_USER_ADDED_YOU_VERB": "siz əlavə olundunuz ",

package/deploy.sh

@@ -1,5 +1,5 @@
 git pull
-npm version patch
+npm version minor
 version=`node -e 'console.log(require("./package.json").version)'`
 echo "version $version"
 

package/models/kb_setting.js

@@ -2,6 +2,31 @@ var mongoose = require('mongoose');
 var Schema = mongoose.Schema;
 var winston = require('../config/winston');
 
+var EngineSchema = new Schema({
+  name: {
+    type: String,
+    required: true
+  },
+  type: {
+    type: String,
+    required: true
+  },
+  apikey: {
+    type: String,
+    required: false
+  },
+  vector_size: {
+    type: Number,
+    required: true
+  },
+  index_name: {
+    type: String,
+    required: true
+  }
+}, {
+  _id: false  // This is schema is always used as an embedded object inside NamespaceSchema
+})
+
 var NamespaceSchema = new Schema({
   id_project: {
     type: String,
@@ -22,6 +47,10 @@ var NamespaceSchema = new Schema({
   default: {
     type: Boolean,
     default: false
+  },
+  engine: {
+    type: EngineSchema,
+    required: false
   }
 }, {
   timestamps: true
@@ -59,12 +88,34 @@ var KBSchema = new Schema({
   status: {
     type: Number,
     required: false
+  },
+  scrape_type: {
+    type: Number,
+    required: false
+  },
+  scrape_options: {
+    type: Object,
+    required: false,
+    default: undefined,
+    tags_to_extract: {
+      type: Array,
+      required: false
+    },
+    unwanted_tags: {
+      type: Array,
+      required: false
+    },
+    unwanted_classnames: {
+      type: Array,
+      required: false
+    }
   }
 }, {
   timestamps: true
 })
 
 
+
 // DEPRECATED !! - Start
 var KBSettingSchema = new Schema({
   id_project: {
@@ -93,7 +144,7 @@ KBSchema.index({ createdAt: -1, updatedAt: -1 })
 
 // DEPRECATED
 const KBSettings = mongoose.model('KBSettings', KBSettingSchema); 
-
+const Engine = mongoose.model('Engine', EngineSchema)
 const Namespace = mongoose.model('Namespace', NamespaceSchema)
 const KB = mongoose.model('KB', KBSchema)
 
@@ -105,5 +156,6 @@ const KB = mongoose.model('KB', KBSchema)
 module.exports = {
   KBSettings: KBSettings,
   Namespace: Namespace,
+  Engine: Engine,
   KB: KB
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.9.32",
+  "version": "2.10.0",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",
@@ -48,7 +48,7 @@
     "@tiledesk/tiledesk-rasa-connector": "^1.0.10",
     "@tiledesk/tiledesk-telegram-connector": "^0.1.14",
     "@tiledesk/tiledesk-train-jobworker": "^0.0.11",
-    "@tiledesk/tiledesk-tybot-connector": "^0.2.107",
+    "@tiledesk/tiledesk-tybot-connector": "^0.2.112",
     "@tiledesk/tiledesk-whatsapp-connector": "^0.1.73",
     "@tiledesk/tiledesk-whatsapp-jobworker": "^0.0.8",
     "@tiledesk/tiledesk-sms-connector": "^0.1.10",

package/routes/kb.js

@@ -1,6 +1,7 @@
 var express = require('express');
-var { Namespace } = require('../models/kb_setting');
-var { KB } = require('../models/kb_setting');
+var { Namespace, KB, Engine } = require('../models/kb_setting');
+// var { KB } = require('../models/kb_setting');
+// var { Engine } = require('../models/kb_setting')
 var router = express.Router();
 var winston = require('../config/winston');
 var multer = require('multer')
@@ -17,6 +18,7 @@ let Integration = require('../models/integrations');
 var parsecsv = require("fast-csv");
 
 const { MODELS_MULTIPLIER } = require('../utils/aiUtils');
+const { body } = require('express-validator');
 
 const AMQP_MANAGER_URL = process.env.AMQP_MANAGER_URL;
 const JOB_TOPIC_EXCHANGE = process.env.JOB_TOPIC_EXCHANGE_TRAIN || 'tiledesk-trainer';
@@ -45,6 +47,13 @@ let default_preview_settings = {
   //context: "You are an awesome AI Assistant."
   context: null
 }
+let default_engine = {
+  name: "pinecone",
+  type: "pod",
+  apikey: "",
+  vector_size: 1536,
+  index_name: process.env.PINECONE_INDEX
+}
 
 //let default_context = "Answer if and ONLY if the answer is contained in the context provided. If the answer is not contained in the context provided ALWAYS answer with <NOANS>\n{context}"
 //let default_context = "You are an helpful assistant for question-answering tasks.\nUse ONLY the following pieces of retrieved context to answer the question.\nIf you don't know the answer, just say that you don't know.\nIf none of the retrieved context answer the question, add this word to the end <NOANS>\n\n{context}";
@@ -68,9 +77,9 @@ router.post('/scrape/single', async (req, res) => {
   let data = req.body;
   winston.debug("/scrape/single data: ", data);
 
-  if (!data.scrape_type) {
-    data.scrape_type = 1;
-  }
+  // if (!data.scrape_type) {
+  //   data.scrape_type = 1;
+  // }
 
   let namespaces = await Namespace.find({ id_project: project_id }).catch((err) => {
     winston.error("find namespaces error: ", err)
@@ -112,11 +121,28 @@ router.post('/scrape/single', async (req, res) => {
         json.content = kb.content;
       }
 
-      json.scrape_type = 1;
-      if (data.scrape_type) {
-        json.scrape_type = data.scrape_type;
+      // json.scrape_type = 1;
+      // if (data.scrape_type) {
+      //   json.scrape_type = data.scrape_type;
+      // }
+
+      if (kb.scrape_type) {
+        json.scrape_type = kb.scrape_type
       }
 
+      if (kb.scrape_options) {
+        json.parameters_scrape_type_4 = {
+          tags_to_extract: kb.scrape_options.tags_to_extract,
+          unwanted_tags: kb.scrape_options.unwanted_tags,
+          unwanted_classnames: kb.scrape_options.unwanted_classnames
+        }
+      }
+
+      let ns = namespaces.find(n => n.id === kb.namespace);
+      json.engine = ns.engine || default_engine;
+
+      winston.verbose("/scrape/single json: ", json);
+
       startScrape(json).then((response) => {
         winston.verbose("startScrape response: ", response);
         res.status(200).send(response);
@@ -132,6 +158,8 @@ router.post('/scrape/single', async (req, res) => {
 
 router.post('/scrape/status', async (req, res) => {
 
+  let project_id = req.projectid;
+  // (EXAMPLE) body: { id, namespace }
   let data = req.body;
   winston.debug("/scrapeStatus req.body: ", req.body);
 
@@ -143,6 +171,25 @@ router.post('/scrape/status', async (req, res) => {
     returnObject = true;
   }
 
+  let namespaces = await Namespace.find({ id_project: project_id }).catch((err) => {
+    winston.error("find namespaces error: ", err)
+    res.status(500).send({ success: false, error: err })
+  })
+
+  if (!namespaces || namespaces.length == 0) {
+    let alert = "No namespace found for the selected project " + project_id + ". Cannot add content to a non-existent namespace."
+    winston.warn(alert);
+    res.status(403).send(alert);
+  }
+
+  let namespaceIds = namespaces.map(namespace => namespace.id);
+  if (!namespaceIds.includes(data.namespace)) {
+    return res.status(403).send({ success: false, error: "Not allowed. The namespace does not belong to the current project." })
+  }
+
+  let ns = namespaces.find(n => n.id === data.namespace);
+  data.engine = ns.engine || default_engine;
+
   openaiService.scrapeStatus(data).then(async (response) => {
 
     winston.debug("scrapeStatus response.data: ", response.data);
@@ -229,7 +276,7 @@ router.post('/qa', async (req, res) => {
   }
 
   // Check if "Advanced Mode" is active. In such case the default_context must be not appended
-  if (!data.advanced_context) {
+  if (!data.advancedPrompt) {
     if (data.system_context) {
       data.system_context = data.system_context + " \n" + contexts[data.model];
     } else {
@@ -237,9 +284,15 @@ router.post('/qa', async (req, res) => {
     }
   }
 
-  // if (process.env.NODE_ENV === 'test') {
-  //   return res.status(200).send({ success: true, message: "Question skipped in test environment"});
-  // }
+  let ns = namespaces.find(n => n.id === data.namespace);
+  data.engine = ns.engine || default_engine;
+  
+  delete data.advancedPrompt;
+  winston.verbose("ask data: ", data);
+  
+  if (process.env.NODE_ENV === 'test') {
+    return res.status(200).send({ success: true, message: "Question skipped in test environment"});
+  }
 
   openaiService.askNamespace(data).then((resp) => {
     winston.debug("qa resp: ", resp.data);
@@ -317,6 +370,9 @@ router.delete('/delete', async (req, res) => {
   if (!namespaceIds.includes(data.namespace)) {
     return res.status(403).send({ success: false, error: "Not allowed. The namespace does not belong to the current project." })
   }
+  
+  let ns = namespaces.find(n => n.id === data.namespace);
+  data.engine = ns.engine || default_engine;
 
   openaiService.deleteIndex(data).then((resp) => {
     winston.debug("delete resp: ", resp.data);
@@ -352,6 +408,11 @@ router.delete('/deleteall', async (req, res) => {
     return res.status(403).send({ success: false, error: "Not allowed. The namespace does not belong to the current project." })
   }
 
+  let ns = namespaces.find(n => n.id === data.namespace);
+  data.engine = ns.engine || default_engine;
+
+  winston.verbose("/deleteall data: ", data);
+
   openaiService.deleteNamespace(data).then((resp) => {
     winston.debug("delete namespace resp: ", resp.data);
     res.status(200).send(resp.data);
@@ -394,7 +455,8 @@ router.get('/namespace/all', async (req, res) => {
         id: project_id,
         name: "Default",
         preview_settings: default_preview_settings,
-        default: true
+        default: true,
+        engine: default_engine
       })
 
       new_namespace.save((err, savedNamespace) => {
@@ -448,7 +510,11 @@ router.get('/namespace/:id/chunks/:content_id', async (req, res) => {
     return res.status(403).send({ success: false, error: "Not allowed. The conten does not belong to the current namespace." })
   }
 
-  openaiService.getContentChunks(namespace_id, content_id).then((resp) => {
+  let ns = namespaces.find(n => n.id === namespace_id);
+  let engine = ns.engine || default_engine;
+  delete engine._id;
+
+  openaiService.getContentChunks(namespace_id, content_id, engine).then((resp) => {
     let chunks = resp.data;
     winston.debug("chunks for content " + content_id);
     winston.debug("chunks found ", chunks);
@@ -492,7 +558,6 @@ router.get('/namespace/:id/chatbots', async (req, res) => {
   let chatbots = intents.map(i => i.id_faq_kb);
   let uniqueChatbots = [...new Set(chatbots)];
 
-  
   let chatbotPromises = uniqueChatbots.map(async (c_id) => {
     try {
       let chatbot = await faq_kb.findOne({ _id: c_id, trashed: false });
@@ -527,6 +592,7 @@ router.post('/namespace', async (req, res) => {
     id: namespace_id,
     name: body.name,
     preview_settings: default_preview_settings,
+    engine: default_engine
   })
 
   let namespaces = await Namespace.find({ id_project: project_id }).catch((err) => {
@@ -589,11 +655,23 @@ router.put('/namespace/:id', async (req, res) => {
 
 router.delete('/namespace/:id', async (req, res) => {
 
-  let id_project = req.projectid;
+  let project_id = req.projectid;
   let namespace_id = req.params.id;
 
+  let namespace = await Namespace.findOne({ id_project: project_id, id: namespace_id }).catch((err) => {
+    winston.error("find namespace error: ", err);
+    res.status(500).send({ success: false, error: err });
+  })
+
+  if (!namespace) {
+    let alert = "No namespace found for id " + namespace_id;
+    winston.warn(alert);
+    res.status(404).send({ success: false, error: alert })
+  }
+
   let data = {
-    namespace: namespace_id
+    namespace: namespace_id,
+    engine: namespace.engine || default_engine
   }
 
   if (req.query.contents_only && (req.query.contents_only === true || req.query.contents_only === 'true')) {
@@ -601,7 +679,7 @@ router.delete('/namespace/:id', async (req, res) => {
     openaiService.deleteNamespace(data).then(async (resp) => {
       winston.debug("delete namespace resp: ", resp.data);
 
-      let deleteResponse = await KB.deleteMany({ id_project: id_project, namespace: namespace_id }).catch((err) => {
+      let deleteResponse = await KB.deleteMany({ id_project: project_id, namespace: namespace_id }).catch((err) => {
         winston.error("deleteMany error: ", err);
         return res.status(500).send({ success: false, error: err });
       })
@@ -635,7 +713,7 @@ router.delete('/namespace/:id', async (req, res) => {
     openaiService.deleteNamespace(data).then(async (resp) => {
       winston.debug("delete namespace resp: ", resp.data);
 
-      let deleteResponse = await KB.deleteMany({ id_project: id_project, namespace: namespace_id }).catch((err) => {
+      let deleteResponse = await KB.deleteMany({ id_project: project_id, namespace: namespace_id }).catch((err) => {
         winston.error("deleteMany error: ", err);
         return res.status(500).send({ success: false, error: err });
       })
@@ -831,7 +909,7 @@ router.post('/', async (req, res) => {
   if (!namespaceIds.includes(body.namespace)) {
     return res.status(403).send({ success: false, error: "Not allowed. The namespace does not belong to the current project." })
   }
-
+  
   let quoteManager = req.app.get('quote_manager');
   let limits = await quoteManager.getPlanLimits(req.project);
   let kbs_limit = limits.kbs;
@@ -848,6 +926,7 @@ router.post('/', async (req, res) => {
   if (total_count > kbs_limit) {
     return res.status(403).send({ success: false, error: "Cannot exceed the number of resources in the current plan", plan_limit: kbs_limit })
   }
+  
   let new_kb = {
     id_project: project_id,
     name: body.name,
@@ -860,6 +939,19 @@ router.post('/', async (req, res) => {
   if (!new_kb.namespace) {
     new_kb.namespace = project_id;
   }
+  if (new_kb.type === 'txt') {
+    new_kb.scrape_type = 1;
+  }
+  if (new_kb.type === 'url') {
+    if (!body.scrape_type || body.scrape_type === 2) {
+      new_kb.scrape_type = 2;
+      new_kb.scrape_options = await setDefaultScrapeOptions();
+    } else {
+      new_kb.scrape_type = body.scrape_type;
+      new_kb.scrape_options = body.scrape_options;
+    }
+  }
+
   winston.debug("adding kb: ", new_kb);
 
   KB.findOneAndUpdate({ id_project: project_id, type: 'url', source: new_kb.source }, new_kb, { upsert: true, new: true, rawResult: true }, async (err, raw) => {
@@ -871,26 +963,37 @@ router.post('/', async (req, res) => {
 
       res.status(200).send(raw);
 
+      let saved_kb = raw.value;
       let webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
 
       let json = {
-        id: raw.value._id,
-        type: raw.value.type,
-        source: raw.value.source,
+        id: saved_kb._id,
+        type: saved_kb.type,
+        source: saved_kb.source,
         content: "",
-        namespace: raw.value.namespace,
+        namespace: saved_kb.namespace,
         webhook: webhook
       }
       winston.debug("json: ", json);
 
-      if (raw.value.content) {
-        json.content = raw.value.content;
+      if (saved_kb.content) {
+        json.content = saved_kb.content;
+      }
+      if (saved_kb.scrape_type) {
+        json.scrape_type = saved_kb.scrape_type;
+      }
+      if (saved_kb.scrape_options) {
+        json.parameters_scrape_type_4 = saved_kb.scrape_options;
       }
+      let ns = namespaces.find(n => n.id === body.namespace);
+      json.engine = ns.engine || default_engine;
 
       let resources = [];
 
       resources.push(json);
-      scheduleScrape(resources);
+      if (!process.env.NODE_ENV) {
+        scheduleScrape(resources);
+      }
 
     }
   })
@@ -908,6 +1011,8 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
   }
 
   let project_id = req.projectid;
+  let scrape_type = req.body.scrape_type;
+  let scrape_options = req.body.scrape_options;
 
   let namespace_id = req.query.namespace;
   if (!namespace_id) {
@@ -956,16 +1061,37 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
   let webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
 
   let kbs = [];
-  list.forEach(url => {
-    kbs.push({
+  list.forEach( async (url) => {
+    let kb = {
       id_project: project_id,
       name: url,
       source: url,
       type: 'url',
       content: "",
       namespace: namespace_id,
-      status: -1
-    })
+      status: -1,
+      scrape_type: scrape_type
+    }
+
+    if (!kb.scrape_type) {
+      scrape_type = 2;
+    }
+
+    if (scrape_type == 2) {
+      kb.scrape_options = {
+        tags_to_extract: ["body"],
+        unwanted_tags: [],
+        unwanted_classnames: []
+      }
+    } else {
+      kb.scrape_options = scrape_options;
+    }
+    // if (scrape_type === 2) {
+    //   kb.scrape_options = await setDefaultScrapeOptions();
+    // } else {
+    //   kb.scrape_options = await setCustomScrapeOptions(scrape_options);
+    // }
+    kbs.push(kb)
   })
 
   let operations = kbs.map(doc => {
@@ -979,13 +1105,20 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
     }
   })
 
+    console.log("kbs: ", kbs);
+
   saveBulk(operations, kbs, project_id).then((result) => {
 
+    let ns = namespaces.find(n => n.id === namespace_id);
+    let engine = ns.engine || default_engine;
+
     let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
-    resources = resources.map(({ _id, ...rest }) => {
-      return { id: _id, webhook: webhook, ...rest };
+    resources = resources.map(({ _id, scrape_options, ...rest }) => {
+      return { id: _id, webhook: webhook, parameters_scrape_type_4: scrape_options, engine: engine, ...rest}
     });
+    console.log("resources to be sent to worker: ", resources);
     winston.verbose("resources to be sent to worker: ", resources);
+
     scheduleScrape(resources);
     res.status(200).send(result);
 
@@ -1085,9 +1218,13 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
       })
 
       saveBulk(operations, kbs, project_id).then((result) => {
+
+        let ns = namespaces.find(n => n.id === namespace_id);
+        let engine = ns.engine || default_engine;
+
         let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project,  ...keepAttrs }) => keepAttrs)
         resources = resources.map(({ _id, ...rest}) => {
-          return { id: _id, webhooh: webhook, ...rest };
+          return { id: _id, webhooh: webhook, engine: engine, ...rest };
         })
         winston.verbose("resources to be sent to worker: ", resources);
         if (!process.env.NODE_ENV) {
@@ -1166,7 +1303,6 @@ router.delete('/:kb_id', async (req, res) => {
   let kb_id = req.params.kb_id;
   winston.verbose("delete kb_id " + kb_id);
 
-
   let kb = await KB.findOne({ id_project: project_id, _id: kb_id}).catch((err) => {
     winston.warn("Unable to find kb. Error: ", err);
     return res.status(500).send({ success: false, error: err })
@@ -1186,6 +1322,16 @@ router.delete('/:kb_id', async (req, res) => {
     data.namespace = project_id;
   }
 
+  let namespaces = await Namespace.find({ id_project: project_id }).catch((err) => {
+    winston.error("find namespaces error: ", err)
+    res.status(500).send({ success: false, error: err })
+  })
+
+  let ns = namespaces.find(n => n.id === data.namespace);
+  data.engine = ns.engine || default_engine;
+
+  winston.verbose("/:delete_id data: ", data);
+
   openaiService.deleteIndex(data).then((resp) => {
     winston.debug("delete resp: ", resp.data);
     if (resp.data.success === true) {
@@ -1369,6 +1515,30 @@ async function getKeyFromIntegrations(project_id) {
     }
   })
 }
+
+async function setDefaultScrapeOptions() {
+  return {
+    tags_to_extract: ["body"],
+    unwanted_tags: [],
+    unwanted_classnames: []
+  }
+}
+
+async function setCustomScrapeOptions(options) {
+  if (!options) {
+    options = await setDefaultScrapeOptions();
+  } else {
+    if (!options.tags_to_extract || options.tags_to_extract.length == 0) {
+      options.tags_to_extract = ["body"];
+    }
+    if (!options.unwanted_tags) {
+      options.unwanted_tags = [];
+    }
+    if (!options.unwanted_classnames) {
+      options.unwanted_classnames = [];
+    }
+  }
+}
 /**
 * ****************************************
 * Utils Methods Section - End

package/services/openaiService.js

@@ -1,11 +1,13 @@
 var winston = require('../config/winston');
 const axios = require("axios").default;
 require('dotenv').config();
+const jwt = require("jsonwebtoken")
 
 let openai_endpoint = process.env.OPENAI_ENDPOINT;
 let kb_endpoint = process.env.KB_ENDPOINT;
 let kb_endpoint_train = process.env.KB_ENDPOINT_TRAIN;
 let kb_endpoint_qa = process.env.KB_ENDPOINT_QA;
+let secret = process.env.JWT_SECRET_KEY;
 
 class OpenaiService {
 
@@ -162,13 +164,15 @@ class OpenaiService {
     })
   }
 
-  getContentChunks(namespace_id, content_id) {
+  getContentChunks(namespace_id, content_id, engine) {
     winston.info("[OPENAI SERVICE] kb endpoint: " + kb_endpoint_train);
     winston.info(kb_endpoint_train + "/id/" + content_id + "/namespace/" + namespace_id)
     return new Promise((resolve, reject) => {
 
+      let payload = { engine: engine };
+      let token = jwt.sign(payload, secret);
       axios({
-        url: kb_endpoint_train + "/id/" + content_id + "/namespace/" + namespace_id,
+        url: kb_endpoint_train + "/id/" + content_id + "/namespace/" + namespace_id + "/" + token,
         headers: {
           'Content-Type': 'application/json'
         },

package/test/kbRoute.js

@@ -58,7 +58,6 @@ describe('KbRoute', () => {
                             expect(res.body.length).to.equal(1);
 
                             let namespace_id = res.body[0].id;
-                            console.log("namespace_id: ", namespace_id);
 
                             let kb = {
                                 name: "example_name5",
@@ -93,6 +92,128 @@ describe('KbRoute', () => {
 
         })
 
+        it('create-new-text-kb', (done) => {
+
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+                projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+                    chai.request(server)
+                        .get('/' + savedProject._id + '/kb/namespace/all')
+                        .auth(email, pwd)
+                        .end((err, res) => {
+
+                            if (err) { console.error("err: ", err); }
+                            if (log) { console.log("get namespaces res.body: ", res.body); }
+
+                            res.should.have.status(200);
+                            expect(res.body.length).to.equal(1);
+
+                            let namespace_id = res.body[0].id;
+                            console.log("namespace_id: ", namespace_id);
+
+                            let kb = {
+                                name: "example_text1",
+                                type: "text",
+                                source: "example_text1",
+                                content: "Example text",
+                                namespace: namespace_id
+                            }
+
+                            chai.request(server)
+                                .post('/' + savedProject._id + '/kb')
+                                .auth(email, pwd)
+                                .send(kb) // can be empty
+                                .end((err, res) => {
+
+                                    if (err) { console.error("err: ", err); }
+                                    if (log) { console.log("create kb res.body: ", res.body); }
+                                    console.log("create kb res.body: ", res.body);
+                                    res.should.have.status(200);
+                                    res.body.should.be.a('object');
+                                    expect(res.body.value.id_project).to.equal(res.body.value.namespace)
+                                    expect(res.body.value.name).to.equal("example_text1")
+                                    expect(res.body.value.type).to.equal("text")
+                                    expect(res.body.value.source).to.equal("example_text1")
+                                    expect(res.body.value.status).to.equal(-1)
+                                    expect(typeof res.body.value.scrape_type === "undefined").to.be.true;
+                                    expect(typeof res.body.value.scrape_options === "undefined").to.be.true;
+
+
+                                    done();
+                                })
+                        })
+                });
+            });
+
+        })
+
+        it('get-kb-chunks', (done) => {
+
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+                projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+                    chai.request(server)
+                        .get('/' + savedProject._id + '/kb/namespace/all')
+                        .auth(email, pwd)
+                        .end((err, res) => {
+
+                            if (err) { console.error("err: ", err); }
+                            if (log) { console.log("get namespaces res.body: ", res.body); }
+
+                            res.should.have.status(200);
+                            expect(res.body.length).to.equal(1);
+
+                            let namespace_id = res.body[0].id;
+
+                            let kb = {
+                                name: "example_text1",
+                                type: "text",
+                                source: "example_text1",
+                                content: "Example text",
+                                namespace: namespace_id
+                            }
+
+                            chai.request(server)
+                                .post('/' + savedProject._id + '/kb')
+                                .auth(email, pwd)
+                                .send(kb) // can be empty
+                                .end((err, res) => {
+
+                                    if (err) { console.error("err: ", err); }
+                                    if (log) { console.log("create kb res.body: ", res.body); }
+                                    console.log("create kb res.body: ", res.body);
+                                    res.should.have.status(200);
+                                    res.body.should.be.a('object');
+
+                                    let content_id = res.body.value._id;
+
+                                    chai.request(server)
+                                        .get('/' + savedProject._id + '/kb/namespace/' + namespace_id + '/chunks/' + content_id)
+                                        .auth(email, pwd)
+                                        .end((err, res) => {
+
+                                            if (err) { console.error("err: ", err )};
+                                            if (log) { console.log("res.body: ", res.body )};
+
+                                            res.should.have.status(200);
+                                            //expect(res.body.length).to.equal(1);
+
+                                            done();
+                                        })
+
+                                })
+                        })
+                });
+            });
+
+        })
+
         it('get-with-queries', (done) => {
 
             var email = "test-signup-" + Date.now() + "@email.com";
@@ -172,14 +293,14 @@ describe('KbRoute', () => {
                                                             if (log) { console.log("create kb3 res.body: ", res.body); }
                                                             res.should.have.status(200);
 
-                                                            let query = "?status=100&type=url&limit=5&page=0&direction=-1&sortField=updatedAt&search=example&namespace=" + namespace_id;
-                                                            //let query = "";
-                                                            console.log("query: ", query);
+                                                            let query = "?status=-1&type=url&limit=5&page=0&direction=-1&sortField=updatedAt&search=example&namespace=" + namespace_id;
+                                                            //let query = "?namespace=" + namespace_id;
 
                                                             chai.request(server)
                                                                 .get('/' + savedProject._id + "/kb" + query)
                                                                 .auth(email, pwd)
                                                                 .end((err, res) => {
+                                                                    if (err) { console.error("err: ", err)}
                                                                     if (log) { console.log("getall res.body: ", res.body); }
                                                                     res.should.have.status(200);
                                                                     res.body.should.be.a('object');
@@ -187,7 +308,7 @@ describe('KbRoute', () => {
                                                                     expect(res.body.kbs.length).to.equal(2);
                                                                     expect(res.body.count).to.equal(2);
                                                                     res.body.query.should.be.a('object');
-                                                                    expect(res.body.query.status).to.equal(100);
+                                                                    expect(res.body.query.status).to.equal(-1);
                                                                     expect(res.body.query.limit).to.equal(5);
                                                                     expect(res.body.query.page).to.equal(0);
                                                                     expect(res.body.query.direction).to.equal(-1);
@@ -255,7 +376,6 @@ describe('KbRoute', () => {
                                     let namespace_id = "fakenamespaceid";
 
                                     let query = "?status=100&type=url&limit=5&page=0&direction=-1&sortField=updatedAt&search=example&namespace=" + namespace_id;
-                                    console.log("query: ", query);
 
                                     chai.request(server)
                                         .get('/' + savedProject._id + "/kb" + query)
@@ -308,7 +428,6 @@ describe('KbRoute', () => {
 
                                     if (err) { console.error("err: ", err); }
                                     if (log) { console.log("res.body: ", res.body) }
-                                    console.log("res.body: ", res.body)
 
                                     res.should.have.status(200);
 
@@ -357,6 +476,7 @@ describe('KbRoute', () => {
 
         }).timeout(10000)
 
+
         /**
          * If you try to add content to a namespace that does not belong to the selected project and 
          * the project has at least one namesapce, it returns 403 forbidden.
@@ -435,6 +555,7 @@ describe('KbRoute', () => {
 
                                     if (err) { console.error("err: ", err); }
                                     if (log) { console.log("res.body: ", res.body) }
+                                    console.log("res.body: ", res.body)
 
                                     res.should.have.status(200);
                                     expect(res.body.length).to.equal(4)
@@ -448,6 +569,97 @@ describe('KbRoute', () => {
 
         }).timeout(10000)
 
+        it('add-multiple-urls-with-scrape-option-success-type-4', (done) => {
+
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+                projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+                    chai.request(server)
+                        .get('/' + savedProject._id + '/kb/namespace/all')
+                        .auth(email, pwd)
+                        .end((err, res) => {
+
+                            if (err) { console.error("err: ", err); }
+                            if (log) { console.log("res.body: ", res.body) }
+
+                            res.should.have.status(200)
+                            expect(res.body.length).to.equal(1);
+
+                            let namespace_id = res.body[0].id;
+
+                            chai.request(server)
+                                .post('/' + savedProject._id + '/kb/multi?namespace=' + namespace_id)
+                                .auth(email, pwd)
+                                .send({ list:["https://gethelp.tiledesk.com/article"], scrape_type: 4,  scrape_options: { tags_to_extract: ["article","p"], unwanted_tags:["script","style"], unwanted_classnames:["header","related-articles"]}})
+                                .end((err, res) => {
+
+                                    if (err) { console.error("err: ", err); }
+                                    if (log) { console.log("res.body: ", res.body) }
+                                    console.log("res.body: ", res.body)
+                                    res.should.have.status(200);
+                                    expect(res.body.length).to.equal(1)
+                                    expect(res.body[0].scrape_type).to.equal(4)
+                                    expect(typeof res.body[0].scrape_options === "undefined").to.be.false;
+                                    expect(res.body[0].scrape_options.tags_to_extract.length).to.equal(2);
+                                    expect(res.body[0].scrape_options.unwanted_tags.length).to.equal(2);
+                                    expect(res.body[0].scrape_options.unwanted_classnames.length).to.equal(2);
+
+                                    done();
+
+                                })
+                        })
+                });
+            });
+
+        }).timeout(10000)
+
+        it('add-multiple-urls-with-scrape-option-success-type-3', (done) => {
+
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+                projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+                    chai.request(server)
+                        .get('/' + savedProject._id + '/kb/namespace/all')
+                        .auth(email, pwd)
+                        .end((err, res) => {
+
+                            if (err) { console.error("err: ", err); }
+                            if (log) { console.log("res.body: ", res.body) }
+
+                            res.should.have.status(200)
+                            expect(res.body.length).to.equal(1);
+
+                            let namespace_id = res.body[0].id;
+
+                            chai.request(server)
+                                .post('/' + savedProject._id + '/kb/multi?namespace=' + namespace_id)
+                                .auth(email, pwd)
+                                .send({ list:["https://gethelp.tiledesk.com/article"], scrape_type: 3 })
+                                .end((err, res) => {
+
+                                    if (err) { console.error("err: ", err); }
+                                    if (log) { console.log("res.body: ", res.body) }
+                                    console.log("res.body: ", res.body)
+                                    res.should.have.status(200);
+                                    expect(res.body.length).to.equal(1)
+                                    expect(res.body[0].scrape_type).to.equal(3)
+                                    expect(typeof res.body[0].scrape_options === "undefined").to.be.true;
+
+                                    done();
+
+                                })
+                        })
+                });
+            });
+
+        }).timeout(10000)
+
         it('expand-sitemap', (done) => {
 
             var email = "test-signup-" + Date.now() + "@email.com";
@@ -1098,41 +1310,41 @@ describe('KbRoute', () => {
         // }).timeout(10000)
 
         // Ask KB
-        // it('askkb-key-from-env', (done) => {
+        it('askkb-key-from-env', (done) => {
 
-        //     var email = "test-signup-" + Date.now() + "@email.com";
-        //     var pwd = "pwd";
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
 
-        //     userService.signup(email, pwd, "Test Firstname", "Test Lastname").then((savedUser) => {
-        //         projectService.create("test-kb-qa", savedUser._id).then((savedProject) => {
+            userService.signup(email, pwd, "Test Firstname", "Test Lastname").then((savedUser) => {
+                projectService.create("test-kb-qa", savedUser._id).then((savedProject) => {
 
-        //             chai.request(server)
-        //                 .get('/' + savedProject._id + '/kb/namespace/all')
-        //                 .auth(email, pwd)
-        //                 .end((err, res) => {
+                    chai.request(server)
+                        .get('/' + savedProject._id + '/kb/namespace/all')
+                        .auth(email, pwd)
+                        .end((err, res) => {
 
-        //                     if (err) { console.error("err: ", err); }
-        //                     if (log) { console.log("get all namespaces res.body: ", res.body); }
-                            
-        //                     console.log("namespace created..")
+                            if (err) { console.error("err: ", err); }
+                            if (log) { console.log("get all namespaces res.body: ", res.body); }
 
-        //                     chai.request(server)
-        //                         .post('/' + savedProject._id + "/kb/qa")
-        //                         .auth(email, pwd)
-        //                         .send({ model: "gpt-4o", namespace: savedProject._id, question: "sample question" })
-        //                         .end((err, res) => {
+                            console.log("namespace created..")
 
-        //                             if (err) { console.error("err: ", err) };
-        //                             if (log) { console.log("res.body: ", res.body) };
-        //                             console.log("res.body: ", res.body)
-        //                             done();
-        //                         })
+                            chai.request(server)
+                                .post('/' + savedProject._id + "/kb/qa")
+                                .auth(email, pwd)
+                                .send({ model: "gpt-4o", namespace: savedProject._id, question: "sample question", advancedPrompt: true, system_context: "You are a robot coming from future" })
+                                .end((err, res) => {
 
+                                    if (err) { console.error("err: ", err) };
+                                    if (log) { console.log("res.body: ", res.body) };
+                                    console.log("res.body: ", res.body)
+                                    done();
+                                })
 
-        //                 })
-        //         })
-        //     })
-        // }).timeout(10000)
+
+                        })
+                })
+            })
+        }).timeout(10000)
 
 
         it('webhook', (done) => {
@@ -1215,7 +1427,7 @@ describe('KbRoute', () => {
          * Get all namespaces of a project.
          * If there isn't namespaces for a project_id, the default namespace is created and returned.
          */
-        it('get-namespaces', (done) => {
+        it('get-namespaces-1', (done) => {
 
             var email = "test-signup-" + Date.now() + "@email.com";
             var pwd = "pwd";
@@ -1238,6 +1450,8 @@ describe('KbRoute', () => {
                             //expect(res.body[0]._id).to.equal(undefined);
                             expect(res.body[0].id).to.equal(savedProject._id.toString());
                             expect(res.body[0].name).to.equal("Default");
+                            should.exist(res.body[0].engine)
+                            expect(res.body[0].engine.name).to.equal('pinecone')
 
                             done();
 
@@ -1318,6 +1532,76 @@ describe('KbRoute', () => {
             });
         })
 
+        it('create-namespaces-with-engine', (done) => {
+
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+                projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+                    // Get all namespaces. Create default namespace and return.
+                    chai.request(server)
+                        .get('/' + savedProject._id + '/kb/namespace/all')
+                        .auth(email, pwd)
+                        .end((err, res) => {
+
+                            if (err) { console.error("err: ", err); }
+                            if (log) { console.log("get all namespaces res.body: ", res.body); }
+
+                            res.should.have.status(200);
+                            res.body.should.be.a('array');
+                            expect(res.body.length).to.equal(1);
+                            should.not.exist(res.body[0]._id);
+                            expect(res.body[0].id).to.equal(savedProject._id.toString());
+                            expect(res.body[0].name).to.equal("Default");
+                            should.exist(res.body[0].engine)
+                            expect(res.body[0].engine.name).to.equal('pinecone');
+                            expect(res.body[0].engine.type).to.equal('pod');
+
+                            // Create another namespace
+                            chai.request(server)
+                                .post('/' + savedProject._id + '/kb/namespace')
+                                .auth(email, pwd)
+                                .send({ name: "MyCustomNamespace" })
+                                .end((err, res) => {
+
+                                    if (err) { console.error("err: ", err) }
+                                    if (log) { console.log("create new namespace res.body: ", res.body) }
+
+                                    res.should.have.status(200);
+                                    res.body.should.be.a('object');
+                                    should.not.exist(res.body._id)
+                                    should.exist(res.body.id)
+                                    expect(res.body.name).to.equal('MyCustomNamespace');
+                                    should.exist(res.body.engine)
+                                    expect(res.body.engine.name).to.equal('pinecone');
+                                    expect(res.body.engine.type).to.equal('pod');
+
+                                    // Get again all namespace. A new default namespace should not be created.
+                                    chai.request(server)
+                                        .get('/' + savedProject._id + '/kb/namespace/all')
+                                        .auth(email, pwd)
+                                        .end((err, res) => {
+
+                                            if (err) { console.error("err: ", err); }
+                                            if (log) { console.log("get all namespaces res.body: ", res.body); }
+
+                                            res.should.have.status(200);
+                                            res.body.should.be.a('array');
+                                            expect(res.body.length).to.equal(2);
+                                            should.not.exist(res.body[0]._id);
+                                            should.not.exist(res.body[1]._id);
+                                            should.exist(res.body[0].id);
+                                            should.exist(res.body[1].id);
+
+                                            done();
+                                        })
+                                })
+                        })
+                });
+            });
+        })
 
         /**
          * Update namespaces
@@ -1346,7 +1630,6 @@ describe('KbRoute', () => {
                             expect(res.body[0].name).to.equal("Default");
 
                             let namespace_id = res.body[0].id;
-                            console.log("namespace_id: ", namespace_id);
 
                             let new_settings = {
                                 model: 'gpt-4o',
@@ -1411,7 +1694,6 @@ describe('KbRoute', () => {
                             expect(res.body[0].name).to.equal("Default");
 
                             let namespace_id = res.body[0].id;
-                            console.log("namespace_id: ", namespace_id);
 
                             // Update namespace
                             chai.request(server)