@tiledesk/tiledesk-server 2.10.86 → 2.10.88

package/CHANGELOG.md

@@ -5,6 +5,21 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.10.88
+- Added: modified field in faq_kb model
+- Added: chatbot template refactoring
+- Added: clearing logic for chatbot
+- Added: ttl when a chatbot is deleted
+- Added: flow logs apis
+- Added: webhooks apis
+- Added: user_phone field in user model
+
+# 2.10.87
+- Updated voice-twilio-connector to 0.1.18
+
+# 2.10.86
+- Updated voice-twilio-connector to 0.1.16
+
 # 2.10.85
 - Updated: default system context for gpt-4.1 models
 

package/app.js

@@ -202,6 +202,10 @@ let whatsappQueue = require('@tiledesk/tiledesk-whatsapp-jobworker');
 winston.info("whatsappQueue");
 jobsManager.listenWhatsappQueue(whatsappQueue);
 
+let multiWorkerQueue = require('@tiledesk/tiledesk-multi-worker');
+winston.info("multiWorkerQueue from App")
+jobsManager.listenMultiWorker(multiWorkerQueue);
+
 var channelManager = require('./channels/channelManager');
 channelManager.listen(); 
 

package/errorCodes.js

@@ -31,6 +31,12 @@ const errorCodes = {
     ERRORS: {
       DUPLICATE_SLUG: 12001
     }
+  },
+  WEBHOOK: {
+    BASE_CODE: 13000,
+    ERRORS: {
+      NO_PRELOADED_DEV_REQUEST: 13001
+    }
   }
   // Aggiungi altre route
 };

package/event/botEvent.js

@@ -2,25 +2,24 @@ const EventEmitter = require('events');
 const messageEvent = require('../event/messageEvent');
 const Faq_kb = require('../models/faq_kb');
 var winston = require('../config/winston');
-
-
 const cacheUtil = require("../utils/cacheUtil");
 const cacheEnabler = require("../services/cacheEnabler");
+var Faq = require("../models/faq");
+const { Webhook } = require('../models/webhook');
 
 // class BotEvent extends EventEmitter {}
 class BotEvent extends EventEmitter {
+    
     constructor() {
         super();
         this.queueEnabled = false;
         this.setMaxListeners(11);
-      }
+    }
 
 
-      listen() {
+    listen() {
         //TODO modify to async
         //messageEvent.on('message.received', function(message) {
-
-
         var messageCreateKey = 'message.create';
         if (messageEvent.queueEnabled) {
             messageCreateKey = 'message.create.queue';
@@ -28,7 +27,7 @@ class BotEvent extends EventEmitter {
 
         winston.info("Listening " + messageCreateKey + " event for Chatbot messages");
 
-        messageEvent.on(messageCreateKey, function(message) {
+        messageEvent.on(messageCreateKey, function (message) {
 
             winston.debug("message", message);
 
@@ -49,7 +48,7 @@ class BotEvent extends EventEmitter {
             // }
 
             if (message.sender === "system") {
-                if (message.text && (message.text=="\\start" || message.text=="/start") ) {
+                if (message.text && (message.text == "\\start" || message.text == "/start")) {
                     winston.debug("it s a start message");
                 } else {
                     winston.debug("it s a message sent from system, exit");
@@ -58,28 +57,28 @@ class BotEvent extends EventEmitter {
             } else {
                 winston.debug("it s a message sent from other let s go");
             }
-            
-            if (message.text && ( message.text.indexOf("\\agent") > -1 || message.text.indexOf("\\close") > -1)) { //not reply to a message containing \\agent
+
+            if (message.text && (message.text.indexOf("\\agent") > -1 || message.text.indexOf("\\close") > -1)) { //not reply to a message containing \\agent
                 return 0;
             }
 
             // if (message.text.startsWith("\\")) { //not reply to a message containing \
             //     return null;
             // }
-            
-        var botId = getBotId(message);
 
-        winston.debug("botId: " + botId);
+            var botId = getBotId(message);
 
-        if (!botId) {
+            winston.debug("botId: " + botId);
+
+            if (!botId) {
                 return null;
-            }else {
-                                                        //loop fix for messages sent from external bot    
+            } else {
+                //loop fix for messages sent from external bot    
                 // botprefix         
-                if (message.sender === 'bot_'+botId || message.sender === botId) {
+                if (message.sender === 'bot_' + botId || message.sender === botId) {
                     winston.debug("it s a message sent from bot, exit");
-                    return null;        
-                }else {
+                    return null;
+                } else {
                     messageEvent.emit('message.received.for.bot', message);  //UNUSED
                 }
 
@@ -88,51 +87,99 @@ class BotEvent extends EventEmitter {
             // qui potresti leggere anche +secret ed evitare prossima query in botNotification
             // let qbot = Faq_kb.findById(botId);  //TODO add cache_bot_here
             let qbot = Faq_kb.findById(botId).select('+secret')
-        //TODO unselect secret. secret is unselectable by default in the model
-
-                if (cacheEnabler.faq_kb) {
-                    winston.debug('message.id_project+":faq_kbs:id:"+botId: '+ message.id_project+":faq_kbs:id:"+botId);  
-                    // qbot.cache(cacheUtil.defaultTTL, message.id_project+":faq_kbs:id:"+botId)
-                    qbot.cache(cacheUtil.defaultTTL, message.id_project+":faq_kbs:id:"+botId+":secret")
-                    winston.debug('faq_kb cache enabled');
-                }
+            //TODO unselect secret. secret is unselectable by default in the model
 
-                qbot.exec(function(err, bot) {
+            if (cacheEnabler.faq_kb) {
+                winston.debug('message.id_project+":faq_kbs:id:"+botId: ' + message.id_project + ":faq_kbs:id:" + botId);
+                // qbot.cache(cacheUtil.defaultTTL, message.id_project+":faq_kbs:id:"+botId)
+                qbot.cache(cacheUtil.defaultTTL, message.id_project + ":faq_kbs:id:" + botId + ":secret")
+                winston.debug('faq_kb cache enabled');
+            }
 
+            qbot.exec(function (err, bot) {
 
-            
                 if (err) {
-                winston.error('Error getting object.', err);
-                return 0;
+                    winston.error('Error getting object.', err);
+                    return 0;
                 }
                 if (!bot) {
-                    winston.warn('Bot not found with id '+botId);
+                    winston.warn('Bot not found with id ' + botId);
                 }
 
                 winston.debug("bot debug", bot);
-                winston.debug('bot debug secret: '+ bot.secret);
+                winston.debug('bot debug secret: ' + bot.secret);
 
                 if (bot) {
-                    if (bot.type==="internal") {
+                    if (bot.type === "internal") {
                         botEvent.emit('bot.message.received.notify.internal', message);
-                    
-                    }else {  //external 
+
+                    } else {  //external 
                         if (bot.url) {
-                            var botNotification = {bot: bot, message: message};
+                            var botNotification = { bot: bot, message: message };
                             botEvent.emit('bot.message.received.notify.external', botNotification);
-                        }else {
+                        } else {
                             winston.warn("bot url is not defined", bot);
                         }
                     }
-                } 
+                }
 
             });
-            
-
 
         });
 
-      }
+        botEvent.on('faqbot.update.virtual.delete', async (chatbot) => {
+            winston.verbose("botEvent ON faqbot.update.virtual.delete: ", chatbot);
+
+            if (chatbot.publishedAt) {
+                // Stop the flow if the chatbot is a published one
+                return;
+            }
+
+            await Faq.updateMany({ id_faq_kb: chatbot._id }, { trashed: true, trashedAt: chatbot.trashedAt }).catch((err) => {
+                winston.error("Event faqbot.update.virtual.delete error updating faqs ", err);
+            })
+
+            let deletedW = await Webhook.findOneAndDelete({ chatbot_id: chatbot._id }).catch((err) => {
+                winston.error("Error deleting webhook on chatbot deleting: ", err);
+            })
+
+            let publishedChatbots = await Faq_kb.find({ root_id: chatbot._id }, { _id: 1 }).catch((err) => {
+                winston.error("Event faqbot.update.virtual.delete error getting all published chatbots ", err);
+            })
+
+            const publishedChatbotIds = publishedChatbots.map(c => c._id);
+
+            if (publishedChatbotIds.length > 0) {
+
+                const batchSize = 20;
+                const sleep_ms = 500;
+                const batches = [];
+                for (let i = 0; i < publishedChatbotIds.length; i += batchSize) {
+                    batches.push(publishedChatbotIds.slice(i, i + batchSize));
+                }
+
+                for (const batch of batches) {
+                    await Faq_kb.updateMany(
+                        { _id: { $in: batch } },
+                        { $set: { trashed: true, trashedAt: chatbot.trashedAt } }
+                    );
+
+                    await Faq.updateMany(
+                        { id_faq_kb: { $in: batch } },
+                        { $set: { trashed: true, trashedAt: chatbot.trashedAt } }
+                    );
+                
+                    await sleep(sleep_ms);
+                }                
+
+            }
+        })
+
+        function sleep(ms) {
+            return new Promise(resolve => setTimeout(resolve, ms));
+          }
+
+    }
 
 }
 

package/jobsManager.js

@@ -12,7 +12,8 @@ class JobsManager {
         this.emailNotificatio = undefined;
         this.activityArchiver = undefined;
         this.whatsappWorker = undefined;
-     
+        this.multiWorkerQueue = undefined;
+
         this.jobWorkerEnabled = jobWorkerEnabled;
         // this.jobWorkerEnabled = false;
         // if (process.env.JOB_WORKER_ENABLED=="true" || process.env.JOB_WORKER_ENABLED == true) {
@@ -93,6 +94,8 @@ class JobsManager {
         if (this.jobWorkerEnabled == true) {
             return winston.info("JobsManager jobWorkerEnabled is enabled. Skipping listener for MultiWorker Queue");  
         }
+        this.multiWorkerQueue = multiWorkerQueue;
+        this.multiWorkerQueue.startJobsWorker();
     }
 
     // listenTrainingQueue(trainingQueue) {

package/models/chatbotTemplates.js

@@ -0,0 +1,22 @@
+module.exports = {
+    chatbot: {
+        default: 'blank',
+        templates: ['empty', 'blank', 'handoff', 'example']
+    },
+    webhook: {
+        default: 'blank_webhook',
+        templates: ['empty', 'blank_webhook']
+    },
+    copilot: {
+        default: 'official_copilot',
+        templates: ['empty', 'blank_copilot', 'official_copilot']
+    },
+    voice: {
+        default: 'blank_voice',
+        templates: ['empty', 'blank_voice']
+    },
+    voice_twilio: {
+        default: 'blank_voice_twilio',
+        templates: ['empty', 'blank_voice_twilio']
+    }
+}

package/models/faq.js

@@ -5,6 +5,7 @@ var { nanoid } = require("nanoid");
 const uuidv4 = require('uuid/v4');
 
 var defaultFullTextLanguage = process.env.DEFAULT_FULLTEXT_INDEX_LANGUAGE || "none";
+let trashExpirationTime = Number(process.env.CHATBOT_TRASH_TTL_SECONDS) || 60 * 60 * 24 * 30; // 30 days
 
 var FaqSchema = new Schema({
   // _id: {
@@ -107,7 +108,15 @@ var FaqSchema = new Schema({
     default: function () {
       return this.isNew ? false : undefined;
     },
-  }
+  },
+  trashed: {
+    type: Boolean,
+    index: true
+  },
+  trashedAt: {
+    type: Date,
+    required: false
+  },
 }, {
   timestamps: true,
   toJSON: { virtuals: true } //used to polulate messages in toJSON// https://mongoosejs.com/docs/populate.html
@@ -183,6 +192,10 @@ FaqSchema.index(
   { partialFilterExpression: { "actions.namespace": { $exists: true } } }
 );
 
+FaqSchema.index(
+  { trashedAt: 1 },
+  { expireAfterSeconds: trashExpirationTime }
+);
 
 var faq = mongoose.model('faq', FaqSchema);
 

package/models/faq_kb.js

@@ -5,7 +5,7 @@ var winston = require('../config/winston');
 const { stringify } = require('uuid');
 
 var defaultFullTextLanguage = process.env.DEFAULT_FULLTEXT_INDEX_LANGUAGE || "none";
-
+let trashExpirationTime = Number(process.env.CHATBOT_TRASH_TTL_SECONDS) || 60 * 60 * 24 * 30; // 30 days
 
 var Faq_kbSchema = new Schema({
   name: {
@@ -58,6 +58,10 @@ var Faq_kbSchema = new Schema({
     type: Boolean,
     index: true
   },
+  trashedAt: {
+    type: Date,
+    required: false
+  },
   secret: {
     type: String,
     required: true,
@@ -107,8 +111,13 @@ var Faq_kbSchema = new Schema({
     index: true,
     default: 0
   },
+  // publishedBy: {
+  //   type: String,
+  // },
   publishedBy: {
-    type: String,
+    type: Schema.Types.ObjectId,
+    ref: 'user',
+    required: false
   },
   publishedAt: {
     type: Date
@@ -140,6 +149,18 @@ var Faq_kbSchema = new Schema({
     type: String,
     required: false,
     index: true
+  },
+  modified: {
+    type: Boolean,
+    required: false
+  },
+  root_id: {
+    type: String,
+    required: false
+  },
+  release_note: {
+    type: String,
+    required: false
   }
 },{
   timestamps: true
@@ -171,6 +192,37 @@ Faq_kbSchema.pre("save", async function (next) {
   next();
 });
 
+Faq_kbSchema.pre('findOneAndUpdate', async function (next) {
+
+  const update = this.getUpdate();
+  const isUnsetSlug = update?.$unset?.slug !== undefined;
+
+  // $unset.slug is used only on publishing. In this case, skip the slug change and the set of trashedAt
+  if (update.trashed === true && !isUnsetSlug) {
+
+    const docToUpdate = await this.model.findOne(this.getQuery());
+    const timestamp = Date.now();
+
+    if (docToUpdate && docToUpdate.slug) {
+      let slug;
+      slug = docToUpdate.slug;
+      update.trashedAt = new Date();
+      update.slug = `${slug || 'undefined'}-trashed-${timestamp}`;
+    }
+    this.setUpdate(update);
+  }
+
+  next();
+
+});
+
+Faq_kbSchema.post('findOneAndUpdate', async function (doc) {
+  if (doc && doc.trashed === true) {
+    botEvent.emit('faqbot.update.virtual.delete', doc)
+  }
+})
+
+
 Faq_kbSchema.virtual('fullName').get(function () {
   // winston.debug("faq_kb fullName virtual called");
   return (this.name);
@@ -188,6 +240,11 @@ Faq_kbSchema.index(
   { unique: true, partialFilterExpression: { slug: { $exists: true } } }
 );
 
+Faq_kbSchema.index(
+  { trashedAt: 1 },
+  { expireAfterSeconds: trashExpirationTime }
+);
+
 
 var faq_kb = mongoose.model('faq_kb', Faq_kbSchema);
 
@@ -209,3 +266,8 @@ function generateSlug(name) {
 }
 
 module.exports = faq_kb
+
+
+
+// Import botEvent after model declaration to avoid circular dependency issues
+const botEvent = require('../event/botEvent');

package/models/flowLogs.js

@@ -0,0 +1,64 @@
+const mongoose = require('mongoose');
+const Schema = mongoose.Schema;
+
+const RowSchema = new Schema(
+  {
+    text: {
+      type: String,
+      required: true
+    },
+    level: {
+      type: String,
+      required: true
+    },
+    timestamp: {
+      type: Date,
+      required: true,
+      default: Date.now
+    }
+  }
+);
+
+const FlowLogsSchema = new Schema(
+  {
+    id_project: {
+      type: String,
+      required: false,
+    },
+    request_id: {
+      type: String,
+      required: false,
+    },
+    level: {
+      type: String,
+      required: true,
+    },
+    rows: {
+      type: [RowSchema],
+      required: false,
+    },
+  }, {
+    timestamps: true
+  }
+);
+
+FlowLogsSchema.index({ request_id: 1 }, { unique: true });
+
+// FlowLogsSchema.pre('findOneAndUpdate', async function (next) {
+//   const update = this.getUpdate();
+
+//   if (update.$push && update.$push.rows) {
+//     const doc = await this.model.findOne(this.getQuery());
+
+//     if (!doc) {
+//       update.$push.rows.index = 0;
+//     } else {
+//       update.$push.rows.index = doc.rows.length;
+//     }
+//   }
+//   next();
+// });
+
+const FlowLogs = mongoose.model('FlowLogs', FlowLogsSchema);
+
+module.exports = { FlowLogs };

package/models/user.js

@@ -65,6 +65,13 @@ var UserSchema = new Schema({
     public_website: {
         type: String,
         required: false
+    },
+    phone: {
+        type: String,
+        required: false,
+        unique: true,
+        trim: true,
+        match: [/^\+?[1-9]\d{1,14}$/, 'Invalid phone number format'], // E.164 format
     }
     // authType: { // update db old data
     //     type: String,

package/models/webhook.js

@@ -13,6 +13,10 @@ const WebhookSchema = mongoose.Schema({
     type: String,
     required: true
   },
+  name: {
+    type: String,
+    required: false
+  },
   chatbot_id: {
     type: String,
     required: true
@@ -29,12 +33,38 @@ const WebhookSchema = mongoose.Schema({
   copilot: {
     type: Boolean,
     required: false
+  },
+  enabled: {
+    type: Boolean,
+    required: true,
+    default: true
   }
 }, {
   timestamps: true
 })
 
+// Indexes
 WebhookSchema.index({ id_project: 1, chatbot_id: 1}, { unique: true })
+WebhookSchema.index({ id_project: 1, webhook_id: 1}, { unique: true })
+
+// Middlewares
+WebhookSchema.pre("save", async function (next) {
+  if (this.isNew) {
+
+    try {
+      const chatbot = await mongoose.model("faq_kb").findById(this.chatbot_id);
+      if (chatbot) {
+        this.name = chatbot.name + "-webhook";
+      }
+
+      next();
+
+    } catch(error) {
+      next(error);
+    }
+  }
+});
+
 
 const Webhook = mongoose.model("Webhook", WebhookSchema);
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.10.86",
+  "version": "2.10.88",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",
@@ -47,13 +47,13 @@
     "@tiledesk/tiledesk-messenger-connector": "^0.1.23",
     "@tiledesk/tiledesk-rasa-connector": "^1.0.10",
     "@tiledesk/tiledesk-telegram-connector": "^0.1.14",
-    "@tiledesk/tiledesk-tybot-connector": "^2.0.11",
+    "@tiledesk/tiledesk-tybot-connector": "^2.0.12",
     "@tiledesk/tiledesk-whatsapp-connector": "^0.1.83",
     "@tiledesk/tiledesk-whatsapp-jobworker": "^0.0.12",
     "@tiledesk/tiledesk-sms-connector": "^0.1.11",
     "@tiledesk/tiledesk-vxml-connector": "^0.1.76",
-    "@tiledesk/tiledesk-voice-twilio-connector": "^0.1.16",
-    "@tiledesk/tiledesk-multi-worker": "^0.1.20",
+    "@tiledesk/tiledesk-voice-twilio-connector": "^0.1.21",
+    "@tiledesk/tiledesk-multi-worker": "^0.3.1",
     "passport-oauth2": "^1.8.0",
     "amqplib": "^0.5.5",
     "app-root-path": "^3.0.0",

package/routes/auth.js

@@ -85,7 +85,7 @@ router.post('/signup',
     //   return res.status(403).send({ success: false, message: "The password does not meet the minimum vulnerability requirements"})
     // }
 
-    return userService.signup(req.body.email, req.body.password, req.body.firstname, req.body.lastname, false)
+    return userService.signup(req.body.email, req.body.password, req.body.firstname, req.body.lastname, false, req.body.phone)
       .then( async function (savedUser) {
         
         winston.debug('-- >> -- >> savedUser ', savedUser.toObject());

package/routes/faq.js

@@ -16,6 +16,7 @@ const axios = require("axios").default;
 var configGlobal = require('../config/global');
 const roleConstants = require('../models/roleConstants');
 const roleChecker = require('../middleware/has-role');
+const { ChatbotService } = require('../services/chatbotService');
 
 const apiUrl = process.env.API_URL || configGlobal.apiUrl;
 
@@ -222,6 +223,7 @@ router.post('/ops_update', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscrip
 
   let id_faq_kb = req.body.id_faq_kb;
   let operations = req.body.operations;
+  let chatbotService = new ChatbotService();
 
   for (let op of operations) {
     let HTTPREQUEST;
@@ -246,6 +248,7 @@ router.post('/ops_update', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscrip
               winston.error("err performing operation: ", err);
             } else {
               winston.debug("\n\nresbody operation: ", resbody);
+              chatbotService.setModified(id_faq_kb, true)
             }
           }
         )
@@ -273,6 +276,7 @@ router.post('/ops_update', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscrip
               winston.error("err performing operation: ", err);
             } else {
               winston.debug("\n\nresbody operation: ", resbody);
+              chatbotService.setModified(id_faq_kb, true)
             }
           }
         )
@@ -296,6 +300,7 @@ router.post('/ops_update', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscrip
               winston.error("err performing operation: ", err);
             } else {
               winston.debug("\n\nresbody operation: ", resbody);
+              chatbotService.setModified(id_faq_kb, true)
             }
           }
         )
@@ -322,6 +327,7 @@ router.post('/ops_update', roleChecker.hasRoleOrTypes('admin', ['bot', 'subscrip
               winston.error("err performing operation: ", err);
             } else {
               winston.debug("\n\nresbody operation: ", resbody);
+              chatbotService.setModified(id_faq_kb, true)
             }
           }
         )