@tiledesk/tiledesk-server 2.10.32 → 2.10.34

package/CHANGELOG.md

@@ -5,6 +5,22 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.10.34
+- added add tags endpoint
+- added tags analytics endpoint
+- added tiledesk-multi-worker 0.1.5
+- updated tybot-connector to 0.2.134
+- updated vxml-connector to 0.1.67
+- updated voice-twilio-connector to 0.1.12
+- updated sms-connector to 0.1.11
+- fix kb source with citations issue (from dashboard)
+
+# 2.10.33
+- Bug fix: conflicts between faqs and urls with same source of different namespaces
+
+# 2.10.32
+- Externalized PINECONE_TYPE variable for kb engine
+
 # 2.10.31
 - updated twilio-voice-connector to 0.1.11
 

package/app.js

@@ -553,7 +553,7 @@ app.use('/:projectid/departments', department);
 channelManager.useUnderProjects(app);
 
 app.use('/:projectid/groups', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('agent', ['bot','subscription'])], group);
-app.use('/:projectid/tags', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRole('agent')], tag);
+app.use('/:projectid/tags', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('agent', ['bot','subscription'])], tag);
 app.use('/:projectid/subscriptions', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRole('admin')], resthook);
 
 //deprecated

package/jobs.js

@@ -112,6 +112,9 @@ async function main()
     let scheduler = require('./pubmodules/scheduler');    
     jobsManager.listenScheduler(scheduler);
 
+    let multiWorkerQueue = require('@tiledesk/tiledesk-multi-worker');
+    jobsManager.listenMultiWorker(multiWorkerQueue);
+
 
     winston.info("Jobs started"); 
 

package/jobsManager.js

@@ -87,6 +87,14 @@ class JobsManager {
         // this.whatsappQueue.listen(); // oppure codice
     }
 
+    listenMultiWorker(multiWorkerQueue) {
+        console.log("JobsManager multiWorkerQueue started");
+        console.log("multiWorkerQueue is: ", multiWorkerQueue)
+        if (this.jobWorkerEnabled == true) {
+            return winston.info("JobsManager jobWorkerEnabled is enabled. Skipping listener for MultiWorker Queue");  
+        }
+    }
+
     // listenTrainingQueue(trainingQueue) {
     //     console.log("JobsManager listenTrainingQueue started");
     //     console.log("trainingQueue is: ", trainingQueue)

package/models/analytics.js

@@ -0,0 +1,29 @@
+var mongoose = require('mongoose');
+var Schema = mongoose.Schema;
+
+var AnalyticsSchema = new Schema({
+  id_project: {
+    type: String,
+    required: true,
+  },
+  type: {
+    type: String,
+    required: true
+  },
+  date: {
+    type: String,
+    required: true
+    //default: () => new Date().setHours(0, 0, 0, 0)
+  },
+  keys: {
+    type: Object,
+    required: true
+  }
+}, {
+  timestamps: true
+})
+
+
+const Analytics = mongoose.model('Analytics', AnalyticsSchema)
+module.exports = Analytics
+

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.10.32",
+  "version": "2.10.34",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",
@@ -51,9 +51,10 @@
     "@tiledesk/tiledesk-tybot-connector": "^0.2.134",
     "@tiledesk/tiledesk-whatsapp-connector": "^0.1.75",
     "@tiledesk/tiledesk-whatsapp-jobworker": "^0.0.10",
-    "@tiledesk/tiledesk-sms-connector": "^0.1.10",
-    "@tiledesk/tiledesk-vxml-connector": "^0.1.65",
-    "@tiledesk/tiledesk-voice-twilio-connector": "^0.1.11",
+    "@tiledesk/tiledesk-sms-connector": "^0.1.11",
+    "@tiledesk/tiledesk-vxml-connector": "^0.1.67",
+    "@tiledesk/tiledesk-voice-twilio-connector": "^0.1.12",
+    "@tiledesk/tiledesk-multi-worker": "^0.1.5",
     "amqplib": "^0.5.5",
     "app-root-path": "^3.0.0",
     "bcrypt-nodejs": "0.0.3",

package/pubmodules/analytics/analytics.js

@@ -9,6 +9,7 @@ var RoleConstants = require("../../models/roleConstants");
 
 var mongoose = require('mongoose');
 var winston = require('../../config/winston');
+var Analytics = require('../../models/analytics');
 var ObjectId = require('mongodb').ObjectId;
 
 
@@ -1742,5 +1743,88 @@ router.get('/messages/count', function(req, res) {
 
   });
 
+router.get('/tags/:type', async (req, res) => {
+
+  let id_project = req.projectid;
+  let type = req.params.type;
+  let startDate = req.query.start_date;
+  let endDate = req.query.end_date;
+
+  if (!type) {
+    return res.status(400).send({ success: false, error: "Missing parameter 'type'" });
+  }
+
+  if (!startDate) {
+    startDate = new Date();
+    startDate.setDate(startDate.getDate() - 6);
+  } else {
+    startDate = new Date(startDate)
+  }
+
+  if (!endDate) {
+    endDate = new Date();
+  } else {
+    endDate = new Date(endDate);
+  }
+  
+  startDate = startDate.setHours(0, 0, 0, 0);
+  endDate = endDate.setHours(0, 0, 0, 0);
+
+  if (startDate > endDate) {
+    return res.status(400).send({ success: false, error: "Invalid dates: start_date can't be greater of end_date"})
+  }
+
+  let query = {
+    id_project: id_project,
+    type: type,
+    date: { $gte: startDate, $lte: endDate }
+  }
+
+  console.log("analytics tags query: ", query)
+
+  let result = await Analytics.find(query).catch((err) => {
+    winston.error("Error finding Analytics: ", err);
+    return res.status(500).send({ success: false, error: "Unable to find analytics" })
+  })
+
+  let parsedDates = result
+    .sort((a, b) => parseInt(a.date) - parseInt(b.date))
+    .map(r => {
+      let date = new Date(parseInt(r.date));
+      let day = String(date.getDate()).padStart(2, '0'); // Day with two digit
+      let month = String(date.getMonth() + 1).padStart(2, '0'); // Month with two digit
+      let year = date.getFullYear(); // Complete year
+      
+      return {
+        formatted: `${day}/${month}/${year}`,
+        original: r
+      };
+  });
+  
+  // Extract and sort all unique dates
+  let dates = [...new Set(parsedDates.map(d => d.formatted))].sort();
+  
+  // Extract all unique keys
+  let allKeys = [...new Set(result.flatMap(r => Object.keys(r.keys)))];
+  
+  // Pre-build a map to optimize data access
+  let dataMap = parsedDates.reduce((acc, { formatted, original }) => {
+    acc[formatted] = original.keys;
+    return acc;
+  }, {});
+  
+  // Build the data series
+  let series = allKeys.map(key => {
+    let values = dates.map(date => dataMap[date]?.[key] || 0); // Default a 0 se il tag non è presente
+    return { name: key, values };
+  });
+  
+  // Assembly final result
+  let data = { dates, series };
+  console.log(data);
+
+  return res.status(200).send(data);
+  
+})
 
 module.exports = router;

package/pubmodules/sms/listener.js

@@ -37,7 +37,6 @@ class Listener {
         sms.startApp({
             MONGODB_URI: config.databaseUri,          
             dbconnection: dbConnection,
-            API_URL: apiUrl,
             BASE_URL: apiUrl + "/modules/sms",
             BRAND_NAME: brand_name,
             REDIS_HOST: host,

package/pubmodules/voice-twilio/listener.js

@@ -37,7 +37,6 @@ class Listener {
         voice_twilio.startApp({
             MONGODB_URI: config.databaseUri,          
             dbconnection: dbConnection,
-            API_URL: apiUrl,
             BASE_URL: apiUrl + "/modules/voice-twilio",                     
             REDIS_HOST: host,
             REDIS_PORT: port,

package/routes/kb.js

@@ -298,41 +298,20 @@ router.post('/qa', async (req, res) => {
     winston.debug("qa resp: ", resp.data);
     let answer = resp.data;
 
-    let id = answer.id;
-    let index = id.indexOf("#");
-    if (index != -1) {
-      id = id.substring(index + 1);
-    }
-
-    KB.findById(id, (err, resource) => {
-
-      if (publicKey === true) {
-        let multiplier = MODELS_MULTIPLIER[data.model];
-        if (!multiplier) {
-          multiplier = 1;
-          winston.info("No multiplier found for AI model")
-        }
-        obj.multiplier = multiplier;
-        obj.tokens = answer.prompt_token_size;
-  
-        let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
-        winston.verbose("incremented_key: ", incremented_key);
-      }
-
-      if (err) {
-        winston.error("Unable to find resource with id " + id + " in namespace " + answer.namespace + ". The standard answer is returned.")
-        return res.status(200).send(resp.data);
-      }
-
-      if (!resource) {
-        winston.error("Resource with id " + id + " not found in namespace " + answer.namespace + ". The standard answer is returned.")
-        return res.status(200).send(resp.data);
+    if (publicKey === true) {
+      let multiplier = MODELS_MULTIPLIER[data.model];
+      if (!multiplier) {
+        multiplier = 1;
+        winston.info("No multiplier found for AI model")
       }
+      obj.multiplier = multiplier;
+      obj.tokens = answer.prompt_token_size;
 
-      answer.source = resource.name;
-      return res.status(200).send(answer);
-    })
-
+      let incremented_key = quoteManager.incrementTokenCount(req.project, obj);
+      winston.verbose("incremented_key: ", incremented_key);
+    }
+  
+    return res.status(200).send(answer);
 
   }).catch((err) => {
     winston.error("qa err: ", err);
@@ -998,7 +977,7 @@ router.post('/', async (req, res) => {
       let resources = [];
 
       resources.push(json);
-      if (!process.env.NODE_ENV) {
+      if (process.env.NODE_ENV !== 'test') {
         scheduleScrape(resources);
       }
 
@@ -1104,7 +1083,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
   let operations = kbs.map(doc => {
     return {
       updateOne: {
-        filter: { id_project: doc.id_project, type: 'url', source: doc.source },
+        filter: { id_project: doc.id_project, type: 'url', source: doc.source, namespace: namespace_id },
         update: doc,
         upsert: true,
         returnOriginal: false
@@ -1123,7 +1102,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
     });
     winston.verbose("resources to be sent to worker: ", resources);
 
-    if (!process.env.NODE_ENV) {
+    if (process.env.NODE_ENV !== 'test') {
       scheduleScrape(resources);
     }
     res.status(200).send(result);
@@ -1214,7 +1193,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
       let operations = kbs.map(doc => {
         return {
           updateOne: {
-            filter: { id_project: doc.id_project, type: 'faq', source: doc.source },
+            filter: { id_project: doc.id_project, type: 'faq', source: doc.source, namespace: namespace_id },
             update: doc,
             upsert: true,
             returnOriginal: false
@@ -1232,7 +1211,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
           return { id: _id, webhooh: webhook, engine: engine, ...rest };
         })
         winston.verbose("resources to be sent to worker: ", resources);
-        if (!process.env.NODE_ENV) {
+        if (process.env.NODE_ENV !== 'test') {
           scheduleScrape(resources);
         }
         res.status(200).send(result);

package/routes/lead.js

@@ -100,6 +100,58 @@ router.put('/:leadid', function (req, res) {
 });
 
 
+router.put('/:leadid/tag', async (req, res) => {
+
+  let lead_id = req.params.leadid;
+  let tags_list = req.body;
+  winston.debug("(Lead) /tag tags_list: ", tags_list)
+  
+  if (tags_list.length == 0) {
+    winston.warn("(Lead) /tag no tag specified")
+    return res.status(400).send({ success: false, message: "No tag specified" })
+  }
+
+  let lead = await Lead.findById(lead_id).catch((err) => {
+    winston.error("(Lead) /tag error getting lead ", err);
+    return res.status(500).send({ success: false, error: "Error getting lead with id " + lead_id});
+  })
+
+  if (!lead) {
+    winston.warn("(Lead) /tag lead not found with lead_id " + lead_id);
+    return res.status(404).send({ success: false, error: "Lead not found with id " + lead_id });
+  }
+
+  let current_tags = lead.tags;
+  tags_list.forEach(t => {
+    // Check if tag already exists in the lead. If true, skip the adding.
+    if (!current_tags.includes(t)) {
+      current_tags.push(t)
+    }
+  })
+
+  let update = {
+    tags: current_tags
+  }
+
+  Lead.findByIdAndUpdate(lead_id, update, { new: true }, (err, updatedLead) => {
+    if (err) {
+      winston.error("(Lead) /tag error finding and update lead ", err);
+      return res.status(500).send({ success: false, error: "Error updating lead with id " + lead_id })
+    }
+
+    if (!updatedLead) {
+      winston.warn("(Lead) /tag The lead was deleted while adding tags for lead " + lead_id);
+      return res.status(404).send({ success: false, error: "The lead was deleted while adding tags for lead " + lead_id })
+    }
+
+    winston.debug("(Lead) /tag Lead updated successfully ", updatedLead);
+    
+    leadEvent.emit('lead.update', updatedLead);
+    res.status(200).send(updatedLead)
+
+  })
+})
+
 
 router.patch('/:leadid/attributes',  function (req, res) {
   var data = req.body;
@@ -274,6 +326,25 @@ router.patch('/:leadid/properties',  function (req, res) {
 //     res.json(updatedLead);
 //   });
 // });
+router.delete('/:leadid/tag/:tag', async (req, res) => {
+
+  let lead_id = req.params.leadid;
+  let tag = req.params.tag;
+
+  Lead.findByIdAndUpdate(lead_id, { $pull: { tags: tag }}, { new: true}).then((updatedLead) => {
+    if (!updatedLead) {
+      winston.warn("(Lead) /removetag lead not found with id " + lead_id);
+      return res.status(404).send({ success: false, error: "Lead not found with id " + lead_id })
+    }
+
+    winston.debug("(Lead) /removetag updatedLead: ", updatedLead)
+    res.status(200).send(updatedLead)
+  }).catch((err) => {
+    winston.error("(Lead) /removetag error updating lead: ", err);
+    res.status(500).send({ success: false, error: err });
+  })
+})
+
 
 router.delete('/:leadid', function (req, res) {
   winston.debug(req.body);

package/routes/request.js

@@ -23,7 +23,7 @@ var cacheEnabler = require("../services/cacheEnabler");
 var Project_user = require("../models/project_user");
 var Lead = require("../models/lead");
 var UIDGenerator = require("../utils/UIDGenerator");
-
+let JobManager = require("@tiledesk/tiledesk-multi-worker");
 
 csv = require('csv-express');
 csv.separator = ';';
@@ -31,9 +31,28 @@ csv.separator = ';';
 const { check, validationResult } = require('express-validator');
 const RoleConstants = require('../models/roleConstants');
 const eventService = require('../pubmodules/events/eventService');
+const { Scheduler } = require('../services/Scheduler');
+//const JobManager = require('../utils/jobs-worker-queue-manager-v2/JobManagerV2');
 
 // var messageService = require('../services/messageService');
 
+const AMQP_MANAGER_URL = process.env.AMQP_MANAGER_URL;
+
+let jobManager = new JobManager(AMQP_MANAGER_URL, {
+  debug: false,
+  queueName: "conversation-tags_queue",
+  exchange: "tiledesk-multi",
+  topic: "conversation-tags",
+})
+
+jobManager.connectAndStartPublisher((status, error) => {
+  if (error) {
+    winston.error("connectAndStartPublisher error: ", error);
+  } else {
+    winston.info("KbRoute - ConnectPublisher done with status: ", status);
+  }
+})
+
 
 
 router.post('/simple', [check('first_text').notEmpty()], async (req, res) => {
@@ -848,6 +867,76 @@ router.put('/:requestid/followers', function (req, res) {
 
 });
 
+
+router.put('/:requestid/tag', async (req, res) => {
+
+  let id_project = req.projectid;
+  let request_id = req.params.requestid;
+  let tags_list = req.body;
+  winston.debug("(Request) /tag tags_list: ", tags_list)
+
+  if (tags_list.length == 0) {
+    winston.warn("(Request) /tag no tag specified")
+    return res.status(400).send({ success: false, message: "No tag specified" })
+  }
+
+  let request = await Request.findOne({ id_project: id_project, request_id: request_id }).catch((err) => {
+    winston.error("(Request) /tag error getting request ", err);
+    return res.status(500).send({ success: false, error: "Error getting request with request id " + request_id});
+  })
+
+  if (!request) {
+    winston.warn("(Request) /tag request not found with request_id " + request_id);
+    return res.status(404).send({ success: false, error: "Request not found with request id " + request_id});
+  }
+
+  let current_tags = request.tags;
+  let adding_tags = [];
+
+  tags_list.forEach(t => {
+    // Check if tag already exists in the conversation. If true, skip the adding.
+    if(!current_tags.some(tag => tag.tag === t.tag)) {
+      current_tags.push(t);
+      adding_tags.push(t);
+    }
+  })
+
+  let update = {
+    tags: current_tags
+  }
+
+  Request.findOneAndUpdate({ id_project: id_project, request_id: request_id }, update, { new: true }).then( async (updatedRequest) => {
+
+    if (!updatedRequest) {
+      winston.warn("(Request) /tag The request was deleted while adding tags for request " + request_id);
+      return res.status(404).send({ success: false, error: "The request was deleted while adding tags for request " + request_id })
+    }
+
+    winston.debug("(Request) /tag Request updated successfully ", updatedRequest);
+
+    const populatedRequest = 
+        await updatedRequest
+          .populate('lead')
+          .populate('department')
+          .populate('participatingBots')
+          .populate('participatingAgents')
+          .populate({ path: 'requester', populate: { path: 'id_user' } })  
+          .execPopulate();
+
+    requestEvent.emit("request.update", populatedRequest)
+    res.status(200).send(updatedRequest)
+    
+    if (process.env.NODE_ENV !== 'test') {
+      scheduleTags(id_project, adding_tags);
+    }
+
+  }).catch((err) => {
+    winston.error("(Request) /tag error finding and update request ", err);
+    return res.status(500).send({ success: false, error: "Error updating request with id " + request_id })
+  })
+
+})
+
 router.delete('/:requestid/followers/:followerid', function (req, res) {
   winston.debug(req.body);
 
@@ -864,7 +953,40 @@ router.delete('/:requestid/followers/:followerid', function (req, res) {
 
 
 
+router.delete('/:requestid/tag/:tag_id', async (req, res) => {
+
+  let id_project = req.projectid;
+  let request_id = req.params.requestid;
+  let tag_id = req.params.tag_id;
+
+
 
+  Request.findOneAndUpdate({ id_project: id_project, request_id: request_id }, { $pull: { tags: { _id: tag_id } } }, { new: true }).then( async (updatedRequest) => {
+    
+    if (!updatedRequest) {
+      winston.warn("(Request) /removetag request not found with id: " + request_id)
+      return res.status(404).send({ success: false, error: "Request not found with id " + request_id})
+    }
+
+    winston.debug("(Request) /removetag updatedRequest: ", updatedRequest)
+
+    const populatedRequest = 
+        await updatedRequest
+          .populate('lead')
+          .populate('department')
+          .populate('participatingBots')
+          .populate('participatingAgents')
+          .populate({ path: 'requester', populate: { path: 'id_user' } })  
+          .execPopulate();
+
+    requestEvent.emit("request.update", populatedRequest)
+    res.status(200).send(updatedRequest);
+    
+  }).catch((err) => {
+    winston.error("(Request) /removetag error updating request: ", err)
+    res.status(500).send({ success: false, error: err })
+  })
+})
 
 
 // TODO make a synchronous chat21 version (with query parameter?) with request.support_group.created
@@ -2166,4 +2288,27 @@ router.get('/:requestid/chatbot/parameters', async (req, res) => {
 
 })
 
+
+async function scheduleTags(id_project, tags) {
+  
+  let scheduler = new Scheduler({ jobManager: jobManager })
+  tags.forEach(t => {
+    let payload = {
+      id_project: id_project,
+      tag: t.tag,
+      type: "conversation-tag",
+      date: new Date()
+    }
+    console.log("payload: ", payload);
+    scheduler.tagSchedule(payload, async (err, result) => {
+      if (err) {
+        winston.error("Scheduling error: ", err);
+      } else {
+        winston.info("Scheduling result: ", result);
+      }
+    })
+  })
+}
+
+
 module.exports = router;

package/services/Scheduler.js

@@ -28,6 +28,18 @@ class Scheduler {
             }
         });
     }
+
+    tagSchedule(data, callback) {
+
+        winston.debug("(tagScheduler) data: ", data);
+        this.jobManager.publish(data, (err, ok) => {
+            let response_data = { success: true, message: "Scheduled" };
+            if (callback) {
+                callback(err, response_data);
+            }
+        })
+    }
+    
 }
 
 module.exports = { Scheduler };

package/services/requestService.js

@@ -20,8 +20,8 @@ var configGlobal = require('../config/global');
 const projectService = require('./projectService');
 const axios = require("axios").default;
 
-
-const apiUrl = process.env.API_URL || configGlobal.apiUrl;
+let port = process.env.PORT || '3000';
+const TILEBOT_ENDPOINT = process.env.TILEBOT_ENDPOINT || "http://localhost:" + port+ "/modules/tilebot/ext/";
 
 let tdCache = new TdCache({
     host: process.env.CACHE_REDIS_HOST,
@@ -2812,7 +2812,7 @@ class RequestService {
 
     return new Promise( async (resolve, reject) => {
       await axios({
-        url: apiUrl + '/modules/tilebot/ext/reserved/parameters/requests/' + request_id,
+        url: TILEBOT_ENDPOINT + 'reserved/parameters/requests/' + request_id,
         headers: {
           'Content-Type': 'application/json'
         },