@tiledesk/tiledesk-server 2.10.100 → 2.10.101

package/CHANGELOG.md

@@ -5,6 +5,13 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.10.101
+- Update: messenger-connector to 0.1.27
+- Update: multi-worker to 0.3.3
+- Added: endpoints for unanswered questions
+- Update: endppoints to static logs
+- Improved knowledge base import/export
+
 # 2.10.100
 - Update: removed verbose logs
 

package/app.js

@@ -127,6 +127,7 @@ var quotes = require('./routes/quotes');
 var integration = require('./routes/integration')
 var kbsettings = require('./routes/kbsettings');
 var kb = require('./routes/kb');
+var unanswered = require('./routes/unanswered');
 
 // var admin = require('./routes/admin');
 var faqpub = require('./routes/faqpub');
@@ -614,6 +615,7 @@ app.use('/:projectid/quotes', [passport.authenticate(['basic', 'jwt'], { session
 app.use('/:projectid/integration', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('admin', ['bot','subscription'])], integration )
 
 app.use('/:projectid/kbsettings', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('agent', ['bot','subscription'])], kbsettings);
+app.use('/:projectid/kb/unanswered', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('admin', ['bot','subscription'])], unanswered);
 app.use('/:projectid/kb', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('admin', ['bot','subscription'])], kb);
 
 app.use('/:projectid/logs', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRole('admin')], logs);

package/middleware/has-role.js

@@ -155,6 +155,7 @@ class RoleChecker {
           }
             q.exec(function (err, project_user) {
               if (err) {
+                winston.error("Error on Request path: " + req.originalUrl);
                 winston.error("Error getting project_user for hasrole",err);
                 return next(err);
               }

package/models/flowLogs.js

@@ -29,6 +29,18 @@ const FlowLogsSchema = new Schema(
       type: String,
       required: false,
     },
+    webhook_id: {
+      type: String,
+      required: false,
+    },
+    shortExp: {
+      type: Date,
+      required: false
+    },
+    longExp: {
+      type: Date,
+      required: false
+    },
     level: {
       type: String,
       required: true,
@@ -43,6 +55,9 @@ const FlowLogsSchema = new Schema(
 );
 
 FlowLogsSchema.index({ request_id: 1 }, { unique: true });
+FlowLogsSchema.index({ webhook_id: 1 });
+FlowLogsSchema.index({ shortExp: 1 }, { expireAfterSeconds: 300 });  // 5 minutes
+FlowLogsSchema.index({ longExp: 1 }, { expireAfterSeconds: 1800 }); // 30 minutes
 
 // FlowLogsSchema.pre('findOneAndUpdate', async function (next) {
 //   const update = this.getUpdate();

package/models/kb_setting.js

@@ -1,6 +1,7 @@
 var mongoose = require('mongoose');
 var Schema = mongoose.Schema;
 var winston = require('../config/winston');
+let expireAfterSeconds = process.env.UNANSWERED_QUESTION_EXPIRATION_TIME || 7 * 24 * 60 * 60; // 7 days
 
 var EngineSchema = new Schema({
   name: {
@@ -122,7 +123,35 @@ var KBSchema = new Schema({
   timestamps: true
 })
 
+const UnansweredQuestionSchema = new Schema({
+  id_project: {
+    type: String,
+    required: true,
+    index: true
+  },
+  namespace: {
+    type: String,
+    required: true,
+    index: true
+  },
+  question: {
+    type: String,
+    required: true
+  },
+  created_at: {
+    type: Date,
+    default: Date.now
+  },
+  updated_at: {
+    type: Date,
+    default: Date.now
+  }
+},{
+  timestamps: true
+});
 
+// Add TTL index to automatically delete documents after 30 days
+UnansweredQuestionSchema.index({ created_at: 1 }, { expireAfterSeconds: expireAfterSeconds }); // 30 days
 
 // DEPRECATED !! - Start
 var KBSettingSchema = new Schema({
@@ -158,15 +187,13 @@ const KBSettings = mongoose.model('KBSettings', KBSettingSchema);
 const Engine = mongoose.model('Engine', EngineSchema)
 const Namespace = mongoose.model('Namespace', NamespaceSchema)
 const KB = mongoose.model('KB', KBSchema)
+const UnansweredQuestion = mongoose.model('UnansweredQuestion', UnansweredQuestionSchema)
 
-// module.exports = {
-//   KBSettings: KBSettings,
-//   KB: KB
-// }
 
 module.exports = {
   KBSettings: KBSettings,
   Namespace: Namespace,
   Engine: Engine,
-  KB: KB
+  KB: KB,
+  UnansweredQuestion: UnansweredQuestion
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.10.100",
+  "version": "2.10.101",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",
@@ -44,7 +44,7 @@
     "@tiledesk/tiledesk-dialogflow-connector": "^1.8.4",
     "@tiledesk/tiledesk-json-rules-engine": "^4.0.3",
     "@tiledesk/tiledesk-kaleyra-proxy": "^0.1.7",
-    "@tiledesk/tiledesk-messenger-connector": "^0.1.24",
+    "@tiledesk/tiledesk-messenger-connector": "^0.1.27",
     "@tiledesk/tiledesk-rasa-connector": "^1.0.10",
     "@tiledesk/tiledesk-telegram-connector": "^0.1.14",
     "@tiledesk/tiledesk-tybot-connector": "^2.0.19",
@@ -53,7 +53,7 @@
     "@tiledesk/tiledesk-sms-connector": "^0.1.11",
     "@tiledesk/tiledesk-vxml-connector": "^0.1.76",
     "@tiledesk/tiledesk-voice-twilio-connector": "^0.1.22",
-    "@tiledesk/tiledesk-multi-worker": "^0.3.2",
+    "@tiledesk/tiledesk-multi-worker": "^0.3.3",
     "passport-oauth2": "^1.8.0",
     "amqplib": "^0.5.5",
     "app-root-path": "^3.0.0",

package/routes/kb.js

@@ -790,7 +790,7 @@ router.post('/namespace/import/:id', upload.single('uploadFile'), async (req, re
     });
   }
 
-  let deleteResponse = await KB.deleteMany({ id_project: id_project, namespace: namespace_id }).catch((err) => {
+  let deleteResponse = await KB.deleteMany({ id_project: id_project, namespace: namespace_id, type: { $in: ['url', 'text', 'faq'] } }).catch((err) => {
     winston.error("deleteMany error: ", err);
     return res.status(500).send({ success: false, error: err });
   })

package/routes/logs.js

@@ -87,35 +87,36 @@ router.post('/whatsapp', async (req, res) => {
 })
 
 
-router.get('/flows/:request_id', async (req, res) => {
+router.get('/flows/:id', async (req, res) => {
+    const id = req.params.id;
+    const { timestamp, direction, logLevel, type } = req.query;
 
-    let request_id = req.params.request_id;
-    const { timestamp, direction, logLevel } = req.query;
-
-    if (!request_id) {
-        return res.status(400).send({ success: false, error: "Missing required parameter 'request_id'." });
+    if (!id) {
+        return res.status(400).send({ success: false, error: "Missing required parameter 'id'." });
     }
 
+    // Determine if we're searching by request_id or webhook_id
+    const isWebhook = type === 'webhook';
+    const queryField = isWebhook ? 'webhook_id' : 'request_id';
+
     let method;
 
     if (!timestamp) {
-        method = logsService.getLastRows(request_id, 20, logLevel);
+        method = logsService.getLastRows(id, 20, logLevel, queryField);
     } else if (direction === 'prev') {
-        logsService.get
-        method = logsService.getOlderRows(request_id, 10, logLevel, new Date(timestamp));
+        method = logsService.getOlderRows(id, 10, logLevel, new Date(timestamp), queryField);
     } else if (direction === 'next') {
-        method = logsService.getNewerRows(request_id, 10, logLevel, new Date(timestamp))
+        method = logsService.getNewerRows(id, 10, logLevel, new Date(timestamp), queryField);
     } else {
-        return res.status(400).send({ success: false, error: "Missing or invalid 'direction' parameter. Use 'prev' or 'next'."})
+        return res.status(400).send({ success: false, error: "Missing or invalid 'direction' parameter. Use 'prev' or 'next'."});
     }
 
     method.then((logs) => {
         res.status(200).send(logs);
     }).catch((err) => {
         res.status(500).send({ success: false, error: "Error fetching logs: " + err.message });
-    })
-
-})
+    });
+});
 
 
 router.get('/flows/auth/:request_id', async (req, res) => {

package/routes/unanswered.js

@@ -0,0 +1,256 @@
+const express = require('express');
+const router = express.Router();
+const { Namespace, UnansweredQuestion } = require('../models/kb_setting');
+var winston = require('../config/winston');
+
+// Add a new unanswered question
+router.post('/', async (req, res) => {
+    try {
+        const { namespace, question } = req.body;
+        const id_project = req.projectid;
+
+        if (!namespace || !question) {
+            return res.status(400).json({
+                success: false,
+                error: "Missing required parameters: namespace and question"
+            });
+        }
+
+        // Check if namespace belongs to project
+        const isValidNamespace = await validateNamespace(id_project, namespace);
+        if (!isValidNamespace) {
+            return res.status(403).json({
+                success: false,
+                error: "Not allowed. The namespace does not belong to the current project."
+            });
+        }
+
+        const unansweredQuestion = new UnansweredQuestion({
+            id_project,
+            namespace,
+            question
+        });
+
+        const savedQuestion = await unansweredQuestion.save();
+        res.status(200).json(savedQuestion);
+
+    } catch (error) {
+        winston.error('Error adding unanswered question:', error);
+        res.status(500).json({
+            success: false,
+            error: "Error adding unanswered question"
+        });
+    }
+});
+
+// Get all unanswered questions for a namespace
+router.get('/:namespace', async (req, res) => {
+    try {
+        const { namespace } = req.params;
+        const id_project = req.projectid;
+
+        if (!namespace) {
+            return res.status(400).json({
+                success: false,
+                error: "Missing required parameter: namespace"
+            });
+        }
+
+        // Check if namespace belongs to project
+        const isValidNamespace = await validateNamespace(id_project, namespace);
+        if (!isValidNamespace) {
+            return res.status(403).json({
+                success: false,
+                error: "Not allowed. The namespace does not belong to the current project."
+            });
+        }
+
+        const page = parseInt(req.query.page) || 0;
+        const limit = parseInt(req.query.limit) || 20;
+        const sortField = req.query.sortField || 'created_at';
+        const direction = parseInt(req.query.direction) || -1;
+
+        const questions = await UnansweredQuestion.find({
+            id_project,
+            namespace
+        })
+        .sort({ [sortField]: direction })
+        .skip(page * limit)
+        .limit(limit);
+
+        const count = await UnansweredQuestion.countDocuments({
+            id_project,
+            namespace
+        });
+
+        res.status(200).json({
+            count,
+            questions,
+            query: {
+                page,
+                limit,
+                sortField,
+                direction
+            }
+        });
+
+    } catch (error) {
+        winston.error('Error getting unanswered questions:', error);
+        res.status(500).json({
+            success: false,
+            error: "Error getting unanswered questions"
+        });
+    }
+});
+
+// Delete a specific unanswered question
+router.delete('/:id', async (req, res) => {
+    try {
+        const { id } = req.params;
+        const id_project = req.projectid;
+
+        const question = await UnansweredQuestion.findOne({ _id: id, id_project });
+        if (!question) {
+            return res.status(404).json({
+                success: false,
+                error: "Question not found"
+            });
+        }
+
+        await UnansweredQuestion.deleteOne({ _id: id });
+        res.status(200).json({
+            success: true,
+            message: "Question deleted successfully"
+        });
+
+    } catch (error) {
+        winston.error('Error deleting unanswered question:', error);
+        res.status(500).json({
+            success: false,
+            error: "Error deleting unanswered question"
+        });
+    }
+});
+
+// Delete all unanswered questions for a namespace
+router.delete('/namespace/:namespace', async (req, res) => {
+    try {
+        const { namespace } = req.params;
+        const id_project = req.projectid;
+
+        // Check if namespace belongs to project
+        const isValidNamespace = await validateNamespace(id_project, namespace);
+        if (!isValidNamespace) {
+            return res.status(403).json({
+                success: false,
+                error: "Not allowed. The namespace does not belong to the current project."
+            });
+        }
+
+        const result = await UnansweredQuestion.deleteMany({ id_project, namespace });
+        res.status(200).json({
+            success: true,
+            count: result.deletedCount,
+            message: "All questions deleted successfully"
+        });
+
+    } catch (error) {
+        winston.error('Error deleting unanswered questions:', error);
+        res.status(500).json({
+            success: false,
+            error: "Error deleting unanswered questions"
+        });
+    }
+});
+
+// Update an unanswered question
+router.put('/:id', async (req, res) => {
+    try {
+        const { id } = req.params;
+        const { question } = req.body;
+        const id_project = req.projectid;
+
+        if (!question) {
+            return res.status(400).json({
+                success: false,
+                error: "Missing required parameter: question"
+            });
+        }
+
+        const updatedQuestion = await UnansweredQuestion.findOneAndUpdate(
+            { _id: id, id_project },
+            { question },
+            { new: true }
+        );
+
+        if (!updatedQuestion) {
+            return res.status(404).json({
+                success: false,
+                error: "Question not found"
+            });
+        }
+
+        res.status(200).json(updatedQuestion);
+
+    } catch (error) {
+        winston.error('Error updating unanswered question:', error);
+        res.status(500).json({
+            success: false,
+            error: "Error updating unanswered question"
+        });
+    }
+});
+
+// Count unanswered questions for a namespace
+router.get('/count/:namespace', async (req, res) => {
+    try {
+        const { namespace } = req.params;
+        const id_project = req.projectid;
+
+        if (!namespace) {
+            return res.status(400).json({
+                success: false,
+                error: "Missing required parameter: namespace"
+            });
+        }
+
+        // Check if namespace belongs to project
+        const isValidNamespace = await validateNamespace(id_project, namespace);
+        if (!isValidNamespace) {
+            return res.status(403).json({
+                success: false,
+                error: "Not allowed. The namespace does not belong to the current project."
+            });
+        }
+
+        const count = await UnansweredQuestion.countDocuments({
+            id_project,
+            namespace
+        });
+
+        res.status(200).json({ count });
+
+    } catch (error) {
+        winston.error('Error counting unanswered questions:', error);
+        res.status(500).json({
+            success: false,
+            error: "Error counting unanswered questions"
+        });
+    }
+});
+
+// Helper function to validate namespace
+async function validateNamespace(id_project, namespace_id) {
+    try {
+        const namespace = await Namespace.findOne({ 
+            id_project: id_project,
+            namespace: namespace_id 
+        });
+        return !!namespace; // return true if namespace exists, false otherwise
+    } catch (err) {
+        winston.error("validate namespace error: ", err);
+        throw err;
+    }
+}
+
+module.exports = router; 

package/services/logsService.js

@@ -7,14 +7,20 @@ const levels = { error: 0, warn: 1, info: 2, debug: 3, native: 4 };
 
 class LogsService {
 
-    async getLastRows(request_id, limit, logLevel) {
+    async getLastRows(id, limit, logLevel, queryField = 'request_id') {
         let level = logLevel || default_log_level;
         if (level === 'default') {
             level = default_log_level
         }
         let nlevel = levels[level];
+
+        // Build match condition based on queryField
+        const matchCondition = queryField === 'webhook_id' 
+            ? { [queryField]: id, longExp: { $exists: true } }
+            : { [queryField]: id };
+
         return FlowLogs.aggregate([
-            { $match: { request_id: request_id } },
+            { $match: matchCondition },
             { $unwind: "$rows" },
             { $match: { "rows.nlevel": { $lte: nlevel } } },
             { $sort: { "rows.timestamp": -1, "rows._id": -1 } },
@@ -22,14 +28,20 @@ class LogsService {
         ]).then(rows => rows.reverse())
     }
 
-    async getOlderRows(request_id, limit, logLevel, timestamp) {
+    async getOlderRows(id, limit, logLevel, timestamp, queryField = 'request_id') {
         let level = logLevel || default_log_level;
         if (level === 'default') {
             level = default_log_level
         }
         let nlevel = levels[level];
+
+        // Build match condition based on queryField
+        const matchCondition = queryField === 'webhook_id' 
+            ? { [queryField]: id, longExp: { $exists: true } }
+            : { [queryField]: id };
+
         return FlowLogs.aggregate([
-            { $match: { request_id: request_id } },
+            { $match: matchCondition },
             { $unwind: "$rows" },
             { $match: { "rows.nlevel": { $lte: nlevel }, "rows.timestamp": { $lt: timestamp } } },
             { $sort: { "rows.timestamp": -1, "rows._id": -1 } },
@@ -37,14 +49,20 @@ class LogsService {
         ]).then(rows => rows.reverse())
     }
 
-    async getNewerRows(request_id, limit, logLevel, timestamp) {
+    async getNewerRows(id, limit, logLevel, timestamp, queryField = 'request_id') {
         let level = logLevel || default_log_level;
         if (level === 'default') {
             level = default_log_level
         }
         let nlevel = levels[level];
+
+        // Build match condition based on queryField
+        const matchCondition = queryField === 'webhook_id' 
+            ? { [queryField]: id, longExp: { $exists: true } }
+            : { [queryField]: id };
+
         return FlowLogs.aggregate([
-            { $match: { request_id: request_id } },
+            { $match: matchCondition },
             { $unwind: "$rows" },
             { $match: { "rows.nlevel": { $lte: nlevel }, "rows.timestamp": { $gt: timestamp } } },
             { $sort: { "rows.timestamp": 1, "rows._id": 1 } },

package/services/webhookService.js

@@ -4,6 +4,7 @@ const uuidv4 = require('uuid/v4');
 var jwt = require('jsonwebtoken');
 var winston = require('../config/winston');
 const errorCodes = require("../errorCodes");
+var ObjectId = require('mongoose').Types.ObjectId;
 
 const port = process.env.PORT || '3000';
 let TILEBOT_ENDPOINT = "http://localhost:" + port + "/modules/tilebot/";;
@@ -45,6 +46,8 @@ class WebhookService {
                 let json_value = JSON.parse(value);
                 payload.preloaded_request_id = json_value.request_id;
                 payload.draft = true;
+            }  else {
+                payload.preloaded_request_id = "automation-request-" + webhook.id_project + "-" + new ObjectId() + "-" + webhook.webhook_id;
             }
 
             let token = await this.generateChatbotToken(chatbot);