@tiledesk/tiledesk-server 2.15.8 → 2.17.2

package/CHANGELOG.md

@@ -5,6 +5,27 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.17.2
+- Added support for situated context in kb route
+- Added RAG context management to KB routes
+- Added support for scrape type 0 (alias: trafilatura)
+
+# 2.16.2
+- Improved multiplier retrieval for model types in quotes route
+
+# 2.16.1
+- Added stream option support to the KB /qa endpoint for real-time responses
+- Enhanced file upload route to correctly handle .webm files
+- Optimized token consumption and management in knowledge base operations
+
+# 2.16.0-hf
+- Fixed bug: issue on audio sent from widget
+
+# 2.16.0
+- Added possibility to update Knowledge Base content
+- Added rated only filter in Conversations History
+- Improved pending requests management
+
 # 2.15.8
 - Updated tybot-connector to 2.0.45
 - Added support for tags management in knowledge base routes

package/config/kb/prompt/rag/PromptManager.js

@@ -0,0 +1,57 @@
+const fs = require('fs');
+const path = require('path');
+
+const modelMap = {
+    "gpt-3.5-turbo":        "gpt-3.5.txt",
+    "gpt-4":                "gpt-4.txt",
+    "gpt-4-turbo-preview":  "gpt-4.txt",
+    "gpt-4o":               "gpt-4o.txt",
+    "gpt-4o-mini":          "gpt-4o.txt",
+    "gpt-4.1":              "gpt-4.1.txt",
+    "gpt-4.1-mini":         "gpt-4.1.txt",
+    "gpt-4.1-nano":         "gpt-4.1.txt",
+    "gpt-5":                "gpt-5.txt",
+    "gpt-5-mini":           "gpt-5.txt",
+    "gpt-5-nano":           "gpt-5.txt",
+    "gpt-5.1":              "gpt-5.x.txt",
+    "gpt-5.2":              "gpt-5.x.txt",
+    "gpt-5.3-chat-latest":  "gpt-5.x.txt",
+    "gpt-5.4":              "gpt-5.x.txt",
+    "gpt-5.4-mini":         "gpt-5.x.txt",
+    "gpt-5.4-nano":         "gpt-5.x.txt",
+    "general":              "general.txt"
+}
+
+
+class PromptManager {
+  
+    constructor(basePath) {
+    this.basePath = basePath;
+    this.cache = new Map();
+  }
+
+  getPrompt(name) {
+    if (this.cache.has(name)) {
+      return this.cache.get(name);
+    }
+
+    const fileName = modelMap[name] || modelMap["general"];
+    const filePath = path.join(this.basePath, fileName);
+
+    let content;
+    try {
+      content = fs.readFileSync(filePath, 'utf-8');
+    } catch (err) {
+      content = fs.readFileSync(
+        path.join(this.basePath, modelMap["general"]),
+        'utf-8'
+      );
+    }
+
+    this.cache.set(name, content);
+    return content;
+  }
+}
+
+PromptManager.modelMap = modelMap;
+module.exports = PromptManager;

package/config/kb/prompt/rag/general.txt

@@ -0,0 +1,9 @@
+You are an helpful assistant for question-answering tasks. Follow these steps carefully:
+
+1. Answer in the same language of the user question, regardless of the retrieved context language
+2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.
+3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer.
+
+==Retrieved context start==
+{context}
+==Retrieved context end==

package/config/kb/prompt/rag/gpt-3.5.txt

@@ -0,0 +1,9 @@
+You are an helpful assistant for question-answering tasks.
+
+Use ONLY the pieces of retrieved context delimited by #### and the chat history to answer the question.
+
+If you don't know the answer, just say: "I don't know<NOANS>"
+
+####
+{context}
+####

package/config/kb/prompt/rag/gpt-4.1.txt

@@ -0,0 +1,9 @@
+You are an helpful assistant for question-answering tasks. Follow these steps carefully:
+
+1. Answer in the same language of the user question, regardless of the retrieved context language
+2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.
+3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, append <NOANS> at the end of the answer
+
+==Retrieved context start==
+{context}
+==Retrieved context end==

package/config/kb/prompt/rag/gpt-4.txt

@@ -0,0 +1,11 @@
+You are an helpful assistant for question-answering tasks.
+
+Use ONLY the pieces of retrieved context delimited by #### and the chat history to answer the question.
+
+If you don't know the answer, just say that you don't know.
+
+If and only if none of the retrieved context is useful for your task, add this word to the end <NOANS>
+
+####
+{context}
+####

package/config/kb/prompt/rag/gpt-4o.txt

@@ -0,0 +1,9 @@
+You are an helpful assistant for question-answering tasks. Follow these steps carefully:
+
+1. Answer in the same language of the user question, regardless of the retrieved context language
+2. Use ONLY the pieces of the retrieved context and the chat history to answer the question.
+3. If the retrieved context does not contain sufficient information to generate an accurate and informative answer, return <NOANS>.
+
+==Retrieved context start==
+{context}
+==Retrieved context end==

package/config/kb/prompt/rag/gpt-5.txt

@@ -0,0 +1,32 @@
+# ROLE
+You are an AI assistant that answers the user's question using only the information contained in the provided context.
+
+# LANGUAGE
+Answer in the same language as the user's question.
+
+# CONTEXT
+You will receive a context delimited by ######:
+######
+{context}
+######
+
+# INSTRUCTIONS
+- Use only the information explicitly contained in the context.
+- Answer the user's question directly, as a human assistant would.
+- Do not mention the context, the document, the source, or the fact that information was provided.
+- Do not say phrases such as:
+  - "according to the context"
+  - "in the provided context"
+  - "the document says"
+  - "based on the information provided"
+- Do not explain your reasoning.
+- Do not repeat the question.
+- Keep the answer concise, clear, and natural.
+- Do not add assumptions, external knowledge, or details not supported by the context.
+
+# FALLBACK
+If the context does not contain enough information to answer the question, reply with exactly:
+<NOANS>
+
+# OUTPUT
+Return only the final answer, with no preamble and no meta-commentary.

package/config/kb/prompt/rag/gpt-5.x.txt

@@ -0,0 +1,32 @@
+# ROLE
+You are an AI assistant that answers the user's question using only the information contained in the provided context.
+
+# LANGUAGE
+Answer in the same language as the user's question.
+
+# CONTEXT
+You will receive a context delimited by ######:
+######
+{context}
+######
+
+# INSTRUCTIONS
+- Use only the information explicitly contained in the context.
+- Answer the user's question directly, as a human assistant would.
+- Do not mention the context, the document, the source, or the fact that information was provided.
+- Do not say phrases such as:
+  - "according to the context"
+  - "in the provided context"
+  - "the document says"
+  - "based on the information provided"
+- Do not explain your reasoning.
+- Do not repeat the question.
+- Keep the answer concise, clear, and natural.
+- Do not add assumptions, external knowledge, or details not supported by the context.
+
+# FALLBACK
+If the context does not contain enough information to answer the question, reply with exactly:
+<NOANS>
+
+# OUTPUT
+Return only the final answer, with no preamble and no meta-commentary.

package/config/kb/situatedContext.js

@@ -0,0 +1,6 @@
+module.exports = {
+    enable: process.env.SITUATED_CONTEXT_ENABLE === "true",
+    provider: process.env.SITUATED_CONTEXT_PROVIDER || "openai",
+    model: process.env.SITUATED_CONTEXT_MODEL || "gpt-4o",
+    api_key: ""
+}

package/event/messageEvent.js

@@ -5,6 +5,7 @@ var Message = require("../models/message");
 var Faq_kb = require("../models/faq_kb");
 var MessageConstants = require("../models/messageConstants");
 var message2Event = require("../event/message2Event");
+var requestEvent = require("../event/requestEvent");
 
 var cacheUtil = require('../utils/cacheUtil');
 var cacheEnabler = require("../services/cacheEnabler");
@@ -178,6 +179,29 @@ function populateMessageWithRequest(message, eventPrefix) {
 messageEvent.on('message.create.simple', populateMessageCreate);
 messageEvent.on('message.update.simple', populateMessageUpdate);
 
+// When the user (lead/requester) sends a message, reopen the conversation if it was pending
+messageEvent.on('message.create.from.requester', function (messageJson) {
+  if (!messageJson.request || messageJson.request.workingStatus !== 'pending') return;
+  var request_id = messageJson.request.request_id;
+  var id_project = messageJson.request.id_project;
+  Request.findOneAndUpdate(
+    { request_id: request_id, id_project: id_project },
+    { $set: { workingStatus: 'open' } },
+    { new: true },
+    function (err, updatedRequest) {
+      if (err) {
+        winston.error("Error updating request workingStatus from pending to open", err);
+        return;
+      }
+      if (updatedRequest) {
+        winston.debug("Request workingStatus set to open (was pending)", { request_id, id_project });
+        requestEvent.emit('request.workingStatus.update', { request: updatedRequest });
+        requestEvent.emit('request.update', updatedRequest);
+      }
+    }
+  );
+});
+
 
 
 // // riattiva commentato per performance

package/middleware/file-type.js

@@ -31,9 +31,10 @@ function areMimeTypesEquivalent(mimeType1, mimeType2) {
     'audio/wave': ['audio/wav', 'audio/x-wav', 'audio/vnd.wave'],
     'audio/x-wav': ['audio/wav', 'audio/wave', 'audio/vnd.wave'],
     'audio/vnd.wave': ['audio/wav', 'audio/wave', 'audio/x-wav'],
-    'audio/mpeg': ['audio/opus', 'audio/mp3'],
-    'audio/mp3': ['audio/mpeg', 'audio/opus'],
-    'audio/opus': ['audio/mpeg', 'audio/mp3'],
+    'audio/mpeg': ['audio/opus', 'audio/mp3', 'audio/webm'],
+    'audio/mp3': ['audio/mpeg', 'audio/opus', 'audio/webm'],
+    'audio/opus': ['audio/mpeg', 'audio/mp3', 'audio/webm'],
+    'audio/webm': ['audio/mpeg', 'audio/mp3', 'audio/opus'],
     'image/jpeg': ['image/jpg'],
     'image/jpg': ['image/jpeg'],
     'application/x-zip-compressed': ['application/zip'],
@@ -47,54 +48,126 @@ function areMimeTypesEquivalent(mimeType1, mimeType2) {
   return false;
 }
 
+// Magic bytes for fallback when file-type throws (e.g. strtok3/token-types Uint8Array vs Buffer)
+const MAGIC_SIGNATURES = {
+  'video/webm': [[0x1A, 0x45, 0xDF, 0xA3]],           // EBML
+  'audio/webm': [[0x1A, 0x45, 0xDF, 0xA3]],
+  'audio/mpeg': [[0xFF, 0xFB], [0xFF, 0xFA], [0xFF, 0xF3], [0xFF, 0xF2], [0x49, 0x44, 0x33]], // ID3 or MP3 frame
+  'audio/mp3': [[0xFF, 0xFB], [0xFF, 0xFA], [0xFF, 0xF3], [0xFF, 0xF2], [0x49, 0x44, 0x33]],
+  'image/png': [[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]],
+  'image/jpeg': [[0xFF, 0xD8, 0xFF]],
+  'image/gif': [[0x47, 0x49, 0x46, 0x38, 0x37, 0x61], [0x47, 0x49, 0x46, 0x38, 0x39, 0x61]],
+  'application/pdf': [[0x25, 0x50, 0x44, 0x46]],
+};
+
+function magicMatches(buf, mimetype) {
+  const signatures = MAGIC_SIGNATURES[mimetype && mimetype.toLowerCase()];
+  if (!signatures) return false;
+  for (const sig of signatures) {
+    if (buf.length < sig.length) continue;
+    let ok = true;
+    for (let i = 0; i < sig.length; i++) {
+      const b = buf[i] !== undefined ? (buf[i] & 0xFF) : -1;
+      if (b !== sig[i]) { ok = false; break; }
+    }
+    if (ok) return true;
+  }
+  return false;
+}
+
+const BASE64_REGEX = /^[A-Za-z0-9+/]+=*$/;
+
+/**
+ * Ensures the input is a Node.js Buffer. file-type (and token-types/strtok3) require
+ * a Buffer with methods like readUInt8; GridFS or other sources may return
+ * Uint8Array, ArrayBuffer, BSON Binary, or (when client sends base64) a string.
+ * We always allocate a new Buffer and copy bytes so file-type never receives
+ * a buffer-like that loses readUInt8 when sliced (e.g. by strtok3).
+ */
+function ensureBuffer(buffer) {
+  if (!buffer) return buffer;
+
+  // Base64 string (e.g. client sends form body as base64): decode to binary
+  if (typeof buffer === 'string' && buffer.length > 0) {
+    const trimmed = buffer.replace(/\s/g, '');
+    if (BASE64_REGEX.test(trimmed)) {
+      return Buffer.from(trimmed, 'base64');
+    }
+    return Buffer.from(buffer, 'utf8');
+  }
+
+  // Copy into a new Buffer so file-type's internal slices are always real Buffers
+  let uint8;
+  if (buffer instanceof Uint8Array) {
+    uint8 = buffer;
+  } else if (buffer instanceof ArrayBuffer) {
+    uint8 = new Uint8Array(buffer);
+  } else if (buffer && typeof buffer.buffer === 'object' && buffer.buffer instanceof ArrayBuffer) {
+    uint8 = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+  } else if (Buffer.isBuffer(buffer)) {
+    uint8 = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+  } else {
+    uint8 = new Uint8Array(Buffer.from(buffer));
+  }
+  return Buffer.from(uint8);
+}
+
 async function verifyFileContent(buffer, mimetype) {
   if (!buffer) throw new Error("No file provided");
 
+  const buf = ensureBuffer(buffer);
+
+  let fileType;
   try {
-    const fileType = await FileType.fromBuffer(buffer);
+    fileType = await FileType.fromBuffer(buf);
+  } catch (err) {
+    // strtok3 uses Uint8Array for numBuffer but token-types expects Buffer.readUInt8 (known compat bug in deps)
+    if (err && typeof err.message === 'string' && err.message.includes('readUInt8')) {
+      if (mimetype && magicMatches(buf, mimetype)) return true;
+      const err2 = new Error(`File content could not be verified. Declared mimetype: ${mimetype}`);
+      err2.source = "FileContentVerification";
+      throw err2;
+    }
+    throw err;
+  }
 
-    // If FileType couldn't detect the file type (returns null/undefined)
-    if (!fileType) {
-      // For text-based MIME types, accept the declared mimetype since FileType can't detect them
-      if (mimetype && TEXT_MIME_TYPES.includes(mimetype)) {
-        // Optionally verify that the content is valid UTF-8 text
-        try {
-          buffer.toString('utf8');
-          return true;
-        } catch (e) {
-          const err = new Error(`File content is not valid text for mimetype: ${mimetype}`);
-          err.source = "FileContentVerification";
-          throw err;
-        }
-      } else if (mimetype && mimetype.startsWith('image/svg')) {
-        // Handle SVG files (can be image/svg+xml or variants)
-        try {
-          buffer.toString('utf8');
-          return true;
-        } catch (e) {
-          const err = new Error(`File content is not valid text for mimetype: ${mimetype}`);
-          err.source = "FileContentVerification";
-          throw err;
-        }
-      } else {
-        // For non-text files, FileType should be able to detect them
-        const err = new Error(`File content does not match mimetype. Detected: unknown, provided: ${mimetype}`);
+  // If FileType couldn't detect the file type (returns null/undefined)
+  if (!fileType) {
+    // For text-based MIME types, accept the declared mimetype since FileType can't detect them
+    if (mimetype && TEXT_MIME_TYPES.includes(mimetype)) {
+      try {
+        buf.toString('utf8');
+        return true;
+      } catch (e) {
+        const err = new Error(`File content is not valid text for mimetype: ${mimetype}`);
         err.source = "FileContentVerification";
         throw err;
       }
     }
-
-    // If FileType detected a type, it must match the declared mimetype (or be equivalent)
-    if (mimetype && !areMimeTypesEquivalent(fileType.mime, mimetype)) {
-        const err = new Error(`File content does not match mimetype. Detected: ${fileType.mime}, provided: ${mimetype}`);
+    if (mimetype && mimetype.startsWith('image/svg')) {
+      try {
+        buf.toString('utf8');
+        return true;
+      } catch (e) {
+        const err = new Error(`File content is not valid text for mimetype: ${mimetype}`);
         err.source = "FileContentVerification";
         throw err;
+      }
     }
+    if (mimetype && magicMatches(buf, mimetype)) return true;
+    const err = new Error(`File content does not match mimetype. Detected: unknown, provided: ${mimetype}`);
+    err.source = "FileContentVerification";
+    throw err;
+  }
 
-    return true;
-  } catch (err) {
+  // If FileType detected a type, it must match the declared mimetype (or be equivalent)
+  if (mimetype && !areMimeTypesEquivalent(fileType.mime, mimetype)) {
+    const err = new Error(`File content does not match mimetype. Detected: ${fileType.mime}, provided: ${mimetype}`);
+    err.source = "FileContentVerification";
     throw err;
   }
+
+  return true;
 }
 
 module.exports = verifyFileContent;

package/models/request.js

@@ -507,6 +507,7 @@ RequestSchema.index({ id_project: 1, createdAt: -1, status: 1 })
 RequestSchema.index({ id_project: 1, preflight: 1, smartAssignment: 1, "snapshot.department.routing": 1, createdAt: 1, status: 1 })
 
 RequestSchema.index({ status: 1, hasBot: 1, updatedAt: 1 }) // For closing unresponsive requests
+RequestSchema.index({ status: 1, hasBot: 1, workingStatus: 1, updatedAt: 1 }) // For closing unresponsive requests
 
 // Contact search by phone / email
 RequestSchema.index({ id_project: 1, 'contact.phone': 1 });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.15.8",
+  "version": "2.17.2",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",

package/pubmodules/queue/reconnect.js

@@ -175,6 +175,11 @@ function startWorker() {
           winston.info("Data queue", oka)
         });
 
+        ch.bindQueue(_ok.queue, exchange, "request_workingStatus_update", {}, function(err3, oka) {
+          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_workingStatus_update");
+          winston.info("Data queue", oka)
+        });
+
         ch.bindQueue(_ok.queue, exchange, "message_create", {}, function(err3, oka) {
               winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: message_create");
               winston.info("Data queue", oka)
@@ -286,6 +291,11 @@ function work(msg, cb) {
     requestEvent.emit('request.close.extended.queue',  JSON.parse(message_string));
   }     
 
+  if (topic === 'request_workingStatus_update') {
+    winston.debug("reconnect here topic:" + topic); 
+    requestEvent.emit('request.workingStatus.update.queue',  JSON.parse(message_string));
+  }     
+
   if (topic === 'message_create') {
     winston.debug("reconnect here topic:" + topic);
     // requestEvent.emit('request.create.queue', msg.content);
@@ -410,6 +420,12 @@ function listen() {
       });
     });
 
+    requestEvent.on('request.workingStatus.update', function(request) {
+      setImmediate(() => {
+        publish(exchange, "request_workingStatus_update", Buffer.from(JSON.stringify(request)));
+      });
+    });
+
     requestEvent.on('request.snapshot.update', function(data) {
       setImmediate(() => {
         winston.debug("reconnect request.snapshot.update")

package/pubmodules/routing-queue/listenerQueued.js

@@ -50,7 +50,7 @@ class Listener {
       return winston.warn("Chatbot is not a project_user. Skip update.")
     }
 
-    return Request.countDocuments({ id_project: id_project, participantsAgents: id_user, status: { $lt: 1000 }, draft: { $in: [null, false] } }, (err, requestsCount) => {
+    return Request.countDocuments({ id_project: id_project, participantsAgents: id_user, status: { $lt: 1000 }, draft: { $in: [null, false] }, workingStatus: { $ne: 'pending' } }, (err, requestsCount) => {
       winston.verbose("requestsCount for id_user: ", id_user, "and project: ", id_project, "-->", requestsCount);
       if (err) {
         return winston.error(err);
@@ -236,8 +236,27 @@ class Listener {
           });
         });
 
-     
-       
+        var requestWorkingStatusUpdateKey = 'request.workingStatus.update';
+        if (requestEvent.queueEnabled) {
+          requestWorkingStatusUpdateKey = 'request.workingStatus.update.queue';
+        }
+        winston.debug('Route queue requestWorkingStatusUpdateKey: ' + requestWorkingStatusUpdateKey);
+
+        requestEvent.on(requestWorkingStatusUpdateKey, async (data) => {
+          winston.debug('Route queue WorkingStatus Update');
+
+          var request = data.request;
+          var participantIds = (request.participantsAgents && request.participantsAgents.length)
+            ? request.participantsAgents
+            : (request.participatingAgents || []).map(u => u._id || u.id);
+          setImmediate(() => {
+            participantIds.forEach(id_user => {
+              if (id_user && !String(id_user).startsWith('bot_')) {
+                this.updateProjectUser(id_user, request.id_project, 0);
+              }
+            });
+          });
+        });
     }
 
 }

package/pubmodules/scheduler/tasks/closeAgentUnresponsiveRequestTask.js

@@ -65,7 +65,8 @@ class CloseAgentUnresponsiveRequestTask {
     const query = {
       hasBot: false,
       status: { $lt: 1000 },
-      updatedAt: { $lte: cutoffDate }
+      updatedAt: { $lte: cutoffDate },
+      workingStatus: { $ne: 'pending' }
     };
 
     if (this.queryProject) {

package/pubmodules/scheduler/tasks/closeBotUnresponsiveRequestTask.js

@@ -73,7 +73,8 @@ class CloseBotUnresponsiveRequestTask {
     const query = { 
       hasBot: true, 
       status: { $lt: 1000 }, 
-      updatedAt: { $lte: cutoffDate } 
+      updatedAt: { $lte: cutoffDate },
+      workingStatus: { $ne: 'pending' }
     };
 
     if (this.queryProject) {

package/routes/filesp.js

@@ -124,9 +124,10 @@ function areMimeTypesEquivalent(mimeType1, mimeType2) {
     'audio/wave': ['audio/wav', 'audio/x-wav', 'audio/vnd.wave'],
     'audio/x-wav': ['audio/wav', 'audio/wave', 'audio/vnd.wave'],
     'audio/vnd.wave': ['audio/wav', 'audio/wave', 'audio/x-wav'],
-    'audio/mpeg': ['audio/opus', 'audio/mp3'],
-    'audio/mp3': ['audio/mpeg', 'audio/opus'],
-    'audio/opus': ['audio/mpeg', 'audio/mp3'],
+    'audio/mpeg': ['audio/opus', 'audio/mp3', 'audio/webm'],
+    'audio/mp3': ['audio/mpeg', 'audio/opus', 'audio/webm'],
+    'audio/opus': ['audio/mpeg', 'audio/mp3', 'audio/webm'],
+    'audio/webm': ['audio/mpeg', 'audio/mp3', 'audio/opus'],
     'image/jpeg': ['image/jpg'],
     'image/jpg': ['image/jpeg'],
     'application/x-zip-compressed': ['application/zip'],