@tiledesk/tiledesk-server 2.18.4 → 2.18.16

package/CHANGELOG.md

@@ -5,6 +5,33 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.18.13
+- Added Analytics tracking
+
+# 2.18.12
+- Added messages filtering on download transcript via csv and pdf
+
+# 2.18.11
+- Added endpoint to recovery urls preview
+
+# 2.18.10
+- Updated tybot-connector to 2.0.51
+
+# 2.18.9
+- Updated tybot-connector to 2.0.49
+
+# 2.18.8
+- Refactored transcript templates
+
+# 2.18.7
+- Added possibility to export Answered and Unanswered questions
+
+# 2.18.6
+- Modified sitemap deletion behavior to also remove all associated URLs.
+
+# 2.18.5
+- Fixed bug on update url content on Knwoledge Base
+
 # 2.18.4
 - Added HyDE support for Knowledge Base Q&A
 - Introduced Cache (cRag) functionality in Knowledge Base Q&A

package/app.js

@@ -125,6 +125,7 @@ var widgets = require('./routes/widget');
 var widgetsLoader = require('./routes/widgetLoader');
 var openai = require('./routes/openai');
 var llm = require('./routes/llm');
+var urlPreview = require('./routes/urlPreview');
 var quotes = require('./routes/quotes');
 var integration = require('./routes/integration')
 var kbsettings = require('./routes/kbsettings');
@@ -636,6 +637,7 @@ app.use('/:projectid/segments',[passport.authenticate(['basic', 'jwt'], { sessio
 
 app.use('/:projectid/llm', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('admin', ['bot','subscription'])], llm);
 app.use('/:projectid/openai', openai);
+app.use('/:projectid/url-preview', urlPreview);
 app.use('/:projectid/quotes', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('agent', ['bot','subscription'])], quotes)
 
 app.use('/:projectid/integration', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('admin', ['bot','subscription'])], integration )
@@ -724,5 +726,4 @@ app.use((err, req, res, next) => {
 });
 
 
-
 module.exports = app;

package/event/kbEvent.js

@@ -0,0 +1,12 @@
+const EventEmitter = require('events');
+
+class KbEvent extends EventEmitter {
+    constructor() {
+        super();
+        this.queueEnabled = false;
+    }
+}
+
+const kbEvent = new KbEvent();
+
+module.exports = kbEvent;

package/event/webhookEvent.js

@@ -0,0 +1,9 @@
+"use strict";
+
+const EventEmitter = require('events');
+
+class WebhookEvent extends EventEmitter {}
+
+const webhookEvent = new WebhookEvent();
+
+module.exports = webhookEvent;

package/jobs.js

@@ -114,6 +114,17 @@ async function main()
     let multiWorkerQueue = require('@tiledesk/tiledesk-multi-worker');
     jobsManager.listenMultiWorker(multiWorkerQueue);
 
+    try {
+        this.analyticsPublisher = require('./pubmodules/analytics-publisher');
+        this.analyticsPublisher.listen();
+        winston.info("AnalyticsPublisher initialized.");
+    } catch(err) {
+        if (err.code === 'MODULE_NOT_FOUND') {
+            winston.info("PubModulesManager init analyticsPublisher module not found");
+        } else {
+            winston.info("PubModulesManager error initializing analyticsPublisher module", err);
+        }
+    }
 
     winston.info("Jobs started"); 
 
@@ -128,4 +139,4 @@ function panic(error)
 }
 
 // https://stackoverflow.com/a/46916601/1478566
-main().catch(panic).finally(clearInterval.bind(null, setInterval(a=>a, 1E9)));
+main().catch(panic).finally(clearInterval.bind(null, setInterval(a=>a, 1E9)));

package/lib/analyticsClient.js

@@ -0,0 +1,60 @@
+"use strict";
+
+const axios = require("axios");
+const crypto = require("crypto");
+const winston = require("../config/winston");
+
+const INGEST_BASE_URL = process.env.ANALYTICS_INGEST_URL;
+const INGEST_API_KEY = process.env.ANALYTICS_INGEST_API_KEY;
+const SOURCE_SERVICE = "tiledesk-server";
+const EVENT_VERSION = "1.0";
+
+const client = axios.create({
+  baseURL: INGEST_BASE_URL || "http://localhost:3099", // placeholder; disabled when INGEST_BASE_URL unset
+  timeout: 5000,
+  headers: INGEST_API_KEY ? { "X-Api-Key": INGEST_API_KEY } : {},
+});
+
+/**
+ * Build a full analytics event envelope.
+ * Mirrors the tiledesk-llm _build_envelope() pattern so both services
+ * produce identical envelope shapes, enabling event_id-based deduplication
+ * in the consumer.
+ */
+function _buildEnvelope(eventType, idProject, payload) {
+  return {
+    event_id: crypto.randomUUID(),
+    event_type: eventType,
+    timestamp: new Date().toISOString(),
+    id_project: idProject,
+    source_service: SOURCE_SERVICE,
+    event_version: EVENT_VERSION,
+    payload: payload,
+  };
+}
+
+/**
+ * Fire-and-forget analytics event. No-op when ANALYTICS_INGEST_URL is unset.
+ *
+ * @param {string} eventType   - e.g. 'conversation.created'
+ * @param {string} idProject   - Tiledesk project ID
+ * @param {object} payload     - event-specific fields
+ */
+function track(eventType, idProject, payload) {
+  if (!INGEST_BASE_URL) return;
+  if (!idProject) return;
+
+  var body = _buildEnvelope(eventType, idProject, payload);
+
+  client.post("/events", body).catch(function (err) {
+    var status = err.response && err.response.status;
+    var detail =
+      (err.response && err.response.data && err.response.data.error) ||
+      err.message;
+    winston.warn(
+      "[analytics] Failed to track " + eventType + ": " + status + " " + detail,
+    );
+  });
+}
+
+module.exports = { track: track };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.18.4",
+  "version": "2.18.16",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",
@@ -49,7 +49,7 @@
     "@tiledesk/tiledesk-rasa-connector": "^1.0.10",
     "@tiledesk/tiledesk-sms-connector": "^0.1.13",
     "@tiledesk/tiledesk-telegram-connector": "^0.1.14",
-    "@tiledesk/tiledesk-tybot-connector": "^2.0.48",
+    "@tiledesk/tiledesk-tybot-connector": "^2.0.52",
     "@tiledesk/tiledesk-voice-twilio-connector": "^0.3.2",
     "@tiledesk/tiledesk-vxml-connector": "^0.1.91",
     "@tiledesk/tiledesk-whatsapp-connector": "1.0.26",

package/pubmodules/analytics-publisher/index.js

@@ -0,0 +1,437 @@
+"use strict";
+
+/**
+ * analytics-publisher pubmodule
+ *
+ * Listens to internal tiledesk-server events and forwards analytics
+ * events to the ingest sidecar via lib/analyticsClient.
+ *
+ * Fire-and-forget: no event is awaited and no error propagates to callers.
+ * Fully disabled (no listeners registered) when ANALYTICS_INGEST_URL is unset.
+ *
+ * All payloads are validated against the @tiledesk-analytics/contracts Zod
+ * schemas (packages/contracts/src/payloads/*.ts).  Field names and nullability
+ * here must match those schemas exactly.
+ */
+
+var requestEvent = require("../../event/requestEvent");
+var messageEvent = require("../../event/messageEvent");
+var authEvent = require("../../event/authEvent");
+var webhookEvent = require("../../event/webhookEvent");
+var kbEvent = require("../../event/kbEvent");
+var departmentEvent = require("../../event/departmentEvent");
+var botEvent = require("../../event/botEvent");
+var { track } = require("../../lib/analyticsClient");
+
+// ─── helpers ────────────────────────────────────────────────────────────────
+
+/**
+ * Convert a boolean user_available value to the analytics status enum.
+ * Returns null for any non-boolean input so callers can guard before emitting.
+ * The contract enum is: 'available' | 'unavailable' | 'busy'.
+ * tiledesk-server only sets available/unavailable; 'busy' is reserved.
+ */
+function availabilityLabel(boolVal) {
+  if (boolVal === true) return "available";
+  if (boolVal === false) return "unavailable";
+  return null;
+}
+
+/**
+ * Derive sender_type enum value from a raw sender string.
+ * Contract enum: 'user' | 'agent' | 'bot'
+ *
+ * Uses request.lead.lead_id (the visitor's ID) to distinguish a human visitor
+ * from a human agent — without this check every non-bot sender falls through
+ * to 'user', misclassifying agent messages.
+ */
+function senderType(sender, request) {
+  let bot, agent = undefined;
+  if(request.participantsBots)
+   bot = request.participantsBots.find(participant => participant.includes(sender));
+  if(request.participantsAgents)
+    agent = request.participantsAgents.find(participant => participant.includes(sender));
+
+  if(bot)
+    return "bot";
+  if(agent)
+    return "agent";
+
+  return "user";
+}
+
+/**
+ * Return the first participant that is not a bot (i.e. the visitor/user).
+ */
+function firstVisitorId(participants) {
+  var list = participants || [];
+  for (var i = 0; i < list.length; i++) {
+    if (!list[i].startsWith("bot_")) return list[i];
+  }
+  return null;
+}
+
+/**
+ * Extract a stable string ID from a department field (ObjectId, object, or string).
+ * Returns null when no department is set.
+ */
+function departmentId(dept) {
+  if (!dept) return null;
+  if (typeof dept === "string") return dept;
+  return (dept._id || dept.id || "").toString() || null;
+}
+
+/**
+ * Extract the human-readable name from a populated department object.
+ * Falls back to the ID string so callers always get a non-null value when
+ * a department exists.
+ */
+function departmentName(dept) {
+  if (!dept) return null;
+  if (typeof dept === "string") return dept; // bare string ID — use as-is
+  return dept.name || (dept._id && dept._id.toString()) || null;
+}
+
+/**
+ * Return the _id of a Mongoose document or plain object as a string.
+ */
+function toStringId(doc) {
+  if (!doc) return null;
+  if (typeof doc === "string") return doc;
+  return (doc._id || doc.id || doc).toString() || null;
+}
+
+// ─── listeners ──────────────────────────────────────────────────────────────
+
+function listen() {
+  // ── 1. conversation.created ────────────────────────────────────────────────
+  // Contract: packages/contracts/src/payloads/conversation-created.ts
+  //   id_request          string   (required)
+  //   request_id          string   (required)
+  //   department          string|null
+  //   channel             string
+  //   first_response_time number|null
+  //   tags                string[] (default [])
+  //   tag                 string|null (optional)
+  //   with_bot            boolean  (default false)
+  //   visitor_id          string|null (optional)
+
+  function trackConversation(request) {
+    if (request.preflight === true) return;
+
+    var dept = request.department;
+
+    track("conversation.created", request.id_project, {
+      id_request: request.request_id || toStringId(request),
+      request_id: request.request_id || toStringId(request),
+      department: departmentId(dept),
+      channel: (request.channel && request.channel.name) || "web",
+      first_response_time: null,
+      with_bot: request.hasBot || false,
+      visitor_id: firstVisitorId(request.participants),
+    });
+  }
+
+  requestEvent.on("request.update.preflight", function (request) {
+    trackConversation(request);
+  })
+
+  requestEvent.on("request.create", function (request) {
+    trackConversation(request);
+  });
+
+  // ── 2. conversation.closed ─────────────────────────────────────────────────
+  // Contract: packages/contracts/src/payloads/conversation-closed.ts
+  //   id_request           string   (required)
+  //   request_id           string   (required)
+  //   closed_by            string   (required, non-null)
+  //   close_reason         string|null
+  //   duration_seconds     number
+  //   waiting_time_seconds number|null
+  //   satisfaction_rating  number 1-5 | null
+  requestEvent.on("request.close", function (request) {
+    var createdAt = new Date(request.createdAt);
+    var closedAt = request.closed_at ? new Date(request.closed_at) : new Date();
+    var durationSeconds =
+      request.duration != null
+        ? Math.round(request.duration / 1000)
+        : Math.round((closedAt - createdAt) / 1000);
+
+    // Normalise rating: must be an integer 1–5 or null.
+    var rawRating = request.rating;
+    var satisfactionRating = null;
+    if (rawRating != null) {
+      var r = parseInt(rawRating, 10);
+      satisfactionRating = r >= 1 && r <= 5 ? r : null;
+    }
+
+    // TODO: da splittare rispetto al rating
+    track("conversation.closed", request.id_project, {
+      id_request: request.request_id || toStringId(request),
+      request_id: request.request_id || toStringId(request),
+      closed_by: request.closed_by || "system",
+      close_reason: null,
+      duration_seconds: durationSeconds,
+      waiting_time_seconds:
+        request.waiting_time != null
+          ? Math.round(request.waiting_time / 1000)
+          : null,
+      satisfaction_rating: satisfactionRating,
+    });
+  });
+
+  // ── 3. message.sent ────────────────────────────────────────────────────────
+  // Contract: packages/contracts/src/payloads/message-sent.ts
+  //   id_message    string   (required)
+  //   id_request    string   (required)
+  //   sender_id     string   (required, non-null)
+  //   sender_type   'user'|'agent'|'bot'
+  //   message_type  string   (required, non-null)
+  //   has_attachment boolean (default false)
+  //   language      string|null
+  messageEvent.on("message.create", function (messageJson) {
+    var sender = messageJson.sender;
+
+    if(sender === 'system') return; // skip system messages
+
+    // recipient is the room/request ID (chat21 convention)
+    var idRequest = messageJson.recipient || null;
+    if (!idRequest) return; // cannot emit without a conversation reference
+
+    track("message.sent", messageJson.id_project, {
+      id_message: (messageJson._id || messageJson.id || "").toString(),
+      id_request: idRequest,
+      sender_id: sender || "unknown", // required non-null — fallback to 'unknown'
+      sender_type: senderType(sender, messageJson.request),
+      message_type: messageJson.type || "text", // required non-null — fallback to 'text'
+      has_attachment: !!(messageJson.metadata && messageJson.metadata.src),
+      language: messageJson.language || null,
+    });
+  });
+
+  // ── 5. handover_to_human ──────────────────────────────────────────────────
+  // Contract: packages/contracts/src/payloads/handover-to-human.ts
+  //   id_request           string   (required)
+  //   human_id             string|null
+  //   reason               string|null
+  //   department_id        string|null
+  //   waiting_time_seconds number int>=0 | null
+  //   agent_id             string|null (optional)
+  //   trigger_intent       string|null (optional)
+  requestEvent.on("request.participants.update", function (data) {
+    var request = data.request || {};
+    var removedParticipants = data.removedParticipants || [];
+    var addedParticipants = data.addedParticipants || [];
+
+    var botRemoved = removedParticipants.some(function (p) {
+      return p.startsWith("bot_");
+    });
+    var humanAdded = addedParticipants.some(function (p) {
+      return !p.startsWith("bot_");
+    });
+    if (!botRemoved || !humanAdded) return;
+
+    var botId =
+      removedParticipants.find(function (p) {
+        return p.startsWith("bot_");
+      }) || null;
+    var humanId =
+      addedParticipants.find(function (p) {
+        return !p.startsWith("bot_");
+      }) || null;
+
+    var waitingTimeSecs = null;
+    if (request.waiting_time != null) {
+      waitingTimeSecs = Math.round(request.waiting_time / 1000);
+    }
+
+    track("handover_to_human", request.id_project, {
+      id_request: request.request_id || toStringId(request),
+      human_id: humanId,
+      reason: null,
+      department_id: departmentId(request.department),
+      waiting_time_seconds: waitingTimeSecs,
+      agent_id: null,
+      trigger_intent: null,
+    });
+  });
+
+
+  // ── 5. project_user.activated ─────────────────────────────────────────────
+  // Contract: packages/contracts/src/payloads/project-user-activated.ts
+  //   id_user    string   (required)
+  //   user_email string   (required, email format — skip if unavailable)
+  //   role       string   (required)
+  //   invited_by string|null
+  // TODO: check
+  authEvent.on("project_user.invite", function (event) {
+    console.log("project_user.invite", event);
+    var pu = event.savedProject_userPopulated || event.updatedPuserPopulated;
+    if (!pu) return;
+
+    var user = pu.id_user;
+    var email = (user && user.email) || null;
+
+    // user_email is required as a valid email string — skip rather than send
+    // a payload that will be rejected with 422 by the ingest sidecar.
+    if (!email) return;
+
+    var userId = pu.uuid_user || (user && toStringId(user)) || null;
+    if (!userId) return;
+
+    track("project_user.activated", pu.id_project, {
+      id_user: userId,
+      user_email: email,
+      role: pu.role || "agent",
+      invited_by: (event.req && event.req.user && event.req.user.id) || null,
+    });
+  });
+
+  // ── 7. human.status_changed ───────────────────────────────────────────────
+  // Contract: packages/contracts/src/payloads/human-status-changed.ts
+  //   human_id        string   (required)
+  //   previous_status 'available'|'unavailable'|'busy'
+  //   new_status      'available'|'unavailable'|'busy'
+  // TODO: check
+  authEvent.on("project_user.update.agent", function (event) {
+    console.log("project_user.update.agent", event);
+    var pu = event.updatedProject_userPopulated;
+    if (!pu) return;
+    if (pu.user_available === undefined) return; // not a status-change update
+
+    var prevBool = event.previousUserAvailable;
+    if (prevBool === pu.user_available) return; // no actual change
+
+    var prevStatus = availabilityLabel(prevBool);
+    var newStatus = availabilityLabel(pu.user_available);
+
+    // Both statuses must be valid enum members — skip if either resolves to null.
+    if (!prevStatus || !newStatus) return;
+
+    var humanId = toStringId(pu.id_user);
+    if (!humanId) return;
+
+    track("human.status_changed", pu.id_project, {
+      human_id: humanId,
+      previous_status: prevStatus,
+      new_status: newStatus,
+    });
+  });
+
+  // ── 7. department.assignment ──────────────────────────────────────────────
+  // Contract: packages/contracts/src/payloads/department-assignment.ts
+  //   id_request      string   (required)
+  //   department_id   string   (required, non-null)
+  //   department_name string   (required, non-null)
+  //   assigned_by     string|null
+  //   routing_type    string
+  requestEvent.on("request.department.update", function (requestComplete) {
+    console.log("request.department.update", requestComplete);
+    var dept = requestComplete.department;
+
+    var deptId = departmentId(dept);
+    var deptName = departmentName(dept) || deptId;
+
+    // Both department_id and department_name are required non-null in the contract.
+    if (!deptId) return;
+
+    track("department.assignment", requestComplete.id_project, {
+      id_request: requestComplete.request_id || toStringId(requestComplete),
+      department_id: deptId,
+      department_name: deptName,
+      assigned_by: requestComplete.updatedBy || null,
+      routing_type: "auto",
+    });
+  });
+
+  // ── 10. webhook.triggered ─────────────────────────────────────────────────
+  // Emitted by routes/webhook.js when a chatbot automation is triggered via
+  // the public webhook URL (production runs only — dev-mode runs are excluded).
+  //   webhook_id  string   (required)
+  //   agent_id    string   (required)
+  //   block_id    string   (required)
+  //   async       boolean
+  //   request_id  string|null  — synthetic automation-request-... identifier
+  webhookEvent.on("webhook.triggered", function ({ webhook, payload }) {
+    if (!webhook || !webhook.id_project) return;
+
+    track("webhook.triggered", webhook.id_project, {
+      webhook_id: webhook.webhook_id,
+      agent_id: webhook.chatbot_id,
+      block_id:   webhook.block_id,
+      async:      webhook.async,
+      request_id:
+        (payload && (payload.preloaded_request_id || payload.request_id)) ||
+        null,
+    });
+  });
+
+  // ── 11. kb.metadata_updated ───────────────────────────────────────────────
+  // Emitted by routes/kb.js on namespace create and update (rename).
+  // kb_id   = Namespace.id  (logical string key, NOT the ObjectId _id)
+  // kb_name = Namespace.name
+  // The consumer writes these into the kb_dimensions ReplacingMergeTree table
+  // so that dashboard queries always resolve the current KB name.
+  kbEvent.on("kb.namespace.create", function ({ savedNamespace, project_id }) {
+    if (!savedNamespace || !project_id) return;
+    track("kb.metadata_updated", project_id, {
+      kb_id:   savedNamespace.id,
+      kb_name: savedNamespace.name,
+    });
+  });
+
+  kbEvent.on("kb.namespace.update", function ({ updatedNamespace }) {
+    if (!updatedNamespace || !updatedNamespace.id_project) return;
+    track("kb.metadata_updated", updatedNamespace.id_project, {
+      kb_id:   updatedNamespace.id,
+      kb_name: updatedNamespace.name,
+    });
+  });
+
+  // ── 12. department.metadata_updated ──────────────────────────────────────
+  // Emitted by routes/department.js on department create, PUT, and PATCH.
+  // department_id   = Department._id.toString()
+  // department_name = Department.name
+  // The consumer writes these into the department_dimensions table so that
+  // dashboard queries always resolve the current department name.
+  departmentEvent.on("department.create", function (savedDepartment) {
+    if (!savedDepartment || !savedDepartment.id_project) return;
+    track("department.metadata_updated", savedDepartment.id_project, {
+      department_id:   savedDepartment._id.toString(),
+      department_name: savedDepartment.name,
+    });
+  });
+
+  departmentEvent.on("department.update", function (updatedDepartment) {
+    if (!updatedDepartment || !updatedDepartment.id_project) return;
+    track("department.metadata_updated", updatedDepartment.id_project, {
+      department_id:   updatedDepartment._id.toString(),
+      department_name: updatedDepartment.name,
+    });
+  });
+
+  // ── 13. agent.metadata_updated ────────────────────────────────────────────
+  // Emitted by routes/faq_kb.js on bot create and update (rename, attribute
+  // changes, language, etc.).
+  // agent_id   = Faq_kb._id.toString()
+  // agent_name = Faq_kb.name
+  // The consumer writes these into the agent_dimensions ReplacingMergeTree so
+  // that dashboard queries always resolve the current bot name.
+  botEvent.on("faqbot.create", function (savedBot) {
+    if (!savedBot || !savedBot.id_project || !savedBot.name) return;
+    track("agent.metadata_updated", savedBot.id_project, {
+      agent_id:   savedBot.root_id || savedBot._id.toString(),
+      agent_name: savedBot.name,
+    });
+  });
+
+  botEvent.on("faqbot.update", function (updatedBot) {
+    if (!updatedBot || !updatedBot.id_project || !updatedBot.name) return;
+    track("agent.metadata_updated", updatedBot.id_project, {
+      agent_id:   updatedBot.root_id || updatedBot._id.toString(),
+      agent_name: updatedBot.name,
+    });
+  });
+}
+
+module.exports = { listen: listen };

package/pubmodules/pubModulesManager.js

@@ -74,6 +74,8 @@ class PubModulesManager {
         this.cache = undefined;
 
         this.dialogFlow = undefined;
+
+        this.analyticsPublisher = undefined;
     }
 
   
@@ -595,6 +597,18 @@ class PubModulesManager {
                 winston.error("PubModulesManager error initializing init dialogFlow module", err);
             }
         }
+
+        try {
+            this.analyticsPublisher = require('./analytics-publisher');
+            this.analyticsPublisher.listen();
+            winston.info("PubModulesManager analyticsPublisher initialized.");
+        } catch(err) {
+            if (err.code == 'MODULE_NOT_FOUND') {
+                winston.info("PubModulesManager init analyticsPublisher module not found");
+            } else {
+                winston.info("PubModulesManager error initializing analyticsPublisher module", err);
+            }
+        }
     }
 
     start() {