@tiledesk/tiledesk-server 2.27.1 → 2.27.2

package/CHANGELOG.md

@@ -5,6 +5,10 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.27.2
+- Improved chat21 sync tracing: traces are buffered in memory and saved in batches (insertMany) instead of one insert per trace
+- Added env variables CHAT21_SYNC_TRACE_FLUSH_INTERVAL_MS (default 1000), CHAT21_SYNC_TRACE_BATCH_SIZE (default 200) and CHAT21_SYNC_TRACE_MAX_BUFFER_SIZE (default 10000)
+
 # 2.27.1
 - Added tracing for chat21 events
 

package/channels/chat21/chat21Handler.js

@@ -689,6 +689,26 @@ class Chat21Handler {
       });
     });
 
+    groupEvent.on('group.deleted', (request_id) => {
+
+      winston.debug("Chat21Handler on group.deleted for request_id: " + request_id);
+
+      setImmediate(() => {
+
+        chat21.auth.setAdminToken(adminToken);
+  
+        return chat21.groups.delete(request_id).then(function (data) {
+          winston.verbose("[Chat21Handler] - group.deleted - Chat21 group deleted: " + JSON.stringify(data));
+          chat21Event.emit('group.deleted', data);
+        }).catch(function (err) {
+          winston.error("[Chat21Handler] - group.deleted - Error deleting chat21 group: ", err);
+          chat21Event.emit('group.deleted.error', err);
+        });
+
+      })
+
+    })
+
   }
 
 }

package/channels/chat21/package-lock.json

@@ -1,24 +1,24 @@
 {
   "name": "@tiledesk/tiledesk-chat21-app",
-  "version": "1.1.7",
+  "version": "1.1.8",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@tiledesk/tiledesk-chat21-app",
-      "version": "1.1.7",
+      "version": "1.1.8",
       "license": "AGPL-3.0",
       "dependencies": {
-        "@chat21/chat21-node-sdk": "^1.1.8",
+        "@chat21/chat21-node-sdk": "^1.1.12",
         "firebase-admin": "^9.5.0",
         "winston": "^3.3.3"
       },
       "devDependencies": {}
     },
     "node_modules/@chat21/chat21-node-sdk": {
-      "version": "1.1.8",
-      "resolved": "https://registry.npmjs.org/@chat21/chat21-node-sdk/-/chat21-node-sdk-1.1.8.tgz",
-      "integrity": "sha512-V6kUMs/3yDUNESMbmHA8C+F0bWFpAqPB4YHvdmTrHDiZfjmb45hotYnIBi7E1YCBaQ4KcKDgSOV8KCS0fY44GA==",
+      "version": "1.1.12",
+      "resolved": "https://registry.npmjs.org/@chat21/chat21-node-sdk/-/chat21-node-sdk-1.1.12.tgz",
+      "integrity": "sha512-KllkQ4HckOAG/l6xIycCCNgM5Y/fDDfEFbK5eVCCMoBJsS4DGJoRB/Zh1Td792E/tdzvn/skTBpGA9+I/hXqGA==",
       "dependencies": {
         "chai": "^4.3.0",
         "dotenv": "^8.2.0",
@@ -199,15 +199,6 @@
         "@firebase/app": "0.x"
       }
     },
-    "node_modules/@firebase/auth-interop-types": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/@firebase/auth-interop-types/-/auth-interop-types-0.1.5.tgz",
-      "integrity": "sha512-88h74TMQ6wXChPA6h9Q3E1Jg6TkTHep2+k63OWg3s0ozyGVMeY+TTOti7PFPzq5RhszQPQOoCi59es4MaRvgCw==",
-      "peerDependencies": {
-        "@firebase/app-types": "0.x",
-        "@firebase/util": "0.x"
-      }
-    },
     "node_modules/@firebase/auth-types": {
       "version": "0.10.3",
       "resolved": "https://registry.npmjs.org/@firebase/auth-types/-/auth-types-0.10.3.tgz",
@@ -261,6 +252,23 @@
         "@firebase/app-types": "0.6.1"
       }
     },
+    "node_modules/@firebase/database/node_modules/@firebase/auth-interop-types": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmjs.org/@firebase/auth-interop-types/-/auth-interop-types-0.1.5.tgz",
+      "integrity": "sha512-88h74TMQ6wXChPA6h9Q3E1Jg6TkTHep2+k63OWg3s0ozyGVMeY+TTOti7PFPzq5RhszQPQOoCi59es4MaRvgCw==",
+      "peerDependencies": {
+        "@firebase/app-types": "0.x",
+        "@firebase/util": "0.x"
+      }
+    },
+    "node_modules/@firebase/database/node_modules/@firebase/util": {
+      "version": "0.3.4",
+      "resolved": "https://registry.npmjs.org/@firebase/util/-/util-0.3.4.tgz",
+      "integrity": "sha512-VwjJUE2Vgr2UMfH63ZtIX9Hd7x+6gayi6RUXaTqEYxSbf/JmehLmAEYSuxS/NckfzAXWeGnKclvnXVibDgpjQQ==",
+      "dependencies": {
+        "tslib": "^1.11.1"
+      }
+    },
     "node_modules/@firebase/firestore": {
       "version": "2.4.1",
       "resolved": "https://registry.npmjs.org/@firebase/firestore/-/firestore-2.4.1.tgz",
@@ -765,13 +773,24 @@
       "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ=="
     },
     "node_modules/@firebase/util": {
-      "version": "0.3.4",
-      "resolved": "https://registry.npmjs.org/@firebase/util/-/util-0.3.4.tgz",
-      "integrity": "sha512-VwjJUE2Vgr2UMfH63ZtIX9Hd7x+6gayi6RUXaTqEYxSbf/JmehLmAEYSuxS/NckfzAXWeGnKclvnXVibDgpjQQ==",
+      "version": "1.15.3",
+      "resolved": "https://registry.npmjs.org/@firebase/util/-/util-1.15.3.tgz",
+      "integrity": "sha512-c/z/gaIlaaLZEuGbE6sLUuJ61tskg1JghvhcNQzW948ASBinbVBBRnZTC4b4yt4LaEtJQkYlyzqLHcutFwEIvA==",
+      "hasInstallScript": true,
+      "peer": true,
       "dependencies": {
-        "tslib": "^1.11.1"
+        "tslib": "^2.1.0"
+      },
+      "engines": {
+        "node": ">=20.0.0"
       }
     },
+    "node_modules/@firebase/util/node_modules/tslib": {
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+      "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
+      "peer": true
+    },
     "node_modules/@firebase/webchannel-wrapper": {
       "version": "0.5.1",
       "resolved": "https://registry.npmjs.org/@firebase/webchannel-wrapper/-/webchannel-wrapper-0.5.1.tgz",
@@ -3645,9 +3664,9 @@
   },
   "dependencies": {
     "@chat21/chat21-node-sdk": {
-      "version": "1.1.8",
-      "resolved": "https://registry.npmjs.org/@chat21/chat21-node-sdk/-/chat21-node-sdk-1.1.8.tgz",
-      "integrity": "sha512-V6kUMs/3yDUNESMbmHA8C+F0bWFpAqPB4YHvdmTrHDiZfjmb45hotYnIBi7E1YCBaQ4KcKDgSOV8KCS0fY44GA==",
+      "version": "1.1.12",
+      "resolved": "https://registry.npmjs.org/@chat21/chat21-node-sdk/-/chat21-node-sdk-1.1.12.tgz",
+      "integrity": "sha512-KllkQ4HckOAG/l6xIycCCNgM5Y/fDDfEFbK5eVCCMoBJsS4DGJoRB/Zh1Td792E/tdzvn/skTBpGA9+I/hXqGA==",
       "requires": {
         "chai": "^4.3.0",
         "dotenv": "^8.2.0",
@@ -3825,12 +3844,6 @@
         "@firebase/auth-types": "0.10.3"
       }
     },
-    "@firebase/auth-interop-types": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/@firebase/auth-interop-types/-/auth-interop-types-0.1.5.tgz",
-      "integrity": "sha512-88h74TMQ6wXChPA6h9Q3E1Jg6TkTHep2+k63OWg3s0ozyGVMeY+TTOti7PFPzq5RhszQPQOoCi59es4MaRvgCw==",
-      "requires": {}
-    },
     "@firebase/auth-types": {
       "version": "0.10.3",
       "resolved": "https://registry.npmjs.org/@firebase/auth-types/-/auth-types-0.10.3.tgz",
@@ -3873,6 +3886,22 @@
         "@firebase/util": "0.3.4",
         "faye-websocket": "0.11.3",
         "tslib": "^1.11.1"
+      },
+      "dependencies": {
+        "@firebase/auth-interop-types": {
+          "version": "0.1.5",
+          "resolved": "https://registry.npmjs.org/@firebase/auth-interop-types/-/auth-interop-types-0.1.5.tgz",
+          "integrity": "sha512-88h74TMQ6wXChPA6h9Q3E1Jg6TkTHep2+k63OWg3s0ozyGVMeY+TTOti7PFPzq5RhszQPQOoCi59es4MaRvgCw==",
+          "requires": {}
+        },
+        "@firebase/util": {
+          "version": "0.3.4",
+          "resolved": "https://registry.npmjs.org/@firebase/util/-/util-0.3.4.tgz",
+          "integrity": "sha512-VwjJUE2Vgr2UMfH63ZtIX9Hd7x+6gayi6RUXaTqEYxSbf/JmehLmAEYSuxS/NckfzAXWeGnKclvnXVibDgpjQQ==",
+          "requires": {
+            "tslib": "^1.11.1"
+          }
+        }
       }
     },
     "@firebase/database-types": {
@@ -4309,11 +4338,20 @@
       "requires": {}
     },
     "@firebase/util": {
-      "version": "0.3.4",
-      "resolved": "https://registry.npmjs.org/@firebase/util/-/util-0.3.4.tgz",
-      "integrity": "sha512-VwjJUE2Vgr2UMfH63ZtIX9Hd7x+6gayi6RUXaTqEYxSbf/JmehLmAEYSuxS/NckfzAXWeGnKclvnXVibDgpjQQ==",
+      "version": "1.15.3",
+      "resolved": "https://registry.npmjs.org/@firebase/util/-/util-1.15.3.tgz",
+      "integrity": "sha512-c/z/gaIlaaLZEuGbE6sLUuJ61tskg1JghvhcNQzW948ASBinbVBBRnZTC4b4yt4LaEtJQkYlyzqLHcutFwEIvA==",
+      "peer": true,
       "requires": {
-        "tslib": "^1.11.1"
+        "tslib": "^2.1.0"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.8.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+          "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
+          "peer": true
+        }
       }
     },
     "@firebase/webchannel-wrapper": {

package/channels/chat21/package.json

@@ -11,7 +11,7 @@
     "url": "https://github.com/Tiledesk/tiledesk-server"
   },
   "dependencies": {
-    "@chat21/chat21-node-sdk": "^1.1.8",
+    "@chat21/chat21-node-sdk": "^1.1.12",
     "winston": "^3.3.3",
     "firebase-admin": "^9.5.0"
   },

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.27.1",
+  "version": "2.27.2",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",

package/services/chat21SyncTracer.js

@@ -6,12 +6,55 @@ const MAX_STRING_LENGTH = 2000;
 const enabled = process.env.CHAT21_SYNC_TRACE_ENABLED === "true";
 const host = os.hostname();
 
+function positiveNumber(value, defaultValue) {
+  const parsed = parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : defaultValue;
+}
+
+const FLUSH_INTERVAL_MS = positiveNumber(process.env.CHAT21_SYNC_TRACE_FLUSH_INTERVAL_MS, 1000);
+const BATCH_SIZE = positiveNumber(process.env.CHAT21_SYNC_TRACE_BATCH_SIZE, 200);
+const MAX_BUFFER_SIZE = positiveNumber(process.env.CHAT21_SYNC_TRACE_MAX_BUFFER_SIZE, 10000);
+
+let buffer = [];
+let flushing = false;
+let dropped = 0;
+
 if (enabled) {
-  winston.info("[Chat21SyncTracer] Chat21 sync tracing enabled");
+  winston.info("[Chat21SyncTracer] Chat21 sync tracing enabled. Flush interval (ms): " + FLUSH_INTERVAL_MS +
+    " batch size: " + BATCH_SIZE + " max buffer size: " + MAX_BUFFER_SIZE);
   // The TTL index must exist even when MONGOOSE_AUTOINDEX is false
   Chat21SyncTrace.createIndexes().catch((err) => {
     winston.error("[Chat21SyncTracer] Error creating indexes: ", err);
   });
+  setInterval(flush, FLUSH_INTERVAL_MS).unref();
+}
+
+/**
+ * Writes the buffered traces in batches. Only one flush runs at a time: while mongo is slow
+ * the buffer grows up to MAX_BUFFER_SIZE, then new traces are dropped.
+ * @returns {Promise<void>} never rejects
+ */
+async function flush() {
+  if (flushing || buffer.length === 0) {
+    return;
+  }
+  flushing = true;
+  try {
+    if (dropped > 0) {
+      winston.warn("[Chat21SyncTracer] Buffer full, dropped " + dropped + " traces");
+      dropped = 0;
+    }
+    while (buffer.length > 0) {
+      const batch = buffer.splice(0, BATCH_SIZE);
+      try {
+        await Chat21SyncTrace.insertMany(batch, { ordered: false });
+      } catch (err) {
+        winston.error("[Chat21SyncTracer] Error saving " + batch.length + " traces: ", err);
+      }
+    }
+  } finally {
+    flushing = false;
+  }
 }
 
 function truncate(value) {
@@ -56,7 +99,7 @@ function describeRequest(request) {
 }
 
 /**
- * Saves a trace document. Never throws and never blocks the caller.
+ * Buffers a trace document, saved by the next flush. Never throws and never blocks the caller.
  * @param {string} event the traced flow, e.g. 'request.close'
  * @param {{request_id: string, id_project: *}} ref
  * @param {string} step
@@ -67,21 +110,24 @@ function trace(event, ref, step, outcome, data) {
   if (!enabled) {
     return;
   }
-  try {
-    Chat21SyncTrace.create({
-      request_id: ref?.request_id,
-      id_project: ref?.id_project ? String(ref.id_project) : undefined,
-      event: event,
-      step: step,
-      outcome: outcome,
-      data: data,
-      host: host,
-      pid: process.pid
-    }).catch((err) => {
-      winston.error("[Chat21SyncTracer] Error saving trace " + event + " " + step + ": ", err);
-    });
-  } catch (err) {
-    winston.error("[Chat21SyncTracer] Error creating trace " + event + " " + step + ": ", err);
+  if (buffer.length >= MAX_BUFFER_SIZE) {
+    dropped++;
+    return;
+  }
+  // createdAt is set here: the document is written later, and the timeline order must reflect the event time
+  buffer.push({
+    request_id: ref?.request_id,
+    id_project: ref?.id_project ? String(ref.id_project) : undefined,
+    event: event,
+    step: step,
+    outcome: outcome,
+    data: data,
+    host: host,
+    pid: process.pid,
+    createdAt: new Date()
+  });
+  if (buffer.length >= BATCH_SIZE) {
+    flush();
   }
 }
 
@@ -104,6 +150,7 @@ module.exports = {
   enabled,
   trace,
   traceCall,
+  flush,
   describeError,
   describeResponse,
   describeRequest