@tiledesk/tiledesk-server 2.26.0 → 2.27.1

package/pubmodules/queue/queueTopicsFilter.js

@@ -0,0 +1,36 @@
+
+function parseList(value) {
+  if (!value) {
+    return [];
+  }
+  return value.split(",").map(function (t) { return t.trim(); }).filter(function (t) { return t.length > 0; });
+}
+
+/**
+ * Selects the routing keys the worker binds to, based on INCLUDE_QUEUE / EXCLUDE_QUEUE
+ * (comma separated routing keys, mutually exclusive).
+ * Returns { enabled, disabled, unknown, filtered }.
+ */
+function resolveTopics(allTopics, includeValue, excludeValue) {
+  const include = parseList(includeValue);
+  const exclude = parseList(excludeValue);
+
+  if (include.length > 0 && exclude.length > 0) {
+    throw new Error("INCLUDE_QUEUE and EXCLUDE_QUEUE are mutually exclusive. Set only one of them.");
+  }
+
+  const selection = include.length > 0 ? include : exclude;
+  const unknown = selection.filter(function (t) { return allTopics.indexOf(t) === -1; });
+
+  let enabled;
+  if (include.length > 0) {
+    enabled = allTopics.filter(function (t) { return include.indexOf(t) !== -1; });
+  } else {
+    enabled = allTopics.filter(function (t) { return exclude.indexOf(t) === -1; });
+  }
+  const disabled = allTopics.filter(function (t) { return enabled.indexOf(t) === -1; });
+
+  return { enabled: enabled, disabled: disabled, unknown: unknown, filtered: selection.length > 0 };
+}
+
+module.exports = { parseList: parseList, resolveTopics: resolveTopics };

package/pubmodules/queue/reconnect.js

@@ -8,6 +8,8 @@ const botEvent = require('../../event/botEvent');
 const authEvent = require('../../event/authEvent');
 const kbEvent = require('../../event/kbEvent');
 const projectUserUpdateContextUtil = require('../../utils/projectUserUpdateContextUtil');
+const queueTopicsFilter = require('./queueTopicsFilter');
+const chat21WebhookWorker = require('./chat21WebhookWorker');
 // https://elements.heroku.com/addons/cloudamqp
 // https://gist.github.com/carlhoerberg/006b01ac17a0a94859ba#file-reconnect-js
 // http://www.rabbitmq.com/tutorials/tutorial-one-javascript.html
@@ -36,6 +38,63 @@ var exchange = process.env.QUEUE_EXCHANGE_TOPIC || 'amq.topic';
 var queueName = process.env.QUEUE_NAME || 'jobs';
 winston.info("Durable queue: " + durable + " Persistent queue: " + persistent + " Exchange topic: " + exchange+ " Queue name: " + queueName);
 
+// Routing keys bound to the jobs queue (QUEUE_NAME) on the tiledesk exchange.
+const JOBS_TOPICS = [
+  "request_create",
+  "request_update_preflight",
+  "request_participants_update",
+  "request_participants_join",
+  "request_participants_leave",
+  "request_assigned",
+  "request_update",
+  "request_close",
+  "request_close_extended",
+  "request_workingStatus_update",
+  "message_create",
+  "project_user_update",
+  "project_user_invite",
+  "project_user_invite_pending",
+  "project_user_delete",
+  "faqbot_update",
+  "faqbot_created",
+  "faqbot_deleted",
+  "faqbot_publish",
+  "kb_namespace_create",
+  "kb_namespace_delete",
+  "kb_contents_add",
+  "kb_contents_delete",
+  "kb_content_delete",
+  "lead_create",
+  "lead_update",
+  "lead_fullname_email_update",
+  "request_snapshot_update"
+];
+// Consumed from its own queue (CHAT21_WEBHOOK_QUEUE_NAME), see chat21WebhookWorker.js.
+const CHAT21_WEBHOOK_TOPIC = chat21WebhookWorker.CHAT21_WEBHOOK_ROUTING_KEY;
+const ALL_WORKER_TOPICS = JOBS_TOPICS.concat([CHAT21_WEBHOOK_TOPIC]);
+
+var topicsSelection = { enabled: ALL_WORKER_TOPICS, disabled: [], unknown: [], filtered: false };
+var enabledJobsTopics = JOBS_TOPICS;
+var disabledJobsTopics = [];
+var chat21WebhookEnabled = true;
+
+// Applies INCLUDE_QUEUE / EXCLUDE_QUEUE so that dedicated worker replicas can be deployed.
+// Replicas with different filters on the jobs topics must use different QUEUE_NAME values:
+// bindings belong to the queue, so replicas sharing a queue share its bindings.
+function resolveWorkerTopics() {
+  topicsSelection = queueTopicsFilter.resolveTopics(ALL_WORKER_TOPICS, process.env.INCLUDE_QUEUE, process.env.EXCLUDE_QUEUE);
+  if (topicsSelection.unknown.length > 0) {
+    winston.warn("INCLUDE_QUEUE/EXCLUDE_QUEUE contain unknown routing keys: " + topicsSelection.unknown.join(","));
+  }
+  enabledJobsTopics = JOBS_TOPICS.filter(function(t) { return topicsSelection.enabled.indexOf(t) !== -1; });
+  disabledJobsTopics = JOBS_TOPICS.filter(function(t) { return topicsSelection.enabled.indexOf(t) === -1; });
+  chat21WebhookEnabled = topicsSelection.enabled.indexOf(CHAT21_WEBHOOK_TOPIC) !== -1;
+  winston.info("Worker topics enabled: " + topicsSelection.enabled.join(","));
+  if (topicsSelection.disabled.length > 0) {
+    winston.info("Worker topics disabled: " + topicsSelection.disabled.join(","));
+  }
+}
+
 
 function start() {
   amqp.connect(url, function(err, conn) {
@@ -122,6 +181,13 @@ function publish(exchange, routingKey, content) {
 // A worker that acks messages only if processed succesfully
 // var channel;
 function startWorker() {
+    if (chat21WebhookEnabled) {
+      chat21WebhookWorker.start(amqpConn, closeOnErr);
+    }
+    if (enabledJobsTopics.length === 0) {
+      winston.info("Queue " + queueName + " not consumed: no jobs topic enabled by INCLUDE_QUEUE/EXCLUDE_QUEUE");
+      return;
+    }
     amqpConn.createChannel(function(err, ch) {
       if (closeOnErr(err)) return;
       ch.on("error", function(err) {
@@ -139,143 +205,21 @@ function startWorker() {
       ch.assertQueue(queueName, { durable: durable }, function(err, _ok) {
       // ch.assertQueue("jobs", { durable: true }, function(err, _ok) {
         if (closeOnErr(err)) return;
-        ch.bindQueue(_ok.queue, exchange, "request_create", {}, function(err3, oka) {
-            winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_create");
-            winston.info("Data queue", oka)
-        });
-        ch.bindQueue(_ok.queue, exchange, "request_update_preflight", {}, function(err3, oka) {
-            winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_update_preflight");
+        enabledJobsTopics.forEach(function(topic) {
+          ch.bindQueue(_ok.queue, exchange, topic, {}, function(err3, oka) {
+            winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: "+topic);
             winston.info("Data queue", oka)
-        });
-        ch.bindQueue(_ok.queue, exchange, "request_participants_update", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_participants_update");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "request_participants_join", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_participants_join");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "request_participants_leave", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_participants_leave");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "request_assigned", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_assigned");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "request_update", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_update");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "request_close", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_close");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "request_close_extended", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_close_extended");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "request_workingStatus_update", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_workingStatus_update");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "message_create", {}, function(err3, oka) {
-              winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: message_create");
-              winston.info("Data queue", oka)
-        });
-        ch.bindQueue(_ok.queue, exchange, "project_user_update", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: project_user_update");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "project_user_invite", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: project_user_invite");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "project_user_invite_pending", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: project_user_invite_pending");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "project_user_delete", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: project_user_delete");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "faqbot_update", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: faqbot_update");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "faqbot_created", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: faqbot_created");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "faqbot_deleted", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: faqbot_deleted");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "faqbot_publish", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: faqbot_publish");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "kb_namespace_create", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: kb_namespace_create");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "kb_namespace_delete", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: kb_namespace_delete");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "kb_contents_add", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: kb_contents_add");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "kb_contents_delete", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: kb_contents_delete");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "kb_content_delete", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: kb_content_delete");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "lead_create", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: lead_create");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "lead_update", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: lead_update");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "lead_fullname_email_update", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: lead_fullname_email_update");
-          winston.info("Data queue", oka)
-        });
-
-        ch.bindQueue(_ok.queue, exchange, "request_snapshot_update", {}, function(err3, oka) {
-          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_snapshot_update");
-          winston.info("Data queue", oka)
+          });
         });
 
+        if (topicsSelection.filtered) {
+          // Remove bindings left on the queue by a previous configuration.
+          disabledJobsTopics.forEach(function(topic) {
+            ch.unbindQueue(_ok.queue, exchange, topic, {}, function(err3) {
+              winston.info("Queue unbind: "+_ok.queue+ " err: "+err3+ " key: "+topic);
+            });
+          });
+        }
 
         ch.consume(queueName, processMsg, { noAck: false });
         winston.info("Worker is started");
@@ -804,6 +748,12 @@ function listen() {
 }
 
 if (process.env.QUEUE_ENABLED === "true") {
+    try {
+      resolveWorkerTopics();
+    } catch (e) {
+      winston.error("Invalid worker queue configuration: " + e.message);
+      process.exit(1);
+    }
     requestEvent.queueEnabled = true;
     messageEvent.queueEnabled = true;
     authEvent.queueEnabled = true;

package/pubmodules/queue/test/queueWorker.test.js

@@ -0,0 +1,131 @@
+// During the test the env variable is set to test
+process.env.NODE_ENV = 'test';
+
+const { expect } = require('chai');
+const queueTopicsFilter = require('../queueTopicsFilter');
+const chat21WebhookWorker = require('../chat21WebhookWorker');
+
+const TOPICS = ['request_create', 'message_create', 'chat21.webhook.event'];
+
+function makeMsg(body, redelivered) {
+  return {
+    content: Buffer.from(typeof body === 'string' ? body : JSON.stringify(body)),
+    fields: { redelivered: !!redelivered }
+  };
+}
+
+function httpError(status) {
+  const err = new Error('Request failed with status code ' + status);
+  err.response = { status: status };
+  return err;
+}
+
+describe('queue worker', function () {
+
+  describe('INCLUDE_QUEUE / EXCLUDE_QUEUE', function () {
+    it('enables every topic when no filter is set', function () {
+      const res = queueTopicsFilter.resolveTopics(TOPICS, undefined, '');
+      expect(res.enabled).to.deep.equal(TOPICS);
+      expect(res.disabled).to.deep.equal([]);
+      expect(res.filtered).to.equal(false);
+    });
+
+    it('INCLUDE_QUEUE enables only the listed topics', function () {
+      const res = queueTopicsFilter.resolveTopics(TOPICS, ' chat21.webhook.event ', undefined);
+      expect(res.enabled).to.deep.equal(['chat21.webhook.event']);
+      expect(res.disabled).to.deep.equal(['request_create', 'message_create']);
+      expect(res.filtered).to.equal(true);
+    });
+
+    it('EXCLUDE_QUEUE disables the listed topics', function () {
+      const res = queueTopicsFilter.resolveTopics(TOPICS, undefined, 'chat21.webhook.event,request_create');
+      expect(res.enabled).to.deep.equal(['message_create']);
+      expect(res.disabled).to.deep.equal(['request_create', 'chat21.webhook.event']);
+    });
+
+    it('reports unknown routing keys', function () {
+      const res = queueTopicsFilter.resolveTopics(TOPICS, 'message_create,not_a_topic', undefined);
+      expect(res.enabled).to.deep.equal(['message_create']);
+      expect(res.unknown).to.deep.equal(['not_a_topic']);
+    });
+
+    it('rejects INCLUDE_QUEUE and EXCLUDE_QUEUE set together', function () {
+      expect(function () {
+        queueTopicsFilter.resolveTopics(TOPICS, 'message_create', 'request_create');
+      }).to.throw(/mutually exclusive/);
+    });
+  });
+
+  describe('chat21 webhook event handling', function () {
+    const event = { event_type: 'message-sent', app_id: 'tilechat', data: { text: 'hello' } };
+
+    it('posts the event body to the chat21 webhook API and acks', async function () {
+      const calls = [];
+      const outcome = await chat21WebhookWorker.handleMessage(makeMsg(event), {
+        apiUrl: 'http://tiledesk/chat21/requests',
+        post: function (url, body) {
+          calls.push({ url: url, body: body });
+          return Promise.resolve({ status: 200 });
+        }
+      });
+      expect(outcome).to.equal('ack');
+      expect(calls).to.deep.equal([{ url: 'http://tiledesk/chat21/requests', body: event }]);
+    });
+
+    it('acks events rejected with a 4xx (e.g. not a support message)', async function () {
+      const outcome = await chat21WebhookWorker.handleMessage(makeMsg(event), {
+        post: function () { return Promise.reject(httpError(400)); }
+      });
+      expect(outcome).to.equal('ack');
+    });
+
+    it('requeues on 5xx the first time and discards on redelivery', async function () {
+      const post = function () { return Promise.reject(httpError(500)); };
+      expect(await chat21WebhookWorker.handleMessage(makeMsg(event, false), { post: post })).to.equal('requeue');
+      expect(await chat21WebhookWorker.handleMessage(makeMsg(event, true), { post: post })).to.equal('ack');
+    });
+
+    it('requeues when the API is unreachable', async function () {
+      const outcome = await chat21WebhookWorker.handleMessage(makeMsg(event, true), {
+        post: function () { return Promise.reject(new Error('connect ECONNREFUSED')); }
+      });
+      expect(outcome).to.equal('requeue');
+    });
+
+    it('discards malformed messages', async function () {
+      let called = false;
+      const outcome = await chat21WebhookWorker.handleMessage(makeMsg('not json'), {
+        post: function () { called = true; return Promise.resolve({ status: 200 }); }
+      });
+      expect(outcome).to.equal('ack');
+      expect(called).to.equal(false);
+    });
+  });
+
+  describe('rate limiter', function () {
+    it('runs immediately when unlimited', function () {
+      let runs = 0;
+      const limiter = chat21WebhookWorker.createRateLimiter(0);
+      for (let i = 0; i < 5; i++) limiter.schedule(function () { runs++; });
+      expect(runs).to.equal(5);
+    });
+
+    it('spreads executions according to the rate', function (done) {
+      const starts = [];
+      const limiter = chat21WebhookWorker.createRateLimiter(20); // one every 50ms
+      const begin = Date.now();
+      for (let i = 0; i < 5; i++) {
+        limiter.schedule(function () {
+          starts.push(Date.now() - begin);
+          if (starts.length === 5) {
+            expect(starts[4]).to.be.at.least(190);
+            for (let j = 1; j < starts.length; j++) {
+              expect(starts[j] - starts[j - 1]).to.be.at.least(40);
+            }
+            done();
+          }
+        });
+      }
+    });
+  });
+});

package/services/chat21SyncTracer.js

@@ -0,0 +1,110 @@
+const os = require('os');
+const winston = require('../config/winston');
+const Chat21SyncTrace = require('../models/chat21SyncTrace');
+
+const MAX_STRING_LENGTH = 2000;
+const enabled = process.env.CHAT21_SYNC_TRACE_ENABLED === "true";
+const host = os.hostname();
+
+if (enabled) {
+  winston.info("[Chat21SyncTracer] Chat21 sync tracing enabled");
+  // The TTL index must exist even when MONGOOSE_AUTOINDEX is false
+  Chat21SyncTrace.createIndexes().catch((err) => {
+    winston.error("[Chat21SyncTracer] Error creating indexes: ", err);
+  });
+}
+
+function truncate(value) {
+  if (typeof value !== 'string' || value.length <= MAX_STRING_LENGTH) {
+    return value;
+  }
+  return value.substring(0, MAX_STRING_LENGTH) + "...[truncated]";
+}
+
+function describeError(err) {
+  if (!err) {
+    return err;
+  }
+  if (err instanceof Error) {
+    return { message: err.message, code: err.code, stack: truncate(err.stack) };
+  }
+  return describeResponse(err);
+}
+
+// Serialized as a string: chat21 responses may contain keys that MongoDB does not accept (e.g. "$" or ".")
+function describeResponse(body) {
+  if (body === undefined) {
+    return "undefined";
+  }
+  try {
+    return truncate(typeof body === 'string' ? body : JSON.stringify(body));
+  } catch (err) {
+    return truncate(String(body));
+  }
+}
+
+function describeRequest(request) {
+  if (!request) {
+    return request;
+  }
+  return {
+    status: request.status,
+    participants: Array.isArray(request.participants) ? Array.from(request.participants) : request.participants,
+    channelOutbound: request.channelOutbound?.name,
+    lead_id: request.lead?.lead_id
+  };
+}
+
+/**
+ * Saves a trace document. Never throws and never blocks the caller.
+ * @param {string} event the traced flow, e.g. 'request.close'
+ * @param {{request_id: string, id_project: *}} ref
+ * @param {string} step
+ * @param {'info'|'success'|'error'|'skipped'} outcome
+ * @param {object} [data]
+ */
+function trace(event, ref, step, outcome, data) {
+  if (!enabled) {
+    return;
+  }
+  try {
+    Chat21SyncTrace.create({
+      request_id: ref?.request_id,
+      id_project: ref?.id_project ? String(ref.id_project) : undefined,
+      event: event,
+      step: step,
+      outcome: outcome,
+      data: data,
+      host: host,
+      pid: process.pid
+    }).catch((err) => {
+      winston.error("[Chat21SyncTracer] Error saving trace " + event + " " + step + ": ", err);
+    });
+  } catch (err) {
+    winston.error("[Chat21SyncTracer] Error creating trace " + event + " " + step + ": ", err);
+  }
+}
+
+/**
+ * Traces a chat21 sdk call as <action>.sent, then <action>.response or <action>.error.
+ * The returned promise settles exactly like the one returned by callFn.
+ */
+function traceCall(event, ref, action, data, callFn) {
+  trace(event, ref, action + '.sent', 'info', data);
+  return callFn().then((response) => {
+    trace(event, ref, action + '.response', 'success', { ...data, response: describeResponse(response) });
+    return response;
+  }, (err) => {
+    trace(event, ref, action + '.error', 'error', { ...data, error: describeError(err) });
+    throw err;
+  });
+}
+
+module.exports = {
+  enabled,
+  trace,
+  traceCall,
+  describeError,
+  describeResponse,
+  describeRequest
+};