mail-time 3.0.0 → 4.0.0

package/adapters/redis.js

@@ -1,16 +1,100 @@
-import { logError } from '../helpers.js';
+import {
+  debug,
+  logError,
+  isSendClaimUpdate,
+  isSendLeaseGuardedUpdate,
+  isAppendMailOptionUpdate,
+  stripInternalUpdateMeta,
+  isSendLeaseRemove,
+} from '../helpers.js';
+
+/**
+ * @typedef {object} RedisClient
+ * @property {(key: string) => Promise<number>} exists
+ * @property {(key: string) => Promise<string|null>} get
+ * @property {(key: string, value: string, options?: object) => Promise<unknown>} set
+ * @property {(key: string|string[]) => Promise<number>} del
+ * @property {() => Promise<string>} ping
+ * @property {(options: object) => AsyncIterable<string|string[]>} scanIterator
+ * @property {(key: string) => Promise<unknown>} [watch]
+ * @property {() => Promise<unknown>} [unwatch]
+ * @property {() => object} [multi]
+ */
+
+/**
+ * @typedef {object} RedisQueueOption
+ * @property {RedisClient} client
+ * @property {string} [prefix]
+ */
+
+const KEY_TYPES = new Set(['letter', 'sendat', 'concatletter']);
+const DEFAULT_PREFIX = 'default';
+
+const canReleaseLease = (currentTask, updateObj) => {
+  return currentTask
+    && currentTask.tries === updateObj.leaseTries
+    && currentTask.isSending === true
+    && (typeof currentTask.sendingAt === 'number' ? currentTask.sendingAt : 0) === updateObj.leaseSendingAt
+    && currentTask.isCancelled !== true
+    && currentTask.isFailed !== true;
+};
+
+const canClaimTask = (currentTask, task, now, sendingTimeout) => {
+  if (!currentTask) {
+    return false;
+  }
+  if (currentTask.isSent === true || currentTask.isFailed === true || currentTask.isCancelled === true) {
+    return false;
+  }
+  if (currentTask.tries !== task.tries) {
+    return false;
+  }
+  if (currentTask.isSending === true) {
+    const sendingAt = typeof currentTask.sendingAt === 'number' ? currentTask.sendingAt : 0;
+    if (sendingAt > now - sendingTimeout) {
+      return false;
+    }
+  }
+  return true;
+};
+
+const isIterateCandidate = (candidate, now, sendingTimeout, maxTries) => {
+  if (!candidate || typeof candidate !== 'object') {
+    return false;
+  }
+  if (candidate.isSent === true || candidate.isFailed === true || candidate.isCancelled === true) {
+    return false;
+  }
+  const tries = typeof candidate.tries === 'number' ? candidate.tries : 0;
+  if (tries >= maxTries) {
+    return false;
+  }
+  if (candidate.isSending === true) {
+    const sendingAt = typeof candidate.sendingAt === 'number' ? candidate.sendingAt : 0;
+    if (sendingAt > now - sendingTimeout) {
+      return false;
+    }
+  }
+  return true;
+};
+
+const parseUuidFromKey = (key, uniqueName) => {
+  const prefix = `${uniqueName}:sendat:`;
+  if (!key.startsWith(prefix)) {
+    return null;
+  }
+  return key.slice(prefix.length);
+};
 
 /** Class representing Redis Queue for MailTime */
 class RedisQueue {
   /**
    * Create a RedisQueue instance
-   * @param {object} opts - configuration object
-   * @param {RedisClient} opts.client - Required, Redis'es `RedisClient` instance, like one returned from `await redis.createClient().connect()` method
-   * @param {string} [opts.prefix] - Optional prefix for scope isolation; use when creating multiple MailTime instances within the single application
+   * @param {RedisQueueOption} opts - configuration object
    */
   constructor (opts) {
     this.name = 'redis-queue';
-    if (!opts || typeof opts !== 'object' || opts === null) {
+    if (!opts || typeof opts !== 'object') {
       throw new TypeError('[mail-time] Configuration object must be passed into RedisQueue constructor');
     }
 
@@ -18,34 +102,41 @@ class RedisQueue {
       throw new Error('[mail-time] [RedisQueue] required {client} option is missing, e.g. returned from `redis.createClient()` or `redis.createCluster()` method');
     }
 
-    this.prefix = (typeof opts.prefix === 'string') ? opts.prefix : 'default';
-    this.uniqueName = `mailtime:${this.prefix}`;
     this.client = opts.client;
+    if (typeof opts.prefix === 'string') {
+      this.__applyPrefix(opts.prefix);
+    }
+  }
+
+  /** @internal */
+  __applyPrefix(prefix) {
+    this.prefix = prefix;
+    this.uniqueName = `mailtime:${prefix}`;
+  }
+
+  /** @internal */
+  __ensurePrefix() {
+    if (typeof this.prefix !== 'string') {
+      this.__applyPrefix(this.mailTimeInstance?.prefix || DEFAULT_PREFIX);
+    }
+  }
 
-    // 3 types of keys are stored in Redis:
-    // 'letter' - JSON with email details
-    // 'sendat' - Timestamp used to scan/find/iterate over scheduled emails
-    // 'concatletter' — uuid of "pending" email for concatenation used when {concatEmails: true}
-
-    // Stored JSON structure:
-    // to          {string|[string]}
-    // uuid        {string}
-    // tries       {number}  - qty of send attempts
-    // sendAt      {number}  - When letter should be sent
-    // isSent      {boolean} - Email status
-    // isCancelled {boolean} - `true` if email was cancelled before it was sent
-    // isFailed    {boolean} - `true` if email has failed to send
-    // template    {string}  - Template for this email
-    // transport   {number}  - Last used transport
-    // concatSubject {string|boolean} - Email concatenation subject
-    // ---
-    // mailOptions         {[object]}  - Array of nodeMailer's `mailOptions`
-    // mailOptions.to      {string|[string]} - [REQUIRED]
-    // mailOptions.from    {string}
-    // mailOptions.text    {string|boolean}
-    // mailOptions.html    {string}
-    // mailOptions.subject {string}
-    // mailOptions.Other nodeMailer `sendMail` options...
+  /** @internal */
+  __debug(...args) {
+    debug(this.mailTimeInstance?.debug === true, `[${this.name}]`, ...args);
+  }
+
+  /**
+   * @async
+   * @memberOf RedisQueue
+   * @name ready
+   * @description Storage adapter has no async setup
+   * @returns {Promise<void 0>}
+   */
+  async ready() {
+    this.__ensurePrefix();
+    this.__debug('[ready]');
+    return void 0;
   }
 
   /**
@@ -56,6 +147,7 @@ class RedisQueue {
    * @returns {Promise<object>}
    */
   async ping() {
+    this.__debug('[ping]');
     if (!this.mailTimeInstance) {
       return {
         status: 'Service Unavailable',
@@ -79,7 +171,7 @@ class RedisQueue {
         status: 'Internal Server Error',
         code: 500,
         statusCode: 500,
-        error: pingError
+        error: pingError,
       };
     }
 
@@ -87,28 +179,64 @@ class RedisQueue {
       status: 'Service Unavailable',
       code: 503,
       statusCode: 503,
-      error: new Error('Service Unavailable')
+      error: new Error('Service Unavailable'),
     };
   }
 
   /**
    * @memberOf RedisQueue
    * @name iterate
-   * @description iterate over queued tasks passing to `mailTimeInstance.___send` method
-   * @returns {void 0}
+   * @description iterate over queued tasks passing each to `mailTimeInstance.___dispatch` (the bounded send pool)
+   * @param {{ limit?: number, sendingTimeout?: number }} [opts] - iteration options
+   * @returns {Promise<void>}
    */
-  async iterate() {
+  async iterate(opts) {
+    this.__debug('[iterate]', opts);
     try {
       const now = Date.now();
+      const sendingTimeout = (opts && typeof opts.sendingTimeout === 'number' && opts.sendingTimeout > 0)
+        ? opts.sendingTimeout
+        : 300000;
+      const limit = (opts && typeof opts.limit === 'number' && Number.isFinite(opts.limit) && opts.limit > 0)
+        ? Math.floor(opts.limit)
+        : 0;
+      const maxTries = (this.mailTimeInstance && typeof this.mailTimeInstance.maxTries === 'number')
+        ? this.mailTimeInstance.maxTries
+        : 60;
+      let dispatched = 0;
+
+      const matchPattern = this.__getKey('*', 'sendat');
       const cursor = this.client.scanIterator({
         TYPE: 'string',
-        MATCH: this.__getKey('*', 'sendat'),
+        MATCH: matchPattern,
         COUNT: 9999,
       });
 
-      for await (const sendatKey of cursor) {
-        if (parseInt(await this.client.get(sendatKey)) <= now) {
-          await this.mailTimeInstance.___send(JSON.parse(await this.client.get(this.__getKey(sendatKey.split(':')[3]))));
+      outer:
+      for await (const cursorValue of cursor) {
+        const sendatKeys = Array.isArray(cursorValue) ? cursorValue : [cursorValue];
+        for (const sendatKey of sendatKeys) {
+          const raw = await this.client.get(sendatKey);
+          if (raw === null || parseInt(raw, 10) > now) {
+            continue;
+          }
+          const uuid = parseUuidFromKey(sendatKey, this.uniqueName);
+          if (!uuid) {
+            continue;
+          }
+          const taskJSON = await this.client.get(this.__getKey(uuid));
+          if (!taskJSON) {
+            continue;
+          }
+          const candidate = JSON.parse(taskJSON);
+          if (!isIterateCandidate(candidate, now, sendingTimeout, maxTries)) {
+            continue;
+          }
+          await this.mailTimeInstance.___dispatch(candidate);
+          dispatched++;
+          if (limit > 0 && dispatched >= limit) {
+            break outer;
+          }
         }
       }
     } catch (iterateError) {
@@ -126,25 +254,25 @@ class RedisQueue {
    * @returns {Promise<object|null>}
    */
   async getPendingTo(to, sendAt) {
+    this.__debug('[getPendingTo]', to, sendAt);
     if (typeof to !== 'string' || typeof sendAt !== 'number') {
       return null;
     }
 
     const concatKey = this.__getKey(to, 'concatletter');
-    let exists = await this.client.exists(concatKey);
-    if (!exists) {
+    const uuid = await this.client.get(concatKey);
+    if (!uuid) {
       return null;
     }
 
-    const uuid = await this.client.get(concatKey);
     const letterKey = this.__getKey(uuid, 'letter');
-    exists = await this.client.exists(letterKey);
-    if (!exists) {
+    const taskJSON = await this.client.get(letterKey);
+    if (!taskJSON) {
       return null;
     }
 
-    const task = JSON.parse(await this.client.get(letterKey));
-    if (!task || task.isSent === true || task.isCancelled === true || task.isFailed === true) {
+    const task = JSON.parse(taskJSON);
+    if (!task || task.isSent === true || task.isCancelled === true || task.isFailed === true || task.sendAt > sendAt || task.isSending === true || task.tries >= this.mailTimeInstance.maxTries) {
       return null;
     }
 
@@ -160,7 +288,8 @@ class RedisQueue {
    * @returns {Promise<void 0>}
    */
   async push(task) {
-    if (typeof task !== 'object') {
+    this.__debug('[push]', task?.uuid);
+    if (!task || typeof task !== 'object') {
       return;
     }
 
@@ -168,11 +297,28 @@ class RedisQueue {
       task.sendAt = +task.sendAt;
     }
 
-    await this.client.set(this.__getKey(task.uuid, 'letter'), JSON.stringify(task));
-    await this.client.set(this.__getKey(task.uuid, 'sendat'), `${task.sendAt}`);
+    const letterKey = this.__getKey(task.uuid, 'letter');
+    const sendatKey = this.__getKey(task.uuid, 'sendat');
+    const taskJSON = JSON.stringify(task);
+
+    if (typeof this.client.multi === 'function') {
+      const multi = this.client.multi();
+      multi.set(letterKey, taskJSON);
+      multi.set(sendatKey, `${task.sendAt}`);
+      if (task.to) {
+        multi.set(this.__getKey(task.to, 'concatletter'), task.uuid, {
+          PXAT: task.sendAt - 128,
+        });
+      }
+      await multi.exec();
+      return;
+    }
+
+    await this.client.set(letterKey, taskJSON);
+    await this.client.set(sendatKey, `${task.sendAt}`);
     if (task.to) {
       await this.client.set(this.__getKey(task.to, 'concatletter'), task.uuid, {
-        PXAT: task.sendAt - 128
+        PXAT: task.sendAt - 128,
       });
     }
   }
@@ -186,18 +332,19 @@ class RedisQueue {
    * @returns {Promise<boolean>} returns `true` if cancelled or `false` if not found, was sent, or was cancelled previously
    */
   async cancel(uuid) {
+    this.__debug('[cancel]', uuid);
     if (typeof uuid !== 'string') {
       return false;
     }
 
     await this.client.del(this.__getKey(uuid, 'sendat'));
     const letterKey = this.__getKey(uuid, 'letter');
-    const exists = await this.client.exists(letterKey);
-    if (!exists) {
+    const taskJSON = await this.client.get(letterKey);
+    if (!taskJSON) {
       return false;
     }
 
-    const task = JSON.parse(await this.client.get(letterKey));
+    const task = JSON.parse(taskJSON);
     if (!task || task.isSent === true || task.isCancelled === true) {
       return false;
     }
@@ -217,23 +364,62 @@ class RedisQueue {
    * @name remove
    * @description remove task from queue
    * @param task {object} - task's object
+   * @param {{ leaseTries: number, leaseSendingAt: number }} [opts] - lease guard: only remove if this worker still holds the lease (tries + sendingAt match, row not cancelled/failed)
    * @returns {Promise<boolean>} returns `true` if removed or `false` if not found
    */
-  async remove(task) {
-    if (typeof task !== 'object' || typeof task.uuid !== 'string') {
+  async remove(task, opts) {
+    this.__debug('[remove]', task?.uuid);
+    if (!task || typeof task !== 'object' || typeof task.uuid !== 'string') {
       return false;
     }
 
     const letterKey = this.__getKey(task.uuid, 'letter');
+    if (isSendLeaseRemove(opts)) {
+      if (typeof this.client.watch !== 'function' || typeof this.client.multi !== 'function') {
+        return false;
+      }
+      try {
+        await this.client.watch(letterKey);
+        const taskJSON = await this.client.get(letterKey);
+        if (!taskJSON) {
+          await this.client.unwatch?.();
+          return false;
+        }
+        const currentTask = JSON.parse(taskJSON);
+        if (currentTask.tries !== opts.leaseTries
+          || currentTask.isSending !== true
+          || (typeof currentTask.sendingAt === 'number' ? currentTask.sendingAt : 0) !== opts.leaseSendingAt
+          || currentTask.isCancelled === true
+          || currentTask.isFailed === true) {
+          await this.client.unwatch?.();
+          return false;
+        }
+        const keysToDelete = [letterKey, this.__getKey(task.uuid, 'sendat')];
+        if (task.to) {
+          keysToDelete.push(this.__getKey(task.to, 'concatletter'));
+        }
+        const multi = this.client.multi();
+        for (const key of keysToDelete) {
+          multi.del(key);
+        }
+        const result = await multi.exec();
+        return result !== null;
+      } catch (opError) {
+        logError('[remove] [lease] [opError]', opError);
+        return false;
+      }
+    }
+
     const exists = await this.client.exists(letterKey);
     if (!exists) {
       return false;
     }
 
-    await this.client.del([
-      letterKey,
-      this.__getKey(task.uuid, 'sendat'),
-    ]);
+    const keysToDelete = [letterKey, this.__getKey(task.uuid, 'sendat')];
+    if (task.to) {
+      keysToDelete.push(this.__getKey(task.to, 'concatletter'));
+    }
+    await this.client.del(keysToDelete);
     return true;
   }
 
@@ -241,31 +427,115 @@ class RedisQueue {
    * @async
    * @memberOf RedisQueue
    * @name update
-   * @description remove task from queue
+   * @description update task in queue
    * @param task {object} - task's object
    * @param updateObj {object} - fields with new values to update
    * @returns {Promise<boolean>} returns `true` if updated or `false` if not found or no changes was made
    */
   async update(task, updateObj) {
-    if (typeof task !== 'object' || typeof task.uuid !== 'string' || typeof updateObj !== 'object') {
+    this.__debug('[update]', task?.uuid);
+    if (!task || typeof task !== 'object' || typeof task.uuid !== 'string' || !updateObj || typeof updateObj !== 'object') {
       return false;
     }
 
     const letterKey = this.__getKey(task.uuid, 'letter');
+    const sendatKey = this.__getKey(task.uuid, 'sendat');
+    const isClaim = isSendClaimUpdate(updateObj);
+    const isAppend = isAppendMailOptionUpdate(updateObj);
+    const isLeaseRelease = isSendLeaseGuardedUpdate(updateObj);
+    const now = isClaim && typeof updateObj.sendingAt === 'number' ? updateObj.sendingAt : Date.now();
+    const sendingTimeout = this.mailTimeInstance?.sendingTimeout || 300000;
 
     try {
-      const exists = await this.client.exists(letterKey);
-      if (!exists) {
+      if (isAppend) {
+        if (typeof this.client.watch !== 'function' || typeof this.client.multi !== 'function') {
+          if (!RedisQueue.__atomicAppendWarned) {
+            RedisQueue.__atomicAppendWarned = true;
+            logError('[update] Redis client without watch()/multi() — concat appendMailOption falls back to non-atomic read-modify-write; concurrent folds into the same row may lose letters');
+          }
+          const taskJSON = await this.client.get(letterKey);
+          if (!taskJSON) {
+            return false;
+          }
+          const currentTask = JSON.parse(taskJSON);
+          if (currentTask.isSending === true || currentTask.isSent === true || currentTask.isFailed === true || currentTask.isCancelled === true) {
+            return false;
+          }
+          currentTask.mailOptions = [...(currentTask.mailOptions || []), updateObj.appendMailOption];
+          await this.client.set(letterKey, JSON.stringify(currentTask));
+          return true;
+        }
+
+        await this.client.watch(letterKey);
+        const taskJSON = await this.client.get(letterKey);
+        if (!taskJSON) {
+          await this.client.unwatch?.();
+          return false;
+        }
+        const currentTask = JSON.parse(taskJSON);
+        if (currentTask.isSending === true || currentTask.isSent === true || currentTask.isFailed === true || currentTask.isCancelled === true) {
+          await this.client.unwatch?.();
+          return false;
+        }
+        currentTask.mailOptions = [...(currentTask.mailOptions || []), updateObj.appendMailOption];
+        const multi = this.client.multi();
+        multi.set(letterKey, JSON.stringify(currentTask));
+        const result = await multi.exec();
+        return result !== null;
+      }
+
+      if ((isClaim || isLeaseRelease) && (typeof this.client.watch !== 'function' || typeof this.client.multi !== 'function')) {
+        if (isClaim && !RedisQueue.__atomicClaimWarned) {
+          RedisQueue.__atomicClaimWarned = true;
+          logError('[update] Redis client must support watch() and multi() for atomic send claims');
+        }
         return false;
       }
-      const updatedTask = { ...task, ...updateObj };
+
+      if (isClaim || isLeaseRelease) {
+        await this.client.watch(letterKey);
+        const taskJSON = await this.client.get(letterKey);
+        if (!taskJSON) {
+          await this.client.unwatch?.();
+          return false;
+        }
+
+        const currentTask = JSON.parse(taskJSON);
+        if (isClaim && !canClaimTask(currentTask, task, now, sendingTimeout)) {
+          await this.client.unwatch?.();
+          return false;
+        }
+        if (isLeaseRelease && !canReleaseLease(currentTask, updateObj)) {
+          await this.client.unwatch?.();
+          return false;
+        }
+
+        const updatedTask = { ...currentTask, ...stripInternalUpdateMeta(updateObj) };
+        const multi = this.client.multi();
+        multi.set(letterKey, JSON.stringify(updatedTask));
+        if (updatedTask.isSent === true || updatedTask.isFailed === true || updatedTask.isCancelled === true) {
+          multi.del(sendatKey);
+        } else if (updatedTask.sendAt) {
+          multi.set(sendatKey, `${+updatedTask.sendAt}`);
+        }
+
+        const result = await multi.exec();
+        return result !== null;
+      }
+
+      const taskJSON = await this.client.get(letterKey);
+      if (!taskJSON) {
+        return false;
+      }
+
+      const currentTask = JSON.parse(taskJSON);
+      const updatedTask = { ...currentTask, ...stripInternalUpdateMeta(updateObj) };
       await this.client.set(letterKey, JSON.stringify(updatedTask));
 
-      const sendatKey = this.__getKey(task.uuid, 'sendat');
       if (updatedTask.isSent === true || updatedTask.isFailed === true || updatedTask.isCancelled === true) {
         await this.client.del(sendatKey);
-      } else if (task.sendAt) {
-        await this.client.set(sendatKey, `${+task.sendAt}`);
+      } else if (updatedTask.sendAt) {
+        await this.client.set(sendatKey, `${+updatedTask.sendAt}`);
       }
       return true;
     } catch (opError) {
@@ -274,29 +544,22 @@ class RedisQueue {
     }
   }
 
-
   /**
+   * @internal
    * @memberOf RedisQueue
    * @name __getKey
    * @description helper to generate scoped key
-   * @param uuid {string} - letter's uuid
+   * @param uuid {string} - letter's uuid (or "to" address for `concatletter` keys)
    * @param type {string} - "letter" or "sendat" or "concatletter"
    * @returns {string} returns key used by Redis
    */
   __getKey(uuid, type = 'letter') {
-    if (!this.__keyTypes.includes(type)) {
+    if (!KEY_TYPES.has(type)) {
       throw new Error(`[mail-time] [RedisQueue] [__getKey] unsupported key "${type}" passed into the second argument`);
     }
+    this.__ensurePrefix();
     return `${this.uniqueName}:${type}:${uuid}`;
   }
-
-  /**
-   * @memberOf RedisQueue
-   * @name __keyTypes
-   * @description list of supported key type
-   * @returns {string} returns key used by Redis
-   */
-  __keyTypes = ['letter', 'sendat', 'concatletter'];
 }
 
 export { RedisQueue };