mail-time 3.0.0 → 4.1.0

package/adapters/mongo.d.ts

@@ -0,0 +1,118 @@
+export type MongoCollection = {
+    collectionName?: string | undefined;
+    createIndex: (keys: object, opts?: object) => Promise<unknown>;
+    indexes: () => Promise<{
+        name: string;
+        key: Record<string, unknown>;
+    }[]>;
+    dropIndex: (name: string) => Promise<unknown>;
+    find: (query: object, opts?: object) => unknown;
+    findOne: (query: object, opts?: object) => Promise<object | null>;
+    insertOne: (doc: object) => Promise<unknown>;
+    deleteOne: (query: object) => Promise<{
+        deletedCount?: number;
+    }>;
+    updateOne: (query: object, update: object) => Promise<{
+        modifiedCount?: number;
+    }>;
+};
+export type Db = {
+    collection: (name: string) => MongoCollection;
+    command: (cmd: object) => Promise<{
+        ok?: number;
+    }>;
+};
+export type MongoQueueOption = {
+    db: Db;
+    prefix?: string | undefined;
+};
+/** Class representing MongoDB Queue for MailTime */
+export class MongoQueue {
+    /**
+     * Create a MongoQueue instance
+     * @param {MongoQueueOption} opts - configuration object
+     */
+    constructor(opts: MongoQueueOption);
+    name: string;
+    db: Db;
+    prefix: any;
+    collection: MongoCollection | undefined;
+    /**
+     * @async
+     * @memberOf MongoQueue
+     * @name ready
+     * @description Wait until indexes are created
+     * @returns {Promise<void 0>}
+     */
+    ready(): Promise<void>;
+    /**
+     * @async
+     * @memberOf MongoQueue
+     * @name ping
+     * @description Check connection to Storage
+     * @returns {Promise<object>}
+     */
+    ping(): Promise<object>;
+    /**
+     * @memberOf MongoQueue
+     * @name iterate
+     * @description iterate over queued tasks passing each to `mailTimeInstance.___dispatch` (the bounded send pool)
+     * @param {{ limit?: number, sendingTimeout?: number }} [opts] - iteration options
+     * @returns {Promise<void>}
+     */
+    iterate(opts?: {
+        limit?: number;
+        sendingTimeout?: number;
+    }): Promise<void>;
+    /**
+     * @async
+     * @memberOf MongoQueue
+     * @name getPendingTo
+     * @description get queued task by `to` field (addressee)
+     * @param to {string} - email address
+     * @param sendAt {number} - timestamp
+     * @returns {Promise<object|null>}
+     */
+    getPendingTo(to: string, sendAt: number): Promise<object | null>;
+    /**
+     * @async
+     * @memberOf MongoQueue
+     * @name push
+     * @description push task to the queue/storage
+     * @param task {object} - task's object
+     * @returns {Promise<void 0>}
+     */
+    push(task: object): Promise<void>;
+    /**
+     * @async
+     * @memberOf MongoQueue
+     * @name cancel
+     * @description cancel scheduled email
+     * @param uuid {string} - email's uuid
+     * @returns {Promise<boolean>} returns `true` if cancelled or `false` if not found, was sent, or was cancelled previously
+     */
+    cancel(uuid: string): Promise<boolean>;
+    /**
+     * @async
+     * @memberOf MongoQueue
+     * @name remove
+     * @description remove task from queue
+     * @param task {object} - task's object
+     * @param {{ leaseTries: number, leaseSendingAt: number }} [opts] - lease guard: only remove if this worker still holds the lease (tries + sendingAt match, row not cancelled/failed)
+     * @returns {Promise<boolean>} returns `true` if removed or `false` if not found
+     */
+    remove(task: object, opts?: {
+        leaseTries: number;
+        leaseSendingAt: number;
+    }): Promise<boolean>;
+    /**
+     * @async
+     * @memberOf MongoQueue
+     * @name update
+     * @description update task in queue
+     * @param task {object} - task's object
+     * @param updateObj {object} - fields with new values to update
+     * @returns {Promise<boolean>} returns `true` if updated or `false` if not found or no changes was made
+     */
+    update(task: object, updateObj: object): Promise<boolean>;
+}

package/adapters/mongo.js

@@ -1,37 +1,62 @@
-import { logError } from '../helpers.js';
+import {
+  debug,
+  logError,
+  isSendClaimUpdate,
+  isSendLeaseGuardedUpdate,
+  isAppendMailOptionUpdate,
+  stripInternalUpdateMeta,
+  isSendLeaseRemove,
+} from '../helpers.js';
+
+const DEFAULT_PREFIX = '';
+
+/**
+ * @typedef {object} MongoCollection
+ * @property {string} [collectionName]
+ * @property {(keys: object, opts?: object) => Promise<unknown>} createIndex
+ * @property {() => Promise<{ name: string, key: Record<string, unknown> }[]>} indexes
+ * @property {(name: string) => Promise<unknown>} dropIndex
+ * @property {(query: object, opts?: object) => unknown} find
+ * @property {(query: object, opts?: object) => Promise<object|null>} findOne
+ * @property {(doc: object) => Promise<unknown>} insertOne
+ * @property {(query: object) => Promise<{ deletedCount?: number }>} deleteOne
+ * @property {(query: object, update: object) => Promise<{ modifiedCount?: number }>} updateOne
+ */
+
+/**
+ * @typedef {object} Db
+ * @property {(name: string) => MongoCollection} collection
+ * @property {(cmd: object) => Promise<{ ok?: number }>} command
+ */
 
 /**
- * Ensure (create) index on MongoDB collection, catch and log exception if thrown
- * @function ensureIndex
- * @param {Collection} collection - Mongo's driver Collection instance
- * @param {object} keys - Field and value pairs where the field is the index key and the value describes the type of index for that field
- * @param {object} opts - Set of options that controls the creation of the index
- * @returns {void 0}
+ * @typedef {object} MongoQueueOption
+ * @property {Db} db
+ * @property {string} [prefix]
  */
+
+/** @internal */
 const ensureIndex = async (collection, keys, opts) => {
   try {
     await collection.createIndex(keys, opts);
   } catch (e) {
-    if (e.code === 85) {
+    if (e?.code === 85) {
       let indexName;
       const indexes = await collection.indexes();
+      const keyNames = Object.keys(keys);
       for (const index of indexes) {
-        let drop = true;
-        for (const indexKey of Object.keys(keys)) {
-          if (typeof index.key[indexKey] === 'undefined') {
-            drop = false;
-            break;
-          }
+        const indexKeys = Object.keys(index.key);
+        if (indexKeys.length !== keyNames.length) {
+          continue;
         }
-
-        for (const indexKey of Object.keys(index.key)) {
-          if (typeof keys[indexKey] === 'undefined') {
-            drop = false;
+        let match = true;
+        for (const k of keyNames) {
+          if (typeof index.key[k] === 'undefined') {
+            match = false;
             break;
           }
         }
-
-        if (drop) {
+        if (match) {
           indexName = index.name;
           break;
         }
@@ -42,7 +67,7 @@ const ensureIndex = async (collection, keys, opts) => {
         await collection.createIndex(keys, opts);
       }
     } else {
-      logError(`[ensureIndex] Can not set ${Object.keys(keys).join(' + ')} index on "${collection._name || 'MongoDB'}" collection`, { keys, opts, details: e });
+      logError(`[ensureIndex] Can not set ${Object.keys(keys).join(' + ')} index on "${collection?.collectionName || 'MongoDB'}" collection`, { keys, opts, details: e });
     }
   }
 };
@@ -51,47 +76,58 @@ const ensureIndex = async (collection, keys, opts) => {
 class MongoQueue {
   /**
    * Create a MongoQueue instance
-   * @param {object} opts - configuration object
-   * @param {Db} opts.db - Required, Mongo's `Db` instance, like one returned from `MongoClient#db()` method
-   * @param {string} [opts.prefix] - Optional prefix for scope isolation; use when creating multiple MailTime instances within the single application
+   * @param {MongoQueueOption} opts - configuration object
    */
   constructor (opts) {
     this.name = 'mongo-queue';
-    if (!opts || typeof opts !== 'object' || opts === null) {
+    if (!opts || typeof opts !== 'object') {
       throw new TypeError('[mail-time] Configuration object must be passed into MongoQueue constructor');
     }
 
     if (!opts.db) {
-      throw new Error('[mail-time] [MongoQueue] requires MongoDB database {db} option, like returned from `MongoClient.connect`');
+      throw new Error('[mail-time] [MongoQueue] requires MongoDB database {db} option, like returned from `MongoClient#db()`');
     }
 
-    this.prefix = (typeof opts.prefix === 'string') ? opts.prefix : '';
     this.db = opts.db;
-    this.collection = opts.db.collection(`__mailTimeQueue__${this.prefix}`);
-    ensureIndex(this.collection, { uuid: 1 }, { background: false });
-    ensureIndex(this.collection, { isSent: 1, isFailed: 1, isCancelled: 1, to: 1, sendAt: 1 }, { background: false });
-    ensureIndex(this.collection, { isSent: 1, isFailed: 1, isCancelled: 1, sendAt: 1, tries: 1 }, { background: false });
-
-    // MongoDB Collection Schema:
-    // _id
-    // uuid        {string}
-    // to          {string|[string]}
-    // tries       {number}  - qty of send attempts
-    // sendAt      {number}  - When letter should be sent
-    // isSent      {boolean} - Email status
-    // isCancelled {boolean} - `true` if email was cancelled before it was sent
-    // isFailed    {boolean} - `true` if email has failed to send
-    // template    {string}  - Template for this email
-    // transport   {number}  - Last used transport
-    // concatSubject {string|boolean} - Email concatenation subject
-    // ---
-    // mailOptions         {[object]}  - Array of nodeMailer's `mailOptions`
-    // mailOptions.to      {string|[string]} - [REQUIRED]
-    // mailOptions.from    {string}
-    // mailOptions.text    {string|boolean}
-    // mailOptions.html    {string}
-    // mailOptions.subject {string}
-    // mailOptions.Other nodeMailer `sendMail` options...
+    if (typeof opts.prefix === 'string') {
+      this.__applyPrefix(opts.prefix);
+    }
+  }
+
+  /** @internal */
+  __applyPrefix(prefix) {
+    this.prefix = prefix;
+    this.collection = this.db.collection(`__mailTimeQueue__${prefix}`);
+    this.__readyPromise = Promise.all([
+      ensureIndex(this.collection, { uuid: 1 }, { background: false }),
+      ensureIndex(this.collection, { isSent: 1, isFailed: 1, isCancelled: 1, to: 1, sendAt: 1 }, { background: false }),
+      ensureIndex(this.collection, { isSent: 1, isFailed: 1, isCancelled: 1, isSending: 1, sendingAt: 1, sendAt: 1, tries: 1 }, { background: false }),
+    ]).then(() => void 0);
+  }
+
+  /** @internal */
+  __ensurePrefix() {
+    if (typeof this.prefix !== 'string') {
+      this.__applyPrefix(this.mailTimeInstance?.prefix || DEFAULT_PREFIX);
+    }
+  }
+
+  /** @internal */
+  __debug(...args) {
+    debug(this.mailTimeInstance?.debug === true, `[${this.name}]`, ...args);
+  }
+
+  /**
+   * @async
+   * @memberOf MongoQueue
+   * @name ready
+   * @description Wait until indexes are created
+   * @returns {Promise<void 0>}
+   */
+  async ready() {
+    this.__ensurePrefix();
+    this.__debug('[ready]');
+    await this.__readyPromise;
   }
 
   /**
@@ -102,6 +138,7 @@ class MongoQueue {
    * @returns {Promise<object>}
    */
   async ping() {
+    this.__debug('[ping]');
     if (!this.mailTimeInstance) {
       return {
         status: 'Service Unavailable',
@@ -125,7 +162,7 @@ class MongoQueue {
         status: 'Internal Server Error',
         code: 500,
         statusCode: 500,
-        error: pingError
+        error: pingError,
       };
     }
 
@@ -133,28 +170,43 @@ class MongoQueue {
       status: 'Service Unavailable',
       code: 503,
       statusCode: 503,
-      error: new Error('Service Unavailable')
+      error: new Error('Service Unavailable'),
     };
   }
 
   /**
    * @memberOf MongoQueue
    * @name iterate
-   * @description iterate over queued tasks passing to `mailTimeInstance.___send` method
-   * @returns {void 0}
+   * @description iterate over queued tasks passing each to `mailTimeInstance.___dispatch` (the bounded send pool)
+   * @param {{ limit?: number, sendingTimeout?: number }} [opts] - iteration options
+   * @returns {Promise<void>}
    */
-  async iterate() {
+  async iterate(opts) {
+    this.__debug('[iterate]', opts);
+    this.__ensurePrefix();
+    const now = Date.now();
+    const sendingTimeout = (opts && typeof opts.sendingTimeout === 'number' && opts.sendingTimeout > 0)
+      ? opts.sendingTimeout
+      : 300000;
+    const limit = (opts && typeof opts.limit === 'number' && Number.isFinite(opts.limit) && opts.limit > 0)
+      ? Math.floor(opts.limit)
+      : 0;
+
     try {
       const cursor = this.collection.find({
         isSent: false,
         isFailed: false,
         isCancelled: false,
         sendAt: {
-          $lte: Date.now()
+          $lte: now,
         },
         tries: {
-          $lt: this.mailTimeInstance.maxTries
-        }
+          $lt: this.mailTimeInstance.maxTries,
+        },
+        $or: [
+          { isSending: { $ne: true } },
+          { sendingAt: { $lte: now - sendingTimeout } },
+        ],
       }, {
         projection: {
           _id: 1,
@@ -165,15 +217,24 @@ class MongoQueue {
           isSent: 1,
           isFailed: 1,
           isCancelled: 1,
+          isSending: 1,
+          sendingAt: 1,
           mailOptions: 1,
           concatSubject: 1,
-        }
+        },
       });
 
-      while (await cursor.hasNext()) {
-        await this.mailTimeInstance.___send(await cursor.next());
+      if (limit > 0) {
+        cursor.limit(limit);
+      }
+
+      try {
+        while (await cursor.hasNext()) {
+          await this.mailTimeInstance.___dispatch(await cursor.next());
+        }
+      } finally {
+        await cursor.close().catch(() => {});
       }
-      await cursor.close();
     } catch (iterateError) {
       logError('[iterate] [while/await] [iterateError]', iterateError);
     }
@@ -189,18 +250,22 @@ class MongoQueue {
    * @returns {Promise<object|null>}
    */
   async getPendingTo(to, sendAt) {
+    this.__debug('[getPendingTo]', to, sendAt);
     if (typeof to !== 'string' || typeof sendAt !== 'number') {
       return null;
     }
+    this.__ensurePrefix();
 
     return await this.collection.findOne({
-      to: to,
+      to,
       isSent: false,
       isFailed: false,
       isCancelled: false,
+      isSending: { $ne: true },
+      tries: { $lt: this.mailTimeInstance.maxTries },
       sendAt: {
-        $lte: sendAt
-      }
+        $lte: sendAt,
+      },
     }, {
       projection: {
         _id: 1,
@@ -211,7 +276,7 @@ class MongoQueue {
         isFailed: 1,
         isCancelled: 1,
         mailOptions: 1,
-      }
+      },
     });
   }
 
@@ -224,9 +289,11 @@ class MongoQueue {
    * @returns {Promise<void 0>}
    */
   async push(task) {
-    if (typeof task !== 'object') {
+    this.__debug('[push]', task?.uuid);
+    if (!task || typeof task !== 'object') {
       return;
     }
+    this.__ensurePrefix();
 
     if (task.sendAt instanceof Date) {
       task.sendAt = +task.sendAt;
@@ -243,9 +310,11 @@ class MongoQueue {
    * @returns {Promise<boolean>} returns `true` if cancelled or `false` if not found, was sent, or was cancelled previously
    */
   async cancel(uuid) {
+    this.__debug('[cancel]', uuid);
     if (typeof uuid !== 'string') {
       return false;
     }
+    this.__ensurePrefix();
 
     const task = await this.collection.findOne({ uuid }, {
       projection: {
@@ -253,7 +322,7 @@ class MongoQueue {
         uuid: 1,
         isSent: 1,
         isCancelled: 1,
-      }
+      },
     });
 
     if (!task || task.isSent === true || task.isCancelled === true) {
@@ -275,35 +344,90 @@ class MongoQueue {
    * @name remove
    * @description remove task from queue
    * @param task {object} - task's object
+   * @param {{ leaseTries: number, leaseSendingAt: number }} [opts] - lease guard: only remove if this worker still holds the lease (tries + sendingAt match, row not cancelled/failed)
    * @returns {Promise<boolean>} returns `true` if removed or `false` if not found
    */
-  async remove(task) {
-    if (typeof task !== 'object') {
+  async remove(task, opts) {
+    this.__debug('[remove]', task?.uuid);
+    if (!task || typeof task !== 'object') {
       return false;
     }
+    this.__ensurePrefix();
 
-    return (await this.collection.deleteOne({ _id: task._id }))?.deletedCount >= 1;
+    const query = { _id: task._id };
+    if (isSendLeaseRemove(opts)) {
+      query.tries = opts.leaseTries;
+      query.isSending = true;
+      query.sendingAt = opts.leaseSendingAt;
+      query.isCancelled = false;
+      query.isFailed = false;
+    }
+
+    const res = await this.collection.deleteOne(query);
+    return (res?.deletedCount || 0) >= 1;
   }
 
   /**
    * @async
    * @memberOf MongoQueue
    * @name update
-   * @description remove task from queue
+   * @description update task in queue
    * @param task {object} - task's object
    * @param updateObj {object} - fields with new values to update
    * @returns {Promise<boolean>} returns `true` if updated or `false` if not found or no changes was made
    */
   async update(task, updateObj) {
-    if (typeof task !== 'object' || typeof updateObj !== 'object') {
+    this.__debug('[update]', task?.uuid);
+    if (!task || typeof task !== 'object' || !updateObj || typeof updateObj !== 'object') {
       return false;
     }
+    this.__ensurePrefix();
 
-    return (await this.collection.updateOne({
-      _id: task._id
-    }, {
-      $set: updateObj
-    }))?.modifiedCount >= 1;
+    if (isAppendMailOptionUpdate(updateObj)) {
+      const res = await this.collection.updateOne({
+        _id: task._id,
+        isSent: false,
+        isFailed: false,
+        isCancelled: false,
+        isSending: { $ne: true },
+      }, {
+        $push: {
+          mailOptions: updateObj.appendMailOption,
+        },
+      });
+      return (res?.modifiedCount || 0) >= 1;
+    }
+
+    const query = {
+      _id: task._id,
+    };
+
+    if (isSendClaimUpdate(updateObj)) {
+      const now = typeof updateObj.sendingAt === 'number' ? updateObj.sendingAt : Date.now();
+      const sendingTimeout = this.mailTimeInstance?.sendingTimeout || 300000;
+      query.isSent = false;
+      query.isFailed = false;
+      query.isCancelled = false;
+      query.tries = task.tries;
+      query.$or = [
+        { isSending: { $ne: true } },
+        { sendingAt: { $lte: now - sendingTimeout } },
+      ];
+    } else if (isSendLeaseGuardedUpdate(updateObj)) {
+      query.tries = updateObj.leaseTries;
+      query.isSending = true;
+      query.sendingAt = updateObj.leaseSendingAt;
+      query.isCancelled = false;
+      query.isFailed = false;
+    }
+
+    const res = await this.collection.updateOne(query, {
+      $set: stripInternalUpdateMeta(updateObj),
+    });
+    if (isSendClaimUpdate(updateObj)) {
+      return (res?.modifiedCount || res?.matchedCount || 0) >= 1;
+    }
+    return (res?.modifiedCount || 0) >= 1;
   }
 }
 

package/adapters/postgres.d.ts

@@ -0,0 +1,100 @@
+export type PostgresQueryResult = {
+    rowCount?: number | null | undefined;
+    rows?: unknown[] | undefined;
+};
+export type PostgresClient = {
+    query: (queryText: string, values?: unknown[]) => Promise<PostgresQueryResult>;
+};
+export type PostgresQueueOption = {
+    client: PostgresClient;
+    prefix?: string | undefined;
+};
+/** Class representing PostgreSQL Queue for MailTime */
+export class PostgresQueue {
+    /**
+     * Create a PostgresQueue instance
+     * @param {PostgresQueueOption} opts - configuration object
+     */
+    constructor(opts: PostgresQueueOption);
+    name: string;
+    client: PostgresClient;
+    prefix: any;
+    /**
+     * @async
+     * @memberOf PostgresQueue
+     * @name ready
+     * @description Wait until PostgreSQL schema is ready
+     * @returns {Promise<void 0>}
+     */
+    ready(): Promise<void>;
+    /**
+     * @async
+     * @memberOf PostgresQueue
+     * @name ping
+     * @description Check connection to Storage
+     * @returns {Promise<object>}
+     */
+    ping(): Promise<object>;
+    /**
+     * @memberOf PostgresQueue
+     * @name iterate
+     * @description iterate over queued tasks passing each to `mailTimeInstance.___dispatch` (the bounded send pool). Postgres reads buffer the full result, so each tick is bounded by `opts.limit` (or 1000 when caller passes `Infinity` / no limit) to keep memory predictable; high-throughput deployments should shard prefixes.
+     * @param {{ limit?: number, sendingTimeout?: number }} [opts] - iteration options
+     * @returns {Promise<void>}
+     */
+    iterate(opts?: {
+        limit?: number;
+        sendingTimeout?: number;
+    }): Promise<void>;
+    /**
+     * @async
+     * @memberOf PostgresQueue
+     * @name getPendingTo
+     * @description get queued task by `to` field (addressee)
+     * @param to {string} - email address
+     * @param sendAt {number} - timestamp
+     * @returns {Promise<object|null>}
+     */
+    getPendingTo(to: string, sendAt: number): Promise<object | null>;
+    /**
+     * @async
+     * @memberOf PostgresQueue
+     * @name push
+     * @description push task to the queue/storage
+     * @param task {object} - task's object
+     * @returns {Promise<void 0>}
+     */
+    push(task: object): Promise<void>;
+    /**
+     * @async
+     * @memberOf PostgresQueue
+     * @name cancel
+     * @description cancel scheduled email
+     * @param uuid {string} - email's uuid
+     * @returns {Promise<boolean>} returns `true` if cancelled or `false` if not found, was sent, or was cancelled previously
+     */
+    cancel(uuid: string): Promise<boolean>;
+    /**
+     * @async
+     * @memberOf PostgresQueue
+     * @name remove
+     * @description remove task from queue
+     * @param task {object} - task's object
+     * @param {{ leaseTries: number, leaseSendingAt: number }} [opts] - lease guard: only remove if this worker still holds the lease (tries + sendingAt match, row not cancelled/failed)
+     * @returns {Promise<boolean>} returns `true` if removed or `false` if not found
+     */
+    remove(task: object, opts?: {
+        leaseTries: number;
+        leaseSendingAt: number;
+    }): Promise<boolean>;
+    /**
+     * @async
+     * @memberOf PostgresQueue
+     * @name update
+     * @description update task in queue
+     * @param task {object} - task's object
+     * @param updateObj {object} - fields with new values to update
+     * @returns {Promise<boolean>} returns `true` if updated or `false` if not found or no changes was made
+     */
+    update(task: object, updateObj: object): Promise<boolean>;
+}