mail-time 4.1.0 → 5.1.0

package/adapters/redis.js

@@ -7,6 +7,7 @@ import {
   stripInternalUpdateMeta,
   isSendLeaseRemove,
 } from '../helpers.js';
+import { createHash } from 'crypto';
 
 /**
  * @typedef {object} RedisClient
@@ -15,7 +16,11 @@ import {
  * @property {(key: string, value: string, options?: object) => Promise<unknown>} set
  * @property {(key: string|string[]) => Promise<number>} del
  * @property {() => Promise<string>} ping
- * @property {(options: object) => AsyncIterable<string|string[]>} scanIterator
+ * @property {(options: object) => AsyncIterable<string|string[]>} [scanIterator]
+ * @property {(key: string, field: string) => Promise<string|null>} [hGet]
+ * @property {(script: string, options: { keys: string[], arguments: string[] }) => Promise<unknown>} [eval]
+ * @property {(sha: string, options: { keys: string[], arguments: string[] }) => Promise<unknown>} [evalSha]
+ * @property {(script: string) => Promise<string>} [scriptLoad]
  * @property {(key: string) => Promise<unknown>} [watch]
  * @property {() => Promise<unknown>} [unwatch]
  * @property {() => object} [multi]
@@ -25,10 +30,168 @@ import {
  * @typedef {object} RedisQueueOption
  * @property {RedisClient} client
  * @property {string} [prefix]
+ * @property {boolean} [useHashTags] - Use Redis Cluster hash-tag keys (`mailtime:{prefix}:*`). Default keeps existing standalone keys.
  */
 
 const KEY_TYPES = new Set(['letter', 'sendat', 'concatletter']);
 const DEFAULT_PREFIX = 'default';
+const VALID_PREFIX = /^[A-Za-z0-9_\-:.]+$/;
+const TAGGED_ITERATE_LIMIT = 100;
+
+const PUSH_TAGGED_TASK_SCRIPT = `
+  redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+  redis.call('ZADD', KEYS[2], tonumber(ARGV[3]), ARGV[1])
+  if #KEYS > 3 then
+    redis.call('SET', KEYS[3], ARGV[1], 'PXAT', tonumber(ARGV[4]))
+    redis.call('ZADD', KEYS[4], tonumber(ARGV[4]), KEYS[3])
+  end
+  return 1
+`;
+
+const ITERATE_TAGGED_TASKS_SCRIPT = `
+  local now = tonumber(ARGV[1])
+  local maxTries = tonumber(ARGV[2])
+  local sendingTimeout = tonumber(ARGV[3])
+  local limit = tonumber(ARGV[4])
+  local scanLimit = tonumber(ARGV[5])
+  local tasks = {}
+  if #KEYS > 2 then
+    local expiredPointers = redis.call('ZRANGEBYSCORE', KEYS[3], '-inf', now, 'LIMIT', 0, 100)
+    for _, pointerKey in ipairs(expiredPointers) do
+      redis.call('DEL', pointerKey)
+      redis.call('ZREM', KEYS[3], pointerKey)
+    end
+  end
+  local due = redis.call('ZRANGEBYSCORE', KEYS[2], '-inf', now, 'LIMIT', 0, scanLimit)
+  for _, uuid in ipairs(due) do
+    local payload = redis.call('HGET', KEYS[1], uuid)
+    if not payload then
+      redis.call('ZREM', KEYS[2], uuid)
+    else
+      local task = cjson.decode(payload)
+      if task.isSent or task.isFailed or task.isCancelled or tonumber(task.tries or 0) >= maxTries then
+        redis.call('ZREM', KEYS[2], uuid)
+      elseif task.isSending then
+        local eligibleAt = tonumber(task.sendingAt or 0) + sendingTimeout
+        if eligibleAt > now then
+          redis.call('ZADD', KEYS[2], eligibleAt, uuid)
+        else
+          table.insert(tasks, task)
+        end
+      elseif tonumber(task.sendAt or 0) <= now then
+        table.insert(tasks, task)
+      else
+        redis.call('ZADD', KEYS[2], tonumber(task.sendAt), uuid)
+      end
+      if #tasks >= limit then
+        break
+      end
+    end
+  end
+
+  return cjson.encode(tasks)
+`;
+
+const UPDATE_TAGGED_TASK_SCRIPT = `
+  local payload = redis.call('HGET', KEYS[1], ARGV[1])
+  if not payload then
+    redis.call('ZREM', KEYS[2], ARGV[1])
+    return 0
+  end
+
+  local task = cjson.decode(payload)
+  local update = cjson.decode(ARGV[2])
+  local mode = ARGV[3]
+  local now = tonumber(ARGV[4])
+  local sendingTimeout = tonumber(ARGV[5])
+  local expectedTries = tonumber(ARGV[6])
+  local leaseSendingAt = tonumber(ARGV[7])
+
+  if mode == 'claim' then
+    if task.isSent or task.isFailed or task.isCancelled or tonumber(task.tries or 0) ~= expectedTries then
+      return 0
+    end
+    if task.isSending and tonumber(task.sendingAt or 0) > now - sendingTimeout then
+      return 0
+    end
+  elseif mode == 'lease' then
+    if task.isCancelled or task.isFailed or not task.isSending
+      or tonumber(task.tries or 0) ~= expectedTries
+      or tonumber(task.sendingAt or 0) ~= leaseSendingAt then
+      return 0
+    end
+  end
+
+  for key, value in pairs(update) do
+    task[key] = value
+  end
+  redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(task))
+
+  if task.isSent or task.isFailed or task.isCancelled then
+    redis.call('ZREM', KEYS[2], ARGV[1])
+    if #KEYS > 3 then
+      local concatUuid = redis.call('GET', KEYS[3])
+      if concatUuid == ARGV[1] then
+        redis.call('DEL', KEYS[3])
+        redis.call('ZREM', KEYS[4], KEYS[3])
+      elseif not concatUuid then
+        redis.call('ZREM', KEYS[4], KEYS[3])
+      end
+    end
+  elseif task.isSending then
+    redis.call('ZADD', KEYS[2], tonumber(task.sendingAt or now) + sendingTimeout, ARGV[1])
+  else
+    redis.call('ZADD', KEYS[2], tonumber(task.sendAt), ARGV[1])
+  end
+  return 1
+`;
+
+const APPEND_TAGGED_MAIL_OPTION_SCRIPT = `
+  local payload = redis.call('HGET', KEYS[1], ARGV[1])
+  if not payload then
+    return 0
+  end
+  local task = cjson.decode(payload)
+  if task.isSending or task.isSent or task.isFailed or task.isCancelled then
+    return 0
+  end
+  task.mailOptions = task.mailOptions or {}
+  table.insert(task.mailOptions, cjson.decode(ARGV[2]))
+  redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(task))
+  return 1
+`;
+
+const REMOVE_TAGGED_TASK_SCRIPT = `
+  local payload = redis.call('HGET', KEYS[1], ARGV[1])
+  if not payload then
+    redis.call('ZREM', KEYS[2], ARGV[1])
+    return 0
+  end
+  local task = cjson.decode(payload)
+  if ARGV[2] == 'lease' and (task.isCancelled or task.isFailed or not task.isSending
+    or tonumber(task.tries or 0) ~= tonumber(ARGV[3])
+    or tonumber(task.sendingAt or 0) ~= tonumber(ARGV[4])) then
+    return 0
+  end
+  redis.call('HDEL', KEYS[1], ARGV[1])
+  redis.call('ZREM', KEYS[2], ARGV[1])
+  if #KEYS > 3 then
+    local concatUuid = redis.call('GET', KEYS[3])
+    if concatUuid == ARGV[1] then
+      redis.call('DEL', KEYS[3])
+      redis.call('ZREM', KEYS[4], KEYS[3])
+    elseif not concatUuid then
+      redis.call('ZREM', KEYS[4], KEYS[3])
+    end
+  end
+  return 1
+`;
+
+const sha1Hex = (string) => createHash('sha1').update(string).digest('hex');
+
+const isNoScriptError = (error) => {
+  return !!error && (error.code === 'NOSCRIPT' || (typeof error.message === 'string' && error.message.includes('NOSCRIPT')));
+};
 
 const canReleaseLease = (currentTask, updateObj) => {
   return currentTask
@@ -102,7 +265,21 @@ class RedisQueue {
       throw new Error('[mail-time] [RedisQueue] required {client} option is missing, e.g. returned from `redis.createClient()` or `redis.createCluster()` method');
     }
 
+    if (opts.useHashTags !== undefined && typeof opts.useHashTags !== 'boolean') {
+      throw new TypeError(`[mail-time] [RedisQueue] {useHashTags} option must be a boolean (received: ${typeof opts.useHashTags})`);
+    }
+
     this.client = opts.client;
+    this.useHashTags = opts.useHashTags === true;
+    this.__scriptSources = {
+      push: PUSH_TAGGED_TASK_SCRIPT,
+      iterate: ITERATE_TAGGED_TASKS_SCRIPT,
+      update: UPDATE_TAGGED_TASK_SCRIPT,
+      append: APPEND_TAGGED_MAIL_OPTION_SCRIPT,
+      remove: REMOVE_TAGGED_TASK_SCRIPT,
+    };
+    this.__scriptShas = Object.fromEntries(Object.entries(this.__scriptSources).map(([name, source]) => [name, sha1Hex(source)]));
+    this.__loadedShas = new Set();
     if (typeof opts.prefix === 'string') {
       this.__applyPrefix(opts.prefix);
     }
@@ -110,8 +287,16 @@ class RedisQueue {
 
   /** @internal */
   __applyPrefix(prefix) {
+    if (this.useHashTags && !VALID_PREFIX.test(prefix)) {
+      throw new Error(`[mail-time] [RedisQueue] {prefix} option must match ${VALID_PREFIX} when {useHashTags} is true (received: "${prefix}")`);
+    }
     this.prefix = prefix;
-    this.uniqueName = `mailtime:${prefix}`;
+    this.uniqueName = this.useHashTags ? `mailtime:{${prefix}}` : `mailtime:${prefix}`;
+    if (this.useHashTags) {
+      this.lettersKey = `${this.uniqueName}:letters`;
+      this.scheduleKey = `${this.uniqueName}:schedule`;
+      this.concatKeysKey = `${this.uniqueName}:concatkeys`;
+    }
   }
 
   /** @internal */
@@ -126,6 +311,49 @@ class RedisQueue {
     debug(this.mailTimeInstance?.debug === true, `[${this.name}]`, ...args);
   }
 
+  /** @internal */
+  async __runScript(scriptKey, options) {
+    const source = this.__scriptSources[scriptKey];
+    const sha = this.__scriptShas[scriptKey];
+    if (!source || !sha) {
+      throw new Error(`[mail-time] [RedisQueue] unknown script "${scriptKey}"`);
+    }
+
+    if (this.__loadedShas.has(sha) && typeof this.client.evalSha === 'function') {
+      try {
+        return await this.client.evalSha(sha, options);
+      } catch (error) {
+        if (!isNoScriptError(error)) {
+          throw error;
+        }
+        this.__loadedShas.delete(sha);
+      }
+    }
+
+    if (typeof this.client.scriptLoad === 'function' && typeof this.client.evalSha === 'function') {
+      try {
+        await this.client.scriptLoad(source);
+        this.__loadedShas.add(sha);
+        return await this.client.evalSha(sha, options);
+      } catch (error) {
+        if (!isNoScriptError(error)) {
+          this.__debug(`[script:${scriptKey}] scriptLoad failed; falling back to EVAL`, error);
+        }
+      }
+    }
+
+    if (typeof this.client.eval !== 'function') {
+      throw new Error('[mail-time] [RedisQueue] Redis Cluster client must support EVAL');
+    }
+    return await this.client.eval(source, options);
+  }
+
+  /** @internal */
+  __getTaggedConcatKey(to) {
+    this.__ensurePrefix();
+    return `${this.uniqueName}:concatletter:${to}`;
+  }
+
   /**
    * @async
    * @memberOf RedisQueue
@@ -203,6 +431,25 @@ class RedisQueue {
       const maxTries = (this.mailTimeInstance && typeof this.mailTimeInstance.maxTries === 'number')
         ? this.mailTimeInstance.maxTries
         : 60;
+
+      if (this.useHashTags) {
+        this.__ensurePrefix();
+        const dispatchLimit = Number.isFinite(limit) && limit > 0 ? Math.floor(limit) : TAGGED_ITERATE_LIMIT;
+        const payload = await this.__runScript('iterate', {
+          keys: [this.lettersKey, this.scheduleKey, this.concatKeysKey],
+          arguments: [`${now}`, `${maxTries}`, `${sendingTimeout}`, `${dispatchLimit}`, `${Math.max(dispatchLimit * 20, 100)}`],
+        });
+        const candidates = payload ? JSON.parse(String(payload)) : [];
+        if (!Array.isArray(candidates)) {
+          return;
+        }
+        for (const candidate of candidates) {
+          if (isIterateCandidate(candidate, now, sendingTimeout, maxTries)) {
+            await this.mailTimeInstance.___dispatch(candidate);
+          }
+        }
+        return;
+      }
       let dispatched = 0;
 
       const matchPattern = this.__getKey('*', 'sendat');
@@ -259,14 +506,15 @@ class RedisQueue {
       return null;
     }
 
-    const concatKey = this.__getKey(to, 'concatletter');
+    const concatKey = this.useHashTags ? this.__getTaggedConcatKey(to) : this.__getKey(to, 'concatletter');
     const uuid = await this.client.get(concatKey);
     if (!uuid) {
       return null;
     }
 
-    const letterKey = this.__getKey(uuid, 'letter');
-    const taskJSON = await this.client.get(letterKey);
+    const taskJSON = this.useHashTags
+      ? await this.client.hGet(this.lettersKey, uuid)
+      : await this.client.get(this.__getKey(uuid, 'letter'));
     if (!taskJSON) {
       return null;
     }
@@ -297,6 +545,19 @@ class RedisQueue {
       task.sendAt = +task.sendAt;
     }
 
+    if (this.useHashTags) {
+      this.__ensurePrefix();
+      const keys = [this.lettersKey, this.scheduleKey];
+      const args = [task.uuid, JSON.stringify(task), `${+task.sendAt}`];
+      if (task.to) {
+        keys.push(this.__getTaggedConcatKey(task.to));
+        keys.push(this.concatKeysKey);
+        args.push(`${task.sendAt - 128}`);
+      }
+      await this.__runScript('push', { keys, arguments: args });
+      return;
+    }
+
     const letterKey = this.__getKey(task.uuid, 'letter');
     const sendatKey = this.__getKey(task.uuid, 'sendat');
     const taskJSON = JSON.stringify(task);
@@ -337,9 +598,14 @@ class RedisQueue {
       return false;
     }
 
-    await this.client.del(this.__getKey(uuid, 'sendat'));
-    const letterKey = this.__getKey(uuid, 'letter');
-    const taskJSON = await this.client.get(letterKey);
+    if (this.useHashTags) {
+      this.__ensurePrefix();
+    } else {
+      await this.client.del(this.__getKey(uuid, 'sendat'));
+    }
+    const taskJSON = this.useHashTags
+      ? await this.client.hGet(this.lettersKey, uuid)
+      : await this.client.get(this.__getKey(uuid, 'letter'));
     if (!taskJSON) {
       return false;
     }
@@ -373,6 +639,30 @@ class RedisQueue {
       return false;
     }
 
+    if (this.useHashTags) {
+      this.__ensurePrefix();
+      try {
+        const keys = [this.lettersKey, this.scheduleKey];
+        if (task.to) {
+          keys.push(this.__getTaggedConcatKey(task.to));
+          keys.push(this.concatKeysKey);
+        }
+        const result = await this.__runScript('remove', {
+          keys,
+          arguments: [
+            task.uuid,
+            isSendLeaseRemove(opts) ? 'lease' : 'plain',
+            `${opts?.leaseTries || 0}`,
+            `${opts?.leaseSendingAt || 0}`,
+          ],
+        });
+        return Number(result) >= 1;
+      } catch (opError) {
+        logError('[remove] [tagged] [opError]', opError);
+        return false;
+      }
+    }
+
     const letterKey = this.__getKey(task.uuid, 'letter');
     if (isSendLeaseRemove(opts)) {
       if (typeof this.client.watch !== 'function' || typeof this.client.multi !== 'function') {
@@ -447,6 +737,36 @@ class RedisQueue {
     const sendingTimeout = this.mailTimeInstance?.sendingTimeout || 300000;
 
     try {
+      if (this.useHashTags) {
+        if (isAppend) {
+          const result = await this.__runScript('append', {
+            keys: [this.lettersKey],
+            arguments: [task.uuid, JSON.stringify(updateObj.appendMailOption)],
+          });
+          return Number(result) >= 1;
+        }
+
+        const mode = isClaim ? 'claim' : (isLeaseRelease ? 'lease' : 'plain');
+        const keys = [this.lettersKey, this.scheduleKey];
+        if (task.to) {
+          keys.push(this.__getTaggedConcatKey(task.to));
+          keys.push(this.concatKeysKey);
+        }
+        const result = await this.__runScript('update', {
+          keys,
+          arguments: [
+            task.uuid,
+            JSON.stringify(stripInternalUpdateMeta(updateObj)),
+            mode,
+            `${now}`,
+            `${sendingTimeout}`,
+            `${isClaim ? task.tries : (updateObj.leaseTries || 0)}`,
+            `${updateObj.leaseSendingAt || 0}`,
+          ],
+        });
+        return Number(result) >= 1;
+      }
+
       if (isAppend) {
         if (typeof this.client.watch !== 'function' || typeof this.client.multi !== 'function') {
           if (!RedisQueue.__atomicAppendWarned) {
@@ -558,6 +878,15 @@ class RedisQueue {
       throw new Error(`[mail-time] [RedisQueue] [__getKey] unsupported key "${type}" passed into the second argument`);
     }
     this.__ensurePrefix();
+    if (this.useHashTags) {
+      if (type === 'letter') {
+        return this.lettersKey;
+      }
+      if (type === 'sendat') {
+        return this.scheduleKey;
+      }
+      return this.__getTaggedConcatKey(uuid);
+    }
     return `${this.uniqueName}:${type}:${uuid}`;
   }
 }

package/helpers.js

@@ -220,6 +220,24 @@ const filterAddressField = (field, acceptedSet) => {
   return field;
 };
 
+const HTML_ESCAPES = {
+  '&': '&',
+  '<': '&lt;',
+  '>': '&gt;',
+  '"': '&quot;',
+  "'": '&#39;',
+};
+
+/**
+ * @name escapeHtml - Escape the five HTML-significant characters.
+ * @function
+ * @param {string} value
+ * @returns {string} `value` safe to interpolate into HTML text or an attribute value.
+ */
+const escapeHtml = (value) => {
+  return `${value}`.replace(/[&<>"']/g, (char) => HTML_ESCAPES[char]);
+};
+
 /**
  * @name isSendClaimUpdate - Detect an atomic send-claim update (`{ isSending: true, tries: N }`).
  * @function
@@ -278,6 +296,7 @@ const isSendLeaseRemove = (opts) => {
 
 export {
   debug,
+  escapeHtml,
   logError,
   hasOwnProp,
   isPlainObject,