@serve.zone/dcrouter 18.0.0 → 18.0.2

package/dist_ts_web/00_commitinfo_data.js

@@ -3,7 +3,7 @@
  */
 export const commitinfo = {
     name: '@serve.zone/dcrouter',
-    version: '18.0.0',
+    version: '18.0.2',
     description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 };
 //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vdHNfd2ViLzAwX2NvbW1pdGluZm9fZGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7R0FFRztBQUNILE1BQU0sQ0FBQyxNQUFNLFVBQVUsR0FBRztJQUN4QixJQUFJLEVBQUUsc0JBQXNCO0lBQzVCLE9BQU8sRUFBRSxRQUFRO0lBQ2pCLFdBQVcsRUFBRSwwRUFBMEU7Q0FDeEYsQ0FBQSJ9

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@serve.zone/dcrouter",
   "private": false,
-  "version": "18.0.0",
+  "version": "18.0.2",
   "description": "A multifaceted routing service handling mail and SMS delivery functions.",
   "type": "module",
   "bin": {

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '18.0.0',
+  version: '18.0.2',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }

package/ts/email/classes.accepted-email-spool.ts

@@ -77,6 +77,19 @@ const ACCEPTED_EMAIL_STOP_DRAIN_TIMEOUT_MS = 30_000;
 /** Retention for catch-all stored inbound mail (30 days). */
 const INBOUND_STORE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
 
+/**
+ * Permanent per-email storage failure: the raw RFC822 payload of an accepted
+ * email is unrecoverable, so delivery can never succeed. The spool marks such
+ * rows failed instead of retrying them, so one lost blob cannot wedge the
+ * whole queue.
+ */
+export class AcceptedEmailRawMessageMissingError extends Error {
+  constructor(messageArg: string) {
+    super(messageArg);
+    this.name = 'AcceptedEmailRawMessageMissingError';
+  }
+}
+
 export type TSmartMtaQueueItemLike = {
   id?: string;
   processingResult?: {
@@ -690,32 +703,55 @@ export class AcceptedEmailSpool {
     for (const item of (emailServerArg as TSmartMtaQueueReader).getQueueItems?.() || []) {
       const cachedEmailId = this.getCachedEmailIdFromQueueItem(item);
       if (!cachedEmailId || !item.smtpTransactions?.length) continue;
-      await this.runCachedEmailUpdate(cachedEmailId, async () => {
-        const cachedEmail = await CachedEmail.findById(cachedEmailId);
-        if (!cachedEmail) return;
-        for (const transaction of item.smtpTransactions || []) {
-          this.appendSmtpTransaction(cachedEmail, transaction);
-        }
-        await cachedEmail.save();
-      });
+      try {
+        await this.runCachedEmailUpdate(cachedEmailId, async () => {
+          const cachedEmail = await CachedEmail.findById(cachedEmailId);
+          if (!cachedEmail) return;
+          for (const transaction of item.smtpTransactions || []) {
+            this.appendSmtpTransaction(cachedEmail, transaction);
+          }
+          await cachedEmail.save();
+        });
+      } catch (error: unknown) {
+        // History reconciliation runs during email-server startup; one row that
+        // cannot be persisted must not abort reconciliation or startup.
+        logger.log('warn', `Unable to reconcile SMTP transaction history for accepted email ${cachedEmailId}: ${(error as Error).message}`);
+      }
     }
   }
 
   /** Requeue emails left in 'queued' state by a previous process as pending. */
   public async recoverQueuedEmails(): Promise<void> {
+    // Rows whose recovery save failed stay 'queued' and are re-served by
+    // findQueuedForRecovery; track them so they are neither retried in a loop
+    // within this run nor able to wedge recovery when they fill a whole batch.
+    const unrecoverableIds = new Set<string>();
     while (true) {
       const queuedEmails = await CachedEmail.findQueuedForRecovery(ACCEPTED_EMAIL_SPOOL_BATCH_SIZE);
-      if (queuedEmails.length === 0) {
+      const recoverableEmails = queuedEmails.filter((queuedEmail) => !unrecoverableIds.has(queuedEmail.id));
+      if (recoverableEmails.length === 0) {
         return;
       }
-      for (const queuedEmail of queuedEmails) {
+      for (const queuedEmail of recoverableEmails) {
         if (this.isCachedEmailTerminal(queuedEmail)) {
           continue;
         }
-        queuedEmail.status = 'pending';
-        queuedEmail.nextAttempt = new Date();
-        await queuedEmail.save();
-        await this.notifyEmailQueuePersisted(queuedEmail, 'email-queue-recovered');
+        const previousStatus = queuedEmail.status;
+        const previousNextAttempt = queuedEmail.nextAttempt;
+        try {
+          queuedEmail.status = 'pending';
+          queuedEmail.nextAttempt = new Date();
+          await queuedEmail.save();
+          await this.notifyEmailQueuePersisted(queuedEmail, 'email-queue-recovered');
+        } catch (error: unknown) {
+          // Recovery runs during email-server startup; a row that cannot be
+          // persisted stays 'queued' for the next startup and must not abort
+          // recovery of the remaining rows.
+          queuedEmail.status = previousStatus;
+          queuedEmail.nextAttempt = previousNextAttempt;
+          unrecoverableIds.add(queuedEmail.id);
+          logger.log('warn', `Unable to recover queued accepted email ${queuedEmail.id}: ${(error as Error).message}`);
+        }
       }
       if (queuedEmails.length < ACCEPTED_EMAIL_SPOOL_BATCH_SIZE) {
         return;
@@ -735,18 +771,54 @@ export class AcceptedEmailSpool {
         if (this.stopping || this.dcRouterRef.emailServer !== emailServer) {
           break;
         }
-        if (await this.postponeLiveSmartMtaOwnedEmail(cachedEmail, emailServer)) {
-          continue;
+        try {
+          if (await this.postponeLiveSmartMtaOwnedEmail(cachedEmail, emailServer)) {
+            continue;
+          }
+          const session = this.buildCachedEmailSession(cachedEmail);
+          const rawMessage = await this.readRawMessage(cachedEmail);
+          await this.processAcceptedCachedEmail(cachedEmail, rawMessage, session, emailServer);
+        } catch (error: unknown) {
+          // One poisoned row must never stall the rest of the batch: record the
+          // failure on that row alone and keep processing the remaining emails.
+          await this.handleSpoolItemFailure(cachedEmail, error);
         }
-        const session = this.buildCachedEmailSession(cachedEmail);
-        const rawMessage = await this.readRawMessage(cachedEmail);
-        await this.processAcceptedCachedEmail(cachedEmail, rawMessage, session, emailServer);
       }
     } finally {
       this.processing = false;
     }
   }
 
+  /**
+   * Record a per-email spool failure without aborting the batch: permanent
+   * raw-message losses are marked failed, everything else is deferred through
+   * the regular retry mechanism (which terminates at maxAttempts).
+   */
+  private async handleSpoolItemFailure(cachedEmail: CachedEmail, errorArg: unknown): Promise<void> {
+    const message = (errorArg as Error).message;
+    try {
+      const currentCachedEmail = await CachedEmail.findById(cachedEmail.id) || cachedEmail;
+      if (this.isCachedEmailTerminal(currentCachedEmail)) {
+        return;
+      }
+      if (errorArg instanceof AcceptedEmailRawMessageMissingError) {
+        currentCachedEmail.markFailed(message);
+        await currentCachedEmail.save();
+        await this.notifyEmailQueuePersisted(currentCachedEmail, 'email-raw-message-missing');
+        logger.log('error', `Accepted email ${currentCachedEmail.id} failed permanently: ${message}`);
+        return;
+      }
+      currentCachedEmail.scheduleRetry(ACCEPTED_EMAIL_RETRY_DELAY_MS);
+      currentCachedEmail.lastError = message;
+      await currentCachedEmail.save();
+      await this.notifyEmailQueuePersisted(currentCachedEmail, 'email-spool-deferred');
+      logger.log('warn', `Accepted email ${currentCachedEmail.id} deferred after spool processing failure: ${message}`);
+    } catch (persistError: unknown) {
+      // Even failing to persist the failure state must not abort the batch.
+      logger.log('error', `Unable to persist spool failure state for accepted email ${cachedEmail.id}: ${(persistError as Error).message} (original error: ${message})`);
+    }
+  }
+
   private async processAcceptedCachedEmail(
     cachedEmail: CachedEmail,
     emailData: Email | plugins.buffer.Buffer,
@@ -1010,11 +1082,16 @@ export class AcceptedEmailSpool {
       }
       const rawMessage = await blobStorage.get(cachedEmailArg.rawContentObjectKey);
       if (!rawMessage) {
-        throw new Error(`Raw RFC822 object is missing for accepted email ${cachedEmailArg.id}`);
+        throw new AcceptedEmailRawMessageMissingError(`Raw RFC822 object is missing for accepted email ${cachedEmailArg.id}`);
       }
       return plugins.buffer.Buffer.from(rawMessage);
     }
-    throw new Error(`Accepted email ${cachedEmailArg.id} has no raw RFC822 payload`);
+    if (cachedEmailArg.rawContent) {
+      // Legacy rows persisted before SmartBucket-backed storage keep their
+      // payload inline; serve it rather than declaring the payload lost.
+      return plugins.buffer.Buffer.from(cachedEmailArg.rawContent, 'utf8');
+    }
+    throw new AcceptedEmailRawMessageMissingError(`Accepted email ${cachedEmailArg.id} has no raw RFC822 payload`);
   }
 
   public async deleteRawMessage(cachedEmailArg: CachedEmail): Promise<void> {

package/ts/email/classes.smartmta-blob-storage-manager.ts

@@ -1,4 +1,5 @@
 import * as plugins from '../plugins.js';
+import { logger } from '../logger.js';
 import type { IBlobStorageManager } from '@push.rocks/smartmta';
 
 const SMARTMTA_BLOB_NAMESPACES = [
@@ -119,11 +120,18 @@ export class SmartMtaBlobStorageManager implements IBlobStorageManager {
     const objectPrefix = this.toObjectKey(prefix);
     const keys: string[] = [];
     for await (const objectKey of this.bucketRef.listAllObjects(objectPrefix)) {
+      // One stray or malformed object under the prefix must not abort the
+      // whole listing (SmartMTA queue recovery lists at startup); skip it.
       if (!objectKey.startsWith(this.objectPrefix)) {
-        throw new Error(`SmartBucket returned an object outside the configured prefix: ${objectKey}`);
+        logger.log('warn', `Skipping SmartBucket object outside the configured prefix during SmartMTA blob listing: ${objectKey}`);
+        continue;
       }
       const key = `/${objectKey.slice(this.objectPrefix.length)}`;
-      keys.push(normalizeBlobKey(key));
+      try {
+        keys.push(normalizeBlobKey(key));
+      } catch (error: unknown) {
+        logger.log('warn', `Skipping unnormalizable SmartBucket object key during SmartMTA blob listing: ${objectKey} (${(error as Error).message})`);
+      }
     }
     return keys.sort();
   }

package/ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '18.0.0',
+  version: '18.0.2',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }