@push.rocks/smartmta 6.5.1 → 7.0.0

package/ts/mail/core/classes.templatemanager.ts

@@ -1,7 +1,6 @@
-import * as plugins from '../../plugins.js';
-import * as paths from '../../paths.js';
 import { logger } from '../../logger.js';
 import { Email, type IEmailOptions, type IAttachment } from './classes.email.js';
+import type { IBlobStorageManager, IStorageManager } from '../interfaces.storage.js';
 
 /**
  * Email template type definition
@@ -18,7 +17,8 @@ export interface IEmailTemplate<T = any> {
   sampleData?: T;
   attachments?: Array<{
     name: string;
-    path: string;
+    contentBase64?: string;
+    storageKey?: string;
     contentType?: string;
   }>;
 }
@@ -45,6 +45,8 @@ export enum TemplateCategory {
  */
 export class TemplateManager {
   private templates: Map<string, IEmailTemplate> = new Map();
+  private readonly templateStorage?: IStorageManager;
+  private readonly attachmentStorage?: IBlobStorageManager;
   private defaultConfig: {
     from: string;
     replyTo?: string;
@@ -57,7 +59,12 @@ export class TemplateManager {
     replyTo?: string;
     footerHtml?: string;
     footerText?: string;
+  }, storage?: {
+    templateStorage?: IStorageManager;
+    attachmentStorage?: IBlobStorageManager;
   }) {
+    this.templateStorage = storage?.templateStorage;
+    this.attachmentStorage = storage?.attachmentStorage;
     // Set default configuration
     this.defaultConfig = {
       from: defaultConfig?.from || 'noreply@mail.lossless.com',
@@ -212,25 +219,26 @@ export class TemplateManager {
     
     if (template.attachments && template.attachments.length > 0) {
       for (const attachment of template.attachments) {
-        try {
-          const attachmentPath = plugins.path.isAbsolute(attachment.path) 
-            ? attachment.path 
-            : plugins.path.join(paths.MtaAttachmentsDir, attachment.path);
-            
-          // Read the file
-          const fileBuffer = await plugins.fs.promises.readFile(attachmentPath);
-          
-          attachments.push({
-            filename: attachment.name,
-            content: fileBuffer,
-            contentType: attachment.contentType || 'application/octet-stream'
-          });
-        } catch (error) {
-          logger.log('error', `Failed to add attachment '${attachment.name}': ${error.message}`);
+        let content: Buffer | null = attachment.contentBase64
+          ? Buffer.from(attachment.contentBase64, 'base64')
+          : null;
+        if (!content && attachment.storageKey) {
+          if (!this.attachmentStorage) {
+            throw new Error(`Template attachment ${attachment.name} requires blob storage`);
+          }
+          content = await this.attachmentStorage.get(attachment.storageKey);
+        }
+        if (!content) {
+          throw new Error(`Template attachment ${attachment.name} has no content`);
         }
+        attachments.push({
+          filename: attachment.name,
+          content,
+          contentType: attachment.contentType || 'application/octet-stream',
+        });
       }
     }
-    
+
     // Create Email instance with template content
     const emailOptions: IEmailOptions = {
       from: template.from || this.defaultConfig.from,
@@ -277,44 +285,21 @@ export class TemplateManager {
   }
   
   
-  /**
-   * Load templates from a directory
-   * @param directory The directory containing template JSON files
-   */
-  public async loadTemplatesFromDirectory(directory: string): Promise<void> {
-    try {
-      // Ensure directory exists
-      if (!plugins.fs.existsSync(directory)) {
-        logger.log('error', `Template directory does not exist: ${directory}`);
-        return;
-      }
-      
-      // Get all JSON files
-      const files = plugins.fs.readdirSync(directory)
-        .filter(file => file.endsWith('.json'));
-      
-      for (const file of files) {
-        try {
-          const filePath = plugins.path.join(directory, file);
-          const content = plugins.fs.readFileSync(filePath, 'utf8');
-          const template = JSON.parse(content) as IEmailTemplate;
-          
-          // Validate template
-          if (!template.id || !template.subject || (!template.bodyHtml && !template.bodyText)) {
-            logger.log('warn', `Invalid template in ${file}: missing required fields`);
-            continue;
-          }
-          
-          this.registerTemplate(template);
-        } catch (error) {
-          logger.log('error', `Error loading template from ${file}: ${error.message}`);
-        }
+  /** Load JSON templates from the configured managed text store. */
+  public async loadTemplatesFromStorage(prefix = '/email/templates/'): Promise<void> {
+    if (!this.templateStorage) {
+      throw new Error('Template storage is not configured');
+    }
+    const keys = await this.templateStorage.list(prefix);
+    for (const key of keys.filter((candidate) => candidate.endsWith('.json'))) {
+      const content = await this.templateStorage.get(key);
+      if (!content) continue;
+      const template = JSON.parse(content) as IEmailTemplate;
+      if (!template.id || !template.subject || (!template.bodyHtml && !template.bodyText)) {
+        throw new Error(`Invalid template in ${key}: missing required fields`);
       }
-      
-      logger.log('info', `Loaded ${this.templates.size} email templates`);
-    } catch (error) {
-      logger.log('error', `Failed to load templates from directory: ${error.message}`);
-      throw error;
+      this.registerTemplate(template);
     }
   }
-}
+
+}

package/ts/mail/delivery/classes.delivery.queue.ts

@@ -1,10 +1,10 @@
-import * as plugins from '../../plugins.js';
 import { EventEmitter } from 'node:events';
-import * as fs from 'node:fs';
-import * as path from 'node:path';
+import { deserialize, serialize } from 'node:v8';
+import type { IBlobStorageManager } from '../interfaces.storage.js';
 import { logger } from '../../logger.js';
 import { type EmailProcessingMode } from './interfaces.js';
 import type { IEmailRoute } from '../routing/interfaces.js';
+import { Email, type IEmailOptions } from '../core/classes.email.js';
 
 /**
  * Queue item status
@@ -17,7 +17,7 @@ export type QueueItemStatus = 'pending' | 'processing' | 'delivered' | 'failed'
 export interface IQueueItem {
   id: string;
   processingMode: EmailProcessingMode;
-  processingResult: any;
+  processingResult: Email;
   route?: IEmailRoute;
   status: QueueItemStatus;
   attempts: number;
@@ -28,13 +28,36 @@ export interface IQueueItem {
   deliveredAt?: Date;
 }
 
+interface IStoredEmail {
+  options: IEmailOptions;
+  messageId: string;
+  envelopeFrom: string;
+}
+
+interface IStoredQueueItem {
+  schemaVersion: 1;
+  id: string;
+  processingMode: EmailProcessingMode;
+  processingResult: IStoredEmail;
+  route?: IEmailRoute;
+  status: QueueItemStatus;
+  attempts: number;
+  nextAttempt: number;
+  lastError?: string;
+  createdAt: number;
+  updatedAt: number;
+  deliveredAt?: number;
+}
+
 /**
  * Queue options interface
  */
 export interface IQueueOptions {
-  // Storage options
-  storageType?: 'memory' | 'disk';
-  persistentPath?: string;
+  /** Persistence is always an explicit caller choice. */
+  storageMode: 'memory' | 'managed';
+  /** Required when storageMode is managed. */
+  storageManager?: IBlobStorageManager;
+  storagePrefix?: string;
   
   // Queue behavior
   checkInterval?: number;
@@ -50,6 +73,22 @@ export interface IQueueOptions {
 /**
  * Queue statistics interface
  */
+function normalizeStoragePrefix(prefixArg = '/email/queue/'): string {
+  if (
+    !prefixArg.startsWith('/') ||
+    !prefixArg.endsWith('/') ||
+    prefixArg.includes('\\') ||
+    prefixArg.includes('\0')
+  ) {
+    throw new Error('Queue storagePrefix must be an absolute managed-storage prefix');
+  }
+  const segments = prefixArg.slice(1, -1).split('/');
+  if (segments.some((segment) => !segment || segment === '.' || segment === '..')) {
+    throw new Error('Queue storagePrefix contains an invalid path segment');
+  }
+  return prefixArg;
+}
+
 export interface IQueueStats {
   queueSize: number;
   status: {
@@ -75,7 +114,17 @@ export interface IQueueStats {
  * A unified queue for all email modes
  */
 export class UnifiedDeliveryQueue extends EventEmitter {
-  private options: Required<IQueueOptions>;
+  private options: {
+    storageMode: 'memory' | 'managed';
+    storageManager?: IBlobStorageManager;
+    storagePrefix: string;
+    checkInterval: number;
+    maxQueueSize: number;
+    maxPerDestination: number;
+    maxRetries: number;
+    baseRetryDelay: number;
+    maxRetryDelay: number;
+  };
   private queue: Map<string, IQueueItem> = new Map();
   private checkTimer?: NodeJS.Timeout;
   private cleanupTimer?: NodeJS.Timeout;
@@ -89,11 +138,18 @@ export class UnifiedDeliveryQueue extends EventEmitter {
    */
   constructor(options: IQueueOptions) {
     super();
-    
-    // Set default options
+
+    if (options.storageMode === 'managed' && !options.storageManager) {
+      throw new Error('Managed queue storage requires an IBlobStorageManager');
+    }
+    if (options.storageMode === 'memory' && options.storageManager) {
+      throw new Error('Memory-only queue storage cannot receive an IBlobStorageManager');
+    }
+
     this.options = {
-      storageType: options.storageType || 'memory',
-      persistentPath: options.persistentPath || path.join(process.cwd(), 'email-queue'),
+      storageMode: options.storageMode,
+      storageManager: options.storageManager,
+      storagePrefix: normalizeStoragePrefix(options.storagePrefix),
       checkInterval: options.checkInterval || 30000, // 30 seconds
       maxQueueSize: options.maxQueueSize || 10000,
       maxPerDestination: options.maxPerDestination || 100,
@@ -130,14 +186,8 @@ export class UnifiedDeliveryQueue extends EventEmitter {
     logger.log('info', 'Initializing UnifiedDeliveryQueue');
     
     try {
-      // Create persistent storage directory if using disk storage
-      if (this.options.storageType === 'disk') {
-        if (!fs.existsSync(this.options.persistentPath)) {
-          fs.mkdirSync(this.options.persistentPath, { recursive: true });
-        }
-        
-        // Load existing items from disk
-        await this.loadFromDisk();
+      if (this.options.storageManager) {
+        await this.loadFromStorage();
       }
       
       // Start the queue processing timer
@@ -237,7 +287,15 @@ export class UnifiedDeliveryQueue extends EventEmitter {
    * @param mode Processing mode
    * @param route Email route
    */
-  public async enqueue(processingResult: any, mode: EmailProcessingMode, route?: IEmailRoute): Promise<string> {
+  public async enqueue(
+    processingResult: Email,
+    mode: EmailProcessingMode,
+    route?: IEmailRoute,
+  ): Promise<string> {
+    if (!(processingResult instanceof Email)) {
+      throw new Error('Queue processingResult must be an Email instance');
+    }
+
     // Check if queue is full
     if (this.queue.size >= this.options.maxQueueSize) {
       throw new Error('Queue is full');
@@ -259,14 +317,10 @@ export class UnifiedDeliveryQueue extends EventEmitter {
       updatedAt: new Date()
     };
     
-    // Add to queue
+    // Commit durable state before exposing the item in memory or emitting acceptance.
+    await this.persistItem(item);
     this.queue.set(id, item);
     
-    // Persist to disk if using disk storage
-    if (this.options.storageType === 'disk') {
-      await this.persistItem(item);
-    }
-    
     // Update statistics
     this.updateStats();
     
@@ -300,15 +354,11 @@ export class UnifiedDeliveryQueue extends EventEmitter {
       return false;
     }
     
-    // Update status
-    item.status = 'processing';
-    item.attempts++;
-    item.updatedAt = new Date();
-    
-    // Persist changes if using disk storage
-    if (this.options.storageType === 'disk') {
-      await this.persistItem(item);
-    }
+    await this.applyPersistedMutation(item, (updatedItem) => {
+      updatedItem.status = 'processing';
+      updatedItem.attempts++;
+      updatedItem.updatedAt = new Date();
+    });
     
     // Update statistics
     this.updateStats();
@@ -331,15 +381,11 @@ export class UnifiedDeliveryQueue extends EventEmitter {
       return false;
     }
     
-    // Update status
-    item.status = 'delivered';
-    item.updatedAt = new Date();
-    item.deliveredAt = new Date();
-    
-    // Persist changes if using disk storage
-    if (this.options.storageType === 'disk') {
-      await this.persistItem(item);
-    }
+    await this.applyPersistedMutation(item, (updatedItem) => {
+      updatedItem.status = 'delivered';
+      updatedItem.updatedAt = new Date();
+      updatedItem.deliveredAt = new Date();
+    });
     
     // Update statistics
     this.totalProcessed++;
@@ -372,30 +418,23 @@ export class UnifiedDeliveryQueue extends EventEmitter {
         this.options.maxRetryDelay
       );
       
-      // Update status
-      item.status = 'deferred';
-      item.lastError = error;
-      item.nextAttempt = new Date(Date.now() + delay);
-      item.updatedAt = new Date();
-      
-      // Persist changes if using disk storage
-      if (this.options.storageType === 'disk') {
-        await this.persistItem(item);
-      }
+      await this.applyPersistedMutation(item, (updatedItem) => {
+        updatedItem.status = 'deferred';
+        updatedItem.lastError = error;
+        updatedItem.nextAttempt = new Date(Date.now() + delay);
+        updatedItem.updatedAt = new Date();
+      });
       
       // Emit event
       this.emit('itemDeferred', item);
       logger.log('info', `Item ${id} deferred for ${delay}ms, attempt ${item.attempts}, error: ${error}`);
     } else {
       // Mark as permanently failed
-      item.status = 'failed';
-      item.lastError = error;
-      item.updatedAt = new Date();
-      
-      // Persist changes if using disk storage
-      if (this.options.storageType === 'disk') {
-        await this.persistItem(item);
-      }
+      await this.applyPersistedMutation(item, (updatedItem) => {
+        updatedItem.status = 'failed';
+        updatedItem.lastError = error;
+        updatedItem.updatedAt = new Date();
+      });
       
       // Update statistics
       this.totalProcessed++;
@@ -422,14 +461,10 @@ export class UnifiedDeliveryQueue extends EventEmitter {
       return false;
     }
     
-    // Remove from queue
+    // Delete durable state before making the removal visible in memory.
+    await this.removeItemFromStorage(id);
     this.queue.delete(id);
     
-    // Remove from disk if using disk storage
-    if (this.options.storageType === 'disk') {
-      await this.removeItemFromDisk(id);
-    }
-    
     // Update statistics
     this.updateStats();
     
@@ -440,82 +475,176 @@ export class UnifiedDeliveryQueue extends EventEmitter {
     return true;
   }
   
-  /**
-   * Persist an item to disk
-   * @param item Item to persist
-   */
+  private storageKey(id: string): string {
+    return `${this.options.storagePrefix}${id}.json`;
+  }
+
   private async persistItem(item: IQueueItem): Promise<void> {
+    if (!this.options.storageManager) return;
+    await this.options.storageManager.set(
+      this.storageKey(item.id),
+      serialize(this.serializeItem(item)),
+    );
+  }
+
+  private serializeItem(item: IQueueItem): IStoredQueueItem {
+    return {
+      schemaVersion: 1,
+      id: item.id,
+      processingMode: item.processingMode,
+      processingResult: {
+        options: item.processingResult.toEmailOptions(),
+        messageId: item.processingResult.getMessageId(),
+        envelopeFrom: item.processingResult.getEnvelopeFrom(),
+      },
+      ...(item.route ? { route: item.route } : {}),
+      status: item.status,
+      attempts: item.attempts,
+      nextAttempt: item.nextAttempt.getTime(),
+      ...(item.lastError !== undefined ? { lastError: item.lastError } : {}),
+      createdAt: item.createdAt.getTime(),
+      updatedAt: item.updatedAt.getTime(),
+      ...(item.deliveredAt ? { deliveredAt: item.deliveredAt.getTime() } : {}),
+    };
+  }
+
+  private async applyPersistedMutation(
+    item: IQueueItem,
+    mutation: (updatedItem: IQueueItem) => void,
+  ): Promise<void> {
+    const updatedItem = { ...item };
+    mutation(updatedItem);
+    await this.persistItem(updatedItem);
+    Object.assign(item, updatedItem);
+  }
+
+  private async removeItemFromStorage(id: string): Promise<void> {
+    await this.options.storageManager?.delete(this.storageKey(id));
+  }
+
+  private deserializeItem(key: string, data: Buffer): IQueueItem {
+    let parsed: Record<string, any>;
     try {
-      const filePath = path.join(this.options.persistentPath, `${item.id}.json`);
-      await fs.promises.writeFile(filePath, JSON.stringify(item, null, 2), 'utf8');
+      parsed = deserialize(data) as Record<string, any>;
     } catch (error) {
-      logger.log('error', `Failed to persist item ${item.id}: ${error.message}`);
-      this.emit('error', error);
+      throw new Error(`Invalid managed queue item at ${key}: ${error.message}`);
     }
-  }
-  
-  /**
-   * Remove an item from disk
-   * @param id Item ID
-   */
-  private async removeItemFromDisk(id: string): Promise<void> {
-    try {
-      const filePath = path.join(this.options.persistentPath, `${id}.json`);
-      
-      if (fs.existsSync(filePath)) {
-        await fs.promises.unlink(filePath);
+    if (!parsed || typeof parsed !== 'object' || typeof parsed.id !== 'string' || !parsed.id) {
+      throw new Error(`Invalid managed queue item at ${key}: missing id`);
+    }
+    if (parsed.schemaVersion !== 1) {
+      throw new Error(`Invalid managed queue item at ${key}: unsupported schema version`);
+    }
+    if (this.storageKey(parsed.id) !== key) {
+      throw new Error(`Invalid managed queue item at ${key}: id does not match object key`);
+    }
+    if (!['pending', 'processing', 'delivered', 'failed', 'deferred'].includes(parsed.status)) {
+      throw new Error(`Invalid managed queue item at ${key}: unsupported status`);
+    }
+    if (!['forward', 'mta', 'process'].includes(parsed.processingMode)) {
+      throw new Error(`Invalid managed queue item at ${key}: unsupported processing mode`);
+    }
+    if (!Number.isInteger(parsed.attempts) || parsed.attempts < 0) {
+      throw new Error(`Invalid managed queue item at ${key}: invalid attempt count`);
+    }
+
+    const parseDate = (value: unknown, field: string): Date => {
+      if (!Number.isFinite(value) || (value as number) < 0) {
+        throw new Error(`Invalid managed queue item at ${key}: invalid ${field}`);
       }
-    } catch (error) {
-      logger.log('error', `Failed to remove item ${id} from disk: ${error.message}`);
-      this.emit('error', error);
+      return new Date(value as number);
+    };
+
+    if (
+      !parsed.processingResult ||
+      typeof parsed.processingResult !== 'object' ||
+      !parsed.processingResult.options ||
+      typeof parsed.processingResult.options !== 'object' ||
+      typeof parsed.processingResult.messageId !== 'string' ||
+      !parsed.processingResult.messageId ||
+      typeof parsed.processingResult.envelopeFrom !== 'string'
+    ) {
+      throw new Error(`Invalid managed queue item at ${key}: invalid email payload`);
     }
-  }
-  
-  /**
-   * Load queue items from disk
-   */
-  private async loadFromDisk(): Promise<void> {
-    try {
-      // Check if directory exists
-      if (!fs.existsSync(this.options.persistentPath)) {
-        return;
+
+    const storedOptions = parsed.processingResult.options as IEmailOptions;
+    if (storedOptions.attachments !== undefined) {
+      if (!Array.isArray(storedOptions.attachments)) {
+        throw new Error(`Invalid managed queue item at ${key}: invalid attachments`);
       }
-      
-      // Get all JSON files
-      const files = fs.readdirSync(this.options.persistentPath).filter(file => file.endsWith('.json'));
-      
-      // Load each file
-      for (const file of files) {
-        try {
-          const filePath = path.join(this.options.persistentPath, file);
-          const data = await fs.promises.readFile(filePath, 'utf8');
-          const item = JSON.parse(data) as IQueueItem;
-          
-          // Convert date strings to Date objects
-          item.createdAt = new Date(item.createdAt);
-          item.updatedAt = new Date(item.updatedAt);
-          item.nextAttempt = new Date(item.nextAttempt);
-          if (item.deliveredAt) {
-            item.deliveredAt = new Date(item.deliveredAt);
-          }
-          
-          // Add to queue
-          this.queue.set(item.id, item);
-        } catch (error) {
-          logger.log('error', `Failed to load item from ${file}: ${error.message}`);
+      for (const attachment of storedOptions.attachments) {
+        if (!attachment || typeof attachment !== 'object' || !Buffer.isBuffer(attachment.content)) {
+          throw new Error(`Invalid managed queue item at ${key}: invalid attachment content`);
         }
       }
-      
-      // Update statistics
-      this.updateStats();
-      
-      logger.log('info', `Loaded ${this.queue.size} items from disk`);
+    }
+
+    let processingResult: Email;
+    try {
+      processingResult = new Email(storedOptions)
+        .setMessageId(parsed.processingResult.messageId)
+        .setEnvelopeFrom(parsed.processingResult.envelopeFrom);
     } catch (error) {
-      logger.log('error', `Failed to load items from disk: ${error.message}`);
-      throw error;
+      throw new Error(`Invalid managed queue item at ${key}: ${error.message}`);
     }
+
+    return {
+      id: parsed.id,
+      processingMode: parsed.processingMode,
+      processingResult,
+      ...(parsed.route ? { route: parsed.route as IEmailRoute } : {}),
+      status: parsed.status,
+      attempts: parsed.attempts,
+      ...(typeof parsed.lastError === 'string' ? { lastError: parsed.lastError } : {}),
+      createdAt: parseDate(parsed.createdAt, 'createdAt'),
+      updatedAt: parseDate(parsed.updatedAt, 'updatedAt'),
+      nextAttempt: parseDate(parsed.nextAttempt, 'nextAttempt'),
+      ...(parsed.deliveredAt
+        ? { deliveredAt: parseDate(parsed.deliveredAt, 'deliveredAt') }
+        : {}),
+    } as IQueueItem;
   }
-  
+
+  private async loadFromStorage(): Promise<void> {
+    const storageManager = this.options.storageManager;
+    if (!storageManager) return;
+    const keys = (await storageManager.list(this.options.storagePrefix)).sort();
+    const recoveredItems: IQueueItem[] = [];
+
+    for (const key of keys) {
+      if (!key.startsWith(this.options.storagePrefix)) {
+        throw new Error(`Blob storage returned a queue key outside ${this.options.storagePrefix}: ${key}`);
+      }
+      if (!key.endsWith('.json')) continue;
+      const data = await storageManager.get(key);
+      if (!data) {
+        throw new Error(`Managed queue item disappeared while loading: ${key}`);
+      }
+      const item = this.deserializeItem(key, data);
+      if (this.queue.has(item.id)) {
+        throw new Error(`Duplicate managed queue item id: ${item.id}`);
+      }
+      if (this.queue.size >= this.options.maxQueueSize) {
+        throw new Error(`Managed queue contains more than ${this.options.maxQueueSize} items`);
+      }
+      if (item.status === 'processing') {
+        item.status = 'deferred';
+        item.nextAttempt = new Date();
+        item.updatedAt = new Date();
+        item.lastError = 'Recovered after process restart during delivery';
+        recoveredItems.push(item);
+      }
+      this.queue.set(item.id, item);
+    }
+
+    await Promise.all(recoveredItems.map((item) => this.persistItem(item)));
+    this.updateStats();
+    logger.log(
+      'info',
+      `Loaded ${this.queue.size} items from managed blob storage; recovered ${recoveredItems.length} interrupted item(s)`,
+    );
+  }
+
   /**
    * Update queue statistics
    */
@@ -639,8 +768,8 @@ export class UnifiedDeliveryQueue extends EventEmitter {
     // Stop processing (clears both check and cleanup timers)
     this.stopProcessing();
     
-    // If using disk storage, make sure all items are persisted
-    if (this.options.storageType === 'disk') {
+    // Persist all items when durable blob storage is configured
+    if (this.options.storageManager) {
       const pendingWrites: Promise<void>[] = [];
       
       for (const item of this.queue.values()) {