@push.rocks/smartmta 5.1.3 → 5.2.0

package/ts/mail/core/classes.bouncemanager.ts

@@ -0,0 +1,731 @@
+import * as plugins from '../../plugins.js';
+import * as paths from '../../paths.js';
+import { logger } from '../../logger.js';
+import { SecurityLogger, SecurityLogLevel, SecurityEventType } from '../../security/index.js';
+import { RustSecurityBridge } from '../../security/classes.rustsecuritybridge.js';
+import { LRUCache } from 'lru-cache';
+import type { Email } from './classes.email.js';
+
+/**
+ * Bounce types for categorizing the reasons for bounces
+ */
+export enum BounceType {
+  // Hard bounces (permanent failures)
+  INVALID_RECIPIENT = 'invalid_recipient',
+  DOMAIN_NOT_FOUND = 'domain_not_found',
+  MAILBOX_FULL = 'mailbox_full',
+  MAILBOX_INACTIVE = 'mailbox_inactive',
+  BLOCKED = 'blocked',
+  SPAM_RELATED = 'spam_related',
+  POLICY_RELATED = 'policy_related',
+  
+  // Soft bounces (temporary failures)
+  SERVER_UNAVAILABLE = 'server_unavailable',
+  TEMPORARY_FAILURE = 'temporary_failure',
+  QUOTA_EXCEEDED = 'quota_exceeded',
+  NETWORK_ERROR = 'network_error',
+  TIMEOUT = 'timeout',
+  
+  // Special cases
+  AUTO_RESPONSE = 'auto_response',
+  CHALLENGE_RESPONSE = 'challenge_response',
+  UNKNOWN = 'unknown'
+}
+
+/**
+ * Hard vs soft bounce classification
+ */
+export enum BounceCategory {
+  HARD = 'hard',
+  SOFT = 'soft',
+  AUTO_RESPONSE = 'auto_response',
+  UNKNOWN = 'unknown'
+}
+
+/**
+ * Bounce data structure
+ */
+export interface BounceRecord {
+  id: string;
+  originalEmailId?: string;
+  recipient: string;
+  sender: string;
+  domain: string;
+  subject?: string;
+  bounceType: BounceType;
+  bounceCategory: BounceCategory;
+  timestamp: number;
+  smtpResponse?: string;
+  diagnosticCode?: string;
+  statusCode?: string;
+  headers?: Record<string, string>;
+  processed: boolean;
+  retryCount?: number;
+  nextRetryTime?: number;
+}
+
+/**
+ * Retry strategy configuration for soft bounces
+ */
+interface RetryStrategy {
+  maxRetries: number;
+  initialDelay: number; // milliseconds
+  maxDelay: number; // milliseconds
+  backoffFactor: number;
+}
+
+/**
+ * Manager for handling email bounces
+ */
+export class BounceManager {
+  // Retry strategy with exponential backoff
+  private retryStrategy: RetryStrategy = {
+    maxRetries: 5,
+    initialDelay: 15 * 60 * 1000, // 15 minutes
+    maxDelay: 24 * 60 * 60 * 1000, // 24 hours
+    backoffFactor: 2
+  };
+  
+  // Store of bounced emails
+  private bounceStore: BounceRecord[] = [];
+  
+  // Cache of recently bounced email addresses to avoid sending to known bad addresses
+  private bounceCache: LRUCache<string, {
+    lastBounce: number;
+    count: number;
+    type: BounceType;
+    category: BounceCategory;
+  }>;
+  
+  // Suppression list for addresses that should not receive emails
+  private suppressionList: Map<string, {
+    reason: string;
+    timestamp: number;
+    expiresAt?: number; // undefined means permanent
+  }> = new Map();
+  
+  private storageManager?: any; // StorageManager instance
+  
+  constructor(options?: {
+    retryStrategy?: Partial<RetryStrategy>;
+    maxCacheSize?: number;
+    cacheTTL?: number;
+    storageManager?: any;
+  }) {
+    // Set retry strategy with defaults
+    if (options?.retryStrategy) {
+      this.retryStrategy = {
+        ...this.retryStrategy,
+        ...options.retryStrategy
+      };
+    }
+    
+    // Initialize bounce cache with LRU (least recently used) caching
+    this.bounceCache = new LRUCache<string, any>({
+      max: options?.maxCacheSize || 10000,
+      ttl: options?.cacheTTL || 30 * 24 * 60 * 60 * 1000, // 30 days default
+    });
+    
+    // Store storage manager reference
+    this.storageManager = options?.storageManager;
+    
+    // Load suppression list from storage
+    // Note: This is async but we can't await in constructor
+    // The suppression list will be loaded asynchronously
+    this.loadSuppressionList().catch(error => {
+      logger.log('error', `Failed to load suppression list on startup: ${error.message}`);
+    });
+  }
+  
+  /**
+   * Process a bounce notification
+   * @param bounceData Bounce data to process
+   * @returns Processed bounce record
+   */
+  public async processBounce(bounceData: Partial<BounceRecord>): Promise<BounceRecord> {
+    try {
+      // Add required fields if missing
+      const bounce: BounceRecord = {
+        id: bounceData.id || plugins.uuid.v4(),
+        recipient: bounceData.recipient,
+        sender: bounceData.sender,
+        domain: bounceData.domain || bounceData.recipient.split('@')[1],
+        subject: bounceData.subject,
+        bounceType: bounceData.bounceType || BounceType.UNKNOWN,
+        bounceCategory: bounceData.bounceCategory || BounceCategory.UNKNOWN,
+        timestamp: bounceData.timestamp || Date.now(),
+        smtpResponse: bounceData.smtpResponse,
+        diagnosticCode: bounceData.diagnosticCode,
+        statusCode: bounceData.statusCode,
+        headers: bounceData.headers,
+        processed: false,
+        originalEmailId: bounceData.originalEmailId,
+        retryCount: bounceData.retryCount || 0,
+        nextRetryTime: bounceData.nextRetryTime
+      };
+      
+      // Determine bounce type and category via Rust bridge if not provided
+      if (!bounceData.bounceType || bounceData.bounceType === BounceType.UNKNOWN) {
+        const bridge = RustSecurityBridge.getInstance();
+        const rustResult = await bridge.detectBounce({
+          smtpResponse: bounce.smtpResponse,
+          diagnosticCode: bounce.diagnosticCode,
+          statusCode: bounce.statusCode,
+        });
+        bounce.bounceType = rustResult.bounce_type as BounceType;
+        bounce.bounceCategory = rustResult.category as BounceCategory;
+      }
+      
+      // Process the bounce based on category
+      switch (bounce.bounceCategory) {
+        case BounceCategory.HARD:
+          // Handle hard bounce - add to suppression list
+          await this.handleHardBounce(bounce);
+          break;
+          
+        case BounceCategory.SOFT:
+          // Handle soft bounce - schedule retry if eligible
+          await this.handleSoftBounce(bounce);
+          break;
+          
+        case BounceCategory.AUTO_RESPONSE:
+          // Handle auto-response - typically no action needed
+          logger.log('info', `Auto-response detected for ${bounce.recipient}`);
+          break;
+          
+        default:
+          // Unknown bounce type - log for investigation
+          logger.log('warn', `Unknown bounce type for ${bounce.recipient}`, {
+            bounceType: bounce.bounceType,
+            smtpResponse: bounce.smtpResponse
+          });
+          break;
+      }
+      
+      // Store the bounce record
+      bounce.processed = true;
+      this.bounceStore.push(bounce);
+      
+      // Update the bounce cache
+      this.updateBounceCache(bounce);
+      
+      // Log the bounce
+      logger.log(
+        bounce.bounceCategory === BounceCategory.HARD ? 'warn' : 'info',
+        `Email bounce processed: ${bounce.bounceCategory} bounce for ${bounce.recipient}`,
+        {
+          bounceType: bounce.bounceType,
+          domain: bounce.domain,
+          category: bounce.bounceCategory
+        }
+      );
+      
+      // Enhanced security logging
+      SecurityLogger.getInstance().logEvent({
+        level: bounce.bounceCategory === BounceCategory.HARD 
+          ? SecurityLogLevel.WARN 
+          : SecurityLogLevel.INFO,
+        type: SecurityEventType.EMAIL_VALIDATION,
+        message: `Email bounce detected: ${bounce.bounceCategory} bounce for recipient`,
+        domain: bounce.domain,
+        details: {
+          recipient: bounce.recipient,
+          bounceType: bounce.bounceType,
+          smtpResponse: bounce.smtpResponse,
+          diagnosticCode: bounce.diagnosticCode,
+          statusCode: bounce.statusCode
+        },
+        success: false
+      });
+      
+      return bounce;
+    } catch (error) {
+      logger.log('error', `Error processing bounce: ${error.message}`, {
+        error: error.message,
+        bounceData
+      });
+      throw error;
+    }
+  }
+  
+  /**
+   * Process an SMTP failure as a bounce
+   * @param recipient Recipient email
+   * @param smtpResponse SMTP error response
+   * @param options Additional options
+   * @returns Processed bounce record
+   */
+  public async processSmtpFailure(
+    recipient: string,
+    smtpResponse: string,
+    options: {
+      sender?: string;
+      originalEmailId?: string;
+      statusCode?: string;
+      headers?: Record<string, string>;
+    } = {}
+  ): Promise<BounceRecord> {
+    // Create bounce data from SMTP failure
+    const bounceData: Partial<BounceRecord> = {
+      recipient,
+      sender: options.sender || '',
+      domain: recipient.split('@')[1],
+      smtpResponse,
+      statusCode: options.statusCode,
+      headers: options.headers,
+      originalEmailId: options.originalEmailId,
+      timestamp: Date.now()
+    };
+    
+    // Process as a regular bounce
+    return this.processBounce(bounceData);
+  }
+  
+  /**
+   * Process a bounce notification email
+   * @param bounceEmail The email containing bounce information
+   * @returns Processed bounce record or null if not a bounce
+   */
+  public async processBounceEmail(bounceEmail: Email): Promise<BounceRecord | null> {
+    try {
+      // Check if this is a bounce notification
+      const subject = bounceEmail.getSubject();
+      const body = bounceEmail.getBody();
+      
+      // Check for common bounce notification subject patterns
+      const isBounceSubject = /mail delivery|delivery (failed|status|notification)|failure notice|returned mail|undeliverable|delivery problem/i.test(subject);
+      
+      if (!isBounceSubject) {
+        // Not a bounce notification based on subject
+        return null;
+      }
+      
+      // Extract original recipient from the body or headers
+      let recipient = '';
+      let originalMessageId = '';
+      
+      // Extract recipient from common bounce formats
+      const recipientMatch = body.match(/(?:failed recipient|to[:=]\s*|recipient:|delivery failed:)\s*<?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})>?/i);
+      if (recipientMatch && recipientMatch[1]) {
+        recipient = recipientMatch[1];
+      }
+      
+      // Extract diagnostic code
+      let diagnosticCode = '';
+      const diagnosticMatch = body.match(/diagnostic(?:-|\\s+)code:\s*(.+)(?:\n|$)/i);
+      if (diagnosticMatch && diagnosticMatch[1]) {
+        diagnosticCode = diagnosticMatch[1].trim();
+      }
+      
+      // Extract SMTP status code
+      let statusCode = '';
+      const statusMatch = body.match(/status(?:-|\\s+)code:\s*([0-9.]+)/i);
+      if (statusMatch && statusMatch[1]) {
+        statusCode = statusMatch[1].trim();
+      }
+      
+      // If recipient not found in standard patterns, try DSN (Delivery Status Notification) format
+      if (!recipient) {
+        // Look for DSN format with Original-Recipient or Final-Recipient fields
+        const originalRecipientMatch = body.match(/original-recipient:.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/i);
+        const finalRecipientMatch = body.match(/final-recipient:.*?([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/i);
+        
+        if (originalRecipientMatch && originalRecipientMatch[1]) {
+          recipient = originalRecipientMatch[1];
+        } else if (finalRecipientMatch && finalRecipientMatch[1]) {
+          recipient = finalRecipientMatch[1];
+        }
+      }
+      
+      // If still no recipient, can't process as bounce
+      if (!recipient) {
+        logger.log('warn', 'Could not extract recipient from bounce notification', {
+          subject,
+          sender: bounceEmail.from
+        });
+        return null;
+      }
+      
+      // Extract original message ID if available
+      const messageIdMatch = body.match(/original[ -]message[ -]id:[ \t]*<?([^>]+)>?/i);
+      if (messageIdMatch && messageIdMatch[1]) {
+        originalMessageId = messageIdMatch[1].trim();
+      }
+      
+      // Create bounce data
+      const bounceData: Partial<BounceRecord> = {
+        recipient,
+        sender: bounceEmail.from,
+        domain: recipient.split('@')[1],
+        subject: bounceEmail.getSubject(),
+        diagnosticCode,
+        statusCode,
+        timestamp: Date.now(),
+        headers: {}
+      };
+      
+      // Process as a regular bounce
+      return this.processBounce(bounceData);
+    } catch (error) {
+      logger.log('error', `Error processing bounce email: ${error.message}`);
+      return null;
+    }
+  }
+  
+  /**
+   * Handle a hard bounce by adding to suppression list
+   * @param bounce The bounce record
+   */
+  private async handleHardBounce(bounce: BounceRecord): Promise<void> {
+    // Add to suppression list permanently (no expiry)
+    this.addToSuppressionList(bounce.recipient, `Hard bounce: ${bounce.bounceType}`, undefined);
+    
+    // Increment bounce count in cache
+    this.updateBounceCache(bounce);
+    
+    // Save to permanent storage
+    await this.saveBounceRecord(bounce);
+    
+    // Log hard bounce for monitoring
+    logger.log('warn', `Hard bounce for ${bounce.recipient}: ${bounce.bounceType}`, {
+      domain: bounce.domain,
+      smtpResponse: bounce.smtpResponse,
+      diagnosticCode: bounce.diagnosticCode
+    });
+  }
+  
+  /**
+   * Handle a soft bounce by scheduling a retry if eligible
+   * @param bounce The bounce record
+   */
+  private async handleSoftBounce(bounce: BounceRecord): Promise<void> {
+    // Check if we've exceeded max retries
+    if (bounce.retryCount >= this.retryStrategy.maxRetries) {
+      logger.log('warn', `Max retries exceeded for ${bounce.recipient}, treating as hard bounce`);
+      
+      // Convert to hard bounce after max retries
+      bounce.bounceCategory = BounceCategory.HARD;
+      await this.handleHardBounce(bounce);
+      return;
+    }
+    
+    // Calculate next retry time with exponential backoff
+    const delay = Math.min(
+      this.retryStrategy.initialDelay * Math.pow(this.retryStrategy.backoffFactor, bounce.retryCount),
+      this.retryStrategy.maxDelay
+    );
+    
+    bounce.retryCount++;
+    bounce.nextRetryTime = Date.now() + delay;
+    
+    // Add to suppression list temporarily (with expiry)
+    this.addToSuppressionList(
+      bounce.recipient,
+      `Soft bounce: ${bounce.bounceType}`,
+      bounce.nextRetryTime
+    );
+    
+    // Log the retry schedule
+    logger.log('info', `Scheduled retry ${bounce.retryCount} for ${bounce.recipient} at ${new Date(bounce.nextRetryTime).toISOString()}`, {
+      bounceType: bounce.bounceType,
+      retryCount: bounce.retryCount,
+      nextRetry: bounce.nextRetryTime
+    });
+  }
+  
+  /**
+   * Add an email address to the suppression list
+   * @param email Email address to suppress
+   * @param reason Reason for suppression
+   * @param expiresAt Expiration timestamp (undefined for permanent)
+   */
+  public addToSuppressionList(
+    email: string,
+    reason: string,
+    expiresAt?: number
+  ): void {
+    this.suppressionList.set(email.toLowerCase(), {
+      reason,
+      timestamp: Date.now(),
+      expiresAt
+    });
+    
+    // Save asynchronously without blocking
+    this.saveSuppressionList().catch(error => {
+      logger.log('error', `Failed to save suppression list after adding ${email}: ${error.message}`);
+    });
+    
+    logger.log('info', `Added ${email} to suppression list`, {
+      reason,
+      expiresAt: expiresAt ? new Date(expiresAt).toISOString() : 'permanent'
+    });
+  }
+  
+  /**
+   * Remove an email address from the suppression list
+   * @param email Email address to remove
+   */
+  public removeFromSuppressionList(email: string): void {
+    const wasRemoved = this.suppressionList.delete(email.toLowerCase());
+    
+    if (wasRemoved) {
+      // Save asynchronously without blocking
+      this.saveSuppressionList().catch(error => {
+        logger.log('error', `Failed to save suppression list after removing ${email}: ${error.message}`);
+      });
+      logger.log('info', `Removed ${email} from suppression list`);
+    }
+  }
+  
+  /**
+   * Check if an email is on the suppression list
+   * @param email Email address to check
+   * @returns Whether the email is suppressed
+   */
+  public isEmailSuppressed(email: string): boolean {
+    const lowercaseEmail = email.toLowerCase();
+    const suppression = this.suppressionList.get(lowercaseEmail);
+    
+    if (!suppression) {
+      return false;
+    }
+    
+    // Check if suppression has expired
+    if (suppression.expiresAt && Date.now() > suppression.expiresAt) {
+      this.suppressionList.delete(lowercaseEmail);
+      // Save asynchronously without blocking
+      this.saveSuppressionList().catch(error => {
+        logger.log('error', `Failed to save suppression list after expiry cleanup: ${error.message}`);
+      });
+      return false;
+    }
+    
+    return true;
+  }
+  
+  /**
+   * Get suppression information for an email
+   * @param email Email address to check
+   * @returns Suppression information or null if not suppressed
+   */
+  public getSuppressionInfo(email: string): {
+    reason: string;
+    timestamp: number;
+    expiresAt?: number;
+  } | null {
+    const lowercaseEmail = email.toLowerCase();
+    const suppression = this.suppressionList.get(lowercaseEmail);
+    
+    if (!suppression) {
+      return null;
+    }
+    
+    // Check if suppression has expired
+    if (suppression.expiresAt && Date.now() > suppression.expiresAt) {
+      this.suppressionList.delete(lowercaseEmail);
+      // Save asynchronously without blocking
+      this.saveSuppressionList().catch(error => {
+        logger.log('error', `Failed to save suppression list after expiry cleanup: ${error.message}`);
+      });
+      return null;
+    }
+    
+    return suppression;
+  }
+  
+  /**
+   * Save suppression list to disk
+   */
+  private async saveSuppressionList(): Promise<void> {
+    try {
+      const suppressionData = JSON.stringify(Array.from(this.suppressionList.entries()));
+      
+      if (this.storageManager) {
+        // Use storage manager
+        await this.storageManager.set('/email/bounces/suppression-list.json', suppressionData);
+      } else {
+        // Fall back to filesystem
+        await plugins.smartfs.file(
+          plugins.path.join(paths.dataDir, 'emails', 'suppression_list.json')
+        ).write(suppressionData);
+      }
+    } catch (error) {
+      logger.log('error', `Failed to save suppression list: ${error.message}`);
+    }
+  }
+  
+  /**
+   * Load suppression list from disk
+   */
+  private async loadSuppressionList(): Promise<void> {
+    try {
+      let entries = null;
+      let needsMigration = false;
+      
+      if (this.storageManager) {
+        // Try to load from storage manager first
+        const suppressionData = await this.storageManager.get('/email/bounces/suppression-list.json');
+        
+        if (suppressionData) {
+          entries = JSON.parse(suppressionData);
+        } else {
+          // Check if data exists in filesystem and migrate
+          const suppressionPath = plugins.path.join(paths.dataDir, 'emails', 'suppression_list.json');
+          
+          if (plugins.fs.existsSync(suppressionPath)) {
+            const data = plugins.fs.readFileSync(suppressionPath, 'utf8');
+            entries = JSON.parse(data);
+            needsMigration = true;
+            
+            logger.log('info', 'Migrating suppression list from filesystem to StorageManager');
+          }
+        }
+      } else {
+        // No storage manager, use filesystem directly
+        const suppressionPath = plugins.path.join(paths.dataDir, 'emails', 'suppression_list.json');
+        
+        if (plugins.fs.existsSync(suppressionPath)) {
+          const data = plugins.fs.readFileSync(suppressionPath, 'utf8');
+          entries = JSON.parse(data);
+        }
+      }
+      
+      if (entries) {
+        this.suppressionList = new Map(entries);
+        
+        // Clean expired entries
+        const now = Date.now();
+        let expiredCount = 0;
+        
+        for (const [email, info] of this.suppressionList.entries()) {
+          if (info.expiresAt && now > info.expiresAt) {
+            this.suppressionList.delete(email);
+            expiredCount++;
+          }
+        }
+        
+        if (expiredCount > 0 || needsMigration) {
+          logger.log('info', `Cleaned ${expiredCount} expired entries from suppression list`);
+          await this.saveSuppressionList();
+        }
+        
+        logger.log('info', `Loaded ${this.suppressionList.size} entries from suppression list`);
+      }
+    } catch (error) {
+      logger.log('error', `Failed to load suppression list: ${error.message}`);
+    }
+  }
+  
+  /**
+   * Save bounce record to disk
+   * @param bounce Bounce record to save
+   */
+  private async saveBounceRecord(bounce: BounceRecord): Promise<void> {
+    try {
+      const bounceData = JSON.stringify(bounce, null, 2);
+      
+      if (this.storageManager) {
+        // Use storage manager
+        await this.storageManager.set(`/email/bounces/records/${bounce.id}.json`, bounceData);
+      } else {
+        // Fall back to filesystem
+        const bouncePath = plugins.path.join(
+          paths.dataDir,
+          'emails',
+          'bounces',
+          `${bounce.id}.json`
+        );
+        
+        // Ensure directory exists
+        const bounceDir = plugins.path.join(paths.dataDir, 'emails', 'bounces');
+        await plugins.smartfs.directory(bounceDir).recursive().create();
+
+        await plugins.smartfs.file(bouncePath).write(bounceData);
+      }
+    } catch (error) {
+      logger.log('error', `Failed to save bounce record: ${error.message}`);
+    }
+  }
+  
+  /**
+   * Update bounce cache with new bounce information
+   * @param bounce Bounce record to update cache with
+   */
+  private updateBounceCache(bounce: BounceRecord): void {
+    const email = bounce.recipient.toLowerCase();
+    const existing = this.bounceCache.get(email);
+    
+    if (existing) {
+      // Update existing cache entry
+      existing.lastBounce = bounce.timestamp;
+      existing.count++;
+      existing.type = bounce.bounceType;
+      existing.category = bounce.bounceCategory;
+    } else {
+      // Create new cache entry
+      this.bounceCache.set(email, {
+        lastBounce: bounce.timestamp,
+        count: 1,
+        type: bounce.bounceType,
+        category: bounce.bounceCategory
+      });
+    }
+  }
+  
+  /**
+   * Check bounce history for an email address
+   * @param email Email address to check
+   * @returns Bounce information or null if no bounces
+   */
+  public getBounceInfo(email: string): {
+    lastBounce: number;
+    count: number;
+    type: BounceType;
+    category: BounceCategory;
+  } | null {
+    return this.bounceCache.get(email.toLowerCase()) || null;
+  }
+  
+  /**
+   * Get all known hard bounced addresses
+   * @returns Array of hard bounced email addresses
+   */
+  public getHardBouncedAddresses(): string[] {
+    const hardBounced: string[] = [];
+    
+    for (const [email, info] of this.bounceCache.entries()) {
+      if (info.category === BounceCategory.HARD) {
+        hardBounced.push(email);
+      }
+    }
+    
+    return hardBounced;
+  }
+  
+  /**
+   * Get suppression list
+   * @returns Array of suppressed email addresses
+   */
+  public getSuppressionList(): string[] {
+    return Array.from(this.suppressionList.keys());
+  }
+  
+  /**
+   * Clear old bounce records (for maintenance)
+   * @param olderThan Timestamp to remove records older than
+   * @returns Number of records removed
+   */
+  public clearOldBounceRecords(olderThan: number): number {
+    let removed = 0;
+    
+    this.bounceStore = this.bounceStore.filter(bounce => {
+      if (bounce.timestamp < olderThan) {
+        removed++;
+        return false;
+      }
+      return true;
+    });
+    
+    return removed;
+  }
+}