@serve.zone/dcrouter 16.5.0 → 17.1.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@serve.zone/dcrouter",
   "private": false,
-  "version": "16.5.0",
+  "version": "17.1.0",
   "description": "A multifaceted routing service handling mail and SMS delivery functions.",
   "type": "module",
   "bin": {
@@ -27,47 +27,48 @@
   "dependencies": {
     "@api.global/typedrequest": "^3.3.2",
     "@api.global/typedrequest-interfaces": "^3.0.19",
-    "@api.global/typedserver": "^8.5.3",
-    "@api.global/typedsocket": "^4.1.4",
+    "@api.global/typedserver": "^8.6.4",
+    "@api.global/typedsocket": "^5.0.0",
     "@apiclient.xyz/cloudflare": "^7.1.0",
     "@design.estate/dees-catalog": "^3.96.3",
-    "@design.estate/dees-element": "^2.2.4",
-    "@idp.global/sdk": "^1.4.0",
+    "@design.estate/dees-element": "^2.4.0",
+    "@idp.global/sdk": "^14.3.0",
     "@push.rocks/lik": "^6.4.1",
     "@push.rocks/projectinfo": "^5.1.0",
     "@push.rocks/qenv": "^6.1.4",
     "@push.rocks/smartacme": "^9.5.0",
+    "@push.rocks/smartbucket": "^4.6.1",
     "@push.rocks/smartchallenge": "^1.1.0",
-    "@push.rocks/smartdata": "^7.2.0",
+    "@push.rocks/smartdata": "^7.3.1",
     "@push.rocks/smartdb": "^2.12.3",
-    "@push.rocks/smartdns": "^7.11.3",
-    "@push.rocks/smartfs": "^1.5.1",
-    "@push.rocks/smartguard": "^3.1.0",
-    "@push.rocks/smartjwt": "^2.2.2",
+    "@push.rocks/smartdns": "^7.12.0",
+    "@push.rocks/smartfs": "^1.6.0",
+    "@push.rocks/smartguard": "^3.1.2",
+    "@push.rocks/smartjwt": "^2.5.1",
     "@push.rocks/smartlog": "^3.2.2",
     "@push.rocks/smartmcp": "^0.2.0",
     "@push.rocks/smartmetrics": "^3.0.3",
     "@push.rocks/smartmigration": "1.4.1",
-    "@push.rocks/smartmta": "^6.5.2",
+    "@push.rocks/smartmta": "^8.0.1",
     "@push.rocks/smartnetwork": "^4.8.0",
     "@push.rocks/smartpath": "^6.0.0",
     "@push.rocks/smartpromise": "^4.2.4",
     "@push.rocks/smartproxy": "^27.17.5",
     "@push.rocks/smartradius": "^1.3.0",
-    "@push.rocks/smartrequest": "^5.0.3",
-    "@push.rocks/smartrx": "^3.0.10",
+    "@push.rocks/smartrequest": "^5.0.4",
+    "@push.rocks/smartrx": "^3.0.12",
     "@push.rocks/smartstate": "^2.3.1",
-    "@push.rocks/smartunique": "^3.0.9",
-    "@push.rocks/smartvpn": "1.20.0",
+    "@push.rocks/smartunique": "^3.0.10",
+    "@push.rocks/smartvpn": "1.21.0",
     "@push.rocks/taskbuffer": "^8.0.2",
-    "@serve.zone/catalog": "^2.14.0",
-    "@serve.zone/interfaces": "^11.2.0",
+    "@serve.zone/catalog": "^2.15.0",
+    "@serve.zone/interfaces": "^16.3.0",
     "@serve.zone/remoteingress": "^5.0.0",
     "@tsclass/tsclass": "^9.5.1",
     "@types/qrcode": "^1.5.6",
-    "lru-cache": "^11.5.1",
+    "lru-cache": "^11.5.2",
     "qrcode": "^1.5.4",
-    "uuid": "^14.0.0"
+    "uuid": "^14.0.1"
   },
   "keywords": [
     "mail service",
@@ -113,6 +114,9 @@
     ".smartconfig.json",
     "readme.md"
   ],
+  "engines": {
+    "node": ">=22"
+  },
   "scripts": {
     "test": "(tstest test/ --verbose --logfile --timeout 60)",
     "start": "(node ./cli.js)",

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '16.5.0',
+  version: '17.1.0',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }

package/ts/classes.dcrouter.ts

@@ -37,7 +37,7 @@ import { DnsManager } from './dns/manager.dns.js';
 import { DnsServerRuntime } from './dns/classes.dns-server-runtime.js';
 import { AcmeConfigManager } from './acme/manager.acme-config.js';
 import { SmartAcmeLifecycle } from './acme/classes.smartacme-lifecycle.js';
-import { AcceptedEmailSpool, EmailDomainManager, EmailRouteBuilder, EmailSettingsManager, MailDnsSync, MailEdgeEligibility, MailEgressCoordinator, ReleasedRemoteIngressEgressIdentitySource, SmartMtaStorageManager, WorkAppMailManager, type TSmartMtaQueueItemLike } from './email/index.js';
+import { AcceptedEmailSpool, EmailDomainManager, EmailRouteBuilder, EmailSettingsManager, MailDnsSync, MailEdgeEligibility, MailEgressCoordinator, ReleasedRemoteIngressEgressIdentitySource, SmartMtaBlobStorageManager, SmartMtaStorageManager, WorkAppMailManager, type ISmartMtaBlobStorageConfig, type TSmartMtaQueueItemLike } from './email/index.js';
 import type { IRoute } from '../ts_interfaces/data/route-management.js';
 import type { IEmailOutboundEgressStatus, IEmailPortConfig, IEmailServerSettings, IEmailServerSettingsSeed, TEmailOutboundMode, TEmailServerSettingsUpdate } from '../ts_interfaces/data/email-settings.js';
 import type { IDcRouterRouteConfig, IRemoteIngressHubSettings, IRemoteIngressPerformanceConfig, TRemoteIngressHubSettingsUpdate } from '../ts_interfaces/data/remoteingress.js';
@@ -62,6 +62,9 @@ export interface IDcRouterOptions {
    */
   emailConfig?: IUnifiedEmailServerOptions;
 
+  /** SmartBucket configuration for durable SmartMTA queue and attachment blobs. */
+  emailBlobStorage?: ISmartMtaBlobStorageConfig;
+
   /** Outbound SMTP connection mode. Defaults to direct unless DB settings override it. */
   emailOutboundMode?: TEmailOutboundMode;
 
@@ -278,7 +281,13 @@ export class DcRouter {
     listener: (...args: any[]) => void;
   }> = [];
 
-  public storageManager: IStorageManagerLike;
+  public storageManager!: IStorageManagerLike;
+  private smartMtaBlobStorageManager?: SmartMtaBlobStorageManager;
+  private databaseMigrationsReady = false;
+
+  public getSmartMtaBlobStorageManager(): SmartMtaBlobStorageManager | undefined {
+    return this.smartMtaBlobStorageManager;
+  }
 
   // Unified database (smartdata + LocalSmartDb or external MongoDB)
   public dcRouterDb?: DcRouterDb;
@@ -372,10 +381,6 @@ export class DcRouter {
 
     // Resolve all data paths from baseDir
     this.resolvedPaths = paths.resolvePaths(this.options.baseDir);
-    paths.ensureDataDirectories(this.resolvedPaths);
-    this.storageManager = new SmartMtaStorageManager(
-      plugins.path.join(this.resolvedPaths.dataDir, 'smartmta-storage')
-    );
     this.workAppMailManager = new WorkAppMailManager(this);
     this.acceptedEmailSpool = new AcceptedEmailSpool(this);
     const mailEgressIdentitySource = new ReleasedRemoteIngressEgressIdentitySource(this);
@@ -426,6 +431,7 @@ export class DcRouter {
             await this.setupDcRouterDb();
           })
           .withStop(async () => {
+            this.databaseMigrationsReady = false;
             if (this.cacheCleaner) {
               this.cacheCleaner.stop();
               this.cacheCleaner = undefined;
@@ -1012,6 +1018,7 @@ export class DcRouter {
    */
   private async setupDcRouterDb(): Promise<void> {
     logger.log('info', 'Setting up DcRouterDb...');
+    this.databaseMigrationsReady = false;
 
     const dbConfig = this.options.dbConfig || {};
 
@@ -1023,33 +1030,70 @@ export class DcRouter {
       debug: false,
     });
 
-    await this.dcRouterDb.start();
+    try {
+      await this.dcRouterDb.start();
+      if (this.options.emailBlobStorage && !this.smartMtaBlobStorageManager) {
+        this.smartMtaBlobStorageManager = await SmartMtaBlobStorageManager.create(
+          this.options.emailBlobStorage,
+        );
+      }
 
-    // Run any pending data migrations before anything else reads from the DB.
-    // This must complete before ConfigManagers loads profiles.
-    const migration = await createMigrationRunner(this.dcRouterDb.getDb(), commitinfo.version, {
-      remoteIngressHubSettings: this.getRemoteIngressHubSettingsMigrationSeed(),
-      emailServerSettings: this.getEmailSettingsMigrationSeed(),
-    });
-    const migrationResult = await migration.run();
-    if (migrationResult.stepsApplied.length > 0) {
-      logger.log('info',
-        `smartmigration: ${migrationResult.currentVersionBefore ?? 'fresh'} → ${migrationResult.currentVersionAfter} ` +
-        `(${migrationResult.stepsApplied.length} step(s) applied in ${migrationResult.totalDurationMs}ms)`,
-      );
-    } else if (migrationResult.wasFreshInstall) {
-      logger.log('info', `smartmigration: fresh install stamped to ${migrationResult.currentVersionAfter}`);
-    }
+      // Run any pending data migrations before anything else reads from the DB.
+      // This must complete before ConfigManagers loads profiles.
+      const migration = await createMigrationRunner(this.dcRouterDb.getDb(), commitinfo.version, {
+        remoteIngressHubSettings: this.getRemoteIngressHubSettingsMigrationSeed(),
+        emailServerSettings: this.getEmailSettingsMigrationSeed(),
+        legacySmartMtaStoragePath: plugins.path.join(
+          this.resolvedPaths.dataDir,
+          'smartmta-storage',
+        ),
+        legacyRawMessageBlobStorage: this.smartMtaBlobStorageManager,
+      });
+      const migrationResult = await migration.run();
+      if (migrationResult.stepsApplied.length > 0) {
+        logger.log('info',
+          `smartmigration: ${migrationResult.currentVersionBefore ?? 'fresh'} → ${migrationResult.currentVersionAfter} ` +
+          `(${migrationResult.stepsApplied.length} step(s) applied in ${migrationResult.totalDurationMs}ms)`,
+        );
+      } else if (migrationResult.wasFreshInstall) {
+        logger.log('info', `smartmigration: fresh install stamped to ${migrationResult.currentVersionAfter}`);
+      }
 
-    // Start the cache cleaner for TTL-based document cleanup
-    const cleanupIntervalMs = (dbConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
-    this.cacheCleaner = new CacheCleaner(this.dcRouterDb, {
-      intervalMs: cleanupIntervalMs,
-      verbose: false,
-    });
-    this.cacheCleaner.start();
+      this.storageManager = new SmartMtaStorageManager(this.dcRouterDb.getDb());
 
-    logger.log('info', `DcRouterDb ready (${this.dcRouterDb.isEmbedded() ? 'embedded' : 'external'})`);
+      // Start the cache cleaner for TTL-based document cleanup
+      const cleanupIntervalMs = (dbConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
+      this.cacheCleaner = new CacheCleaner(this.dcRouterDb, {
+        intervalMs: cleanupIntervalMs,
+        verbose: false,
+        beforeDeleteCachedEmail: async (cachedEmail) => {
+          await this.acceptedEmailSpool.deleteRawMessage(cachedEmail);
+        },
+      });
+      this.cacheCleaner.start();
+
+      this.databaseMigrationsReady = true;
+      logger.log('info', `DcRouterDb ready (${this.dcRouterDb.isEmbedded() ? 'embedded' : 'external'})`);
+    } catch (error) {
+      this.databaseMigrationsReady = false;
+      if (this.cacheCleaner) {
+        this.cacheCleaner.stop();
+        this.cacheCleaner = undefined;
+      }
+      const failedDb = this.dcRouterDb;
+      this.dcRouterDb = undefined;
+      try {
+        await failedDb?.stop();
+      } catch (stopError: unknown) {
+        logger.log(
+          'warn',
+          `Failed to stop DcRouterDb after startup migration failure: ${(stopError as Error).message}`,
+        );
+      } finally {
+        DcRouterDb.resetInstance();
+      }
+      throw error;
+    }
   }
 
   /**
@@ -1632,6 +1676,9 @@ export class DcRouter {
     }
 
     // Apply port mapping if behind SmartProxy
+    if (!this.databaseMigrationsReady) {
+      throw new Error('DcRouterDb migrations must complete before email acceptance');
+    }
     const portMapping = this.options.emailPortConfig?.portMapping || {
       25: 10025,   // SMTP
       587: 10587,  // Submission
@@ -1648,14 +1695,28 @@ export class DcRouter {
       : mappedEmailPorts.includes(configuredSecurePort) && !baseEmailConfig.ports.includes(configuredSecurePort)
         ? configuredSecurePort
         : portMapping[configuredSecurePort] || configuredSecurePort + 10000;
-    const emailConfig: IUnifiedEmailServerOptions = await this.workAppMailManager.applyStoredIdentitiesToEmailConfig({
+    const queueBehavior = { ...baseEmailConfig.queue } as Record<string, unknown>;
+    delete queueBehavior.storageMode;
+    delete queueBehavior.storageManager;
+    delete queueBehavior.storageType;
+    delete queueBehavior.persistentPath;
+    if (!this.smartMtaBlobStorageManager) {
+      throw new Error(
+        'emailBlobStorage is required for durable SmartMTA queue and raw RFC822 persistence',
+      );
+    }
+    const queueOptions: IUnifiedEmailServerOptions['queue'] = {
+      ...queueBehavior,
+      storageMode: 'managed',
+      storageManager: this.smartMtaBlobStorageManager,
+    };
+
+    let emailConfig: IUnifiedEmailServerOptions = await this.workAppMailManager.applyStoredIdentitiesToEmailConfig({
       ...this.options.emailConfig,
       ports: mappedEmailPorts,
+      dkimKeyProvisioning: 'caller-managed',
       persistRoutes: this.options.emailConfig.persistRoutes ?? false,
-      queue: {
-        ...this.options.emailConfig.queue,
-        storageType: 'memory',
-      },
+      queue: queueOptions,
       outbound: {
         ...baseEmailConfig.outbound,
         connectionProxyProvider: (context) => this.mailEgressCoordinator.provideConnectionProxy(context),
@@ -1699,9 +1760,28 @@ export class DcRouter {
     });
     
     // Create unified email server
-    const emailServer = new UnifiedEmailServer(this, emailConfig);
+    let emailServer = new UnifiedEmailServer(this, emailConfig);
     this.emailServer = emailServer;
     this.clearEmailEventSubscriptions();
+    try {
+      const repairResult = await this.emailDomainManager?.repairManagedDkimBeforeEmailStart();
+      if (repairResult && repairResult.failedDomainIds.length > 0) {
+        emailConfig = await this.workAppMailManager.applyStoredIdentitiesToEmailConfig({
+          ...emailConfig,
+          domains: [...(this.options.emailConfig?.domains || [])],
+        });
+        // DkimManager captures its DomainRegistry at construction. Rebuild the
+        // unstarted server so per-domain repair failures are absent from
+        // caller-managed DKIM startup validation.
+        emailServer = new UnifiedEmailServer(this, emailConfig);
+        this.emailServer = emailServer;
+      }
+    } catch (error) {
+      if (this.emailServer === emailServer) {
+        this.emailServer = undefined;
+      }
+      throw error;
+    }
     
     // Set up error handling
     this.addEmailEventSubscription(emailServer, 'error', (err: Error) => {
@@ -1724,6 +1804,10 @@ export class DcRouter {
     // Start the server
     try {
       await emailServer.start();
+      // Restore failed managed domains as inbound-only after DKIM validation.
+      // Outbound routes and SMTP users stay absent through readiness gating.
+      await this.emailDomainManager?.syncManagedDomainsToRuntime();
+      await this.workAppMailManager.applyStoredIdentitiesToRuntime();
     } catch (error: unknown) {
       this.clearEmailEventSubscriptions();
       try {
@@ -1738,18 +1822,25 @@ export class DcRouter {
     }
 
     try {
-      this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemEnqueued', (item: TSmartMtaQueueItemLike) => {
-        this.acceptedEmailSpool.trackQueueUpdate(item, 'queued', 'Unable to update accepted email after queue enqueue');
+      this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemEnqueued', async (item: TSmartMtaQueueItemLike) => {
+        await this.acceptedEmailSpool.trackQueueUpdate(item, 'queued', 'Unable to update accepted email after queue enqueue');
       });
-      this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemDelivered', (item: TSmartMtaQueueItemLike) => {
-        this.acceptedEmailSpool.trackQueueUpdate(item, 'delivered', 'Unable to mark accepted email delivered');
+      this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemDelivered', async (item: TSmartMtaQueueItemLike) => {
+        await this.acceptedEmailSpool.trackQueueUpdate(item, 'delivered', 'Unable to mark accepted email delivered');
       });
-      this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemDeferred', (item: TSmartMtaQueueItemLike) => {
-        this.acceptedEmailSpool.trackQueueUpdate(item, 'deferred', 'Unable to defer accepted email');
+      this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemDeferred', async (item: TSmartMtaQueueItemLike) => {
+        await this.acceptedEmailSpool.trackQueueUpdate(item, 'deferred', 'Unable to defer accepted email');
       });
-      this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemFailed', (item: TSmartMtaQueueItemLike) => {
-        this.acceptedEmailSpool.trackQueueUpdate(item, 'failed', 'Unable to mark accepted email failed');
+      this.addEmailEventSubscription(emailServer.deliveryQueue, 'itemFailed', async (item: TSmartMtaQueueItemLike) => {
+        await this.acceptedEmailSpool.trackQueueUpdate(item, 'failed', 'Unable to mark accepted email failed');
       });
+      this.addEmailEventSubscription(
+        emailServer.deliverySystem,
+        'smtpTransactionCompleted',
+        async (transaction: import('./db/documents/classes.cached.email.js').ICachedEmailSmtpTransaction) => {
+          await this.acceptedEmailSpool.trackSmtpTransaction(transaction, emailServer);
+        },
+      );
 
       // Wire delivery events to MetricsManager and logger using smartmta's public queue APIs.
       if (this.metricsManager) {
@@ -1801,6 +1892,7 @@ export class DcRouter {
         updateQueueSize();
       }
 
+      await this.acceptedEmailSpool.recoverSmtpTransactionHistory(emailServer);
       await this.acceptedEmailSpool.recoverQueuedEmails();
       this.acceptedEmailSpool.start();
     } catch (error: unknown) {

package/ts/db/classes.cache.cleaner.ts

@@ -14,6 +14,8 @@ export interface ICacheCleanerOptions {
   intervalMs?: number;
   /** Enable verbose logging */
   verbose?: boolean;
+  /** Delete externally stored raw payloads before removing a CachedEmail row. */
+  beforeDeleteCachedEmail?: (cachedEmailArg: CachedEmail) => Promise<void>;
 }
 
 /**
@@ -25,7 +27,11 @@ export interface ICacheCleanerOptions {
 export class CacheCleaner {
   private cleanupInterval: ReturnType<typeof setInterval> | null = null;
   private isRunning: boolean = false;
-  private options: Required<ICacheCleanerOptions>;
+  private options: {
+    intervalMs: number;
+    verbose: boolean;
+    beforeDeleteCachedEmail?: (cachedEmailArg: CachedEmail) => Promise<void>;
+  };
   private dcRouterDb: DcRouterDb;
 
   constructor(dcRouterDb: DcRouterDb, options: ICacheCleanerOptions = {}) {
@@ -33,6 +39,7 @@ export class CacheCleaner {
     this.options = {
       intervalMs: options.intervalMs || 60 * 60 * 1000, // 1 hour default
       verbose: options.verbose || false,
+      beforeDeleteCachedEmail: options.beforeDeleteCachedEmail,
     };
   }
 
@@ -95,7 +102,11 @@ export class CacheCleaner {
     const results: { collection: string; deleted: number }[] = [];
 
     try {
-      const emailsDeleted = await this.cleanExpiredDocuments(CachedEmail, now);
+      const emailsDeleted = await this.cleanExpiredDocuments(
+        CachedEmail,
+        now,
+        this.options.beforeDeleteCachedEmail,
+      );
       results.push({ collection: 'CachedEmail', deleted: emailsDeleted });
 
       const ipReputationDeleted = await this.cleanExpiredDocuments(CachedIPReputation, now);
@@ -124,7 +135,8 @@ export class CacheCleaner {
    */
   private async cleanExpiredDocuments<T extends { delete: () => Promise<void> }>(
     documentClass: { getInstances: (filter: any) => Promise<T[]> },
-    now: Date
+    now: Date,
+    beforeDeleteArg?: (documentArg: T) => Promise<void>,
   ): Promise<number> {
     try {
       // Find all expired documents
@@ -136,6 +148,9 @@ export class CacheCleaner {
       let deletedCount = 0;
       for (const doc of expiredDocs) {
         try {
+          if (beforeDeleteArg) {
+            await beforeDeleteArg(doc);
+          }
           await doc.delete();
           deletedCount++;
         } catch (deleteError: unknown) {

package/ts/db/classes.dcrouter-db.ts

@@ -80,8 +80,6 @@ export class DcRouterDb {
         logger.log('info', `DcRouterDb connecting to external MongoDB`);
       } else {
         // Embedded LocalSmartDb mode
-        await plugins.fsUtils.ensureDir(this.options.storagePath);
-
         this.localSmartDb = new plugins.smartdb.LocalSmartDb({
           folderPath: this.options.storagePath,
         });

package/ts/db/documents/classes.cached.email.ts

@@ -14,6 +14,54 @@ export type TCachedEmailStatus = 'pending' | 'processing' | 'queued' | 'delivere
  */
 export type TCachedEmailDirection = 'inbound' | 'outbound';
 
+export type TCachedEmailSmtpDeliveryPhase =
+  | 'connect'
+  | 'greeting'
+  | 'ehlo'
+  | 'starttls'
+  | 'tls_handshake'
+  | 'post_tls_ehlo'
+  | 'auth'
+  | 'mail_from'
+  | 'rcpt_to'
+  | 'data_command'
+  | 'message_body'
+  | 'final_response'
+  | 'rset'
+  | 'quit'
+  | 'unknown';
+
+export interface ICachedEmailSmtpTransaction {
+  id: string;
+  queueItemId: string;
+  queueAttempt: number;
+  targetHost: string;
+  targetPort: number;
+  recipientDomain?: string;
+  recipients: string[];
+  startedAt: string;
+  completedAt: string;
+  durationMs: number;
+  outcome: 'succeeded' | 'failed';
+  retryable: boolean;
+  errorType?: string;
+  smtpCode?: number;
+  recipientResults?: Array<{
+    recipient: string;
+    accepted: boolean;
+    responseCode: number;
+  }>;
+  failurePhase?: TCachedEmailSmtpDeliveryPhase;
+  error?: string;
+  transcript: Array<{
+    timestampMs: number;
+    phase: TCachedEmailSmtpDeliveryPhase;
+    direction: 'client' | 'server' | 'system';
+    text: string;
+    responseCode?: number;
+  }>;
+}
+
 /**
  * Helper to get the smartdata database instance
  */
@@ -70,11 +118,21 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
   @plugins.smartdata.svDb()
   public subject!: string;
 
-  /**
-   * Raw RFC822 email content
-   */
+  /** Legacy inline RFC822 content. New messages use rawContentObjectKey. */
+  @plugins.smartdata.svDb()
+  public rawContent?: string;
+
+  /** SmartBucket object key containing the raw RFC822 message. */
+  @plugins.smartdata.svDb()
+  public rawContentObjectKey?: string;
+
+  /** Raw RFC822 message size in bytes. */
   @plugins.smartdata.svDb()
-  public rawContent!: string;
+  public rawContentSize: number = 0;
+
+  /** Bounded durable SMTP transaction history copied from SmartMTA. */
+  @plugins.smartdata.svDb()
+  public smtpTransactions: ICachedEmailSmtpTransaction[] = [];
 
   /**
    * Current status of the email
@@ -164,6 +222,14 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
     this.bcc = [];
   }
 
+  public override async createSavableObject(): Promise<CachedEmail> {
+    const savableObject = await super.createSavableObject();
+    if (this.rawContentObjectKey) {
+      delete savableObject.rawContent;
+    }
+    return savableObject;
+  }
+
   /**
    * Create a new CachedEmail with a unique ID
    */
@@ -286,6 +352,14 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
     this.lastError = error;
   }
 
+  public appendSmtpTransaction(transactionArg: ICachedEmailSmtpTransaction): void {
+    const transactions = (this.smtpTransactions || []).filter(
+      (transaction) => transaction.id !== transactionArg.id,
+    );
+    transactions.push(structuredClone(transactionArg));
+    this.smtpTransactions = transactions.slice(-20);
+  }
+
   /**
    * Increment attempt counter and schedule next attempt
    */

package/ts/db/documents/classes.dns-record.doc.ts

@@ -61,6 +61,7 @@ export class DnsRecordDoc extends plugins.smartdata.SmartDataDbDoc<DnsRecordDoc,
     if (!this.managedBy) delete (savable as any).managedBy;
     if (!this.managedOwnerId) delete (savable as any).managedOwnerId;
     if (!this.managedRecordKey) delete (savable as any).managedRecordKey;
+    if (!this.providerRecordId?.trim()) delete (savable as any).providerRecordId;
     return savable;
   }
 

package/ts/db/documents/classes.smartmta-storage.doc.ts

@@ -0,0 +1,25 @@
+import * as plugins from '../../plugins.js';
+import { DcRouterDb } from '../classes.dcrouter-db.js';
+
+const getDb = () => DcRouterDb.getInstance().getDb();
+
+@plugins.smartdata.Collection(() => getDb())
+export class SmartMtaStorageDoc extends plugins.smartdata.SmartDataDbDoc<
+  SmartMtaStorageDoc,
+  SmartMtaStorageDoc
+> {
+  @plugins.smartdata.svDb()
+  public id!: string;
+
+  @plugins.smartdata.svDb()
+  public key!: string;
+
+  @plugins.smartdata.svDb()
+  public value!: string;
+
+  @plugins.smartdata.svDb()
+  public createdAt!: string;
+
+  @plugins.smartdata.svDb()
+  public updatedAt!: string;
+}

package/ts/db/documents/index.ts

@@ -41,3 +41,4 @@ export * from './classes.acme-config.doc.js';
 // Email domain management
 export * from './classes.email-domain.doc.js';
 export * from './classes.email-server-settings.doc.js';
+export * from './classes.smartmta-storage.doc.js';