@serve.zone/dcrouter 16.4.1 → 17.0.0

package/ts/classes.dcrouter.ts

@@ -37,7 +37,7 @@ import { DnsManager } from './dns/manager.dns.js';
 import { DnsServerRuntime } from './dns/classes.dns-server-runtime.js';
 import { AcmeConfigManager } from './acme/manager.acme-config.js';
 import { SmartAcmeLifecycle } from './acme/classes.smartacme-lifecycle.js';
-import { AcceptedEmailSpool, EmailDomainManager, EmailRouteBuilder, EmailSettingsManager, MailDnsSync, MailEgressCoordinator, SmartMtaStorageManager, WorkAppMailManager, type TSmartMtaQueueItemLike } from './email/index.js';
+import { AcceptedEmailSpool, EmailDomainManager, EmailRouteBuilder, EmailSettingsManager, MailDnsSync, MailEdgeEligibility, MailEgressCoordinator, ReleasedRemoteIngressEgressIdentitySource, SmartMtaBlobStorageManager, SmartMtaStorageManager, WorkAppMailManager, type ISmartMtaBlobStorageConfig, type TSmartMtaQueueItemLike } from './email/index.js';
 import type { IRoute } from '../ts_interfaces/data/route-management.js';
 import type { IEmailOutboundEgressStatus, IEmailPortConfig, IEmailServerSettings, IEmailServerSettingsSeed, TEmailOutboundMode, TEmailServerSettingsUpdate } from '../ts_interfaces/data/email-settings.js';
 import type { IDcRouterRouteConfig, IRemoteIngressHubSettings, IRemoteIngressPerformanceConfig, TRemoteIngressHubSettingsUpdate } from '../ts_interfaces/data/remoteingress.js';
@@ -62,6 +62,9 @@ export interface IDcRouterOptions {
    */
   emailConfig?: IUnifiedEmailServerOptions;
 
+  /** SmartBucket configuration for durable SmartMTA queue and attachment blobs. */
+  emailBlobStorage?: ISmartMtaBlobStorageConfig;
+
   /** Outbound SMTP connection mode. Defaults to direct unless DB settings override it. */
   emailOutboundMode?: TEmailOutboundMode;
 
@@ -278,7 +281,8 @@ export class DcRouter {
     listener: (...args: any[]) => void;
   }> = [];
 
-  public storageManager: IStorageManagerLike;
+  public storageManager!: IStorageManagerLike;
+  private smartMtaBlobStorageManager?: SmartMtaBlobStorageManager;
 
   // Unified database (smartdata + LocalSmartDb or external MongoDB)
   public dcRouterDb?: DcRouterDb;
@@ -311,6 +315,7 @@ export class DcRouter {
   public acceptedEmailSpool: AcceptedEmailSpool;
   public mailEgressCoordinator: MailEgressCoordinator;
   public mailDnsSync: MailDnsSync;
+  public mailEdgeEligibility: MailEdgeEligibility;
   public securityPolicyManager?: SecurityPolicyManager;
 
   // Auto-discovered public IP (populated by generateAuthoritativeRecords)
@@ -372,13 +377,12 @@ export class DcRouter {
     // Resolve all data paths from baseDir
     this.resolvedPaths = paths.resolvePaths(this.options.baseDir);
     paths.ensureDataDirectories(this.resolvedPaths);
-    this.storageManager = new SmartMtaStorageManager(
-      plugins.path.join(this.resolvedPaths.dataDir, 'smartmta-storage')
-    );
     this.workAppMailManager = new WorkAppMailManager(this);
     this.acceptedEmailSpool = new AcceptedEmailSpool(this);
-    this.mailEgressCoordinator = new MailEgressCoordinator(this);
-    this.mailDnsSync = new MailDnsSync(this);
+    const mailEgressIdentitySource = new ReleasedRemoteIngressEgressIdentitySource(this);
+    this.mailEdgeEligibility = new MailEdgeEligibility(this, mailEgressIdentitySource);
+    this.mailEgressCoordinator = new MailEgressCoordinator(this, this.mailEdgeEligibility);
+    this.mailDnsSync = new MailDnsSync(this, this.mailEdgeEligibility);
     this.smartAcmeLifecycle = new SmartAcmeLifecycle(this);
     this.vpnAccessResolver = new VpnAccessResolver(this);
     this.remoteIngressHubLifecycle = new RemoteIngressHubLifecycle(this);
@@ -803,6 +807,22 @@ export class DcRouter {
           })
           .withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
       );
+
+      // Mail DNS reconciliation starts last: it needs persistence, provider
+      // state, the SMTP DKIM creator, and live tunnel topology. Earlier events
+      // only mark the reconciler dirty and are consumed here.
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('MailDnsSync')
+          .optional()
+          .dependsOn('DnsManager', 'EmailDomainManager', 'EmailServer', 'RemoteIngress')
+          .withStart(async () => {
+            await this.mailDnsSync.start();
+          })
+          .withStop(async () => {
+            await this.mailDnsSync.stop();
+          })
+          .withRetry({ maxRetries: 1, baseDelayMs: 1000 }),
+      );
     }
 
     // VPN Server: optional, depends on SmartProxy
@@ -1011,6 +1031,10 @@ export class DcRouter {
     const migration = await createMigrationRunner(this.dcRouterDb.getDb(), commitinfo.version, {
       remoteIngressHubSettings: this.getRemoteIngressHubSettingsMigrationSeed(),
       emailServerSettings: this.getEmailSettingsMigrationSeed(),
+      legacySmartMtaStoragePath: plugins.path.join(
+        this.resolvedPaths.dataDir,
+        'smartmta-storage',
+      ),
     });
     const migrationResult = await migration.run();
     if (migrationResult.stepsApplied.length > 0) {
@@ -1022,6 +1046,8 @@ export class DcRouter {
       logger.log('info', `smartmigration: fresh install stamped to ${migrationResult.currentVersionAfter}`);
     }
 
+    this.storageManager = new SmartMtaStorageManager(this.dcRouterDb.getDb());
+
     // Start the cache cleaner for TTL-based document cleanup
     const cleanupIntervalMs = (dbConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
     this.cacheCleaner = new CacheCleaner(this.dcRouterDb, {
@@ -1629,14 +1655,32 @@ export class DcRouter {
       : mappedEmailPorts.includes(configuredSecurePort) && !baseEmailConfig.ports.includes(configuredSecurePort)
         ? configuredSecurePort
         : portMapping[configuredSecurePort] || configuredSecurePort + 10000;
+    const queueBehavior = { ...baseEmailConfig.queue } as Record<string, unknown>;
+    delete queueBehavior.storageMode;
+    delete queueBehavior.storageManager;
+    delete queueBehavior.storageType;
+    delete queueBehavior.persistentPath;
+    if (this.options.emailBlobStorage && !this.smartMtaBlobStorageManager) {
+      this.smartMtaBlobStorageManager = await SmartMtaBlobStorageManager.create(
+        this.options.emailBlobStorage,
+      );
+    }
+    const queueOptions: IUnifiedEmailServerOptions['queue'] = this.smartMtaBlobStorageManager
+      ? {
+          ...queueBehavior,
+          storageMode: 'managed',
+          storageManager: this.smartMtaBlobStorageManager,
+        }
+      : {
+          ...queueBehavior,
+          storageMode: 'memory',
+        };
+
     const emailConfig: IUnifiedEmailServerOptions = await this.workAppMailManager.applyStoredIdentitiesToEmailConfig({
       ...this.options.emailConfig,
       ports: mappedEmailPorts,
       persistRoutes: this.options.emailConfig.persistRoutes ?? false,
-      queue: {
-        ...this.options.emailConfig.queue,
-        storageType: 'memory',
-      },
+      queue: queueOptions,
       outbound: {
         ...baseEmailConfig.outbound,
         connectionProxyProvider: (context) => this.mailEgressCoordinator.provideConnectionProxy(context),

package/ts/db/documents/classes.dns-record.doc.ts

@@ -44,6 +44,26 @@ export class DnsRecordDoc extends plugins.smartdata.SmartDataDbDoc<DnsRecordDoc,
   @plugins.smartdata.svDb()
   public createdBy!: string;
 
+  @plugins.smartdata.svDb()
+  public managedBy?: string;
+
+  @plugins.smartdata.svDb()
+  public managedOwnerId?: string;
+
+  @plugins.smartdata.svDb()
+  public managedRecordKey?: string;
+
+  public async createSavableObject(): Promise<DnsRecordDoc> {
+    const savable = await super.createSavableObject() as DnsRecordDoc;
+    // SmartData persists decorated `undefined` values through `$set`. Managed
+    // metadata must be truly absent so partial indexes and ownership checks do
+    // not accidentally classify operator/provider records as reconciler-owned.
+    if (!this.managedBy) delete (savable as any).managedBy;
+    if (!this.managedOwnerId) delete (savable as any).managedOwnerId;
+    if (!this.managedRecordKey) delete (savable as any).managedRecordKey;
+    return savable;
+  }
+
   constructor() {
     super();
   }

package/ts/db/documents/classes.email-domain.doc.ts

@@ -5,6 +5,8 @@ import type {
   IEmailDomainRateLimits,
   IEmailDomainDnsStatus,
   IEmailDomainRemoteIngress,
+  IEmailDomainReconciliation,
+  IEmailDkimMaterial,
 } from '../../../ts_interfaces/data/email-domain.js';
 
 const getDb = () => DcRouterDb.getInstance().getDb();
@@ -36,6 +38,18 @@ export class EmailDomainDoc extends plugins.smartdata.SmartDataDbDoc<EmailDomain
   @plugins.smartdata.svDb()
   public dnsStatus!: IEmailDomainDnsStatus;
 
+  @plugins.smartdata.svDb()
+  public reconciliation?: IEmailDomainReconciliation;
+
+  @plugins.smartdata.svDb()
+  public activeDkim?: IEmailDkimMaterial;
+
+  @plugins.smartdata.svDb()
+  public pendingDkim?: IEmailDkimMaterial;
+
+  @plugins.smartdata.svDb()
+  public retiringDkim?: IEmailDkimMaterial[];
+
   @plugins.smartdata.svDb()
   public createdAt!: string;
 

package/ts/db/documents/classes.smartmta-storage.doc.ts

@@ -0,0 +1,25 @@
+import * as plugins from '../../plugins.js';
+import { DcRouterDb } from '../classes.dcrouter-db.js';
+
+const getDb = () => DcRouterDb.getInstance().getDb();
+
+@plugins.smartdata.Collection(() => getDb())
+export class SmartMtaStorageDoc extends plugins.smartdata.SmartDataDbDoc<
+  SmartMtaStorageDoc,
+  SmartMtaStorageDoc
+> {
+  @plugins.smartdata.svDb()
+  public id!: string;
+
+  @plugins.smartdata.svDb()
+  public key!: string;
+
+  @plugins.smartdata.svDb()
+  public value!: string;
+
+  @plugins.smartdata.svDb()
+  public createdAt!: string;
+
+  @plugins.smartdata.svDb()
+  public updatedAt!: string;
+}

package/ts/db/documents/index.ts

@@ -41,3 +41,4 @@ export * from './classes.acme-config.doc.js';
 // Email domain management
 export * from './classes.email-domain.doc.js';
 export * from './classes.email-server-settings.doc.js';
+export * from './classes.smartmta-storage.doc.js';

package/ts/dns/manager.dns.ts

@@ -88,10 +88,12 @@ export class DnsManager {
     let registered = 0;
     for (const domain of dcrouterDomains) {
       const records = await DnsRecordDoc.findByDomainId(domain.id);
-      for (const rec of records) {
-        this.registerRecordWithDnsServer(rec);
-        registered++;
+      const rrsets = new Set(records.map((record) => this.rrsetKey(record.name, record.type)));
+      for (const rrset of rrsets) {
+        const [name, type] = rrset.split('|') as [string, TDnsRecordType];
+        await this.refreshLocalRrset(domain.id, name, type);
       }
+      registered += records.length;
       await this.registerAuthoritativeZoneDefaults(domain);
     }
     logger.log(
@@ -167,6 +169,26 @@ export class DnsManager {
     });
   }
 
+  private rrsetKey(name: string, type: TDnsRecordType): string {
+    return `${name.toLowerCase()}|${type}`;
+  }
+
+  /** Rebuild one authoritative RRset atomically from persisted rows. */
+  private async refreshLocalRrset(
+    domainId: string,
+    name: string,
+    type: TDnsRecordType,
+  ): Promise<void> {
+    if (!this.dnsServer) return;
+    this.dnsServer.unregisterHandler(name, [type]);
+    const records = await DnsRecordDoc.findByDomainId(domainId);
+    for (const record of records) {
+      if (record.name.toLowerCase() === name.toLowerCase() && record.type === type) {
+        this.registerRecordWithDnsServer(record);
+      }
+    }
+  }
+
   private parseRecordData(type: TDnsRecordType, value: string): any {
     switch (type) {
       case 'A':
@@ -608,9 +630,16 @@ export class DnsManager {
     const doc = await DomainDoc.findById(id);
     if (!doc) return false;
     const records = await DnsRecordDoc.findByDomainId(id);
+    const localRrsets = new Set(records.map((record) => this.rrsetKey(record.name, record.type)));
     for (const r of records) {
       await r.delete();
     }
+    if (doc.source === 'dcrouter' && this.dnsServer) {
+      for (const rrset of localRrsets) {
+        const [name, type] = rrset.split('|') as [string, TDnsRecordType];
+        this.dnsServer.unregisterHandler(name, [type]);
+      }
+    }
     await doc.delete();
     return true;
   }
@@ -631,13 +660,34 @@ export class DnsManager {
     }
     const providerRecords = await client.listRecords(doc.name);
 
-    // Drop existing records and replace
+    // Merge by provider id, then exact tuple. Replacing the collection would
+    // erase explicit ownership metadata from reconciler-managed rows.
     const existing = await DnsRecordDoc.findByDomainId(id);
-    for (const r of existing) {
-      await r.delete();
-    }
+    const remaining = new Set(existing);
     for (const pr of providerRecords) {
-      await this.createSyncedRecord(id, pr, doc.createdBy);
+      const match = existing.find((record) => record.providerRecordId === pr.providerRecordId)
+        || existing.find((record) => (
+          record.name.toLowerCase() === pr.name.toLowerCase()
+          && record.type === pr.type
+          && record.value.trim() === pr.value.trim()
+        ));
+      if (!match) {
+        await this.createSyncedRecord(id, pr, doc.createdBy);
+        continue;
+      }
+      remaining.delete(match);
+      match.name = pr.name.toLowerCase();
+      match.type = pr.type;
+      match.value = pr.value;
+      match.ttl = pr.ttl;
+      match.proxied = pr.proxied;
+      match.source = 'synced';
+      match.providerRecordId = pr.providerRecordId;
+      match.updatedAt = Date.now();
+      await match.save();
+    }
+    for (const staleMirror of remaining) {
+      await staleMirror.delete();
     }
     doc.lastSyncedAt = Date.now();
     doc.updatedAt = doc.lastSyncedAt;
@@ -811,6 +861,9 @@ export class DnsManager {
     ttl?: number;
     proxied?: boolean;
     createdBy: string;
+    managedBy?: string;
+    managedOwnerId?: string;
+    managedRecordKey?: string;
   }): Promise<{ success: boolean; id?: string; message?: string }> {
     const domain = await DomainDoc.findById(args.domainId);
     if (!domain) return { success: false, message: 'Domain not found' };
@@ -828,6 +881,9 @@ export class DnsManager {
     doc.createdAt = now;
     doc.updatedAt = now;
     doc.createdBy = args.createdBy;
+    doc.managedBy = args.managedBy;
+    doc.managedOwnerId = args.managedOwnerId;
+    doc.managedRecordKey = args.managedRecordKey;
 
     if (domain.source === 'provider') {
       // Push to provider first; only persist locally on success
@@ -849,12 +905,12 @@ export class DnsManager {
       } catch (err: unknown) {
         return { success: false, message: `Provider rejected record: ${(err as Error).message}` };
       }
-    } else {
-      // dcrouter-hosted / authoritative — register with embedded DnsServer immediately
-      this.registerRecordWithDnsServer(doc);
     }
 
     await doc.save();
+    if (domain.source === 'dcrouter') {
+      await this.refreshLocalRrset(doc.domainId, doc.name, doc.type);
+    }
     return { success: true, id: doc.id };
   }
 
@@ -870,6 +926,8 @@ export class DnsManager {
     const domain = await DomainDoc.findById(doc.domainId);
     if (!domain) return { success: false, message: 'Parent domain not found' };
 
+    const previousName = doc.name;
+    const previousType = doc.type;
     if (args.name !== undefined) doc.name = args.name.toLowerCase();
     if (args.value !== undefined) doc.value = args.value;
     if (args.ttl !== undefined) doc.ttl = args.ttl;
@@ -893,12 +951,15 @@ export class DnsManager {
       } catch (err: unknown) {
         return { success: false, message: `Provider rejected update: ${(err as Error).message}` };
       }
-    } else {
-      // Re-register the local record so the new closure picks up the updated fields
-      this.registerRecordWithDnsServer(doc);
     }
 
     await doc.save();
+    if (domain.source === 'dcrouter') {
+      await this.refreshLocalRrset(doc.domainId, previousName, previousType);
+      if (previousName.toLowerCase() !== doc.name.toLowerCase()) {
+        await this.refreshLocalRrset(doc.domainId, doc.name, doc.type);
+      }
+    }
     return { success: true };
   }
 
@@ -909,24 +970,23 @@ export class DnsManager {
     if (!domain) return { success: false, message: 'Parent domain not found' };
 
     if (domain.source === 'provider') {
-      if (domain.providerId && doc.providerRecordId) {
-        const client = await this.getProviderClientById(domain.providerId);
-        if (client) {
-          try {
-            await client.deleteRecord(domain.name, doc.providerRecordId);
-          } catch (err: unknown) {
-            return { success: false, message: `Provider rejected delete: ${(err as Error).message}` };
-          }
-        }
+      if (!domain.providerId || !doc.providerRecordId) {
+        return { success: false, message: 'Provider record metadata missing' };
+      }
+      const client = await this.getProviderClientById(domain.providerId);
+      if (!client) {
+        return { success: false, message: 'Provider client unavailable' };
+      }
+      try {
+        await client.deleteRecord(domain.name, doc.providerRecordId);
+      } catch (err: unknown) {
+        return { success: false, message: `Provider rejected delete: ${(err as Error).message}` };
       }
     }
-    // For dcrouter-hosted records: unregister the handler from the embedded DnsServer
-    // so the record stops being served immediately (not just after restart).
-    if (domain.source === 'dcrouter' && this.dnsServer) {
-      this.unregisterRecordFromDnsServer(doc);
-    }
-
     await doc.delete();
+    if (domain.source === 'dcrouter') {
+      await this.refreshLocalRrset(doc.domainId, doc.name, doc.type);
+    }
     return { success: true };
   }
 

package/ts/dns/providers/cloudflare.provider.ts

@@ -49,17 +49,21 @@ export class CloudflareDnsProvider implements IDnsProviderClient {
   }
 
   public async listRecords(domain: string): Promise<IProviderRecord[]> {
-    const records = await this.cfAccount.recordManager.listRecords(domain);
+    const zoneId = await this.cfAccount.zoneManager.getZoneId(domain);
+    const records: any[] = [];
+    // RecordManager.listRecords() intentionally converts provider failures to
+    // an empty array. DNS reconciliation must distinguish an empty zone from
+    // an unavailable provider, so use the raw paginated iterator and let
+    // errors propagate to the durable reconciliation outcome.
+    for await (const record of (this.cfAccount as any).apiAccount.dns.records.list({
+      zone_id: zoneId,
+      per_page: 100,
+    })) {
+      records.push(record);
+    }
     return records
       .filter((r) => this.isSupportedType(r.type))
-      .map((r) => ({
-        providerRecordId: r.id,
-        name: r.name,
-        type: r.type as TDnsRecordType,
-        value: r.content,
-        ttl: r.ttl,
-        proxied: r.proxied,
-      }));
+      .map((r) => this.fromApiRecord(r));
   }
 
   public async createRecord(
@@ -71,21 +75,14 @@ export class CloudflareDnsProvider implements IDnsProviderClient {
       zone_id: zoneId,
       type: record.type,
       name: record.name,
-      content: record.value,
       ttl: record.ttl ?? 1, // 1 = automatic
+      ...this.toApiRecordValue(record),
     };
     if (record.proxied !== undefined) {
       apiRecord.proxied = record.proxied;
     }
     const created = await (this.cfAccount as any).apiAccount.dns.records.create(apiRecord);
-    return {
-      providerRecordId: created.id,
-      name: created.name,
-      type: created.type as TDnsRecordType,
-      value: created.content,
-      ttl: created.ttl,
-      proxied: created.proxied,
-    };
+    return this.fromApiRecord(created);
   }
 
   public async updateRecord(
@@ -98,8 +95,8 @@ export class CloudflareDnsProvider implements IDnsProviderClient {
       zone_id: zoneId,
       type: record.type,
       name: record.name,
-      content: record.value,
       ttl: record.ttl ?? 1,
+      ...this.toApiRecordValue(record),
     };
     if (record.proxied !== undefined) {
       apiRecord.proxied = record.proxied;
@@ -108,14 +105,7 @@ export class CloudflareDnsProvider implements IDnsProviderClient {
       providerRecordId,
       apiRecord,
     );
-    return {
-      providerRecordId: updated.id,
-      name: updated.name,
-      type: updated.type as TDnsRecordType,
-      value: updated.content,
-      ttl: updated.ttl,
-      proxied: updated.proxied,
-    };
+    return this.fromApiRecord(updated);
   }
 
   public async deleteRecord(domain: string, providerRecordId: string): Promise<void> {
@@ -128,4 +118,33 @@ export class CloudflareDnsProvider implements IDnsProviderClient {
   private isSupportedType(type: string): boolean {
     return ['A', 'AAAA', 'CNAME', 'MX', 'TXT', 'NS', 'SOA', 'CAA'].includes(type);
   }
+
+  private toApiRecordValue(record: IProviderRecordInput): { content: string; priority?: number } {
+    if (record.type !== 'MX') {
+      return { content: record.value };
+    }
+    const match = record.value.trim().match(/^(\d+)\s+(.+)$/);
+    if (!match) {
+      throw new Error(`CloudflareDnsProvider: invalid MX value "${record.value}"; expected "<priority> <exchange>"`);
+    }
+    const priority = Number.parseInt(match[1], 10);
+    if (!Number.isInteger(priority) || priority < 0 || priority > 65535) {
+      throw new Error(`CloudflareDnsProvider: invalid MX priority in "${record.value}"`);
+    }
+    return { content: match[2], priority };
+  }
+
+  private fromApiRecord(record: any): IProviderRecord {
+    const value = record.type === 'MX'
+      ? `${Number(record.priority)} ${record.content}`
+      : String(record.content);
+    return {
+      providerRecordId: String(record.id),
+      name: String(record.name),
+      type: record.type as TDnsRecordType,
+      value,
+      ttl: Number(record.ttl),
+      ...(record.proxied !== undefined ? { proxied: Boolean(record.proxied) } : {}),
+    };
+  }
 }