@serve.zone/dcrouter 17.1.1 → 17.1.3

package/ts/email/classes.email-domain.manager.ts

@@ -12,6 +12,10 @@ import type {
   IEmailDomainOperationError,
   TEmailDomainLifecycleStatus,
 } from '../../ts_interfaces/data/email-domain.js';
+import {
+  MAIL_DNS_MAX_RETRY_INTERVAL_MS,
+  MAIL_DNS_RETRY_INTERVAL_MS,
+} from './classes.mail-dns-sync.js';
 
 export interface IEmailDomainManagerActionResult {
   success: boolean;
@@ -28,9 +32,20 @@ export interface IEmailDomainManagerActionResult {
 
 export interface IManagedDkimRepairResult {
   repairedDomainIds: string[];
+  repairedConfiguredDomains: string[];
+  failedConfiguredDomains: string[];
   failedDomainIds: string[];
 }
 
+interface IDkimMaterialRepairResult {
+  regenerated: boolean;
+}
+
+interface IDkimDocumentRepairOutcome {
+  outcome?: 'repaired' | 'failed';
+  failureMessage?: string;
+}
+
 /**
  * Persists managed email domains and projects them into SmartMTA. DNS mutation
  * is delegated exclusively to MailDnsSync.
@@ -38,6 +53,7 @@ export interface IManagedDkimRepairResult {
 export class EmailDomainManager {
   private dcRouter: any; // DcRouter — avoids circular import
   private baseEmailDomains: IEmailDomainConfig[] = [];
+  private failedBaseDkimDomainNames = new Set<string>();
   private mutationChain: Promise<unknown> = Promise.resolve();
   /** Explicit upstream capability marker; absence is deliberately fail-closed. */
   private get selectorCorrectSigningCapability(): boolean {
@@ -52,6 +68,10 @@ export class EmailDomainManager {
   public setBaseEmailDomains(domains: IEmailDomainConfig[] | undefined): void {
     this.baseEmailDomains = (domains || [])
       .map((domainConfig) => JSON.parse(JSON.stringify(domainConfig)) as IEmailDomainConfig);
+    const configuredNames = new Set(this.baseEmailDomains.map((domainConfig) => domainConfig.domain.toLowerCase()));
+    this.failedBaseDkimDomainNames = new Set(
+      [...this.failedBaseDkimDomainNames].filter((domainName) => configuredNames.has(domainName)),
+    );
   }
 
   private get dkimCreator(): any | undefined {
@@ -381,6 +401,8 @@ export class EmailDomainManager {
   public async repairManagedDkimBeforeEmailStart(): Promise<IManagedDkimRepairResult> {
     const repairedDomainIds: string[] = [];
     const failedDomainIds: string[] = [];
+    const repairedConfiguredDomains: string[] = [];
+    const failedConfiguredDomains: string[] = [];
 
     await this.runMutationExclusive(async () => {
       const dkimCreator = this.dkimCreator;
@@ -388,85 +410,10 @@ export class EmailDomainManager {
         throw new Error('DKIM key creator is not ready for managed-domain repair');
       }
 
-      for (const doc of await EmailDomainDoc.findAll()) {
+      const managedDocs = await EmailDomainDoc.findAll();
+      for (const doc of managedDocs) {
         if (doc.reconciliation?.lifecycleStatus === 'deleting') continue;
-        const currentActive = this.getActiveDkim(doc);
-        const selector = currentActive?.selector || doc.dkim?.selector || 'default';
-        const keySize = currentActive?.keySize || doc.dkim?.keySize || 2048;
-        let regenerated = false;
-        let outcome: 'repaired' | 'failed' | undefined;
-        let failureMessage: string | undefined;
-
-        try {
-          try {
-            await dkimCreator.readValidatedDKIMKeysForSelector(doc.domain, selector);
-          } catch {
-            await dkimCreator.createAndStoreDKIMKeysForSelector(doc.domain, selector, keySize);
-            await dkimCreator.readValidatedDKIMKeysForSelector(doc.domain, selector);
-            regenerated = true;
-          }
-
-          const record = await dkimCreator.getDNSRecordForSelector(doc.domain, selector);
-          const publicKey = this.extractDkimPublicKey(record?.value);
-          if (!publicKey) {
-            throw new Error('validated DKIM material produced no usable public key');
-          }
-
-          if (
-            !regenerated
-            && !this.hasDkimRepairFailure(doc)
-            && currentActive?.selector === selector
-            && currentActive.publicKey === publicKey
-            && doc.dkim?.selector === selector
-            && doc.dkim.publicKey === publicKey
-          ) {
-            continue;
-          }
-
-          const now = new Date().toISOString();
-          doc.activeDkim = {
-            selector,
-            keySize,
-            publicKey,
-            createdAt: regenerated ? now : currentActive?.createdAt || now,
-            validatedAt: now,
-          };
-          doc.dkim = {
-            ...doc.dkim,
-            selector,
-            keySize,
-            publicKey,
-            ...(regenerated ? { lastRotatedAt: now } : {}),
-          };
-          doc.dnsStatus = {
-            ...doc.dnsStatus,
-            dkim: 'unchecked',
-          };
-          doc.reconciliation = this.invalidateDkimReconciliation(
-            doc,
-            regenerated ? 'DKIM_KEY_MATERIAL_REGENERATED' : 'DKIM_KEY_MATERIAL_REPAIRED',
-            regenerated
-              ? 'Caller-managed DKIM key material was regenerated; DNS deployment must be revalidated'
-              : 'Caller-managed DKIM metadata was repaired; DNS deployment must be revalidated',
-          );
-          doc.updatedAt = now;
-          outcome = 'repaired';
-        } catch (error: unknown) {
-          const message = `Failed to repair caller-managed DKIM for ${selector}._domainkey.${doc.domain}: ${(error as Error).message}`;
-          doc.reconciliation = this.invalidateDkimReconciliation(
-            doc,
-            'DKIM_KEY_REPAIR_FAILED',
-            message,
-            false,
-          );
-          doc.dnsStatus = {
-            ...doc.dnsStatus,
-            dkim: 'invalid',
-          };
-          doc.updatedAt = new Date().toISOString();
-          outcome = 'failed';
-          failureMessage = message;
-        }
+        const { outcome, failureMessage } = await this.repairManagedDkimDocument(doc, dkimCreator);
 
         // Persistence failures are global prerequisites, not per-domain key
         // failures. Keep this save outside the material-repair catch.
@@ -479,6 +426,28 @@ export class EmailDomainManager {
         }
       }
 
+      const managedNames = new Set(
+        managedDocs
+          .filter((doc) => doc.reconciliation?.lifecycleStatus !== 'deleting')
+          .map((doc) => doc.domain.toLowerCase()),
+      );
+      for (const domainConfig of this.baseEmailDomains) {
+        const domainName = domainConfig.domain.toLowerCase();
+        if (managedNames.has(domainName)) continue;
+        const selector = domainConfig.dkim?.selector || 'default';
+        const keySize = domainConfig.dkim?.keySize || 2048;
+        try {
+          const result = await this.ensureValidatedDkimMaterial(domainName, selector, keySize, dkimCreator);
+          const wasFailed = this.failedBaseDkimDomainNames.delete(domainName);
+          if (result.regenerated || wasFailed) repairedConfiguredDomains.push(domainName);
+        } catch (error: unknown) {
+          const message = `Failed to repair caller-managed DKIM for ${selector}._domainkey.${domainName}: ${(error as Error).message}`;
+          this.failedBaseDkimDomainNames.add(domainName);
+          failedConfiguredDomains.push(domainName);
+          logger.log('error', message);
+        }
+      }
+
       await this.syncManagedDomainsToRuntime({ excludeDkimRepairFailures: true });
     });
 
@@ -488,7 +457,137 @@ export class EmailDomainManager {
         [...new Set(repairedDomainIds)],
       );
     }
-    return { repairedDomainIds, failedDomainIds };
+    return {
+      repairedDomainIds,
+      repairedConfiguredDomains,
+      failedConfiguredDomains,
+      failedDomainIds,
+    };
+  }
+
+  /** Retry a previously failed managed-domain repair when its bounded retry is due. */
+  public async repairManagedDkimIfDue(id: string, now = Date.now()): Promise<EmailDomainDoc | undefined> {
+    return await this.runMutationExclusive(async () => {
+      const doc = await EmailDomainDoc.findById(id);
+      if (!doc || !this.hasDkimRepairFailure(doc)) return doc || undefined;
+      const retryAt = Date.parse(doc.reconciliation?.retryAt || '');
+      if (Number.isFinite(retryAt) && retryAt > now) return doc;
+      const dkimCreator = this.dkimCreator;
+      if (!dkimCreator) return doc;
+
+      const result = await this.repairManagedDkimDocument(doc, dkimCreator);
+      await doc.save();
+      await this.syncManagedDomainsToRuntime({ excludeDkimRepairFailures: true });
+      if (result.outcome === 'failed') logger.log('error', result.failureMessage!);
+      return doc;
+    });
+  }
+
+  private async repairManagedDkimDocument(
+    doc: EmailDomainDoc,
+    dkimCreator: any,
+  ): Promise<IDkimDocumentRepairOutcome> {
+    const currentActive = this.getActiveDkim(doc);
+    const selector = currentActive?.selector || doc.dkim?.selector || 'default';
+    const keySize = currentActive?.keySize || doc.dkim?.keySize || 2048;
+    try {
+      const { regenerated } = await this.ensureValidatedDkimMaterial(
+        doc.domain,
+        selector,
+        keySize,
+        dkimCreator,
+      );
+      const record = await dkimCreator.getDNSRecordForSelector(doc.domain, selector);
+      const publicKey = this.extractDkimPublicKey(record?.value);
+      if (!publicKey) throw new Error('validated DKIM material produced no usable public key');
+
+      if (
+        !regenerated
+        && !this.hasDkimRepairFailure(doc)
+        && currentActive?.selector === selector
+        && currentActive.publicKey === publicKey
+        && doc.dkim?.selector === selector
+        && doc.dkim.publicKey === publicKey
+      ) return {};
+
+      const timestamp = new Date().toISOString();
+      doc.activeDkim = {
+        selector,
+        keySize,
+        publicKey,
+        createdAt: regenerated ? timestamp : currentActive?.createdAt || timestamp,
+        validatedAt: timestamp,
+      };
+      doc.dkim = {
+        ...doc.dkim,
+        selector,
+        keySize,
+        publicKey,
+        ...(regenerated ? { lastRotatedAt: timestamp } : {}),
+      };
+      doc.dnsStatus = { ...doc.dnsStatus, dkim: 'unchecked' };
+      doc.reconciliation = this.invalidateDkimReconciliation(
+        doc,
+        regenerated ? 'DKIM_KEY_MATERIAL_REGENERATED' : 'DKIM_KEY_MATERIAL_REPAIRED',
+        regenerated
+          ? 'Caller-managed DKIM key material was regenerated; DNS deployment must be revalidated'
+          : 'Caller-managed DKIM metadata was repaired; DNS deployment must be revalidated',
+      );
+      doc.updatedAt = timestamp;
+      return { outcome: 'repaired' };
+    } catch (error: unknown) {
+      const message = `Failed to repair caller-managed DKIM for ${selector}._domainkey.${doc.domain}: ${(error as Error).message}`;
+      const previousAttempt = doc.reconciliation?.retryAttempt || 0;
+      const retryAttempt = previousAttempt + 1;
+      const retryDelay = Math.min(
+        MAIL_DNS_RETRY_INTERVAL_MS * (2 ** Math.max(0, retryAttempt - 1)),
+        MAIL_DNS_MAX_RETRY_INTERVAL_MS,
+      );
+      doc.reconciliation = this.invalidateDkimReconciliation(doc, 'DKIM_KEY_REPAIR_FAILED', message);
+      doc.reconciliation.retryAttempt = retryAttempt;
+      doc.reconciliation.retryAt = new Date(Date.now() + retryDelay).toISOString();
+      doc.dnsStatus = { ...doc.dnsStatus, dkim: 'invalid' };
+      doc.updatedAt = new Date().toISOString();
+      return { outcome: 'failed', failureMessage: message };
+    }
+  }
+
+  private async ensureValidatedDkimMaterial(
+    domain: string,
+    selector: string,
+    keySize: number,
+    dkimCreator: any,
+  ): Promise<IDkimMaterialRepairResult> {
+    let initialError: unknown;
+    try {
+      await dkimCreator.readValidatedDKIMKeysForSelector(domain, selector);
+      return { regenerated: false };
+    } catch (error: unknown) {
+      initialError = error;
+    }
+
+    let creationError: unknown;
+    try {
+      await dkimCreator.createAndStoreDKIMKeysForSelector(domain, selector, keySize);
+    } catch (error: unknown) {
+      creationError = error;
+    }
+
+    try {
+      await dkimCreator.readValidatedDKIMKeysForSelector(domain, selector);
+      // Creation may have stored a complete key pair before a metadata write
+      // failed. Treat any attempted regeneration as a DNS-changing operation.
+      return { regenerated: true };
+    } catch (finalError: unknown) {
+      const parts = [
+        `initial validation failed: ${(initialError as Error).message}`,
+        creationError
+          ? `regeneration failed: ${(creationError as Error).message}`
+          : 'regeneration completed',
+        `final validation failed: ${(finalError as Error).message}`,
+      ];
+      throw new Error(parts.join('; '));
+    }
   }
 
   public async requestDkimRotation(id: string): Promise<IEmailDomainManagerActionResult> {
@@ -623,6 +722,7 @@ export class EmailDomainManager {
       errors: _errors,
       lastSuccessAt: _lastSuccessAt,
       retryAt: _retryAt,
+      retryAttempt: _retryAttempt,
       ...preserved
     } = previous;
     const alreadyInvalidated = (
@@ -763,9 +863,10 @@ export class EmailDomainManager {
     const mergedDomains = new Map<string, IEmailDomainConfig>();
     for (const domainConfig of this.baseEmailDomains) {
       const key = domainConfig.domain.toLowerCase();
-      if (failedDomainNames.has(key) && optionsArg.excludeDkimRepairFailures) continue;
+      const dkimRepairFailed = failedDomainNames.has(key) || this.failedBaseDkimDomainNames.has(key);
+      if (dkimRepairFailed && optionsArg.excludeDkimRepairFailures) continue;
       const projectedConfig = JSON.parse(JSON.stringify(domainConfig)) as IEmailDomainConfig;
-      if (failedDomainNames.has(key)) delete projectedConfig.dkim;
+      if (dkimRepairFailed) delete projectedConfig.dkim;
       mergedDomains.set(key, projectedConfig);
     }
     for (const managedConfig of await this.buildManagedDomainConfigs(optionsArg)) {

package/ts/email/classes.mail-dns-sync.ts

@@ -32,6 +32,11 @@ export const MAIL_DNS_SYNC_CREATED_BY = 'mail-dns-reconciler';
 export const MAIL_DNS_MANAGED_BY = 'mail-dns-reconciler';
 export const MAIL_DNS_RETRY_INTERVAL_MS = 5 * 60_000;
 export const MAIL_DNS_MAX_RETRY_INTERVAL_MS = 60 * 60_000;
+export const MAIL_DNS_PUBLIC_RESOLVER_OPTIONS = {
+  strategy: 'doh' as const,
+  allowDohFallback: false,
+  timeoutMs: 5_000,
+};
 
 export interface IMailDnsVerificationResult {
   valid: boolean;
@@ -150,7 +155,7 @@ export class SmartDnsMailVerifier implements IMailDnsVerifier {
   private client?: plugins.smartdns.dnsClientMod.Smartdns;
 
   private getClient(): plugins.smartdns.dnsClientMod.Smartdns {
-    this.client ||= new plugins.smartdns.dnsClientMod.Smartdns({});
+    this.client ||= new plugins.smartdns.dnsClientMod.Smartdns(MAIL_DNS_PUBLIC_RESOLVER_OPTIONS);
     return this.client;
   }
 
@@ -476,8 +481,20 @@ export class MailDnsSync {
     const plans: IPlannedDomain[] = [];
     const deletingDocs: EmailDomainDoc[] = [];
     for (const loadedDoc of docs) {
-      const doc = await this.dcRouterRef.emailDomainManager?.stageDkimRotationIfDue?.(loadedDoc.id)
+      if (loadedDoc.reconciliation?.lifecycleStatus === 'deleting') {
+        await this.persistDeletingAttempt(loadedDoc);
+        deletingDocs.push(loadedDoc);
+        continue;
+      }
+      const repairedDoc = await this.dcRouterRef.emailDomainManager?.repairManagedDkimIfDue?.(loadedDoc.id)
         || loadedDoc;
+      if (repairedDoc.reconciliation?.errors.some((error) => error.code === 'DKIM_KEY_REPAIR_FAILED')) {
+        // Key repair owns this retry schedule. DNS planning here would publish
+        // stale DKIM material and overwrite the bounded repair backoff.
+        continue;
+      }
+      const doc = await this.dcRouterRef.emailDomainManager?.stageDkimRotationIfDue?.(repairedDoc.id)
+        || repairedDoc;
       if (doc.reconciliation?.lifecycleStatus === 'deleting') {
         await this.persistDeletingAttempt(doc);
         deletingDocs.push(doc);
@@ -1243,9 +1260,16 @@ export class MailDnsSync {
           ? {}
           : { retryAt: new Date(Date.now() + retryDelay).toISOString() }),
     };
+    const mailPublicationReady = plan.egressIdentitySupported
+      && plan.revision.edgeIdentities.length > 0
+      && plan.revision.mxSlots.length > 0;
     plan.doc.dnsStatus = {
-      mx: this.statusForIntent(plan.intents, (intent) => intent.type === 'MX'),
-      spf: this.statusForIntent(plan.intents, (intent) => intent.type === 'TXT' && intent.value.startsWith('v=spf1')),
+      mx: mailPublicationReady
+        ? this.statusForIntent(plan.intents, (intent) => intent.type === 'MX')
+        : 'unchecked',
+      spf: mailPublicationReady
+        ? this.statusForIntent(plan.intents, (intent) => intent.type === 'TXT' && intent.value.startsWith('v=spf1'))
+        : 'unchecked',
       dkim: this.statusForIntent(plan.intents, (intent) => intent.type === 'TXT' && intent.name.includes('._domainkey.')),
       dmarc: this.statusForIntent(plan.intents, (intent) => intent.type === 'TXT' && intent.value.startsWith('v=DMARC1')),
       lastCheckedAt: now,

package/ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '17.1.1',
+  version: '17.1.3',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }