@serve.zone/dcrouter 18.6.2 → 18.7.1

package/ts/email/classes.mail-dns-sync.ts

@@ -34,6 +34,7 @@ export const MAIL_DNS_SYNC_CREATED_BY = 'mail-dns-reconciler';
 export const MAIL_DNS_MANAGED_BY = 'mail-dns-reconciler';
 export const MAIL_DNS_RETRY_INTERVAL_MS = 5 * 60_000;
 export const MAIL_DNS_MAX_RETRY_INTERVAL_MS = 60 * 60_000;
+export const MAIL_DNS_RECONCILIATION_INTERVAL_MS = 5 * 60_000;
 export const MAIL_DNS_EDGE_ELIGIBILITY_DEBOUNCE_MS = 250;
 export const MAIL_DNS_EDGE_ELIGIBILITY_COOLDOWN_MS = 5_000;
 export const MAIL_DNS_PUBLIC_RESOLVER_OPTIONS = {
@@ -379,6 +380,9 @@ export class MailDnsSync {
   private dirty = false;
   private runPromise?: Promise<void>;
   private retryTimer?: ReturnType<typeof setTimeout>;
+  private schedulePromises = new Set<Promise<void>>();
+  private scheduleGeneration = 0;
+  private lastScheduledReconciliationAt = 0;
   private requestedGeneration = 0;
   private completedGeneration = 0;
   private forceAll = false;
@@ -416,13 +420,19 @@ export class MailDnsSync {
     try {
       await this.kick('startup');
     } catch (error: unknown) {
+      this.stopped = true;
+      this.started = false;
+      this.dirty = false;
+      this.scheduleGeneration++;
       if (this.retryTimer) clearTimeout(this.retryTimer);
       this.retryTimer = undefined;
       if (this.edgeEligibilityCheckTimer) clearTimeout(this.edgeEligibilityCheckTimer);
       this.edgeEligibilityCheckTimer = undefined;
       this.pendingEdgeEligibilityReason = undefined;
-      this.started = false;
-      this.dirty = false;
+      await this.edgeEligibilityCheckPromise?.catch(() => undefined);
+      await this.runPromise?.catch(() => undefined);
+      await Promise.all([...this.schedulePromises]);
+      this.verifier.destroy?.();
       throw error;
     }
   }
@@ -431,6 +441,7 @@ export class MailDnsSync {
     this.stopped = true;
     this.started = false;
     this.dirty = false;
+    this.scheduleGeneration++;
     if (this.retryTimer) {
       clearTimeout(this.retryTimer);
       this.retryTimer = undefined;
@@ -442,6 +453,7 @@ export class MailDnsSync {
     this.pendingEdgeEligibilityReason = undefined;
     await this.edgeEligibilityCheckPromise?.catch(() => undefined);
     await this.runPromise?.catch(() => undefined);
+    await Promise.all([...this.schedulePromises]);
     this.verifier.destroy?.();
   }
 
@@ -474,13 +486,14 @@ export class MailDnsSync {
       this.lastEdgeEligibilitySyncAt + this.edgeEligibilityCooldownMs - Date.now(),
     );
     const delay = Math.max(this.edgeEligibilityDebounceMs, cooldownRemaining);
-    this.edgeEligibilityCheckTimer = setTimeout(() => {
+    const timer = setTimeout(() => {
       this.edgeEligibilityCheckTimer = undefined;
       this.runEdgeEligibilityCheck().catch((error: unknown) => {
         logger.log('error', `MailDnsSync edge eligibility check failed: ${(error as Error).message}`);
       });
-    }, delay);
-    this.edgeEligibilityCheckTimer.unref?.();
+    }, delay) as ReturnType<typeof setTimeout> & { unref?: () => void };
+    this.edgeEligibilityCheckTimer = timer;
+    timer.unref?.();
   }
 
   private async runEdgeEligibilityCheck(): Promise<void> {
@@ -518,8 +531,10 @@ export class MailDnsSync {
     if (this.stopped) throw new Error('MailDnsSync is stopped');
     const ticket = this.queueRequest(domainIds, domainIds === undefined);
     while (this.completedGeneration < ticket) {
+      if (this.stopped) throw new Error('MailDnsSync is stopped');
       await this.kick(reason);
     }
+    if (this.stopped) throw new Error('MailDnsSync is stopped');
   }
 
   public async syncDomain(domainId: string, reason: string): Promise<{
@@ -531,8 +546,10 @@ export class MailDnsSync {
     if (this.stopped) throw new Error('MailDnsSync is stopped');
     const requestedGeneration = this.queueRequest([domainId], false);
     while (this.completedGeneration < requestedGeneration) {
+      if (this.stopped) throw new Error('MailDnsSync is stopped');
       await this.kick(reason);
     }
+    if (this.stopped) throw new Error('MailDnsSync is stopped');
     const domain = await EmailDomainDoc.findById(domainId);
     return {
       requestedGeneration,
@@ -551,6 +568,7 @@ export class MailDnsSync {
   }
 
   private async kick(reason: string): Promise<void> {
+    if (this.stopped) throw new Error('MailDnsSync is stopped');
     if (this.runPromise) return await this.runPromise;
     // Defer execution one microtask so runPromise is assigned before the first
     // awaited provider/eligibility hook can synchronously enqueue another event.
@@ -561,9 +579,7 @@ export class MailDnsSync {
           logger.log('error', `MailDnsSync follow-up failed: ${(error as Error).message}`);
         });
       }
-      this.scheduleNextDue().catch((error: unknown) => {
-        logger.log('error', `MailDnsSync scheduling failed: ${(error as Error).message}`);
-      });
+      this.queueScheduleNextDue();
     });
     return await this.runPromise;
   }
@@ -593,7 +609,15 @@ export class MailDnsSync {
       logger.log('warn', `MailDnsSync (${reason}): DnsManager unavailable; retry intent retained`);
       return;
     }
+    await dnsManager.runManagedMailDnsMutationExclusive(
+      async () => await this.reconcileExclusive(reason, work),
+    );
+  }
 
+  private async reconcileExclusive(
+    reason: string,
+    work: { forceAll: boolean; forcedDomainIds: Set<string> },
+  ): Promise<void> {
     const zones = await DomainDoc.findAll();
     const allDocs = await EmailDomainDoc.findAll();
     const now = Date.now();
@@ -663,8 +687,9 @@ export class MailDnsSync {
       new Set(plans.map((plan) => `domain:${plan.doc.id}`).concat(
         deletingDocs.map((doc) => `domain:${doc.id}`),
       )),
-      work.forceAll,
+      work.forceAll || deletingDocs.length > 0,
     );
+    await this.pruneRetiredDkimMaterial(plans.map((plan) => plan.doc));
     await this.finalizeDeletingDomains(deletingDocs, zones);
     await this.dcRouterRef.emailDomainManager?.syncManagedDomainsToRuntime();
     await this.dcRouterRef.workAppMailManager?.applyStoredIdentitiesToRuntime();
@@ -677,9 +702,17 @@ export class MailDnsSync {
   private isDomainDue(doc: EmailDomainDoc, now: number): boolean {
     const retryAt = Date.parse(doc.reconciliation?.retryAt || '');
     if (Number.isFinite(retryAt) && retryAt <= now) return true;
+    if (Number.isFinite(retryAt)) return false;
     if (!doc.reconciliation?.lastAttemptAt) return true;
+    const lastAttemptAt = Date.parse(doc.reconciliation.lastAttemptAt);
+    if (
+      Number.isFinite(lastAttemptAt)
+      && lastAttemptAt + MAIL_DNS_RECONCILIATION_INTERVAL_MS <= now
+    ) return true;
     const rotationDueAt = this.getDkimRotationDueAt(doc, now);
-    return rotationDueAt !== undefined && rotationDueAt <= now;
+    if (rotationDueAt !== undefined && rotationDueAt <= now) return true;
+    const retirementDueAt = this.getDkimRetirementDueAt(doc);
+    return retirementDueAt !== undefined && retirementDueAt <= now;
   }
 
   private getDkimRotationDueAt(doc: EmailDomainDoc, now: number): number | undefined {
@@ -702,29 +735,65 @@ export class MailDnsSync {
     return Math.max(dueAt, retryReference + MAIL_DNS_RETRY_INTERVAL_MS);
   }
 
+  private getDkimRetirementDueAt(doc: EmailDomainDoc): number | undefined {
+    const dueTimes = (doc.retiringDkim || [])
+      .map((material) => Date.parse(material.retireAfter || ''))
+      .filter(Number.isFinite);
+    return dueTimes.length > 0 ? Math.min(...dueTimes) : undefined;
+  }
+
   private async scheduleNextDue(): Promise<void> {
+    const generation = ++this.scheduleGeneration;
     if (this.retryTimer) clearTimeout(this.retryTimer);
     this.retryTimer = undefined;
     if (!this.started || this.stopped) return;
     const dueTimes: number[] = [];
     const now = Date.now();
-    for (const doc of await EmailDomainDoc.findAll()) {
+    const docs = await EmailDomainDoc.findAll();
+    if (generation !== this.scheduleGeneration || !this.started || this.stopped) return;
+    for (const doc of docs) {
       const retryAt = Date.parse(doc.reconciliation?.retryAt || '');
-      if (Number.isFinite(retryAt)) dueTimes.push(retryAt);
+      if (Number.isFinite(retryAt)) {
+        dueTimes.push(retryAt);
+        continue;
+      }
+      const lastAttemptAt = Date.parse(doc.reconciliation?.lastAttemptAt || '');
+      dueTimes.push(Number.isFinite(lastAttemptAt)
+        ? lastAttemptAt + MAIL_DNS_RECONCILIATION_INTERVAL_MS
+        : now);
       const rotationDueAt = this.getDkimRotationDueAt(doc, now);
       if (rotationDueAt !== undefined) dueTimes.push(rotationDueAt);
+      const retirementDueAt = this.getDkimRetirementDueAt(doc);
+      if (retirementDueAt !== undefined) dueTimes.push(retirementDueAt);
     }
     if (dueTimes.length === 0) return;
-    const delay = Math.max(1, Math.min(2_147_483_647, Math.min(...dueTimes) - now));
+    const nextDueAt = Math.max(
+      Math.min(...dueTimes),
+      this.lastScheduledReconciliationAt + MAIL_DNS_RECONCILIATION_INTERVAL_MS,
+    );
+    const delay = Math.max(1, Math.min(2_147_483_647, nextDueAt - now));
     this.retryTimer = setTimeout(() => {
-      if (this.stopped) return;
-      this.queueRequest([], false);
-      this.kick('scheduled retry').catch((error: unknown) => {
+      this.retryTimer = undefined;
+      if (this.stopped || !this.started) return;
+      this.lastScheduledReconciliationAt = Date.now();
+      this.queueRequest(undefined, true);
+      this.kick('scheduled desired-state reconciliation').catch((error: unknown) => {
         logger.log('error', `MailDnsSync scheduled retry failed: ${(error as Error).message}`);
       });
     }, delay);
   }
 
+  private queueScheduleNextDue(): void {
+    const schedulePromise = this.scheduleNextDue()
+      .catch((error: unknown) => {
+        logger.log('error', `MailDnsSync scheduling failed: ${(error as Error).message}`);
+      })
+      .finally(() => {
+        this.schedulePromises.delete(schedulePromise);
+      });
+    this.schedulePromises.add(schedulePromise);
+  }
+
   private async buildPlan(doc: EmailDomainDoc, zones: DomainDoc[]): Promise<IPlannedDomain> {
     const now = new Date().toISOString();
     const eligibility = await this.edgeEligibility.resolveForDomain(doc);
@@ -746,6 +815,12 @@ export class MailDnsSync {
         message: eligibility.reason || 'No eligible mail edge is connected',
         retryable: true,
       });
+    } else if (eligibility.edges.length < 2) {
+      preliminaryErrors.push({
+        code: 'INSUFFICIENT_ELIGIBLE_MAIL_EDGES',
+        message: 'Two eligible RemoteIngress mail edges are required; direct hub fallback is disabled',
+        retryable: true,
+      });
     }
 
     const activeDkim = this.getActiveDkim(doc);
@@ -767,19 +842,26 @@ export class MailDnsSync {
       });
     }
 
-    const edgeIdentities = eligibility.edges.map((candidate) => this.toDomainEdgeIdentity(candidate));
     const mxSlots = this.allocateMxSlots(doc, eligibility.edges);
     if (eligibility.edges.length >= 2 && mxSlots.length < 2) {
       preliminaryErrors.push({
         code: 'INSUFFICIENT_DISTINCT_MX_IDENTITIES',
         message: 'At least two eligible mail edges exist but they do not provide two distinct MX hostnames',
-        retryable: false,
+        retryable: true,
       });
     }
+    const candidatesById = new Map<string, IEligibleMailEdge>(
+      eligibility.edges.map((candidate) => [candidate.edge.id, candidate]),
+    );
+    const selectedEdges: IEligibleMailEdge[] = mxSlots.flatMap((slot) => {
+      const candidate = candidatesById.get(slot.edgeId);
+      return candidate ? [candidate] : [];
+    });
+    const edgeIdentities = selectedEdges.map((candidate) => this.toDomainEdgeIdentity(candidate));
     const revisionId = plugins.smartunique.shortId();
     const intents: IEmailDnsRecordIntent[] = [];
 
-    for (const candidate of eligibility.edges) {
+    for (const candidate of selectedEdges) {
       for (const identity of candidate.identities) {
         const type = identity.family === 4 ? 'A' : 'AAAA';
         intents.push(this.intentForName(zones, {
@@ -794,7 +876,7 @@ export class MailDnsSync {
       }
     }
 
-    if (eligibility.edges.length > 0) {
+    if (mxSlots.length === 2) {
       for (const slot of mxSlots) {
         intents.push(this.intentForName(zones, {
           key: `mail-domain:${doc.id}:mx:${slot.priority}:${slot.edgeId}:${slot.hostname}`,
@@ -922,13 +1004,7 @@ export class MailDnsSync {
   }
 
   private allocateMxSlots(doc: EmailDomainDoc, candidates: IEligibleMailEdge[]): IEmailDomainMxSlot[] {
-    if (candidates.length === 1) {
-      return [{
-        priority: 10,
-        edgeId: candidates[0].edge.id,
-        hostname: candidates[0].hostname,
-      }];
-    }
+    if (candidates.length < 2) return [];
     const byId = new Map(candidates.map((candidate) => [candidate.edge.id, candidate]));
     const result: IEmailDomainMxSlot[] = [];
     const usedEdges = new Set<string>();
@@ -956,7 +1032,9 @@ export class MailDnsSync {
       usedEdges.add(candidate.edge.id);
       usedHostnames.add(candidate.hostname);
     }
-    return result.sort((left, right) => left.priority - right.priority);
+    return result.length === 2
+      ? result.sort((left, right) => left.priority - right.priority)
+      : [];
   }
 
   private buildSpfValue(identities: IEmailDomainEdgeIdentity[]): string {
@@ -975,7 +1053,7 @@ export class MailDnsSync {
     const generation = (previous?.desiredGeneration || 0) + (previousSignature === nextSignature ? 0 : 1);
     const desiredGeneration = Math.max(generation, 1);
     plan.revision.generation = desiredGeneration;
-    const providerZoneIds = new Set(previous?.providerZoneIds || []);
+    const providerZoneIds = new Set<string>();
     const providerZones = new Set(zones.filter((zone) => zone.source === 'provider').map((zone) => zone.id));
     for (const intent of plan.intents) {
       if (providerZones.has(intent.domainId)) providerZoneIds.add(intent.domainId);
@@ -1010,6 +1088,34 @@ export class MailDnsSync {
   }
 
   private async persistDeletingAttempt(doc: EmailDomainDoc): Promise<void> {
+    const deletingProviderRecordIds = new Set(doc.reconciliation?.deletingProviderRecordIds || []);
+    const deletingRecordKeys = new Set([
+      ...(doc.reconciliation?.activeRevision?.recordKeys || []),
+      ...(doc.reconciliation?.pendingRevision?.recordKeys || []),
+    ]);
+    const retainedTopologyKeys = new Set<string>();
+    for (const otherDoc of await EmailDomainDoc.findAll()) {
+      if (otherDoc.id === doc.id || otherDoc.reconciliation?.lifecycleStatus === 'deleting') continue;
+      for (const intent of otherDoc.reconciliation?.intents || []) retainedTopologyKeys.add(intent.key);
+      for (const key of otherDoc.reconciliation?.activeRevision?.recordKeys || []) retainedTopologyKeys.add(key);
+      for (const key of otherDoc.reconciliation?.pendingRevision?.recordKeys || []) retainedTopologyKeys.add(key);
+    }
+    const hasKnownProviderProvenance = doc.reconciliation?.providerZoneIds !== undefined;
+    const providerZoneIds = new Set(doc.reconciliation?.providerZoneIds || []);
+    for (const record of await DnsRecordDoc.findAll()) {
+      const ownedByDomain = record.managedOwnerId === `domain:${doc.id}`;
+      const unsharedDeletingTopology = record.managedOwnerId?.startsWith('topology:')
+        && Boolean(record.managedRecordKey && deletingRecordKeys.has(record.managedRecordKey))
+        && Boolean(record.managedRecordKey && !retainedTopologyKeys.has(record.managedRecordKey));
+      if (
+        record.managedBy === MAIL_DNS_MANAGED_BY
+        && (ownedByDomain || unsharedDeletingTopology)
+        && record.providerRecordId?.trim()
+      ) {
+        deletingProviderRecordIds.add(record.providerRecordId.trim());
+        if (hasKnownProviderProvenance) providerZoneIds.add(record.domainId);
+      }
+    }
     doc.reconciliation = {
       ...(doc.reconciliation || {
         lifecycleStatus: 'deleting',
@@ -1022,6 +1128,8 @@ export class MailDnsSync {
       errors: (doc.reconciliation?.errors || []).filter((error) => error.code !== 'DNS_CLEANUP_FAILED'),
       lastAttemptAt: new Date().toISOString(),
       retryAt: new Date(Date.now() + MAIL_DNS_RETRY_INTERVAL_MS).toISOString(),
+      ...(hasKnownProviderProvenance ? { providerZoneIds: [...providerZoneIds].sort() } : {}),
+      deletingProviderRecordIds: [...deletingProviderRecordIds].sort(),
     };
     await doc.save();
   }
@@ -1095,16 +1203,35 @@ export class MailDnsSync {
     const rrset = records.filter((record) => (
       record.name.toLowerCase() === intent.name.toLowerCase() && record.type === intent.type
     ));
+    const ownedByKey = records.filter((record) => (
+      record.managedBy === MAIL_DNS_MANAGED_BY
+      && record.managedOwnerId === intent.ownerId
+      && record.managedRecordKey === intent.key
+    ));
+    if (ownedByKey.length > 1) {
+      intent.status = 'conflict';
+      intent.verificationOutcome = 'duplicate';
+      intent.error = {
+        code: 'DNS_MANAGED_KEY_DUPLICATE',
+        message: `Multiple managed DNS records share ownership key ${intent.key}`,
+        recordKey: intent.key,
+        retryable: false,
+      };
+      return;
+    }
+    const reconciliationRecords = [...new Map(
+      [...rrset, ...ownedByKey].map((record) => [record.id, record]),
+    ).values()];
     if (await reconcileMailDnsSingleton({
       intent,
-      rrset,
+      rrset: reconciliationRecords,
       dnsManager: this.dcRouterRef.dnsManager!,
       managedBy: MAIL_DNS_MANAGED_BY,
       createdBy: MAIL_DNS_SYNC_CREATED_BY,
       evaluate: async (candidate, values) => await this.evaluateTxtRecords(candidate, values),
     })) return;
 
-    const exactOwner = rrset
+    const exactOwner = reconciliationRecords
       .filter((record) => (
         record.managedBy === MAIL_DNS_MANAGED_BY
         && record.managedOwnerId === intent.ownerId
@@ -1175,10 +1302,16 @@ export class MailDnsSync {
     const managed = exactOwner.find((record) => this.valuesEqual(intent.type, record.value, intent.value))
       || exactOwner[0];
     if (managed) {
-      if (!this.valuesEqual(intent.type, managed.value, intent.value) || managed.name !== intent.name) {
+      if (
+        !this.valuesEqual(intent.type, managed.value, intent.value)
+        || managed.name !== intent.name
+        || managed.type !== intent.type
+        || managed.ttl !== intent.ttl
+      ) {
         const result = await this.dcRouterRef.dnsManager!.updateRecord({
           id: managed.id,
           name: intent.name,
+          type: intent.type,
           value: intent.value,
           ttl: intent.ttl,
         });
@@ -1347,10 +1480,8 @@ export class MailDnsSync {
     }
 
     const canActivate = plan.egressIdentitySupported
-      && plan.revision.edgeIdentities.length > 0
-      && (plan.revision.edgeIdentities.length >= 2
-        ? plan.revision.mxSlots.length === 2
-        : plan.revision.mxSlots.length === 1)
+      && plan.revision.edgeIdentities.length === 2
+      && plan.revision.mxSlots.length === 2
       && plan.selectorCorrectSigningSupported
       && Boolean(this.getActiveDkim(plan.doc))
       && plan.intents.length > 0
@@ -1415,8 +1546,8 @@ export class MailDnsSync {
           : { retryAt: new Date(Date.now() + retryDelay).toISOString() }),
     };
     const mailPublicationReady = plan.egressIdentitySupported
-      && plan.revision.edgeIdentities.length > 0
-      && plan.revision.mxSlots.length > 0;
+      && plan.revision.edgeIdentities.length === 2
+      && plan.revision.mxSlots.length === 2;
     plan.doc.dnsStatus = {
       mx: mailPublicationReady
         ? projectMailDnsIntentAggregateStatus(plan.intents, (intent) => intent.type === 'MX')
@@ -1523,45 +1654,93 @@ export class MailDnsSync {
     for (const doc of docs) {
       if (doc.reconciliation?.lifecycleStatus === 'deleting') continue;
       for (const intent of doc.reconciliation?.intents || []) keep.add(intent.key);
-      if (doc.reconciliation?.lifecycleStatus !== 'active') {
+      const hasCompleteDesiredMxSet = (doc.reconciliation?.intents || [])
+        .filter((intent) => intent.type === 'MX').length === 2;
+      if (doc.reconciliation?.lifecycleStatus !== 'active' && hasCompleteDesiredMxSet) {
         for (const key of doc.reconciliation?.activeRevision?.recordKeys || []) keep.add(key);
       }
     }
     const managed = (await DnsRecordDoc.findAll()).filter((record) => record.managedBy === MAIL_DNS_MANAGED_BY);
-    for (const record of managed) {
-      if (!record.managedOwnerId || !record.managedRecordKey) continue;
+    const staleRecords = managed.filter((record) => {
+      if (!record.managedOwnerId || !record.managedRecordKey) return false;
       if (
         !selectedOwnerIds.has(record.managedOwnerId)
         && !(allowTopologyCleanup && record.managedOwnerId.startsWith('topology:'))
-      ) continue;
-      if (record.managedRecordKey && keep.has(record.managedRecordKey)) continue;
+      ) return false;
+      if (keep.has(record.managedRecordKey)) return false;
       const zone = zones.find((candidate) => candidate.id === record.domainId);
-      if (zone?.source === 'provider' && !provenProviderZoneIds.has(zone.id)) continue;
-      try {
-        const result = await this.dcRouterRef.dnsManager!.deleteRecord(record.id);
-        if (!result.success) throw new Error(result.message || 'delete failed');
-      } catch (error: unknown) {
-        const ownerId = record.managedOwnerId?.replace(/^domain:/, '');
-        const owner = ownerId ? docs.find((doc) => doc.id === ownerId) : undefined;
-        if (owner?.reconciliation) {
+      return zone?.source !== 'provider' || provenProviderZoneIds.has(zone.id);
+    });
+    if (staleRecords.length === 0) return;
+    const results = await this.dcRouterRef.dnsManager!.deleteRecords(
+      staleRecords.map((record) => record.id),
+    );
+    for (let index = 0; index < staleRecords.length; index++) {
+      const record = staleRecords[index];
+      const result = results[index];
+      if (result?.success) continue;
+      const error = new Error(result?.message || 'delete failed');
+      const domainOwnerId = record.managedOwnerId?.startsWith('domain:')
+        ? record.managedOwnerId.slice('domain:'.length)
+        : undefined;
+      const affectedOwners = domainOwnerId
+        ? docs.filter((doc) => doc.id === domainOwnerId)
+        : docs.filter((doc) => selectedOwnerIds.has(`domain:${doc.id}`));
+      if (affectedOwners.length > 0) {
+        for (const owner of affectedOwners) {
+          if (!owner.reconciliation) continue;
           owner.reconciliation.errors.push({
             code: 'DNS_CLEANUP_FAILED',
-            message: `Failed to delete ${record.type} ${record.name}: ${(error as Error).message}`,
+            message: `Failed to delete ${record.type} ${record.name}: ${error.message}`,
             recordKey: record.managedRecordKey,
             retryable: true,
           });
           owner.reconciliation.retryAt = new Date(Date.now() + MAIL_DNS_RETRY_INTERVAL_MS).toISOString();
+          if (owner.reconciliation.lifecycleStatus !== 'deleting') {
+            owner.reconciliation.lifecycleStatus = 'pending';
+          }
           await owner.save();
         }
+      } else {
+        logger.log(
+          'error',
+          `MailDnsSync failed to delete orphaned topology record ${record.type} ${record.name}: ${error.message}`,
+        );
       }
     }
   }
 
+  private async pruneRetiredDkimMaterial(docs: EmailDomainDoc[]): Promise<void> {
+    const now = Date.now();
+    const records = await DnsRecordDoc.findAll();
+    for (const plannedDoc of docs) {
+      const doc = await EmailDomainDoc.findById(plannedDoc.id);
+      if (!doc) continue;
+      const retiring = doc.retiringDkim || [];
+      const retained = retiring.filter((material) => {
+        const retireAt = Date.parse(material.retireAfter || '');
+        if (!Number.isFinite(retireAt) || retireAt > now) return true;
+        const key = `mail-domain:${doc.id}:dkim:${material.selector}`;
+        return records.some((record) => (
+          record.managedBy === MAIL_DNS_MANAGED_BY
+          && record.managedOwnerId === `domain:${doc.id}`
+          && record.managedRecordKey === key
+        ));
+      });
+      if (retained.length === retiring.length) continue;
+      doc.retiringDkim = retained;
+      doc.updatedAt = new Date().toISOString();
+      await doc.save();
+    }
+  }
+
   private async finalizeDeletingDomains(
     deletingDocs: EmailDomainDoc[],
     zones: DomainDoc[],
   ): Promise<void> {
-    for (const doc of deletingDocs) {
+    for (const deletingDoc of deletingDocs) {
+      const doc = await EmailDomainDoc.findById(deletingDoc.id);
+      if (!doc) continue;
       if (doc.reconciliation?.lifecycleStatus !== 'deleting') continue;
       if (doc.reconciliation.providerZoneIds === undefined) continue;
       if (doc.reconciliation.errors.some((error) => (
@@ -1587,6 +1766,20 @@ export class MailDnsSync {
           await doc.save();
           break;
         }
+        const stillPublishedIds = new Set(confirmation.listedProviderRecordIds || []);
+        const lingeringProviderId = (doc.reconciliation.deletingProviderRecordIds || [])
+          .find((providerRecordId) => stillPublishedIds.has(providerRecordId));
+        if (lingeringProviderId) {
+          providerProofComplete = false;
+          doc.reconciliation.errors.push({
+            code: 'DNS_CLEANUP_FAILED',
+            message: `Provider zone ${zone.name} still publishes deleting record ${lingeringProviderId}`,
+            retryable: true,
+          });
+          doc.reconciliation.retryAt = new Date(Date.now() + MAIL_DNS_RETRY_INTERVAL_MS).toISOString();
+          await doc.save();
+          break;
+        }
       }
       if (!providerProofComplete) continue;
       const records = await DnsRecordDoc.findAll();

package/ts/email/classes.mail-edge-eligibility.ts

@@ -2,6 +2,7 @@ import { REMOTE_INGRESS_MAIL_TAG } from '../../ts_interfaces/data/remoteingress.
 import type { IRemoteIngress } from '../../ts_interfaces/data/remoteingress.js';
 import type { EmailDomainDoc } from '../db/documents/classes.email-domain.doc.js';
 import type { DcRouter } from '../classes.dcrouter.js';
+import type { IEmailDomainEdgeIdentity } from '../../ts_interfaces/data/email-domain.js';
 
 export const MAIL_EGRESS_IDENTITY_MAX_AGE_MS = 45_000;
 
@@ -146,13 +147,22 @@ export class MailEdgeEligibility {
       const hostname = edge.mailHostname?.toLowerCase().replace(/\.$/, '');
       if (!hostname) continue;
 
-      const identities = (identitiesByEdge.get(edge.id) || []).filter((identity) => {
-        if (identity.heloHostname.toLowerCase().replace(/\.$/, '') !== hostname) return false;
-        return identity.family === 4
-          ? Boolean(edge.publicIp && edge.publicIp === identity.address)
-          : Boolean(edge.publicIpV6 && edge.publicIpV6 === identity.address);
+      const reported = (identitiesByEdge.get(edge.id) || []).filter((identity) => (
+        identity.heloHostname.toLowerCase().replace(/\.$/, '') === hostname
+      ));
+      const expected = [
+        ...(edge.publicIp ? [{ family: 4 as const, address: edge.publicIp }] : []),
+        ...(edge.publicIpV6 ? [{ family: 6 as const, address: edge.publicIpV6 }] : []),
+      ];
+      const identities = expected.flatMap((configured) => {
+        const matches = reported.filter((identity) => (
+          identity.family === configured.family && identity.address === configured.address
+        ));
+        return matches.length === 1 ? matches : [];
       });
-      if (identities.length > 0) {
+      // Every configured source family must have one fresh source-bound proof.
+      // A matching IPv4 observation never excuses an unproven configured IPv6.
+      if (expected.length > 0 && identities.length === expected.length) {
         edges.push({ edge, hostname, identities });
       }
     }
@@ -179,27 +189,55 @@ export class MailEdgeEligibility {
 
   public async resolveActiveForDomain(doc: EmailDomainDoc): Promise<IMailEdgeEligibilityResult> {
     const live = await this.resolveForDomain(doc);
-    const active = doc.reconciliation?.activeRevision;
-    if (!live.supported || !active) {
+    const reconciliation = doc.reconciliation;
+    const active = reconciliation?.activeRevision;
+    const slots = active?.mxSlots || [];
+    const completeSlots = slots.length === 2
+      && slots[0]?.priority === 10
+      && slots[1]?.priority === 20
+      && slots[0].edgeId !== slots[1].edgeId
+      && slots[0].hostname !== slots[1].hostname;
+    if (
+      !live.supported
+      || reconciliation?.lifecycleStatus !== 'active'
+      || reconciliation.errors.length > 0
+      || !active
+      || active.generation !== reconciliation.desiredGeneration
+      || !completeSlots
+    ) {
       return {
         supported: live.supported,
-        reason: live.reason || 'email domain has no validated active DNS revision',
+        reason: live.reason || 'email domain has no current complete validated DNS revision',
         edges: [],
       };
     }
-    const activeByEdge = new Map(active.edgeIdentities.map((identity) => [identity.edgeId, identity]));
-    const edges = live.edges.filter((candidate) => {
+    const activeByEdge = new Map<string, IEmailDomainEdgeIdentity>(
+      active.edgeIdentities.map((identity) => [identity.edgeId, identity]),
+    );
+    const liveByEdge = new Map(live.edges.map((candidate) => [candidate.edge.id, candidate]));
+    const edges = slots.flatMap((slot) => {
+      const candidate = liveByEdge.get(slot.edgeId);
+      if (!candidate || candidate.hostname !== slot.hostname) return [];
       const activeIdentity = activeByEdge.get(candidate.edge.id);
-      if (!activeIdentity || activeIdentity.hostname !== candidate.hostname) return false;
-      return candidate.identities.length > 0 && candidate.identities.every((identity) => (
+      if (!activeIdentity || activeIdentity.hostname !== candidate.hostname) return [];
+      const expectedFamilies = [
+        ...(activeIdentity.ipv4 ? [{ family: 4 as const, address: activeIdentity.ipv4 }] : []),
+        ...(activeIdentity.ipv6 ? [{ family: 6 as const, address: activeIdentity.ipv6 }] : []),
+      ];
+      const exact = candidate.identities.length === expectedFamilies.length
+        && expectedFamilies.every((expected) => candidate.identities.some((identity) => (
+          identity.family === expected.family && identity.address === expected.address
+        )))
+        && candidate.identities.every((identity) => (
         (identity.family === 4 && identity.address === activeIdentity.ipv4)
         || (identity.family === 6 && identity.address === activeIdentity.ipv6)
       ));
+      return exact ? [candidate] : [];
     });
     return {
       supported: live.supported,
-      ...(edges.length === 0 ? { reason: 'no live edge is present in the validated active DNS revision' } : {}),
-      edges,
+      ...(edges.length !== 2 ? { reason: 'both validated MX edges must be live with exact source-bound identities' } : {}),
+      edges: edges.length === 2 ? edges : [],
     };
   }
 }