@serve.zone/dcrouter 17.2.4 → 17.4.0

package/ts/dns/classes.gateway-route-dns-reconciler.ts

@@ -0,0 +1,801 @@
+import * as plugins from '../plugins.js';
+import type { DcRouter } from '../classes.dcrouter.js';
+import type { DnsRecordDoc, DomainDoc } from '../db/index.js';
+import type * as interfaces from '../../ts_interfaces/index.js';
+import { logger } from '../logger.js';
+import { normalizeGatewayHostname } from '../opsserver/helpers/gateway-client-auth.js';
+
+export const GATEWAY_ROUTE_DNS_MANAGED_BY = 'gateway-route-dns';
+const GATEWAY_ROUTE_DNS_TTL = 3600;
+const RECONCILE_INTERVAL_MS = 60_000;
+const RECONCILE_COOLDOWN_MS = 15_000;
+
+type TGatewayDnsResult = plugins.servezoneInterfaces.data.IGatewayRouteDnsResult;
+
+export interface IGatewayRouteDnsReconcileOptions {
+  externalKey: string;
+  gatewayClientId: string;
+  hostname?: string;
+  mode?: plugins.servezoneInterfaces.data.TGatewayRouteDnsMode;
+  delete?: boolean;
+}
+
+interface IGatewayDnsReconcileExecutionOptions {
+  respectCooldown?: boolean;
+  providerZoneRefreshes?: Map<string, Promise<IProviderZoneRefreshResult>>;
+}
+
+interface IProviderZoneRefreshResult {
+  success: boolean;
+  message?: string;
+}
+
+interface IGatewayDnsLastAttempt {
+  workKey: string;
+  generation: number;
+  attemptedAt: number;
+  result?: TGatewayDnsResult;
+}
+
+interface IGatewayDnsLatestIntent {
+  workKey: string;
+  generation: number;
+  promise: Promise<TGatewayDnsResult>;
+}
+
+export interface IGatewayDnsActiveClaim {
+  gatewayClientId: string;
+  routeExternalKeys: string[];
+}
+
+export function getActiveGatewayDnsClaims(
+  dcRouterRefArg: Pick<DcRouter, 'routeConfigManager'>,
+  hostnameArg: string,
+): IGatewayDnsActiveClaim[] {
+  const hostname = normalizeGatewayHostname(hostnameArg);
+  const claims = new Map<string, string[]>();
+  const routes = dcRouterRefArg.routeConfigManager?.getMergedRoutes().routes || [];
+  for (const routeArg of routes) {
+    const metadata = routeArg.metadata;
+    if (!routeArg.enabled || metadata?.ownerType !== 'gatewayClient'
+      || metadata.gatewayDnsMode !== 'reconcile' || !metadata.gatewayClientId
+      || !metadata.externalKey) continue;
+    const domains = routeArg.route.match?.domains;
+    const firstDomain = Array.isArray(domains)
+      ? domains[0]
+      : typeof domains === 'string'
+        ? domains.split(',')[0]
+        : undefined;
+    if (!firstDomain) continue;
+    try {
+      if (normalizeGatewayHostname(firstDomain) !== hostname) continue;
+    } catch {
+      continue;
+    }
+    const gatewayClientId = metadata.gatewayClientId.trim().toLowerCase();
+    const keys = claims.get(gatewayClientId) || [];
+    keys.push(metadata.externalKey);
+    claims.set(gatewayClientId, keys);
+  }
+  return Array.from(claims.entries())
+    .map(([gatewayClientId, routeExternalKeys]) => ({
+      gatewayClientId,
+      routeExternalKeys: routeExternalKeys.sort((left, right) => left.localeCompare(right)),
+    }))
+    .sort((left, right) => left.gatewayClientId.localeCompare(right.gatewayClientId));
+}
+
+export class GatewayRouteDnsReconciler {
+  private interval?: ReturnType<typeof setInterval> & { unref?: () => void };
+  private fullReconcilePromise?: Promise<void>;
+  private lastAttemptByHostname = new Map<string, IGatewayDnsLastAttempt>();
+  private latestIntentByHostname = new Map<string, IGatewayDnsLatestIntent>();
+  private hostnameChains = new Map<string, Promise<void>>();
+  private intentGeneration = 0;
+
+  constructor(private dcRouterRef: DcRouter) {}
+
+  public async start(): Promise<void> {
+    await this.runFullReconcile(true);
+    this.interval = setInterval(() => {
+      this.runFullReconcile(false).catch((error: unknown) => {
+        logger.log('warn', `Gateway route DNS reconcile failed: ${(error as Error).message}`);
+      });
+    }, RECONCILE_INTERVAL_MS);
+    this.interval.unref?.();
+  }
+
+  public async stop(): Promise<void> {
+    if (this.interval) clearInterval(this.interval);
+    this.interval = undefined;
+    await this.fullReconcilePromise?.catch(() => undefined);
+  }
+
+  public async reconcileRoute(optionsArg: IGatewayRouteDnsReconcileOptions): Promise<TGatewayDnsResult> {
+    return await this.reconcileRouteWithExecution(optionsArg, {});
+  }
+
+  private async reconcileRouteWithExecution(
+    optionsArg: IGatewayRouteDnsReconcileOptions,
+    executionArg: IGatewayDnsReconcileExecutionOptions,
+  ): Promise<TGatewayDnsResult> {
+    const checkedAt = Date.now();
+    const mode = optionsArg.mode || 'skip';
+    if (mode === 'skip') {
+      return { success: true, retryable: false, mode, status: 'skipped', checkedAt };
+    }
+    if (!optionsArg.hostname) {
+      return { success: true, retryable: false, mode, status: 'not-applicable', checkedAt };
+    }
+    const hostname = normalizeGatewayHostname(optionsArg.hostname);
+    const targetIp = optionsArg.delete ? undefined : this.getDesiredTargetIp();
+    const workKey = this.buildWorkKey(optionsArg, hostname, mode, targetIp);
+    const latestIntent = this.latestIntentByHostname.get(hostname);
+    if (latestIntent?.workKey === workKey) return await latestIntent.promise;
+
+    const lastAttempt = this.lastAttemptByHostname.get(hostname);
+    if (
+      executionArg.respectCooldown
+      && lastAttempt?.workKey === workKey
+      && lastAttempt.result
+      && checkedAt - lastAttempt.attemptedAt < RECONCILE_COOLDOWN_MS
+    ) {
+      return lastAttempt.result;
+    }
+
+    const generation = ++this.intentGeneration;
+    this.lastAttemptByHostname.set(hostname, { workKey, generation, attemptedAt: checkedAt });
+    const run = this.withHostnameLock(hostname, async () => {
+      return await this.reconcileRouteUnderHostnameLock(
+        optionsArg,
+        hostname,
+        mode,
+        checkedAt,
+        targetIp,
+        executionArg.providerZoneRefreshes,
+      );
+    });
+    this.latestIntentByHostname.set(hostname, {
+      workKey,
+      generation,
+      promise: run,
+    });
+    try {
+      const result = await run;
+      const currentAttempt = this.lastAttemptByHostname.get(hostname);
+      if (currentAttempt?.generation === generation) currentAttempt.result = result;
+      return result;
+    } finally {
+      const currentIntent = this.latestIntentByHostname.get(hostname);
+      if (currentIntent?.generation === generation) {
+        this.latestIntentByHostname.delete(hostname);
+      }
+    }
+  }
+
+  private async reconcileRouteUnderHostnameLock(
+    optionsArg: IGatewayRouteDnsReconcileOptions,
+    hostname: string,
+    mode: Exclude<plugins.servezoneInterfaces.data.TGatewayRouteDnsMode, 'skip'>,
+    checkedAt: number,
+    targetIp: string | undefined,
+    providerZoneRefreshes?: Map<string, Promise<IProviderZoneRefreshResult>>,
+  ): Promise<TGatewayDnsResult> {
+    const claimOwnerId = this.buildClaimOwnerId(optionsArg.gatewayClientId, hostname);
+    if (optionsArg.delete) {
+      return await this.deleteOwnedRecords(
+        claimOwnerId,
+        optionsArg.externalKey,
+        optionsArg.gatewayClientId,
+        hostname,
+        mode,
+        checkedAt,
+        providerZoneRefreshes,
+      );
+    }
+
+    if (!this.getActiveClaimsForHostname(hostname).some((claimArg) => {
+      return claimArg.gatewayClientId === optionsArg.gatewayClientId.trim().toLowerCase();
+    })) {
+      return {
+        success: true,
+        retryable: false,
+        mode,
+        status: 'not-applicable',
+        hostname,
+        targetIp,
+        checkedAt,
+      };
+    }
+
+    const ipFamily = targetIp ? plugins.net.isIP(targetIp) : 0;
+    if (!targetIp || !ipFamily) {
+      return {
+        success: false,
+        retryable: true,
+        mode,
+        status: 'target-invalid',
+        hostname,
+        checkedAt,
+        message: 'Gateway DNS target requires a valid proxyIps[0], publicIp, or detectedPublicIp',
+      };
+    }
+    const recordType = ipFamily === 6 ? 'AAAA' : 'A';
+    const dnsManager = this.dcRouterRef.dnsManager;
+    if (!dnsManager) {
+      return {
+        success: false,
+        retryable: true,
+        mode,
+        status: 'zone-unavailable',
+        hostname,
+        targetIp,
+        recordType,
+        checkedAt,
+        message: 'DnsManager is not initialized',
+      };
+    }
+    const zone = await dnsManager.findDomainForFqdn(hostname);
+    if (!zone) {
+      return {
+        success: false,
+        retryable: false,
+        mode,
+        status: 'zone-unmanaged',
+        hostname,
+        targetIp,
+        recordType,
+        checkedAt,
+        message: `No managed DNS zone covers ${hostname}`,
+      };
+    }
+    if (zone.source === 'provider' && !zone.providerId) {
+      return this.failureForZone(mode, checkedAt, hostname, targetIp, recordType, zone, 'zone-unavailable', 'Managed provider zone has no providerId');
+    }
+    if (zone.source === 'provider') {
+      const syncResult = await this.refreshProviderZone(zone.id, providerZoneRefreshes);
+      if (!syncResult.success) {
+        return this.failureForZone(
+          mode,
+          checkedAt,
+          hostname,
+          targetIp,
+          recordType,
+          zone,
+          'zone-unavailable',
+          syncResult.message || 'Provider zone synchronization failed',
+        );
+      }
+    }
+
+    const records = (await dnsManager.listRecordsForDomain(zone.id)).filter((recordArg) => {
+      return this.isAddressRecord(recordArg)
+        && this.normalizeStoredHostname(recordArg.name) === hostname;
+    });
+    const competing = records.find((recordArg) => recordArg.managedBy
+      && (recordArg.managedBy !== GATEWAY_ROUTE_DNS_MANAGED_BY || recordArg.managedOwnerId !== claimOwnerId));
+    if (competing) {
+      return this.failureForZone(
+        mode,
+        checkedAt,
+        hostname,
+        targetIp,
+        recordType,
+        zone,
+        'record-conflict',
+        `DNS hostname ${hostname} is managed by ${competing.managedBy}:${competing.managedOwnerId || 'unknown'}`,
+        records,
+      );
+    }
+
+    const desired = records.find((recordArg) => recordArg.type === recordType
+      && recordArg.value === targetIp
+      && recordArg.ttl === GATEWAY_ROUTE_DNS_TTL
+      && recordArg.proxied === false);
+    if (mode === 'observe') {
+      const satisfied = records.some((recordArg) => recordArg.type === recordType && recordArg.value === targetIp);
+      return {
+        success: satisfied,
+        retryable: !satisfied,
+        mode,
+        status: satisfied ? 'observed-satisfied' : 'record-missing',
+        hostname,
+        targetIp,
+        recordType,
+        zoneId: zone.id,
+        zoneName: zone.name,
+        nameservers: zone.nameservers,
+        records: records.map((recordArg) => this.toEvidence(recordArg)),
+        checkedAt,
+      };
+    }
+
+    const ownerRecords = records.filter((recordArg) => recordArg.managedBy === GATEWAY_ROUTE_DNS_MANAGED_BY
+      && recordArg.managedOwnerId === claimOwnerId);
+    const manualRecords = records.filter((recordArg) => !recordArg.managedBy);
+    const stableKey = this.buildManagedRecordKey(claimOwnerId);
+    if (
+      records.length === 1
+      && manualRecords.length === 1
+      && desired === manualRecords[0]
+    ) {
+      const adopted = await dnsManager.adoptRecordManagement({
+        id: desired.id,
+        name: hostname,
+        type: recordType,
+        value: targetIp,
+        ttl: GATEWAY_ROUTE_DNS_TTL,
+        proxied: false,
+        managedBy: GATEWAY_ROUTE_DNS_MANAGED_BY,
+        managedOwnerId: claimOwnerId,
+        managedRecordKey: stableKey,
+      });
+      if (!adopted.success) {
+        return this.failureForZone(
+          mode,
+          checkedAt,
+          hostname,
+          targetIp,
+          recordType,
+          zone,
+          'mutation-failed',
+          adopted.message || 'Failed to adopt existing DNS record',
+          records,
+        );
+      }
+      const adoptedRecords = (await dnsManager.listRecordsForDomain(zone.id)).filter((recordArg) => {
+        return this.isAddressRecord(recordArg)
+          && this.normalizeStoredHostname(recordArg.name) === hostname;
+      });
+      return this.successForZone('managed-updated', mode, checkedAt, hostname, targetIp, recordType, zone, adoptedRecords);
+    }
+    if (
+      records.length === 1
+      && ownerRecords.length === 1
+      && desired === ownerRecords[0]
+      && desired.managedRecordKey === stableKey
+    ) {
+      return this.successForZone('managed-unchanged', mode, checkedAt, hostname, targetIp, recordType, zone, records);
+    }
+
+    const overwrittenRecords = manualRecords.map((recordArg) => this.toEvidence(recordArg));
+    const rollbackRecords = records.map((recordArg) => this.snapshotRecord(recordArg));
+    try {
+      for (const record of records) {
+        const result = await dnsManager.deleteRecord(record.id);
+        if (!result.success) throw new Error(result.message || `Failed to delete DNS record ${record.id}`);
+      }
+      const created = await dnsManager.createRecord({
+        domainId: zone.id,
+        name: hostname,
+        type: recordType,
+        value: targetIp,
+        ttl: GATEWAY_ROUTE_DNS_TTL,
+        proxied: false,
+        createdBy: GATEWAY_ROUTE_DNS_MANAGED_BY,
+        managedBy: GATEWAY_ROUTE_DNS_MANAGED_BY,
+        managedOwnerId: claimOwnerId,
+        managedRecordKey: stableKey,
+      });
+      if (!created.success) throw new Error(created.message || `Failed to create ${recordType} record for ${hostname}`);
+      const finalRecords = (await dnsManager.listRecordsForDomain(zone.id)).filter((recordArg) => {
+        return this.isAddressRecord(recordArg) && this.normalizeStoredHostname(recordArg.name) === hostname;
+      });
+      if (manualRecords.length > 0) {
+        await this.recordManualClaimEvent(
+          zone,
+          hostname,
+          targetIp,
+          claimOwnerId,
+          this.getContributingRouteKeys(optionsArg.gatewayClientId, hostname),
+          manualRecords,
+        );
+      }
+      const status = manualRecords.length > 0
+        ? 'managed-overwrote-manual'
+        : ownerRecords.length > 0
+          ? 'managed-updated'
+          : 'managed-created';
+      return {
+        ...this.successForZone(status, mode, checkedAt, hostname, targetIp, recordType, zone, finalRecords),
+        ...(overwrittenRecords.length ? { overwrittenRecords } : {}),
+      };
+    } catch (error) {
+      let status: TGatewayDnsResult['status'] = 'mutation-failed';
+      let message = (error as Error).message;
+      try {
+        await this.restoreRecords(zone, hostname, rollbackRecords);
+      } catch (restoreError) {
+        status = 'compensation-failed';
+        message = `${message}; DNS restore failed: ${(restoreError as Error).message}`;
+        logger.log('error', `Gateway DNS compensation failed for ${hostname}: ${message}`);
+      }
+      return this.failureForZone(
+        mode,
+        checkedAt,
+        hostname,
+        targetIp,
+        recordType,
+        zone,
+        status,
+        message,
+      );
+    }
+  }
+
+  public async runFullReconcile(force = false): Promise<void> {
+    if (this.fullReconcilePromise) return await this.fullReconcilePromise;
+    const run = this.reconcileAll(force);
+    this.fullReconcilePromise = run;
+    try {
+      await run;
+    } finally {
+      if (this.fullReconcilePromise === run) this.fullReconcilePromise = undefined;
+    }
+  }
+
+  private async reconcileAll(force: boolean): Promise<void> {
+    const routes = this.dcRouterRef.routeConfigManager?.getMergedRoutes().routes || [];
+    const activeOwners = new Set<string>();
+    const providerZoneRefreshes = new Map<string, Promise<IProviderZoneRefreshResult>>();
+    for (const route of routes) {
+      const externalKey = route.metadata?.externalKey;
+      if (!route.enabled || route.metadata?.ownerType !== 'gatewayClient'
+        || route.metadata.gatewayDnsMode !== 'reconcile' || !externalKey) continue;
+      const hostname = this.getRouteHostname(route.route);
+      if (!hostname) continue;
+      const gatewayClientId = route.metadata.gatewayClientId;
+      if (!gatewayClientId) continue;
+      const claimOwnerId = this.buildClaimOwnerId(gatewayClientId, hostname);
+      const isFirstRouteForClaim = !activeOwners.has(claimOwnerId);
+      activeOwners.add(claimOwnerId);
+      if (!isFirstRouteForClaim) continue;
+      const result = await this.reconcileRouteWithExecution(
+        { externalKey, gatewayClientId, hostname, mode: 'reconcile' },
+        { respectCooldown: !force, providerZoneRefreshes },
+      );
+      if (!result.success) {
+        logger.log(
+          'warn',
+          `Gateway route DNS reconcile returned ${result.status} for ${hostname}: ${result.message || 'no detail'}`,
+        );
+      }
+    }
+
+    const dnsManager = this.dcRouterRef.dnsManager;
+    if (!dnsManager) return;
+    for (const record of await (await import('../db/documents/classes.dns-record.doc.js')).DnsRecordDoc.findAll()) {
+      if (record.managedBy !== GATEWAY_ROUTE_DNS_MANAGED_BY || !record.managedOwnerId) continue;
+      if (activeOwners.has(record.managedOwnerId)) continue;
+      const hostname = this.normalizeStoredHostname(record.name);
+      if (!hostname) continue;
+      await this.withHostnameLock(hostname, async () => {
+        const liveClaimOwnerIds = this.getActiveClaimsForHostname(hostname).map((claimArg) => {
+          return this.buildClaimOwnerId(claimArg.gatewayClientId, hostname);
+        });
+        if (liveClaimOwnerIds.includes(record.managedOwnerId!)) return;
+        const current = await (await import('../db/documents/classes.dns-record.doc.js')).DnsRecordDoc.findById(record.id);
+        if (!current || current.managedBy !== GATEWAY_ROUTE_DNS_MANAGED_BY
+          || current.managedOwnerId !== record.managedOwnerId
+          || this.normalizeStoredHostname(current.name) !== hostname) return;
+        const result = await dnsManager.deleteRecord(record.id);
+        if (!result.success) {
+          logger.log('warn', `Failed to delete orphan gateway DNS record ${record.id}: ${result.message || 'unknown error'}`);
+        }
+      });
+    }
+  }
+
+  private async deleteOwnedRecords(
+    claimOwnerIdArg: string,
+    excludedExternalKeyArg: string,
+    gatewayClientIdArg: string,
+    hostnameArg: string,
+    modeArg: plugins.servezoneInterfaces.data.TGatewayRouteDnsMode,
+    checkedAtArg: number,
+    providerZoneRefreshes?: Map<string, Promise<IProviderZoneRefreshResult>>,
+  ): Promise<TGatewayDnsResult> {
+    const dnsManager = this.dcRouterRef.dnsManager;
+    if (!dnsManager) {
+      return { success: false, retryable: true, mode: modeArg, status: 'zone-unavailable', hostname: hostnameArg, checkedAt: checkedAtArg };
+    }
+    const zone = await dnsManager.findDomainForFqdn(hostnameArg);
+    if (!zone) {
+      return { success: true, retryable: false, mode: modeArg, status: 'managed-deleted', hostname: hostnameArg, checkedAt: checkedAtArg };
+    }
+    if (zone.source === 'provider') {
+      const syncResult = await this.refreshProviderZone(zone.id, providerZoneRefreshes);
+      if (!syncResult.success) {
+        return this.failureForZone(
+          modeArg,
+          checkedAtArg,
+          hostnameArg,
+          undefined,
+          undefined,
+          zone,
+          'zone-unavailable',
+          syncResult.message || 'Provider zone synchronization failed',
+        );
+      }
+    }
+    const remainingRouteKeys = this.getContributingRouteKeys(
+      gatewayClientIdArg,
+      hostnameArg,
+      excludedExternalKeyArg,
+    );
+    const owned = (await dnsManager.listRecordsForDomain(zone.id)).filter((recordArg) => {
+      return this.normalizeStoredHostname(recordArg.name) === hostnameArg
+        && recordArg.managedBy === GATEWAY_ROUTE_DNS_MANAGED_BY
+        && recordArg.managedOwnerId === claimOwnerIdArg;
+    });
+    if (modeArg === 'observe') {
+      return this.successForZone('observed-satisfied', modeArg, checkedAtArg, hostnameArg, undefined, undefined, zone, owned);
+    }
+    if (remainingRouteKeys.length > 0) {
+      return this.successForZone('managed-unchanged', modeArg, checkedAtArg, hostnameArg, undefined, undefined, zone, owned);
+    }
+    for (const record of owned) {
+      const result = await dnsManager.deleteRecord(record.id);
+      if (!result.success) {
+        return this.failureForZone(modeArg, checkedAtArg, hostnameArg, undefined, undefined, zone, 'mutation-failed', result.message || 'DNS cleanup failed', owned);
+      }
+    }
+    return this.successForZone('managed-deleted', modeArg, checkedAtArg, hostnameArg, undefined, undefined, zone, []);
+  }
+
+  private getDesiredTargetIp(): string | undefined {
+    return this.dcRouterRef.options.proxyIps?.[0]
+      || this.dcRouterRef.options.publicIp
+      || this.dcRouterRef.detectedPublicIp
+      || undefined;
+  }
+
+  private getRouteHostname(routeArg: interfaces.data.IDcRouterRouteConfig): string | undefined {
+    const domains = routeArg.match?.domains;
+    const first = Array.isArray(domains) ? domains[0] : typeof domains === 'string' ? domains.split(',')[0] : undefined;
+    return first ? normalizeGatewayHostname(first) : undefined;
+  }
+
+  private buildWorkKey(
+    optionsArg: IGatewayRouteDnsReconcileOptions,
+    hostnameArg: string,
+    modeArg: Exclude<plugins.servezoneInterfaces.data.TGatewayRouteDnsMode, 'skip'>,
+    targetIpArg: string | undefined,
+  ): string {
+    return JSON.stringify({
+      hostname: hostnameArg,
+      gatewayClientId: optionsArg.gatewayClientId.trim().toLowerCase(),
+      externalKey: optionsArg.externalKey,
+      mode: modeArg,
+      delete: optionsArg.delete === true,
+      targetIp: targetIpArg || null,
+      claims: this.getActiveClaimsForHostname(hostnameArg),
+    });
+  }
+
+  private async refreshProviderZone(
+    zoneIdArg: string,
+    providerZoneRefreshesArg?: Map<string, Promise<IProviderZoneRefreshResult>>,
+  ): Promise<IProviderZoneRefreshResult> {
+    const dnsManager = this.dcRouterRef.dnsManager;
+    if (!dnsManager) return { success: false, message: 'DnsManager is not initialized' };
+    if (!providerZoneRefreshesArg) return await dnsManager.syncDomain(zoneIdArg);
+    let refresh = providerZoneRefreshesArg.get(zoneIdArg);
+    if (!refresh) {
+      refresh = dnsManager.syncDomain(zoneIdArg);
+      providerZoneRefreshesArg.set(zoneIdArg, refresh);
+    }
+    return await refresh;
+  }
+
+  public buildClaimOwnerId(gatewayClientIdArg: string, hostnameArg: string): string {
+    const gatewayClientId = gatewayClientIdArg.trim().toLowerCase();
+    const hostname = normalizeGatewayHostname(hostnameArg);
+    return `gateway-client:${gatewayClientId}:${hostname}`;
+  }
+
+  public getActiveClaimsForHostname(hostnameArg: string): IGatewayDnsActiveClaim[] {
+    return getActiveGatewayDnsClaims(this.dcRouterRef, hostnameArg);
+  }
+
+  private buildManagedRecordKey(claimOwnerIdArg: string): string {
+    return `${GATEWAY_ROUTE_DNS_MANAGED_BY}:${Buffer.from(claimOwnerIdArg, 'utf8').toString('base64url')}`;
+  }
+
+  private getContributingRouteKeys(
+    gatewayClientIdArg: string,
+    hostnameArg: string,
+    excludedExternalKeyArg?: string,
+  ): string[] {
+    const claim = this.getActiveClaimsForHostname(hostnameArg)
+      .find((claimArg) => claimArg.gatewayClientId === gatewayClientIdArg.trim().toLowerCase());
+    return (claim?.routeExternalKeys || []).filter((externalKeyArg) => externalKeyArg !== excludedExternalKeyArg);
+  }
+
+  private isAddressRecord(recordArg: DnsRecordDoc): boolean {
+    return recordArg.type === 'A' || recordArg.type === 'AAAA' || recordArg.type === 'CNAME';
+  }
+
+  private normalizeStoredHostname(hostnameArg: string): string {
+    try {
+      return normalizeGatewayHostname(hostnameArg);
+    } catch {
+      return '';
+    }
+  }
+
+  private toEvidence(recordArg: DnsRecordDoc): plugins.servezoneInterfaces.data.IGatewayRouteDnsRecord {
+    return {
+      id: recordArg.id,
+      type: recordArg.type as 'A' | 'AAAA' | 'CNAME',
+      name: recordArg.name,
+      value: recordArg.value,
+      ttl: recordArg.ttl,
+      proxied: recordArg.proxied,
+      managed: recordArg.managedBy === GATEWAY_ROUTE_DNS_MANAGED_BY,
+    };
+  }
+
+  private snapshotRecord(recordArg: DnsRecordDoc) {
+    return {
+      name: recordArg.name,
+      type: recordArg.type,
+      value: recordArg.value,
+      ttl: recordArg.ttl,
+      proxied: recordArg.proxied,
+      createdBy: recordArg.createdBy,
+      managedBy: recordArg.managedBy,
+      managedOwnerId: recordArg.managedOwnerId,
+      managedRecordKey: recordArg.managedRecordKey,
+    };
+  }
+
+  private async restoreRecords(
+    zoneArg: DomainDoc,
+    hostnameArg: string,
+    snapshotsArg: ReturnType<GatewayRouteDnsReconciler['snapshotRecord']>[],
+  ): Promise<void> {
+    const dnsManager = this.dcRouterRef.dnsManager;
+    if (!dnsManager) throw new Error('DnsManager is not initialized');
+    const current = (await dnsManager.listRecordsForDomain(zoneArg.id)).filter((recordArg) => {
+      return this.normalizeStoredHostname(recordArg.name) === hostnameArg && this.isAddressRecord(recordArg);
+    });
+    for (const record of current) {
+      const deleted = await dnsManager.deleteRecord(record.id);
+      if (!deleted.success) throw new Error(deleted.message || `Failed to remove partial DNS record ${record.id}`);
+    }
+    for (const snapshot of snapshotsArg) {
+      const created = await dnsManager.createRecord({ domainId: zoneArg.id, ...snapshot });
+      if (!created.success) throw new Error(created.message || `Failed to restore ${snapshot.type} ${snapshot.name}`);
+    }
+    const restored = (await dnsManager.listRecordsForDomain(zoneArg.id)).filter((recordArg) => {
+      return this.normalizeStoredHostname(recordArg.name) === hostnameArg && this.isAddressRecord(recordArg);
+    });
+    const normalize = (valueArg: ReturnType<GatewayRouteDnsReconciler['snapshotRecord']>) => JSON.stringify({
+      name: this.normalizeStoredHostname(valueArg.name),
+      type: valueArg.type,
+      value: valueArg.value,
+      ttl: valueArg.ttl,
+      proxied: valueArg.proxied,
+      managedBy: valueArg.managedBy,
+      managedOwnerId: valueArg.managedOwnerId,
+      managedRecordKey: valueArg.managedRecordKey,
+    });
+    const expected = snapshotsArg.map(normalize).sort();
+    const actual = restored.map((recordArg) => normalize(this.snapshotRecord(recordArg))).sort();
+    if (JSON.stringify(actual) !== JSON.stringify(expected)) {
+      throw new Error('Restored DNS records do not match the pre-mutation snapshot');
+    }
+  }
+
+  private async withHostnameLock<T>(hostnameArg: string, taskArg: () => Promise<T>): Promise<T> {
+    const hostname = normalizeGatewayHostname(hostnameArg);
+    const previous = this.hostnameChains.get(hostname) || Promise.resolve();
+    const run = previous.catch(() => undefined).then(taskArg);
+    const settled = run.then(
+      () => undefined,
+      () => undefined,
+    );
+    this.hostnameChains.set(hostname, settled);
+    try {
+      return await run;
+    } finally {
+      if (this.hostnameChains.get(hostname) === settled) {
+        this.hostnameChains.delete(hostname);
+      }
+    }
+  }
+
+  private async recordManualClaimEvent(
+    zoneArg: DomainDoc,
+    hostnameArg: string,
+    targetIpArg: string,
+    claimOwnerIdArg: string,
+    contributingRouteKeysArg: string[],
+    recordsArg: DnsRecordDoc[],
+  ): Promise<void> {
+    if (this.dcRouterRef.opsEventManager) {
+      try {
+        await this.dcRouterRef.opsEventManager.recordEvent({
+          severity: 'error',
+          category: 'dns-conflict',
+          title: 'Gateway route DNS automation claimed a manual hostname',
+          detail: `Gateway DNS owner ${claimOwnerIdArg} replaced ${recordsArg.length} manual A/AAAA/CNAME record(s) for ${hostnameArg}.`,
+          context: {
+            domain: zoneArg.name,
+            recordName: hostnameArg,
+            previousValue: recordsArg.map((recordArg) => `${recordArg.type} ${recordArg.value}`).join(', '),
+            newValue: targetIpArg,
+            routeRef: contributingRouteKeysArg.join(', ') || claimOwnerIdArg,
+            actor: GATEWAY_ROUTE_DNS_MANAGED_BY,
+          },
+        });
+      } catch (error) {
+        logger.log('warn', `Failed to record gateway DNS conflict event for ${hostnameArg}: ${(error as Error).message}`);
+      }
+    }
+    try {
+      this.dcRouterRef.opsServer?.invalidateRealtime?.('configEvents', {
+        reason: 'gateway-dns-manual-claim',
+      });
+    } catch (error) {
+      logger.log('warn', `Failed to invalidate config events after gateway DNS claim for ${hostnameArg}: ${(error as Error).message}`);
+    }
+  }
+
+  private successForZone(
+    statusArg: TGatewayDnsResult['status'],
+    modeArg: TGatewayDnsResult['mode'],
+    checkedAtArg: number,
+    hostnameArg: string,
+    targetIpArg: string | undefined,
+    recordTypeArg: 'A' | 'AAAA' | undefined,
+    zoneArg: DomainDoc,
+    recordsArg: DnsRecordDoc[],
+  ): TGatewayDnsResult {
+    return {
+      success: true,
+      retryable: false,
+      mode: modeArg,
+      status: statusArg,
+      hostname: hostnameArg,
+      targetIp: targetIpArg,
+      recordType: recordTypeArg,
+      zoneId: zoneArg.id,
+      zoneName: zoneArg.name,
+      nameservers: zoneArg.nameservers,
+      records: recordsArg.map((recordArg) => this.toEvidence(recordArg)),
+      checkedAt: checkedAtArg,
+    };
+  }
+
+  private failureForZone(
+    modeArg: TGatewayDnsResult['mode'],
+    checkedAtArg: number,
+    hostnameArg: string,
+    targetIpArg: string | undefined,
+    recordTypeArg: 'A' | 'AAAA' | undefined,
+    zoneArg: DomainDoc,
+    statusArg: TGatewayDnsResult['status'],
+    messageArg: string,
+    recordsArg: DnsRecordDoc[] = [],
+  ): TGatewayDnsResult {
+    return {
+      success: false,
+      retryable: statusArg === 'zone-unavailable'
+        || statusArg === 'mutation-failed'
+        || statusArg === 'compensation-failed',
+      mode: modeArg,
+      status: statusArg,
+      hostname: hostnameArg,
+      targetIp: targetIpArg,
+      recordType: recordTypeArg,
+      zoneId: zoneArg.id,
+      zoneName: zoneArg.name,
+      nameservers: zoneArg.nameservers,
+      records: recordsArg.map((recordArg) => this.toEvidence(recordArg)),
+      checkedAt: checkedAtArg,
+      message: messageArg,
+    };
+  }
+}