@serve.zone/dcrouter 18.7.1 → 18.8.0

package/ts/dns/classes.dns-server-runtime.ts

@@ -10,16 +10,7 @@ import {
 
 type TDnsRecordSeed = { name: string; type: string; value: string; ttl?: number; useIngressProxy?: boolean };
 
-/**
- * The zone smartdns is handed when the authority set is empty.
- *
- * smartdns falls back to `[dnssecZone]` whenever `authoritativeZones` is empty,
- * so this value decides what an *unauthoritative* dcrouter would claim. It has
- * to be a name that can never match a real question, or a database that has not
- * been seeded yet would quietly start asserting authority over whatever
- * placeholder we picked. `.invalid` is reserved by RFC 2606 and is guaranteed
- * never to be delegated to anybody.
- */
+/** The required DNSSEC keying zone when dcrouter currently claims no DNS zone. */
 export const NO_AUTHORITY_SENTINEL_ZONE = 'no-authority.invalid';
 
 /**
@@ -38,28 +29,12 @@ export class DnsServerRuntime {
   private privateRouteHostnames = new Set<string>();
   private privateRouteTargetIp?: string;
 
-  /**
-   * The array instance handed to smartdns as `authoritativeZones`.
-   *
-   * smartdns keeps the options object by reference and re-reads this array on
-   * every query, so mutating it in place changes the authority set of a
-   * *running* server.
-   *
-   * **This is a consumer-side shim, and it is sequenced for replacement.**
-   * `authoritativeZones` is a public option, but "the array is retained by
-   * reference and re-read per query" is storage behaviour, not a documented
-   * contract. The proper fix is a `setAuthoritativeZones()` on smartdns' own
-   * `DnsServer`; it cannot be written here because that release does not exist
-   * yet, and the only alternative available today — re-creating the server to
-   * change its zones — drops UDP/TCP 53, which is the 30–60 s public outage
-   * this entire path exists to remove.
-   *
-   * Guarded two ways until the setter ships: `assertLiveAuthorityZones()` logs
-   * at `error` if the reference is no longer shared after construction, and a
-   * test asserts that identity against the installed smartdns so a release that
-   * starts copying the array fails the suite rather than production.
-   */
+  /** Local snapshot of the zone set successfully applied to smartdns. */
   private readonly authoritativeZones: string[] = [];
+  private attachedDnsServer?: plugins.smartdns.dnsServerMod.DnsServer;
+  private cleanupPendingDnsServer?: plugins.smartdns.dnsServerMod.DnsServer;
+  private cleanupPendingDnsServerPromise?: Promise<void>;
+  private lifecycleTail: Promise<void> = Promise.resolve();
 
   constructor(private dcRouterRef: DcRouter) {}
 
@@ -72,7 +47,15 @@ export class DnsServerRuntime {
    * Create the DNS server, start it on UDP, wire metrics/logging, and
    * register all generated records.
    */
-  public async setup(): Promise<void> {
+  public setup(): Promise<void> {
+    return this.enqueueLifecycle(() => this.setupInternal());
+  }
+
+  private async setupInternal(): Promise<void> {
+    await this.retryPendingDnsServerCleanup();
+    if (this.dcRouterRef.dnsServer) {
+      throw new Error('DNS server setup refused because another server instance is still owned');
+    }
     const options = this.dcRouterRef.options;
     if (!options.dnsNsDomains || options.dnsNsDomains.length === 0) {
       throw new Error('dnsNsDomains is required for DNS server setup');
@@ -154,21 +137,20 @@ export class DnsServerRuntime {
       // authority set — sorted, therefore stable across restarts rather than
       // dependent on database insertion order.
       dnssecZone: bootAuthorityZones[0] || NO_AUTHORITY_SENTINEL_ZONE,
-      // Live array, mutated in place by syncAuthorityZones(). See the field.
-      authoritativeZones: this.authoritativeZones,
+      // SmartDNS copies this boot snapshot; later changes use its live setter.
+      authoritativeZones: bootAuthorityZones,
       primaryNameserver: primaryNameserver, // Automatically generates correct SOA records
       // For now, use self-signed cert until we integrate with Let's Encrypt
       httpsKey: '',
       httpsCert: ''
     });
-    this.assertLiveAuthorityZones(dnsServer);
     this.dcRouterRef.dnsServer = dnsServer;
     this.registerPrivateRouteHandler(dnsServer);
 
-    // Start the DNS server. smartdns owns UDP and DNS-over-TCP; dcrouter
-    // keeps DNS-over-HTTPS routed through SmartProxy manual HTTPS mode.
-    await dnsServer.start();
     try {
+      // SmartDNS owns UDP and DNS-over-TCP; dcrouter keeps DNS-over-HTTPS
+      // routed through SmartProxy manual HTTPS mode.
+      await dnsServer.start();
       logger.log('info', `DNS server started on UDP/TCP ${vmIpAddress}:53`);
 
       // Wire DNS query events to MetricsManager and logger with adaptive rate limiting
@@ -244,26 +226,125 @@ export class DnsServerRuntime {
       // dcrouter-hosted domains get registered too.
       await this.attachDnsServer(dnsServer);
     } catch (error) {
+      this.detachDnsServer(dnsServer);
       dnsServer.removeAllListeners();
-      await dnsServer.stop().catch((stopError: Error) => {
+      let stopError: Error | undefined;
+      try {
+        await dnsServer.stop();
+      } catch (errorDuringStop: unknown) {
+        stopError = errorDuringStop instanceof Error
+          ? errorDuringStop
+          : new Error(String(errorDuringStop));
+        this.cleanupPendingDnsServer = dnsServer;
+        void this.ensureDnsServerCleanup(dnsServer, true);
         logger.log('warn', `Failed to stop DNS server after setup failure: ${stopError.message}`);
-      });
-      if (this.dcRouterRef.dnsServer === dnsServer) {
+      }
+      if (!stopError && this.dcRouterRef.dnsServer === dnsServer) {
         this.dcRouterRef.dnsServer = undefined;
       }
       this.flushQueryLogBatch();
+      if (stopError) {
+        throw new AggregateError(
+          [error, stopError],
+          'DNS server setup failed and process cleanup remains pending',
+        );
+      }
       throw error;
     }
   }
 
+  private async retryPendingDnsServerCleanup(): Promise<void> {
+    const pendingServer = this.cleanupPendingDnsServer;
+    if (!pendingServer) return;
+    await this.ensureDnsServerCleanup(pendingServer);
+  }
+
+  /**
+   * Keep retrying until SmartDNS confirms Rust process closure. Taskbuffer
+   * swallows service-stop errors, so returning after a failed termination would
+   * otherwise strand the child and its port bindings with no future retry.
+   */
+  private ensureDnsServerCleanup(
+    dnsServerArg: plugins.smartdns.dnsServerMod.DnsServer,
+    delayFirstAttemptArg = false,
+  ): Promise<void> {
+    if (this.cleanupPendingDnsServerPromise) {
+      if (this.cleanupPendingDnsServer !== dnsServerArg) {
+        throw new Error('Cannot clean up two DNS server instances concurrently');
+      }
+      return this.cleanupPendingDnsServerPromise;
+    }
+
+    this.cleanupPendingDnsServer = dnsServerArg;
+    let cleanupPromise: Promise<void>;
+    cleanupPromise = this.cleanupDnsServerUntilStopped(dnsServerArg, delayFirstAttemptArg)
+      .finally(() => {
+        if (this.cleanupPendingDnsServerPromise === cleanupPromise) {
+          this.cleanupPendingDnsServerPromise = undefined;
+        }
+      });
+    this.cleanupPendingDnsServerPromise = cleanupPromise;
+    return cleanupPromise;
+  }
+
+  private async cleanupDnsServerUntilStopped(
+    dnsServerArg: plugins.smartdns.dnsServerMod.DnsServer,
+    delayFirstAttemptArg: boolean,
+  ): Promise<void> {
+    let failedAttempts = 0;
+    if (delayFirstAttemptArg) {
+      await this.waitForDnsServerCleanupRetry(1);
+    }
+
+    while (this.cleanupPendingDnsServer === dnsServerArg) {
+      try {
+        await dnsServerArg.stop();
+        if (this.dcRouterRef.dnsServer === dnsServerArg) {
+          this.dcRouterRef.dnsServer = undefined;
+        }
+        if (this.cleanupPendingDnsServer === dnsServerArg) {
+          this.cleanupPendingDnsServer = undefined;
+        }
+        return;
+      } catch (error: unknown) {
+        failedAttempts++;
+        const message = error instanceof Error ? error.message : String(error);
+        logger.log(
+          'warn',
+          `DNS server process cleanup attempt ${failedAttempts} failed; retrying until closure is confirmed: ${message}`,
+        );
+        await this.waitForDnsServerCleanupRetry(failedAttempts);
+      }
+    }
+  }
+
+  private async waitForDnsServerCleanupRetry(failedAttemptsArg: number): Promise<void> {
+    const delayMs = Math.min(250 * (2 ** Math.min(failedAttemptsArg - 1, 5)), 5000);
+    await new Promise<void>((resolve) => setTimeout(resolve, delayMs));
+  }
+
+  private enqueueLifecycle(operationArg: () => Promise<void>): Promise<void> {
+    const operation = this.lifecycleTail.then(operationArg, operationArg);
+    this.lifecycleTail = operation.then(() => undefined, () => undefined);
+    return operation;
+  }
+
   private async attachDnsServer(
     dnsServer: plugins.smartdns.dnsServerMod.DnsServer,
   ): Promise<void> {
     if (!this.dcRouterRef.dnsManager) return;
     await this.dcRouterRef.dnsManager.attachDnsServer(dnsServer);
+    this.attachedDnsServer = dnsServer;
     this.dcRouterRef.mailDnsSync?.requestSync('DNS server attached');
   }
 
+  private detachDnsServer(dnsServerArg: plugins.smartdns.dnsServerMod.DnsServer): void {
+    this.dcRouterRef.dnsManager?.detachDnsServer(dnsServerArg);
+    if (this.attachedDnsServer === dnsServerArg) {
+      this.attachedDnsServer = undefined;
+    }
+  }
+
   /**
    * Re-derive the running server's authoritative zone set from the database.
    *
@@ -286,42 +367,18 @@ export class DnsServerRuntime {
     const nextZones = this.effectiveAuthorityZones();
     const changed = nextZones.length !== this.authoritativeZones.length
       || nextZones.some((zone, index) => zone !== this.authoritativeZones[index]);
-    // In place: smartdns holds this exact array and re-reads it per query.
-    this.authoritativeZones.splice(0, this.authoritativeZones.length, ...nextZones);
-    if (changed) {
-      logger.log(
-        'info',
-        `DNS authoritative zone set is now [${nextZones.join(', ') || 'empty'}] (${reasonArg})`,
-        { zone: 'dns' },
-      );
+    if (!changed) {
+      return nextZones;
     }
-    return nextZones;
-  }
-
-  /**
-   * Prove smartdns kept our array rather than copying it.
-   *
-   * If this ever fails, authority silently freezes at its boot-time value and
-   * every runtime verification stops taking effect until a restart — a failure
-   * that is invisible from the outside because the server keeps answering
-   * correctly for the zones it started with. Logged rather than thrown: a frozen
-   * authority set is a degradation, and taking DNS down entirely over it would
-   * be worse than the fault it reports.
-   */
-  private assertLiveAuthorityZones(
-    dnsServerArg: plugins.smartdns.dnsServerMod.DnsServer,
-  ): void {
-    const heldZones = (dnsServerArg as unknown as {
-      options?: { authoritativeZones?: string[] };
-    }).options?.authoritativeZones;
-    if (heldZones === this.authoritativeZones) return;
+    // Commit the local snapshot only after the live server accepts the update.
+    this.dcRouterRef.dnsServer?.setAuthoritativeZones(nextZones);
+    this.authoritativeZones.splice(0, this.authoritativeZones.length, ...nextZones);
     logger.log(
-      'error',
-      'smartdns did not retain the authoritative zone array by reference, so DNS authority can no longer be '
-        + 'reconciled without restarting the DNS server. Zones verified at runtime will register handlers but '
-        + 'keep answering REFUSED for every qtype they do not handle until dcrouter is restarted.',
+      'info',
+      `DNS authoritative zone set is now [${nextZones.join(', ') || 'empty'}] (${reasonArg})`,
       { zone: 'dns' },
     );
+    return nextZones;
   }
 
   /**
@@ -329,12 +386,10 @@ export class DnsServerRuntime {
    * compiled source policy is private or whose only ingress is SmartVPN.
    * Public routes compile without clientIp restrictions and are never added.
    *
-   * Every hostname must clear the domain-ownership gate first. This overlay is
-   * only "internal" by intent, never by mechanism: smartdns marks any answer a
-   * registered handler produces authoritative, and the server listens on a public
-   * UDP/TCP 53. Ungated, a single route was enough to make dcrouter publicly hand
-   * out an RFC1918 address, with the `aa` flag, for a domain whose real
-   * delegation belonged to somebody else.
+   * Every hostname must clear the domain-ownership gate first. The handler uses
+   * SmartDNS' explicit non-authoritative mode, but the server still listens on a
+   * public UDP/TCP 53. Ungated, a private-route definition could publicly hand
+   * out an RFC1918 address for a domain whose real delegation belonged elsewhere.
    */
   public async syncPrivateRouteOverrides(routesArg: IDcRouterRouteConfig[]): Promise<void> {
     const candidateHostnames = new Set<string>();
@@ -425,23 +480,7 @@ export class DnsServerRuntime {
     return ownedHostnames;
   }
 
-  /**
-   * SEQUENCED BEHIND A SMARTDNS RELEASE — this registration must become
-   * `{ authority: 'non-authoritative', owner: 'private-route-overlay' }`.
-   *
-   * The overlay is a split-horizon answer by intent: it hands out an internal
-   * address for a hostname whose zone we own, and it deliberately matches `*`
-   * so it can cover any owned hostname. Under the hardened smartdns authority
-   * model, a handler left at the default `authoritative` mode is *suppressed*
-   * for any name outside every configured `authoritativeZones` entry. Overlay
-   * hostnames proven by `provider-zone` are exactly that — owned, but not in
-   * the delegation-verified set — so on that smartdns they would stop being
-   * answered entirely, silently.
-   *
-   * The options parameter does not exist in the installed smartdns (7.12.1) and
-   * the change adding it is unreleased, so this cannot be written yet. It must
-   * land in the same change that takes the smartdns bump.
-   */
+  /** Register the split-horizon overlay without claiming DNS authority. */
   private registerPrivateRouteHandler(
     dnsServerArg: plugins.smartdns.dnsServerMod.DnsServer,
   ): void {
@@ -462,7 +501,7 @@ export class DnsServerRuntime {
         ttl: 60,
         data: this.privateRouteTargetIp,
       };
-    });
+    }, { authority: 'non-authoritative', owner: 'private-route-overlay' });
   }
 
   private normalizePrivateRouteHostname(hostnameArg: string): string | undefined {
@@ -475,9 +514,7 @@ export class DnsServerRuntime {
     return hostname;
   }
 
-  /**
-   * Create the DoH socket handler SmartProxy routes hand TLS sockets to.
-   */
+  /** Create the DoH handler for cleartext sockets after SmartProxy terminates TLS. */
   public createSocketHandler(): (socket: plugins.net.Socket) => Promise<void> {
     return async (socket: plugins.net.Socket) => {
       if (!this.dcRouterRef.dnsServer) {
@@ -497,9 +534,7 @@ export class DnsServerRuntime {
       logger.log('debug', 'DNS socket handler: passing socket to DnsServer');
 
       try {
-        // Use the built-in socket handler from smartdns
-        // This handles HTTP/2, DoH protocol, etc.
-        await (this.dcRouterRef.dnsServer as any).handleHttpsSocket(socket);
+        this.dcRouterRef.dnsServer.handleHttpSocket(socket);
       } catch (error: unknown) {
         logger.log('error', `DNS socket handler error: ${(error as Error).message}`);
         if (!socket.destroyed) {
@@ -523,9 +558,23 @@ export class DnsServerRuntime {
     }
   }
 
-  /** Stop runtime-owned bookkeeping. smartdns owns network listener cleanup. */
-  public async stop(): Promise<void> {
+  /** Detach handlers and retain termination ownership until process closure. */
+  public stop(): Promise<void> {
+    return this.enqueueLifecycle(() => this.stopInternal());
+  }
+
+  private async stopInternal(): Promise<void> {
     this.flushQueryLogBatch();
+    const ownedServers = new Set<plugins.smartdns.dnsServerMod.DnsServer>();
+    if (this.attachedDnsServer) ownedServers.add(this.attachedDnsServer);
+    if (this.cleanupPendingDnsServer) ownedServers.add(this.cleanupPendingDnsServer);
+    if (this.dcRouterRef.dnsServer) ownedServers.add(this.dcRouterRef.dnsServer);
+
+    for (const dnsServer of ownedServers) {
+      this.detachDnsServer(dnsServer);
+      dnsServer.removeAllListeners();
+      await this.ensureDnsServerCleanup(dnsServer);
+    }
   }
 
   private registerRecords(records: TDnsRecordSeed[]): void {

package/ts/dns/domain-ownership.ts

@@ -20,11 +20,10 @@ import type { TDomainSource } from '../../ts_interfaces/data/domain.js';
  *    provisioning budget was consumed against a cause no retry can fix and the
  *    certificates silently expired.
  *  - `DomainDoc`s created through the ops API set `authoritative = true`
- *    unconditionally, and the embedded smartdns server marks *any* answer a
- *    registered handler produces as authoritative (`aa`) regardless of
- *    `authoritativeZones`. dcrouter therefore served apex NS records and an
- *    RFC1918 A record, publicly, for zones whose real delegation belonged to
- *    third parties.
+ *    unconditionally, and older embedded SmartDNS releases treated a handler
+ *    answer as authoritative regardless of configured zones. dcrouter therefore
+ *    served apex NS records and an RFC1918 A record publicly for zones whose
+ *    real delegation belonged to third parties.
  *
  * There are exactly two proofs available in-process, neither of which an ops-API
  * caller can forge: