@serfab/cadre-host 1.13.0 → 1.14.0

package/dist/nat/nat-service.js

@@ -2,286 +2,309 @@ import debug from 'debug';
 import { NatError, } from './types.js';
 import { NatStore } from './nat-store.js';
 import { ExternalIpDetector } from './external-ip.js';
-import { PortMapperService, createDefaultPortMapper, } from './port-mapper.js';
-import { evaluateReachability } from './reachability.js';
+import { DEFAULT_LEASE_TTL_MS, DEFAULT_REFRESH_MS, GATEWAY_DISCOVERY_TIMEOUT_MS, UpnpPortMapper, } from './port-mapper.js';
+import { evaluateHostReachability, evaluateNodeReachability } from './reachability.js';
 import { DdnsUpdater } from './ddns/updater.js';
 import { getProvider, listProviders } from './ddns/index.js';
-import { buildInviteAddresses, } from './address-resolver.js';
+import { buildPublicAddresses, isPublicIpv4 } from './address-resolver.js';
 import { createSecretsStore, ddnsAccount } from './secrets/index.js';
-import { OwnerNodeUnavailableError } from '../owner/owner-node-client.js';
+import { AddressWatch } from './address-watch.js';
 const log = debug('cadre:host:nat-service');
-/** Sleep helper for the initial-push retry loop; unref'd so it never holds the process open. */
-function sleep(ms) {
-    return new Promise((res) => {
-        const t = setTimeout(res, ms);
-        if (typeof t.unref === 'function')
-            t.unref();
-    });
+/**
+ * How long a node may sit `stopped` before its mappings are released. Longer
+ * than the hosted-node supervisor's whole respawn backoff (150 s), so a crash
+ * followed by a respawn keeps its mapping.
+ */
+export const NAT_UNMAP_GRACE_MS = 3 * 60000;
+/** The backstop reconcile cadence; state-change events trigger passes between ticks. */
+export const NAT_RECONCILE_INTERVAL_MS = 60000;
+/**
+ * How often the external IP is re-detected, and how often gateway discovery is
+ * retried while UPnP is on and no router has answered.
+ */
+export const NAT_IP_REDETECT_INTERVAL_MS = 5 * 60000;
+const PORT_KINDS = ['tcp', 'ws'];
+function noRoute(internalPort) {
+    return { internalPort, externalPort: null, source: null, leaseExpiresAt: null, error: null };
+}
+function manualRoute(internalPort, externalPort) {
+    return { internalPort, externalPort, source: 'manual', leaseExpiresAt: null, error: null };
+}
+function portsOf(ports) {
+    // A handle read back from an older `state.json` lacks `ws` whatever the type says.
+    const ws = typeof ports.ws === 'number' ? ports.ws : null;
+    return { tcp: ports.p2p, ws };
+}
+function samePorts(a, b) {
+    return a.tcp === b.tcp && a.ws === b.ws;
+}
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
 }
 /**
- * NatService — orchestrates port mapping, external-IP detection, DDNS updates,
- * and address resolution for invites.
+ * NatService — one mapping table for every node the host runs, keyed by node
+ * id, with a route per port (TCP and WebSocket); external-IP detection; DDNS;
+ * and the public addresses a node announces.
  *
  * Lifecycle:
- *   - `start()`: load settings, install port mapping (if upnpEnabled), detect
- *     external IP, kick off DDNS loop.
- *   - `stop()`: release port mapping, stop DDNS timer.
- *   - `getStatus()`: build a fresh status snapshot from internal state (cheap).
- *   - `testReachability()`: re-run the IP + verify cycle and the heuristic.
- *   - `getInviteAddresses()`: build the multiaddrs to embed in invites.
+ *   - `start()`: load settings, discover the router (bounded), detect the
+ *     external IP, start DDNS, then map the running nodes in the background.
+ *   - `stop()`: clear timers only — mappings outlive the host process.
+ *   - `getStatus()`: snapshot from cached state (cheap).
+ *   - `publicAddressesFor(nodeId, ports)`: a node's public multiaddrs, also
+ *     from cached state, predicting the identity mapping for a port that has
+ *     no route yet.
+ *   - `onNodeAddressesStale(listener)`: asked to restart a running node whose
+ *     announced addresses no longer match `publicAddressesFor`, checked after
+ *     every pass and every settings write (see `AddressWatch`).
  *
- * NatService is library + lifecycle only. The long-running process that owns
- * it is `cadre-host-local-ui` (see docs/cadre-host.md and the ticket header).
+ * One reconcile pass against the node source behind three triggers (start,
+ * node state change, a 1-minute timer), plus one renewal pass for every lease
+ * every `DEFAULT_REFRESH_MS`. Passes are serialized on one promise tail, so a
+ * state change during a timer pass can neither map a port twice nor unmap a
+ * port the other pass is mapping.
+ *
+ * NOTE: this file holds the settings/DDNS glue, the mapping table and the status
+ * assembly together (about 890 lines; the stale-address check lives in
+ * `address-watch.ts` for that reason); when the next capability lands here, move
+ * the mapping table (`NodeEntry` through `releaseUpnpRoutes`) into its own module.
  */
 export class NatService {
     constructor(opts) {
-        // Built once start() completes.
-        this.portService = null;
+        this.detector = null;
         this.ddnsUpdater = null;
+        this.gateway = { found: false, lanAddress: null, routerExternalIp: null, lastError: null };
         this.latestIp = null;
         this.lastTestedAt = null;
+        this.table = new Map();
         this.started = false;
-        /** Listeners notified when the invite addresses may have changed. */
-        this.addressListeners = new Set();
-        this.cadreNode = opts.cadreNode;
+        this.tail = Promise.resolve();
+        /** True while an event-triggered pass is queued and not yet started (crash storms coalesce into it). */
+        this.eventPassQueued = false;
+        this.timers = [];
+        this.unsubscribe = null;
+        this.changeListeners = new Set();
+        this.lastSignature = null;
+        this.nodeSource = opts.nodeSource;
         this.store = opts.store ?? new NatStore(opts.rootDir);
         this.nowFn = opts.now ?? (() => new Date());
         this.fetchImpl = opts.fetch ?? fetch;
+        this.portMapper = opts.portMapper ?? new UpnpPortMapper();
+        this.leaseTtlMs = opts.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
         this.secretsStore = opts.secretsStore ?? null;
-        this.portMapper = opts.portMapper ?? null;
         this.injectedDetector = opts.externalIpDetector ?? null;
         this.secretsRootDir = opts.rootDir;
-        this.initialPushRetryMs = opts.initialPushRetryMs ?? 250;
-        this.initialPushTimeoutMs = opts.initialPushTimeoutMs ?? 15000;
         this.currentSettings = this.store.load();
+        this.addressWatch = new AddressWatch(() => this.nowFn().getTime());
     }
-    /** Initial start: port mapping → IP detect → DDNS. Idempotent. */
+    // --- lifecycle ---
+    /**
+     * Discover the router and detect the external IP (both awaited, both
+     * bounded), start DDNS, then map re-attached running nodes in the background
+     * so the management API comes up promptly. Idempotent.
+     */
     async start() {
         if (this.started)
             return;
         this.started = true;
-        // 1. Secrets store (lazy if not injected).
         if (!this.secretsStore) {
             this.secretsStore = await createSecretsStore(this.secretsRootDir);
         }
-        // 2. Port mapping.
-        await this.startPortService();
-        // 3. External IP detection.
+        if (this.currentSettings.upnpEnabled)
+            await this.discoverGateway();
         await this.detectIp();
-        // 4. DDNS updater.
         this.ddnsUpdater = new DdnsUpdater({
             settings: this.currentSettings.ddns,
             secrets: this.secretsStore,
-            getExternalIp: () => this.latestIp?.publicIp ?? this.latestIp?.routerIp ?? null,
+            getExternalIp: () => this.detectedIp(),
             fetch: this.fetchImpl,
             now: this.nowFn,
         });
         await this.ddnsUpdater.start();
-        // Initial invite-address push. Retries on `node_unavailable` until the
-        // freshly spawned owner node accepts it (bounded; best-effort after).
-        await this.pushInitialAddresses();
+        // Subscribe before the first pass so a spawn during it is not missed.
+        this.unsubscribe = this.nodeSource.onStateChange(() => { this.queueEventPass(); });
+        this.timers = [
+            setInterval(() => { this.sweep('reconcile', () => this.reconcile()); }, NAT_RECONCILE_INTERVAL_MS),
+            setInterval(() => { this.sweep('renewal', () => this.renewMappings()); }, DEFAULT_REFRESH_MS),
+            setInterval(() => { this.sweep('probe', () => this.redetect()); }, NAT_IP_REDETECT_INTERVAL_MS),
+        ];
+        for (const t of this.timers)
+            t.unref();
+        this.sweep('startup', () => this.reconcile());
     }
-    /** Release the port mapping and clear timers. Idempotent. */
+    /**
+     * Clear timers and release client resources. Idempotent.
+     *
+     * NOTE: deliberately unmaps nothing. Hosted children are detached and keep
+     * running across a host restart (an update restart included), so their
+     * mappings must outlive this process; they expire on their own within the
+     * lease TTL if the host stays down, and the next `start()` re-maps the
+     * re-attached nodes (re-mapping the same internal port is idempotent on the
+     * router).
+     */
     async stop() {
         if (!this.started)
             return;
         this.started = false;
+        for (const t of this.timers)
+            clearInterval(t);
+        this.timers = [];
+        this.unsubscribe?.();
+        this.unsubscribe = null;
         try {
             this.ddnsUpdater?.stop();
         }
         catch (err) {
-            log('ddns stop failed: %s', err.message);
+            log('ddns stop failed: %s', errorMessage(err));
         }
         this.ddnsUpdater = null;
         try {
-            await this.portService?.stop();
+            await this.portMapper.stop();
         }
         catch (err) {
-            log('port-service stop failed: %s', err.message);
+            log('port mapper stop failed: %s', errorMessage(err));
         }
-        this.portService = null;
     }
+    // --- reads ---
     /** Cheap read of the current snapshot. */
     getStatus() {
-        const portState = this.portService?.getState() ?? this.disabledPortState();
-        const externalIp = this.latestIp?.publicIp ?? this.latestIp?.routerIp ?? null;
-        const ddns = this.ddnsUpdater?.getStatus() ?? {
-            providerId: this.currentSettings.ddns.providerId,
-            hostname: this.currentSettings.ddns.hostname,
-            externallyManaged: this.currentSettings.ddns.externallyManaged,
-            lastUpdateAt: null,
-            lastUpdateOk: null,
-            lastError: null,
-        };
+        const cgnatDetected = this.latestIp?.cgnat ?? false;
+        const nodes = [...this.table.values()]
+            .sort((a, b) => a.nodeId.localeCompare(b.nodeId))
+            .map((entry) => this.describeNode(entry, cgnatDetected));
         return {
-            portMode: this.currentSettings.upnpEnabled ? portState.mode : 'disabled',
-            externalPort: portState.externalPort,
-            internalPort: portState.internalPort,
-            routerExternalIp: portState.routerExternalIp,
-            mappingLeaseExpiresAt: portState.leaseExpiresAt?.toISOString() ?? null,
-            externalIp,
+            upnpEnabled: this.currentSettings.upnpEnabled,
+            gateway: { ...this.gateway, routerExternalIp: this.latestIp?.routerIp ?? null },
+            externalIp: this.detectedIp(),
             externalIpDetectedAt: this.latestIp?.detectedAt.toISOString() ?? null,
-            cgnatDetected: this.latestIp?.cgnat ?? false,
-            directReachability: evaluateReachability({
-                portMode: portState.mode,
-                cgnatDetected: this.latestIp?.cgnat ?? false,
-            }),
+            cgnatDetected,
+            directReachability: evaluateHostReachability(nodes, cgnatDetected),
             lastTestedAt: this.lastTestedAt?.toISOString() ?? null,
-            ddns,
+            ddns: this.ddnsStatus(),
+            nodes,
         };
     }
-    /** Re-run the IP probe + port-mapper verify, then return a fresh snapshot. */
-    async testReachability() {
-        if (this.portService) {
-            await this.portService.refresh().catch((err) => log('refresh err: %s', err.message));
-        }
-        await this.detectIp();
-        this.lastTestedAt = this.nowFn();
-        // Best-effort DDNS push when the IP just changed.
-        if (this.ddnsUpdater) {
-            await this.ddnsUpdater.forceUpdate().catch((err) => log('ddns forceUpdate err: %s', err.message));
-        }
-        // Reachability/IP may have changed → invite addresses may have changed.
-        await this.fireAddressesChanged();
-        return this.getStatus();
+    /** Current settings (cheap read; for handlers to surface to UI). */
+    getSettings() {
+        return this.currentSettings;
     }
     /**
-     * Build the multiaddrs to embed in invites. The peer ID and libp2p
-     * fallback addresses are fetched over the owner node's admin channel; a
-     * node-unavailable failure surfaces as `NatError('node_unavailable')`.
+     * The public multiaddrs for a node, from cached state. A port with no route
+     * yet is predicted to land on the identity mapping when UPnP is enabled and
+     * a gateway was discovered — callers ask at spawn time, before the mapping
+     * for a brand-new node exists. A port whose mapping attempt failed is not
+     * predicted.
      */
-    async getInviteAddresses() {
-        let peerId;
-        let libp2pAddrs;
-        try {
-            peerId = await this.cadreNode.getPeerId();
-            libp2pAddrs = await this.cadreNode.getMultiaddrs();
-        }
-        catch (err) {
-            if (err instanceof OwnerNodeUnavailableError) {
-                throw new NatError('node_unavailable', `Owner node unavailable: ${err.message}`);
-            }
-            throw err;
-        }
-        // A reachable node that hasn't yet established its libp2p peer ID reports
-        // an empty peerId. Building addresses now would mint malformed `…/p2p/`
-        // suffixes (and push them to the node), so treat it as not-ready.
-        if (!peerId) {
-            throw new NatError('node_unavailable', 'Owner node has no peer ID yet (not ready)');
-        }
-        const status = this.getStatus();
-        const input = {
-            peerId,
-            externalPort: status.externalPort,
-            ddnsHostname: status.ddns.hostname,
-            externalIp: status.externalIp,
-            reachable: status.directReachability === 'reachable',
-            libp2pAddrs,
-        };
-        return buildInviteAddresses(input);
+    publicAddressesFor(nodeId, ports) {
+        const pair = portsOf(ports);
+        const entry = this.table.get(nodeId);
+        const tcp = this.routeOrPrediction(nodeId, 'tcp', pair.tcp, entry);
+        const ws = pair.ws === null ? null : this.routeOrPrediction(nodeId, 'ws', pair.ws, entry);
+        return this.addressesFor(tcp, ws);
     }
     /**
-     * Register a listener fired when the invite addresses may have changed
-     * (after `start`, `putSettings`, and `testReachability`). `cadre-host start`
-     * wires this to `OwnerNodeClient.pushInviteAddresses`. Returns an
-     * unsubscribe fn.
+     * Register a listener fired whenever a node's routes, the gateway, the
+     * external IP, the CGNAT flag, the UPnP toggle or the DDNS settings change.
+     * Returns an unsubscribe fn.
      */
-    onAddressesChanged(listener) {
-        this.addressListeners.add(listener);
-        return () => { this.addressListeners.delete(listener); };
+    onChange(listener) {
+        this.changeListeners.add(listener);
+        return () => { this.changeListeners.delete(listener); };
     }
     /**
-     * Recompute invite addresses and notify listeners. Best-effort: when the
-     * owner node is unreachable (or the build fails), the notification is
-     * skipped — the push is retried on the next NAT event.
+     * Register a listener asked to restart a running node whose announced addresses
+     * (`ManagedNodeInfo.announcedAddrs`) differ from `publicAddressesFor` — at most
+     * once per node per `NAT_ADDRESS_RESTART_MIN_INTERVAL_MS`. Returns an unsubscribe fn.
      */
-    async fireAddressesChanged() {
-        if (this.addressListeners.size === 0)
-            return;
-        let addresses;
-        try {
-            addresses = await this.getInviteAddresses();
-        }
-        catch (err) {
-            log('onAddressesChanged skipped: %s', err.message);
-            return;
-        }
-        await this.notifyAddressListeners(addresses);
+    onNodeAddressesStale(listener) {
+        return this.addressWatch.subscribe(listener);
     }
-    /** Deliver an address set to every registered listener (errors are logged, not thrown). */
-    async notifyAddressListeners(addresses) {
-        for (const listener of this.addressListeners) {
-            try {
-                await listener(addresses);
-            }
-            catch (err) {
-                log('address listener threw: %s', err.message);
-            }
-        }
+    /** List available DDNS providers (UI listing for configuration). */
+    listDdnsProviders() {
+        return listProviders().map((p) => ({
+            id: p.id,
+            displayName: p.displayName,
+            configFields: p.configFields,
+        }));
     }
+    // --- passes (public so tests and `testReachability` can drive them) ---
     /**
-     * Initial invite-address push. The owner node's admin channel is a freshly
-     * spawned detached child that may not be bound yet (and may not have a libp2p
-     * peer ID yet), so a single attempt races readiness and silently drops the first
-     * NAT-resolved address set. Retry on `node_unavailable` until the node accepts it
-     * or the bounded budget elapses (best-effort thereafter). Awaited by start() so
-     * the management API — the invite-minting path — does not come up first.
+     * One reconcile pass against the node source: map running nodes, release
+     * nodes stopped past the grace, forget terminated ones. Serialized against
+     * every other pass, so a caller may get a queued pass rather than an
+     * immediate one.
      */
-    async pushInitialAddresses() {
-        if (this.addressListeners.size === 0)
-            return;
-        const deadline = this.nowFn().getTime() + this.initialPushTimeoutMs;
-        for (;;) {
-            if (!this.started)
-                return; // stop() ran during a retry
-            try {
-                const addresses = await this.getInviteAddresses();
-                await this.notifyAddressListeners(addresses);
-                return;
-            }
-            catch (err) {
-                const code = err instanceof NatError ? err.code : undefined;
-                // Only retry transient not-ready; surface anything unexpected via log + give up.
-                if (code !== 'node_unavailable' || this.nowFn().getTime() >= deadline) {
-                    log('initial invite-address push not delivered: %s', err.message);
-                    return;
-                }
-                await sleep(this.initialPushRetryMs);
-            }
+    reconcile() {
+        return this.enqueue(() => this.reconcileOnce({ mapMissing: true }));
+    }
+    /**
+     * One renewal pass: re-request every UPnP-eligible mapping of every running
+     * node, failed ones included. Each renewal is isolated; one failure does not
+     * stop the pass. A router that answers with another external port updates
+     * the route.
+     */
+    renewMappings() {
+        return this.enqueue(() => this.renewOnce());
+    }
+    /**
+     * The 5-minute probe pass: while UPnP is on and no router has answered,
+     * search again and map the running nodes as soon as one does — a host that
+     * boots before its network, or a router that comes up later, is found here
+     * rather than at the next restart; then re-detect the external IP.
+     */
+    async redetect() {
+        if (this.currentSettings.upnpEnabled && !this.gateway.found) {
+            await this.enqueue(async () => {
+                await this.discoverGateway();
+                if (this.gateway.found)
+                    await this.reconcileOnce({ mapMissing: true });
+            });
         }
+        await this.detectIp();
+        this.publish();
     }
-    /** Current settings (cheap read; for handlers to surface to UI). */
-    getSettings() {
-        return this.currentSettings;
+    // --- writes ---
+    /** Re-discover the router if needed, re-map everything, re-detect the IP, then return a fresh snapshot. */
+    async testReachability() {
+        await this.enqueue(async () => {
+            if (this.currentSettings.upnpEnabled && !this.gateway.found)
+                await this.discoverGateway();
+            await this.reconcileOnce({ mapMissing: false });
+            await this.renewOnce();
+        });
+        await this.detectIp();
+        this.lastTestedAt = this.nowFn();
+        if (this.ddnsUpdater) {
+            await this.ddnsUpdater.forceUpdate().catch((err) => log('ddns forceUpdate err: %s', errorMessage(err)));
+        }
+        this.publish();
+        return this.getStatus();
     }
     /** Replace settings; persists to disk and reconfigures the runtime. */
     async putSettings(patch) {
         const next = this.store.update(patch);
         const upnpChanged = next.upnpEnabled !== this.currentSettings.upnpEnabled;
-        const portChanged = next.externalPort !== this.currentSettings.externalPort ||
-            next.internalPort !== this.currentSettings.internalPort;
         const ddnsChanged = next.ddns.providerId !== this.currentSettings.ddns.providerId ||
             next.ddns.hostname !== this.currentSettings.ddns.hostname ||
             next.ddns.externallyManaged !== this.currentSettings.ddns.externallyManaged ||
             next.ddns.intervalMs !== this.currentSettings.ddns.intervalMs;
         this.currentSettings = next;
         if (this.started) {
-            if (upnpChanged || portChanged) {
-                try {
-                    await this.portService?.stop();
-                }
-                catch { /* ignore */ }
-                this.portService = null;
-                await this.startPortService();
-            }
             if (ddnsChanged && this.ddnsUpdater) {
                 await this.ddnsUpdater.updateSettings(next.ddns);
             }
-            if (upnpChanged || portChanged || ddnsChanged) {
-                // Port / DDNS changes alter the advertised invite addresses.
-                await this.fireAddressesChanged();
+            if (upnpChanged) {
+                // Turning UPnP on with no gateway known yet: look for one before the
+                // pass, so the pass can map. Turning it off: the pass releases every
+                // `upnp` route and keeps the manual ones.
+                await this.enqueue(async () => {
+                    if (next.upnpEnabled && !this.gateway.found)
+                        await this.discoverGateway();
+                    await this.reconcileOnce({ mapMissing: true });
+                });
             }
         }
+        this.publish();
         return this.getStatus();
     }
     /**
@@ -296,7 +319,6 @@ export class NatService {
         if (!this.secretsStore) {
             throw new NatError('secrets_unavailable', 'secrets store not initialised; call start() first');
         }
-        // Persist secrets.
         for (const field of provider.configFields) {
             if (!field.secret)
                 continue;
@@ -306,84 +328,426 @@ export class NatService {
             }
             await this.secretsStore.set(ddnsAccount(provider.id, field.key), value);
         }
-        // Update nat.json.
         const externallyManaged = body.externallyManaged ?? false;
         return await this.putSettings({
             ddns: {
                 providerId: provider.id,
-                hostname: body.hostname,
+                hostname: provider.qualifyHostname(body.hostname),
                 externallyManaged,
                 intervalMs: this.currentSettings.ddns.intervalMs,
             },
         });
     }
-    /** List available DDNS providers (UI listing for configuration). */
-    listDdnsProviders() {
-        return listProviders().map((p) => ({
-            id: p.id,
-            displayName: p.displayName,
-            configFields: p.configFields,
-        }));
+    /**
+     * Record the external ports the user forwarded by hand for one node (`null`
+     * clears a port). A port that became manual releases its UPnP mapping; a
+     * port whose entry was cleared is re-requested over UPnP.
+     */
+    async putForward(nodeId, patch) {
+        validateForwardPatch(patch);
+        if (!this.nodeSource.listNodes().some((n) => n.id === nodeId)) {
+            throw new NatError('unknown_node', `no hosted node with id "${nodeId}"`);
+        }
+        this.currentSettings = this.store.setForward(nodeId, patch);
+        if (this.started)
+            await this.reconcile();
+        this.publish();
+        return this.getStatus();
+    }
+    // --- reconcile internals ---
+    enqueue(fn) {
+        const next = this.tail.then(fn, fn);
+        // The stored tail swallows outcomes — a failed pass must not reject the
+        // next caller's wait, only sequence after it.
+        this.tail = next.then(() => undefined, () => undefined);
+        return next;
     }
-    // --- private ---
-    async startPortService() {
-        if (!this.currentSettings.upnpEnabled) {
-            log('UPnP disabled by settings; skipping port mapping');
+    /** Fire-and-forget pass for the timer / event / startup triggers. */
+    sweep(trigger, run) {
+        void run().catch((err) => { log('%s pass failed: %s', trigger, errorMessage(err)); });
+    }
+    /**
+     * One pass per burst of events: an event while a pass is queued joins it,
+     * and an event during a running pass queues exactly one more, so a node that
+     * spawned after the running pass listed the nodes is mapped by the next pass
+     * rather than by the timer.
+     */
+    queueEventPass() {
+        if (this.eventPassQueued)
             return;
-        }
-        let mapper = this.portMapper;
-        if (!mapper) {
+        this.eventPassQueued = true;
+        void this.enqueue(() => {
+            this.eventPassQueued = false;
+            return this.reconcileOnce({ mapMissing: true });
+        }).catch((err) => { log('event pass failed: %s', errorMessage(err)); });
+    }
+    async reconcileOnce(opts) {
+        if (!this.started)
+            return;
+        const listed = new Map(this.nodeSource.listNodes().map((n) => [n.id, n]));
+        await this.forgetMissingNodes(listed);
+        for (const node of listed.values()) {
+            if (!this.started)
+                break;
             try {
-                mapper = await createDefaultPortMapper();
-                this.portMapper = mapper;
+                await this.reconcileNode(node, opts.mapMissing);
             }
             catch (err) {
-                log('default port mapper unavailable: %s', err.message);
-                return;
+                // One bad node must never end the pass — the others still need their routes.
+                log('reconcile of node %s failed: %s', node.id, errorMessage(err));
             }
         }
-        this.portService = new PortMapperService({
-            mapper,
-            externalPort: this.currentSettings.externalPort,
-            internalPort: this.currentSettings.internalPort,
-            now: this.nowFn,
-        });
-        await this.portService.start();
+        this.publish();
+    }
+    /**
+     * A node no longer listed was terminated (`removeContainer`): release its
+     * mappings at once and drop its manual forward. A forward left behind for a
+     * node that vanished while the host was down goes the same way.
+     */
+    async forgetMissingNodes(listed) {
+        for (const [nodeId, entry] of this.table) {
+            if (listed.has(nodeId))
+                continue;
+            await this.releaseUpnpRoutes(entry);
+            this.table.delete(nodeId);
+        }
+        for (const nodeId of Object.keys(this.currentSettings.forwards)) {
+            if (listed.has(nodeId))
+                continue;
+            this.currentSettings = this.store.deleteForward(nodeId);
+        }
+    }
+    async reconcileNode(node, mapMissing) {
+        const ports = portsOf(node.ports);
+        let entry = this.table.get(node.id);
+        if (entry && !samePorts(entry.ports, ports)) {
+            // Ports changed across a respawn: the old mappings point at ports nothing binds.
+            await this.releaseUpnpRoutes(entry);
+            entry = undefined;
+        }
+        if (!entry) {
+            entry = {
+                nodeId: node.id,
+                running: false,
+                stoppedSince: null,
+                ports,
+                routes: { tcp: noRoute(ports.tcp), ws: ports.ws === null ? null : noRoute(ports.ws) },
+            };
+            this.table.set(node.id, entry);
+        }
+        if (node.status === 'running') {
+            entry.running = true;
+            entry.stoppedSince = null;
+            await this.reconcileRoutes(entry, mapMissing);
+            return;
+        }
+        entry.running = false;
+        entry.stoppedSince ?? (entry.stoppedSince = this.nowFn().getTime());
+        if (this.nowFn().getTime() - entry.stoppedSince >= NAT_UNMAP_GRACE_MS) {
+            await this.releaseUpnpRoutes(entry);
+        }
+        // Manual forwards still follow the settings while the node is down; no router call is needed.
+        await this.reconcileRoutes(entry, false);
+    }
+    /**
+     * Bring one node's routes in line with the settings: manual wins over UPnP;
+     * UPnP off releases every `upnp` route; with `mapMissing`, a port with no
+     * route and no recorded failure (or an expired lease) is requested.
+     */
+    async reconcileRoutes(entry, mapMissing) {
+        for (const kind of this.kindsOf(entry)) {
+            const route = entry.routes[kind];
+            const forward = this.currentSettings.forwards[entry.nodeId]?.[kind];
+            if (forward !== undefined) {
+                if (route.source === 'upnp')
+                    await this.unmapRoute(entry, kind);
+                entry.routes[kind] = manualRoute(route.internalPort, forward);
+                continue;
+            }
+            if (route.source === 'manual') {
+                entry.routes[kind] = noRoute(route.internalPort);
+            }
+            if (!this.currentSettings.upnpEnabled) {
+                if (entry.routes[kind].source === 'upnp')
+                    await this.unmapRoute(entry, kind);
+                entry.routes[kind] = noRoute(route.internalPort);
+                continue;
+            }
+            if (!mapMissing || !this.gateway.found)
+                continue;
+            const current = entry.routes[kind];
+            const untried = current.source === null && current.error === null;
+            if (untried || this.leaseExpired(current)) {
+                await this.mapRoute(entry, kind);
+            }
+        }
+    }
+    async renewOnce() {
+        if (!this.started)
+            return;
+        if (!this.currentSettings.upnpEnabled || !this.gateway.found)
+            return;
+        for (const entry of this.table.values()) {
+            if (!this.started)
+                break;
+            if (!entry.running)
+                continue;
+            for (const kind of this.kindsOf(entry)) {
+                if (entry.routes[kind].source === 'manual')
+                    continue;
+                await this.mapRoute(entry, kind);
+            }
+        }
+        this.publish();
+    }
+    /**
+     * Ask the router for one port. A refusal records `error` on that port only.
+     *
+     * NOTE: a router that stops answering costs each attempt its 10 s timeout, so a
+     * pass over N mapped ports can take 10 s × N; passes are serialized, so they queue
+     * rather than overlap. If that ever shows, skip ports whose last failure was a
+     * timeout until the next renewal pass.
+     */
+    async mapRoute(entry, kind) {
+        const previous = entry.routes[kind];
+        const internalPort = previous.internalPort;
+        try {
+            const result = await this.portMapper.map({ internalPort, protocol: 'tcp', ttlMs: this.leaseTtlMs });
+            entry.routes[kind] = {
+                internalPort,
+                externalPort: result.externalPort,
+                source: 'upnp',
+                leaseExpiresAt: result.leaseExpiresAt.toISOString(),
+                error: null,
+            };
+            log('mapped %s %s port %d → external %d', entry.nodeId, kind, internalPort, result.externalPort);
+        }
+        catch (err) {
+            const message = errorMessage(err);
+            // A failed renewal of a live mapping keeps the route: the router still
+            // holds it until the lease expires, and `leaseExpired` drops it then.
+            const live = previous.source === 'upnp' && !this.leaseExpired(previous);
+            entry.routes[kind] = live ? { ...previous, error: message } : { ...noRoute(internalPort), error: message };
+            log('mapping %s %s port %d failed: %s', entry.nodeId, kind, internalPort, message);
+        }
+    }
+    async unmapRoute(entry, kind) {
+        const route = entry.routes[kind];
+        await this.portMapper.unmap(route.internalPort, 'tcp');
+        entry.routes[kind] = noRoute(route.internalPort);
+        log('unmapped %s %s port %d', entry.nodeId, kind, route.internalPort);
+    }
+    async releaseUpnpRoutes(entry) {
+        for (const kind of this.kindsOf(entry)) {
+            if (entry.routes[kind].source === 'upnp')
+                await this.unmapRoute(entry, kind);
+        }
+    }
+    kindsOf(entry) {
+        return PORT_KINDS.filter((kind) => entry.routes[kind] !== null);
+    }
+    leaseExpired(route) {
+        if (route.source !== 'upnp' || !route.leaseExpiresAt)
+            return false;
+        return Date.parse(route.leaseExpiresAt) <= this.nowFn().getTime();
+    }
+    /** A UPnP route whose lease ran out reads as no route, keeping any recorded failure. */
+    effectiveRoute(route) {
+        if (!this.leaseExpired(route))
+            return route;
+        return { ...noRoute(route.internalPort), error: route.error };
     }
+    // --- gateway / external IP ---
+    async discoverGateway() {
+        try {
+            const info = await this.portMapper.discover();
+            this.gateway = info
+                ? { found: true, lanAddress: info.lanAddress, routerExternalIp: null, lastError: null }
+                : {
+                    found: false,
+                    lanAddress: null,
+                    routerExternalIp: null,
+                    lastError: `no UPnP gateway answered within ${GATEWAY_DISCOVERY_TIMEOUT_MS / 1000} s`,
+                };
+        }
+        catch (err) {
+            this.gateway = { found: false, lanAddress: null, routerExternalIp: null, lastError: errorMessage(err) };
+        }
+        if (!this.gateway.found)
+            log('gateway discovery: %s', this.gateway.lastError);
+    }
+    /**
+     * Detect the external IP. A detection that finds nothing (or throws), or that
+     * lost the public echo the previous one had, keeps the previous result: one
+     * failed probe must not drop every node's public address, and a router-only
+     * answer must not flip the CGNAT flag the two probes had agreed on.
+     */
     async detectIp() {
-        // If the caller injected a detector (typically a test stub), use it as-is —
-        // don't second-guess them by stitching in the port mapper's router probe.
-        const detector = this.injectedDetector ?? (this.portMapper
-            ? new ExternalIpDetector({
-                fetch: this.fetchImpl,
-                now: this.nowFn,
-                routerProbe: async () => this.portMapper.externalIp(),
-            })
-            : new ExternalIpDetector({
-                fetch: this.fetchImpl,
-                now: this.nowFn,
-            }));
         try {
-            this.latestIp = await detector.detect();
+            const result = await this.ipDetector().detect();
+            if (this.knowsLessThanPrevious(result)) {
+                log('external IP detection found less than before; keeping the previous result');
+                return;
+            }
+            this.latestIp = result;
         }
         catch (err) {
-            log('detectIp failed: %s', err.message);
+            log('detectIp failed: %s', errorMessage(err));
         }
     }
-    disabledPortState() {
+    knowsLessThanPrevious(result) {
+        if (result.publicIp === null && result.routerIp === null)
+            return true;
+        return result.publicIp === null && this.latestIp?.publicIp != null;
+    }
+    ipDetector() {
+        if (this.injectedDetector)
+            return this.injectedDetector;
+        this.detector ?? (this.detector = new ExternalIpDetector({
+            fetch: this.fetchImpl,
+            now: this.nowFn,
+            routerProbe: async () => (this.gateway.found ? await this.portMapper.externalIp() : null),
+        }));
+        return this.detector;
+    }
+    /** The detected external IP for display and DDNS: the public probe's, else the router's. */
+    detectedIp() {
+        return this.latestIp?.publicIp ?? this.latestIp?.routerIp ?? null;
+    }
+    /** The IP used in `/ip4/` addresses: only a public IPv4 counts. */
+    publicIpv4() {
+        const ip = this.detectedIp();
+        return ip && isPublicIpv4(ip) ? ip : null;
+    }
+    // --- status assembly ---
+    describeNode(entry, cgnatDetected) {
+        const tcp = this.effectiveRoute(entry.routes.tcp);
+        const ws = entry.routes.ws ? this.effectiveRoute(entry.routes.ws) : null;
+        const { verdict, reason } = evaluateNodeReachability({
+            tcp,
+            ws,
+            hostKnown: this.hostKnown(),
+            cgnatDetected,
+            upnpEnabled: this.currentSettings.upnpEnabled,
+            gatewayFound: this.gateway.found,
+            lanAddress: this.gateway.lanAddress,
+        });
         return {
-            mode: this.currentSettings.upnpEnabled ? 'failed' : 'disabled',
-            externalPort: this.currentSettings.externalPort,
-            internalPort: this.currentSettings.internalPort,
-            routerExternalIp: null,
-            leaseExpiresAt: null,
+            nodeId: entry.nodeId,
+            running: entry.running,
+            verdict,
+            reason,
+            tcp,
+            ws,
+            publicAddrs: this.addressesFor(tcp, ws),
+        };
+    }
+    hostKnown() {
+        return this.currentSettings.ddns.hostname !== null || this.publicIpv4() !== null;
+    }
+    addressesFor(tcp, ws) {
+        return buildPublicAddresses({
+            ddnsHostname: this.currentSettings.ddns.hostname,
+            externalIp: this.publicIpv4(),
+            cgnatDetected: this.latestIp?.cgnat ?? false,
+            tcp,
+            ws,
+        });
+    }
+    routeOrPrediction(nodeId, kind, internalPort, entry) {
+        const forward = this.currentSettings.forwards[nodeId]?.[kind];
+        if (forward !== undefined)
+            return manualRoute(internalPort, forward);
+        const existing = entry?.routes[kind];
+        if (existing && existing.internalPort === internalPort) {
+            const route = this.effectiveRoute(existing);
+            if (route.source !== null || route.error !== null)
+                return route;
+        }
+        if (this.currentSettings.upnpEnabled && this.gateway.found) {
+            return { internalPort, externalPort: internalPort, source: 'upnp', leaseExpiresAt: null, error: null };
+        }
+        return noRoute(internalPort);
+    }
+    ddnsStatus() {
+        return this.ddnsUpdater?.getStatus() ?? {
+            providerId: this.currentSettings.ddns.providerId,
+            hostname: this.currentSettings.ddns.hostname,
+            externallyManaged: this.currentSettings.ddns.externallyManaged,
+            lastUpdateAt: null,
+            lastUpdateOk: null,
             lastError: null,
         };
     }
+    // --- change notification ---
+    /**
+     * The end of every pass and every settings write: tell the change listeners
+     * when the snapshot moved, then ask for a restart of every running node whose
+     * announced addresses went stale. Running the check here rather than only on a
+     * change is what lets a difference deferred by the restart rate limit fire on the
+     * first timer pass after the limit ends.
+     */
+    publish() {
+        this.notifyIfChanged();
+        if (!this.started)
+            return;
+        this.addressWatch.check(this.nodeSource.listNodes(), (node) => this.publicAddressesFor(node.id, node.ports));
+    }
+    /** Fire the change listeners when the snapshot differs (timestamps aside) from the last one they saw. */
+    notifyIfChanged() {
+        const snapshot = this.getStatus();
+        const signature = signatureOf(snapshot);
+        if (signature === this.lastSignature)
+            return;
+        this.lastSignature = signature;
+        for (const listener of this.changeListeners) {
+            try {
+                listener(snapshot);
+            }
+            catch (err) {
+                log('change listener threw: %s', errorMessage(err));
+            }
+        }
+    }
+}
+/** The snapshot minus the fields that move on their own (timestamps, lease expiry). */
+function signatureOf(s) {
+    return JSON.stringify({
+        upnpEnabled: s.upnpEnabled,
+        gateway: s.gateway,
+        externalIp: s.externalIp,
+        cgnatDetected: s.cgnatDetected,
+        directReachability: s.directReachability,
+        ddns: { providerId: s.ddns.providerId, hostname: s.ddns.hostname, externallyManaged: s.ddns.externallyManaged },
+        nodes: s.nodes.map((n) => ({
+            nodeId: n.nodeId,
+            running: n.running,
+            verdict: n.verdict,
+            tcp: routeSignature(n.tcp),
+            ws: n.ws ? routeSignature(n.ws) : null,
+            publicAddrs: n.publicAddrs,
+        })),
+    });
+}
+function routeSignature(r) {
+    return { internalPort: r.internalPort, externalPort: r.externalPort, source: r.source, error: r.error };
+}
+function validateForwardPatch(patch) {
+    if (!patch || typeof patch !== 'object' || Array.isArray(patch)) {
+        throw new NatError('invalid_config', 'a forward patch must be an object with tcp and/or ws');
+    }
+    for (const kind of PORT_KINDS) {
+        const value = patch[kind];
+        if (value === undefined || value === null)
+            continue;
+        if (!Number.isInteger(value) || value < 1 || value > 65535) {
+            throw new NatError('invalid_config', `${kind} must be an integer 1-65535 or null, got ${String(value)}`);
+        }
+    }
 }
 /**
- * Wrap a NatService into the typed handler shape consumed by
- * `cadre-host-local-ui`. Mirrors `createTrustCircleHandlers`.
+ * Wrap a NatService into the typed handler shape consumed by the local UI
+ * server. Same shape as `createUpdateHandlers`.
  */
 export function createNatHandlers(service) {
     return {
@@ -392,6 +756,7 @@ export function createNatHandlers(service) {
         async listDdnsProviders() { return service.listDdnsProviders(); },
         async putDdns(body) { return await service.putDdns(body); },
         async putSettings(body) { return await service.putSettings(body); },
+        async putForward(nodeId, patch) { return await service.putForward(nodeId, patch); },
     };
 }
 //# sourceMappingURL=nat-service.js.map