@wpelevator/tug 0.7.0 → 0.9.0

package/dist/control.js

@@ -2,6 +2,7 @@
 import express from 'express';
 import { resolveImage, parseRegistryCredentials, imageLocation, REGISTRY_AUTH_SECRET, } from "./registry.js";
 import multer from 'multer';
+import crypto from 'node:crypto';
 import fs from 'node:fs/promises';
 import { createReadStream } from 'node:fs';
 import { pipeline } from 'node:stream/promises';
@@ -15,17 +16,21 @@ import * as tar from 'tar';
 import { openDb, openSnapshotDb } from "./db.js";
 import { isUniqueViolation } from "./db-driver.js";
 import { encrypt, decrypt, loadOrCreateMasterKey } from "./secrets.js";
-import { buildDesiredState, loopbackIngressServices, serviceSecretNames, validateCompose, } from "./compose.js";
+import { buildDesiredState, ingressHostsOf, ingressHostsOverlap, loopbackIngressServices, serviceSecretNames, validateCompose, } from "./compose.js";
 import { buildImage, unpackContext, run as runCommand } from "./build.js";
 import { sentinelNames } from "./normalize.js";
-import { ensureDir, errMessage, hashToken, id, now, token } from "./util.js";
-import { DEFAULT_MESH_CIDR, DEFAULT_WG_PORT, WG_INTERFACE, WG_KEEPALIVE, allocateOverlayIp, isWgKey, parseCidr, wgEndpoint, } from "./mesh.js";
+import { ensureDir, errMessage, id, now } from "./util.js";
+import { ENROLL_DEFAULT_TTL_MS, NODE_LIFETIME_MS, Pki, SERVER_LIFETIME_MS, encodeCredential, identityUri, namesKey, parseIdentityUri, peerIdentity, serverNames, writeFileDurable, } from "./pki.js";
+import { EXPIRY_WARN_DAYS, certificateExpiry, diagnoseNode } from "./diagnostics.js";
+import { serverTlsOptions } from "./tls-server.js";
+import { DEFAULT_MESH_CIDR, DEFAULT_WG_PORT, WG_INTERFACE, WG_KEEPALIVE, allocateOverlayIp, isWgKey, parseCidr, wgEndpoint, isNodeEndpoint, isNodeHost, } from "./mesh.js";
 import { buildIngressWork, resolveServiceEndpoints } from "./ingress.js";
+import { parseStatusReport } from "./status-report.js";
 import { ACTIVE_DEPLOYMENT_STATES, LIVE_MAX_FRAME_BYTES, LIVE_MAX_BUFFERED_BYTES, LIVE_MAX_SESSIONS, LIVE_START_TIMEOUT_MS, BACKUP_COMMAND_TYPES, } from "./protocol.js";
 import { BACKUP_SECRET_PROJECT, isReservedProject, parseBackupDestination, projectRepository, repositoryPassword, redactSecrets, } from "./backup-destination.js";
 import { createBackupService } from "./backup-scheduler.js";
 import { createRestoreService } from "./backup-restore.js";
-import { AUTO_ROLLBACK_ACTOR, ROUTE_ROLES, PROJECT_NAME_PATTERN, authenticate, hasRole, assertProject, bodyProject, createGuard, createToken, listTokens, parseTokenRequest, queryProject, recordEvent, revokeToken, } from "./tokens.js";
+import { AUTO_ROLLBACK_ACTOR, ROUTE_ROLES, PROJECT_NAME_PATTERN, authenticate, hasRole, assertProject, bodyProject, createGuard, createCredential, listCredentials, parseCredentialRequest, queryProject, recordEvent, revokeCredential, } from "./credentials.js";
 class WakeHub {
     cursors = new Map();
     projects = new Map();
@@ -398,14 +403,14 @@ function deploymentFailureMessage(rows) {
                 parts.push(`${service}: ${oneLine(state.rolloutError)}`);
         }
         const detail = parts.join('; ');
-        return detail ? `${row.node_id}: ${detail}` : row.node_id;
+        return detail ? `${row.node}: ${detail}` : row.node;
     });
     return `one or more assigned nodes reported failed health: ${details.join('; ')}`;
 }
 /** Versions of rollback deployments start with this; `tug deploy` may not use it, which is how automatic rollback tells them apart. */
 const ROLLBACK_VERSION_PREFIX = 'rollback-';
 const ACTIVE_STATES_SQL = ACTIVE_DEPLOYMENT_STATES.map((s) => `'${s}'`).join(',');
-const PORT = Number(process.env.PORT || 8080);
+const PORT = Number(process.env.PORT || 8443);
 export async function createApp(options = {}) {
     const images = {
         resolve: resolveImage,
@@ -430,7 +435,7 @@ export async function createApp(options = {}) {
     const MESH = process.env.TUG_MESH === 'true';
     const MESH_CIDR = process.env.TUG_MESH_CIDR || DEFAULT_MESH_CIDR;
     const WG_PORT = Number(process.env.TUG_WG_PORT || DEFAULT_WG_PORT);
-    const SERVICE_DNS = (process.env.TUG_SERVICE_DNS ?? String(MESH)) === 'true';
+    const SERVICE_DNS = (process.env.TUG_SERVICE_NAMES ?? String(MESH)) === 'true';
     const ACME_EMAIL = process.env.TUG_ACME_EMAIL || undefined;
     if (MESH)
         parseCidr(MESH_CIDR);
@@ -472,23 +477,94 @@ export async function createApp(options = {}) {
         console.error(`event pruning failed: ${errMessage(e)}`);
     }
     const masterKey = await loadOrCreateMasterKey(masterKeyFile);
-    const operator = db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash');
-    if (!operator) {
-        const t = process.env.TUG_OPERATOR_TOKEN || token('tug_op');
-        db.run('INSERT INTO meta(key,value) VALUES(?,?)', 'operator_token_hash', hashToken(t));
-        await fs
-            .writeFile(path.join(DATA_DIR, 'operator.token'), `${t}\n`, { mode: 0o600 })
-            .catch(() => { });
-        console.log(`operator token: ${t}`);
-    }
+    // A CA is created only for a fresh installation; existing state with missing or mismatched CA material is a startup error.
+    const pki = await Pki.load(path.join(DATA_DIR, 'pki'), !db.get('SELECT 1 FROM operator_credentials') && !db.get('SELECT 1 FROM nodes'));
+    const advertiseUrl = () => {
+        const url = options.advertiseUrl ?? process.env.TUG_ADVERTISE_URL;
+        if (!url)
+            throw httpError(500, 'TUG_ADVERTISE_URL is not set');
+        return new URL(url).origin;
+    };
     const app = express();
-    const authenticateAgent = (raw) => raw
-        ? db.get('SELECT * FROM nodes WHERE agent_token_hash=?', hashToken(raw))
-        : undefined;
+    app.locals.pki = pki;
+    app.locals.dataDir = DATA_DIR;
+    /** Resolves a presented certificate (principal URI and key hash) to a principal from current database state. */
+    function resolvePrincipal(uri, keyHash) {
+        const parsed = parseIdentityUri(uri);
+        if (!parsed)
+            return null;
+        if (parsed.klass === 'node') {
+            const node = db.get('SELECT * FROM nodes WHERE id=?', parsed.id);
+            return node && node.key_hash === keyHash && !node.disabled_at
+                ? { klass: 'node', node }
+                : null;
+        }
+        if (parsed.klass === 'operator') {
+            const actor = authenticate(db, parsed.id, keyHash);
+            return actor ? { klass: 'operator', actor, credentialId: parsed.id } : null;
+        }
+        const row = db.get('SELECT * FROM enrollment_credentials WHERE id=?', parsed.id);
+        return row && row.key_hash === keyHash && !row.revoked_at && row.expires_at > now()
+            ? { klass: 'enroll', row }
+            : null;
+    }
+    const liveSockets = new Set();
+    app.locals.liveSockets = liveSockets;
+    /** Closes connections whose principal is disabled, revoked, replaced, removed or expired. */
+    const reconcileSockets = () => {
+        for (const entry of [...liveSockets]) {
+            if (entry.socket.destroyed)
+                liveSockets.delete(entry);
+            else if (entry.notAfter <= Date.now() || !resolvePrincipal(entry.uri, entry.keyHash)) {
+                entry.socket.destroy();
+                liveSockets.delete(entry);
+            }
+        }
+    };
+    const closeAllSockets = () => {
+        for (const entry of liveSockets)
+            entry.socket.destroy();
+        liveSockets.clear();
+    };
+    app.locals.reconcileSockets = () => {
+        if (!restartRequired)
+            reconcileSockets();
+    };
+    /** The certificate a request or upgrade arrived with, as a (uri, keyHash) pair; null if none can be established. */
+    const presented = (request) => {
+        if (options.identity)
+            return options.identity(request, db);
+        const socket = request.socket;
+        if (typeof socket.getPeerCertificate !== 'function' || socket.isSessionReused())
+            return null;
+        return peerIdentity(socket);
+    };
+    const identify = (request) => {
+        const cert = presented(request);
+        return cert ? resolvePrincipal(cert.uri, cert.keyHash) : null;
+    };
     let restartRequired = false;
     app.use((_req, res, next) => restartRequired
         ? res.status(503).json({ error: 'control restored; restart required', restartRequired: true })
         : next());
+    // Route class gate: runs before body parsing and every handler. There is no unauthenticated route.
+    app.use((req, res, next) => {
+        const identity = identify(req);
+        if (!identity)
+            return void res.status(401).json({ error: 'unauthorized' });
+        const enrollRoute = req.method === 'POST' && req.path === '/agent/v1/enroll';
+        const allowed = identity.klass === 'enroll'
+            ? enrollRoute
+            : identity.klass === 'node'
+                ? req.path.startsWith('/agent/v1/') && !enrollRoute
+                : req.path.startsWith('/api/v1/');
+        if (!allowed)
+            return void res
+                .status(403)
+                .json({ error: `forbidden: ${identity.klass} credentials cannot use this route` });
+        req.identity = identity;
+        next();
+    });
     app.use(express.json({ limit: '20mb' }));
     // Hold the writer for the operation lifetime, including after an HTTP disconnect.
     let writer = Promise.resolve();
@@ -534,8 +610,8 @@ export async function createApp(options = {}) {
     const route = (method, url, handlers) => app[method](url, ...handlers.map(safe));
     const get = (url, ...handlers) => route('get', url, handlers);
     const post = (url, ...handlers) => route('post', url, handlers);
-    // Operator-facing routes go through api(): the role table in tokens.ts is the only place a route's access is declared.
-    const guard = createGuard(db);
+    // Operator-facing routes go through api(): the role table in credentials.ts is the only place a route's access is declared.
+    const guard = createGuard();
     const apiRoutes = new Map();
     app.locals.apiRoutes = apiRoutes;
     const api = (method, url, resolve, ...handlers) => {
@@ -546,10 +622,9 @@ export async function createApp(options = {}) {
         apiRoutes.set(key, role);
         route(method, url, [guard(role, resolve ?? undefined), ...handlers]);
     };
-    const deploymentProject = (req) => db.get('SELECT project_id FROM deployments WHERE id=?', req.params.id)
-        ?.project_id;
+    const deploymentProject = (req) => db.get('SELECT project_id FROM deployments WHERE id=?', String(req.params.id))?.project_id;
     const commandProject = (req) => {
-        const row = db.get('SELECT project_id FROM commands WHERE id=?', req.params.id);
+        const row = db.get('SELECT project_id FROM commands WHERE id=?', String(req.params.id));
         return row ? row.project_id : undefined;
     };
     /** Audit trail entry for a control action; bookkeeping failures are logged, never returned to the caller. */
@@ -568,11 +643,9 @@ export async function createApp(options = {}) {
         }
         : { where: '1=1', args: [] };
     const authAgent = async (req, res, next) => {
-        const raw = (req.headers.authorization || '').replace(/^Bearer /, '');
-        const node = authenticateAgent(raw);
-        if (!node)
-            return res.status(401).json({ error: 'unauthorized' });
-        req.node = node;
+        if (req.identity?.klass !== 'node')
+            return res.status(403).json({ error: 'node credentials required' });
+        req.node = req.identity.node;
         next();
     };
     const wakeAll = (project) => {
@@ -594,8 +667,9 @@ export async function createApp(options = {}) {
                 required.add(node);
         if (!required.size)
             return;
-        const statuses = db.all('SELECT node_id,observed_version,health,status_json FROM node_statuses WHERE project_id=?', projectId);
-        const byNode = new Map(statuses.map((s) => [s.node_id, s]));
+        // Placement names nodes; reports are keyed by the immutable node ID, so join to the current name.
+        const statuses = db.all('SELECT n.name AS node,s.observed_version,s.health,s.status_json FROM node_statuses s JOIN nodes n ON n.id=s.node_id WHERE s.project_id=?', projectId);
+        const byNode = new Map(statuses.map((s) => [s.node, s]));
         const reportsRevision = (node) => state.revision
             ? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision
             : byNode.get(node)?.observed_version === state.version;
@@ -623,6 +697,23 @@ export async function createApp(options = {}) {
      * new revision. Shared by `POST /api/v1/rollback` and automatic rollback; the caller holds the writer queue.
      * `auto` only considers `succeeded` deployments and requires `failedDeploymentId` to still be current.
      */
+    /**
+     * Ingress hostnames belong to the first project that publishes them: another project's current deployment
+     * that already routes a host blocks any project from taking it, on deploy and on rollback alike.
+     */
+    function assertIngressHostsFree(project, compose) {
+        const wanted = ingressHostsOf(compose);
+        if (!wanted.size)
+            return;
+        for (const other of currentDesiredStates()) {
+            if (other.project === project)
+                continue;
+            for (const [host, service] of ingressHostsOf(other.compose))
+                for (const [wantedHost, mine] of wanted)
+                    if (ingressHostsOverlap(wantedHost, host))
+                        throw httpError(409, `${mine}: ingress.host ${wantedHost} ${wantedHost === host ? 'is' : `overlaps ${host},`} already routed by project ${other.project} (${service})`);
+        }
+    }
     function createRollback(project, opts) {
         let lock;
         try {
@@ -641,6 +732,7 @@ export async function createApp(options = {}) {
             const ts = now();
             const depId = id('dep');
             const previousState = JSON.parse(previousDs.state_json);
+            assertIngressHostsFree(project, previousState.compose);
             previousState.revision = depId;
             db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, `${ROLLBACK_VERSION_PREFIX}${previousState.version}-${Date.now()}`, 'rolling_out', ts, ts, opts.actor);
             db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, previousState.version, JSON.stringify(previousState), ts);
@@ -734,9 +826,10 @@ export async function createApp(options = {}) {
     }, options.liveStartTimeoutMs);
     streamRuntimes.set(app, {
         hub: wakeHub,
-        authenticate: authenticateAgent,
-        authenticateOperator: (raw) => authenticate(db, raw),
+        identify,
+        resolvePrincipal,
         live,
+        isRestoring: () => restartRequired,
     });
     api('get', '/api/v1/live', null, (_req, res) => {
         res.status(426).json({ error: 'WebSocket upgrade required' });
@@ -799,7 +892,7 @@ export async function createApp(options = {}) {
         const state = JSON.parse(dsRow.state_json);
         if (!state.services?.[service])
             throw httpError(404, `service ${service} not found in project ${project}`);
-        const nodeRows = db.all('SELECT name,labels_json,last_seen_at FROM nodes ORDER BY created_at,name');
+        const nodeRows = db.all('SELECT name,labels_json,last_seen_at FROM nodes WHERE disabled_at IS NULL ORDER BY created_at,name');
         const nodeInfos = nodeRows
             .filter((row) => !isStale(row))
             .map((row) => ({
@@ -809,7 +902,7 @@ export async function createApp(options = {}) {
         const composeSvc = state.compose?.services?.[service] || {};
         const ext = composeSvc['x-tug'] || {};
         const placement = ext.placement || {};
-        const global = ext.global === true || placement.global === true;
+        const global = ext.global === true;
         const labels = Array.isArray(placement.labels) ? placement.labels : [];
         const matched = nodeInfos.filter((node) => labels.every((label) => node.labels.includes(label)));
         const pool = labels.length ? matched : nodeInfos;
@@ -834,14 +927,13 @@ export async function createApp(options = {}) {
             nodes: selected.map((node) => node.name),
         });
     });
-    get('/healthz', (req, res) => res.json({ ok: true }));
     api('post', '/api/v1/projects', (req) => String(req.body?.name || 'site'), async (req, res) => {
         const name = req.body.name || 'site';
         const ts = now();
         db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', name, name, ts, ts);
         res.json(db.get('SELECT * FROM projects WHERE id=?', name));
     });
-    api('get', '/api/v1/projects/:project', (req) => req.params.project, async (req, res) => res.json(db.get('SELECT * FROM projects WHERE id=?', req.params.project)));
+    api('get', '/api/v1/projects/:project', (req) => String(req.params.project), async (req, res) => res.json(db.get('SELECT * FROM projects WHERE id=?', String(req.params.project))));
     /** A node whose agent has been seen before but not within TUG_STALE_SECONDS; never-seen (just registered) nodes stay schedulable. */
     const isStale = (row) => !!row.last_seen_at &&
         Date.parse(row.last_seen_at) < Date.now() - Number(process.env.TUG_STALE_SECONDS || 60) * 1000;
@@ -880,20 +972,12 @@ export async function createApp(options = {}) {
         db.run(`INSERT INTO wg_peers(node_id,public_key,overlay_ip,endpoint,allowed_ips,updated_at) VALUES(?,?,?,?,?,?)
       ON CONFLICT(node_id) DO UPDATE SET public_key=excluded.public_key,overlay_ip=excluded.overlay_ip,endpoint=excluded.endpoint,allowed_ips=excluded.allowed_ips,updated_at=excluded.updated_at`, nodeId, key, node.overlay_ip, wgEndpoint(node.public_address, WG_PORT) ?? null, `${node.overlay_ip}/32`, now());
     }
-    async function registerNode({ name, address, publicAddress, wgPublicKey, labels, }) {
-        if (wgPublicKey !== undefined && !isWgKey(wgPublicKey))
-            throw httpError(400, 'wgPublicKey must be a base64 WireGuard public key');
-        const agentToken = token('tug_agent');
-        const ts = now();
-        db.run(`INSERT INTO nodes(id,name,address,public_address,status,labels_json,agent_token_hash,created_at,updated_at)
-      VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET name=excluded.name,address=excluded.address,public_address=excluded.public_address,labels_json=excluded.labels_json,agent_token_hash=excluded.agent_token_hash,updated_at=excluded.updated_at`, name, name, address || null, publicAddress || null, 'registered', JSON.stringify(Array.isArray(labels) ? labels : []), hashToken(agentToken), ts, ts);
-        ensureOverlayIp(name);
-        syncWgPeer(name, wgPublicKey);
-        wakeAll();
-        const row = db.get(NODE_INFO_SQL, name);
+    const NODE_RENEW_MIN_INTERVAL_MS = () => Number(process.env.TUG_RENEW_MIN_INTERVAL_MS ?? 60 * 60 * 1000);
+    const nodeInfo = (nodeId) => {
+        const row = db.get(NODE_INFO_SQL, nodeId);
         if (!row)
-            throw httpError(500, 'node not found after registration');
-        const node = {
+            throw httpError(500, 'node not found');
+        return {
             id: row.id,
             name: row.name,
             address: row.address,
@@ -904,75 +988,198 @@ export async function createApp(options = {}) {
             created_at: row.created_at,
             updated_at: row.updated_at,
         };
-        return { node, agentToken };
+    };
+    const NODE_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,62}$/;
+    /** Inserts a node bound to a key. Synchronous so the existence check and the write cannot interleave. */
+    function insertNode(input) {
+        if (!NODE_NAME.test(input.name))
+            throw httpError(400, 'node name must be 1-63 letters, numbers, dots, underscores or hyphens');
+        if (input.address && !isNodeHost(input.address))
+            throw httpError(400, 'address must be an IP address or hostname without a port');
+        if (input.publicAddress && !isNodeEndpoint(input.publicAddress))
+            throw httpError(400, 'publicAddress must be an IP address or hostname with an optional port');
+        if (input.wgPublicKey !== undefined && !isWgKey(input.wgPublicKey))
+            throw httpError(400, 'wgPublicKey must be a base64 WireGuard public key');
+        const ts = now();
+        db.run(`INSERT INTO nodes(id,name,address,public_address,status,labels_json,key_hash,cert_not_after,created_at,updated_at)
+      VALUES(?,?,?,?,?,?,?,?,?,?)`, input.id, input.name, input.address || null, input.publicAddress || null, 'registered', JSON.stringify(Array.isArray(input.labels) ? input.labels : []), input.keyHash, input.notAfter.toISOString(), ts, ts);
+        ensureOverlayIp(input.id);
+        syncWgPeer(input.id, input.wgPublicKey);
+        wakeAll();
     }
-    api('post', '/api/v1/nodes/register', null, async (req, res) => {
-        const name = req.body.name;
-        if (!name)
-            return res.status(400).json({ error: 'name required' });
+    const identityResponse = (nodeId, cert) => ({
+        node: nodeInfo(nodeId),
+        certificate: cert,
+        ca: pki.caPem,
+        url: advertiseUrl(),
+    });
+    // The node generates its key and CSR; an admin posts the CSR here. The private key is never transmitted.
+    api('post', '/api/v1/nodes/:name/identity', null, async (req, res) => {
+        const body = req.body ?? {};
+        const name = String(req.params.name);
+        if (body.mode !== 'create' && body.mode !== 'replace')
+            return res.status(400).json({ error: "mode must be 'create' or 'replace'" });
         try {
-            res.json(await registerNode({
-                name,
-                address: req.body.address,
-                publicAddress: req.body.publicAddress,
-                wgPublicKey: req.body.wgPublicKey,
-                labels: req.body.labels,
-            }));
+            // The certificate names the immutable node ID: the existing node's, or a fresh one for a new name. A name freed by
+            // node-remove therefore never inherits the old ID, and certificates for the removed node cannot match the new one.
+            let nodeId = db.get('SELECT * FROM nodes WHERE name=?', name)?.id ?? id('node');
+            let signed = await pki.signClientCsr(body.csr, identityUri('node', nodeId), NODE_LIFETIME_MS);
+            // Signing is asynchronous: re-read, and re-sign once if a concurrent request created the node meanwhile.
+            const current = db.get('SELECT * FROM nodes WHERE name=?', name);
+            if (current && current.id !== nodeId) {
+                nodeId = current.id;
+                signed = await pki.signClientCsr(body.csr, identityUri('node', nodeId), NODE_LIFETIME_MS);
+            }
+            const existing = db.get('SELECT * FROM nodes WHERE name=?', name);
+            if (existing && existing.id !== nodeId)
+                throw httpError(409, `node ${name} was created concurrently; retry`);
+            if (body.mode === 'create') {
+                if (existing) {
+                    if (existing.key_hash !== signed.keyHash)
+                        return res.status(409).json({
+                            error: `node ${name} already exists; use node-replace-identity to install a new key`,
+                        });
+                }
+                else {
+                    insertNode({
+                        id: nodeId,
+                        name,
+                        keyHash: signed.keyHash,
+                        notAfter: signed.notAfter,
+                        address: body.address,
+                        publicAddress: body.publicAddress,
+                        labels: body.labels,
+                    });
+                }
+            }
+            else {
+                if (!existing)
+                    return res.status(404).json({ error: 'node not found' });
+                db.run("UPDATE nodes SET key_hash=?,disabled_at=NULL,status='registered',cert_not_after=?,updated_at=? WHERE id=?", signed.keyHash, signed.notAfter.toISOString(), now(), existing.id);
+                audit(req.actor, {
+                    type: 'node.replaced',
+                    message: `node ${name} key replaced by ${req.actor.name}`,
+                    metadata: { node: name },
+                });
+                reconcileSockets();
+            }
+            res.json(identityResponse(nodeId, signed.cert));
         }
         catch (e) {
             sendError(res, e);
         }
     });
-    // Self-service enrollment: operator mints a short-lived, limited-use key; the agent
-    // exchanges it for durable credentials on first boot (Portainer Edge-style join).
-    api('post', '/api/v1/nodes/enroll-token', null, async (req, res) => {
-        const name = req.body.name || null;
-        const ttlSeconds = Math.max(1, Number(req.body.ttlSeconds ?? 3600));
-        const uses = Math.max(1, Number(req.body.uses ?? 1));
-        const enrollToken = token('tug_enroll');
-        const expiresAt = new Date(Date.now() + ttlSeconds * 1000).toISOString();
-        db.run('INSERT INTO enroll_tokens(token_hash,name,uses_remaining,expires_at,created_at) VALUES(?,?,?,?,?)', hashToken(enrollToken), name, uses, expiresAt, now());
-        res.json({ enrollToken, name, usesRemaining: uses, expiresAt });
-    });
-    post('/agent/v1/enroll', async (req, res) => {
-        const body = req.body;
-        const raw = String(body.enrollToken || (req.headers.authorization || '').replace(/^Bearer /, '')).trim();
-        if (!raw)
-            return res.status(400).json({ error: 'enrollToken required' });
-        const row = db.get('SELECT * FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
-        if (!row || row.uses_remaining < 1 || row.expires_at < now()) {
-            if (row)
-                db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
-            return res.status(401).json({ error: 'invalid or expired enroll token' });
-        }
-        const name = row.name || body.name;
-        if (!name)
+    // Self-enrollment: an admin mints a short-lived, single-use, name-bound bootstrap certificate that reaches one route.
+    api('post', '/api/v1/nodes/enrollment-credential', null, async (req, res) => {
+        const name = req.body?.name;
+        if (typeof name !== 'string' || !NODE_NAME.test(name))
             return res
                 .status(400)
-                .json({ error: 'node name required (bind one to the enroll token or send name)' });
-        let result;
+                .json({ error: 'name required: a valid node name to bind the join key to' });
+        const labels = req.body.labels;
+        if (labels !== undefined &&
+            (!Array.isArray(labels) || labels.some((l) => typeof l !== 'string')))
+            return res.status(400).json({ error: 'labels must be a list of strings' });
+        const ttlSeconds = Math.max(1, Number(req.body.ttlSeconds ?? ENROLL_DEFAULT_TTL_MS / 1000));
+        const rowId = id('enr');
+        const issued = await pki.issueClient(identityUri('enroll', rowId), ttlSeconds * 1000, advertiseUrl());
+        const expiresAt = issued.notAfter.toISOString();
+        db.run('DELETE FROM enrollment_credentials WHERE expires_at < ?', new Date(Date.now() - 86_400_000).toISOString());
+        // Minting again for a name revokes its previous unconsumed key.
+        db.run('UPDATE enrollment_credentials SET revoked_at=? WHERE name=? AND node_id IS NULL AND revoked_at IS NULL', now(), name);
+        db.run('INSERT INTO enrollment_credentials(id,key_hash,name,labels_json,expires_at,reserved_node_id,created_at) VALUES(?,?,?,?,?,?,?)', rowId, issued.keyHash, name, JSON.stringify(labels ?? []), expiresAt, id('node'), now());
+        reconcileSockets();
+        const out = {
+            credential: issued.credential,
+            name,
+            labels: labels ?? [],
+            expiresAt,
+        };
+        res.json(out);
+    });
+    post('/agent/v1/enroll', async (req, res) => {
+        const row = req.identity?.klass === 'enroll' ? req.identity.row : null;
+        if (!row)
+            return res.status(403).json({ error: 'bootstrap credentials required' });
+        const body = req.body ?? {};
         try {
-            result = await registerNode({
-                name,
-                address: body.address,
-                publicAddress: body.publicAddress,
-                wgPublicKey: body.wgPublicKey,
-                labels: body.labels,
-            });
+            const signed = await pki.signClientCsr(body.csr, identityUri('node', row.reserved_node_id), NODE_LIFETIME_MS);
+            // Re-read the row: the checks and writes below are synchronous, so concurrent enrollments cannot both create a node.
+            const fresh = db.get('SELECT * FROM enrollment_credentials WHERE id=?', row.id);
+            if (!fresh || fresh.revoked_at || fresh.expires_at <= now())
+                return res.status(401).json({ error: 'join key is expired or revoked' });
+            if (fresh.node_id) {
+                if (fresh.node_key_hash !== signed.keyHash)
+                    return res.status(403).json({ error: 'join key was already used by a different key' });
+            }
+            else {
+                if (db.get('SELECT 1 FROM nodes WHERE name=?', fresh.name))
+                    return res
+                        .status(409)
+                        .json({ error: `node ${fresh.name} already exists; a join key cannot replace a node` });
+                insertNode({
+                    id: fresh.reserved_node_id,
+                    name: fresh.name,
+                    keyHash: signed.keyHash,
+                    notAfter: signed.notAfter,
+                    address: body.address,
+                    publicAddress: body.publicAddress,
+                    wgPublicKey: body.wgPublicKey,
+                    labels: fresh.labels_json ? JSON.parse(fresh.labels_json) : [],
+                });
+                db.run('UPDATE enrollment_credentials SET node_id=?,node_key_hash=? WHERE id=?', fresh.reserved_node_id, signed.keyHash, fresh.id);
+                recordEvent(db, {
+                    actor: `enroll:${fresh.id}`,
+                    type: 'node.enrolled',
+                    message: `node ${fresh.name} enrolled`,
+                });
+            }
+            const enrolled = identityResponse(fresh.reserved_node_id, signed.cert);
+            res.json(enrolled);
         }
         catch (e) {
-            return sendError(res, e);
+            sendError(res, e);
+        }
+    });
+    // Same-key renewal: another certificate for the stored key. Nothing is stored beyond an informational expiry.
+    post('/agent/v1/renew', authAgent, async (req, res) => {
+        const node = req.node;
+        try {
+            const signed = await pki.signClientCsr(req.body?.csr, identityUri('node', node.id), NODE_LIFETIME_MS);
+            if (signed.keyHash !== node.key_hash)
+                return res.status(403).json({
+                    error: 'renewal requires a CSR for the node key; use node-replace-identity to change keys',
+                });
+            const last = node.last_renewed_at ? Date.parse(node.last_renewed_at) : 0;
+            if (Date.now() - last < NODE_RENEW_MIN_INTERVAL_MS())
+                return res.status(429).json({ error: 'certificate renewed too recently' });
+            db.run('UPDATE nodes SET cert_not_after=?,last_renewed_at=? WHERE id=?', signed.notAfter.toISOString(), now(), node.id);
+            const renewed = { certificate: signed.cert };
+            res.json(renewed);
+        }
+        catch (e) {
+            sendError(res, e);
         }
-        const remaining = row.uses_remaining - 1;
-        if (remaining < 1)
-            db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
-        else
-            db.run('UPDATE enroll_tokens SET uses_remaining=? WHERE token_hash=?', remaining, hashToken(raw));
-        const enrolled = { ...result };
-        res.json(enrolled);
+    });
+    // Blocks a node's key without deleting its metadata or changing placement. It returns only through node-replace-identity.
+    api('post', '/api/v1/nodes/:name/disable', null, async (req, res) => {
+        const node = findNode(String(req.params.name));
+        if (!node)
+            return res.status(404).json({ error: 'node not found' });
+        db.run("UPDATE nodes SET disabled_at=COALESCE(disabled_at,?),status='disabled',updated_at=? WHERE id=?", now(), now(), node.id);
+        audit(req.actor, {
+            type: 'node.disabled',
+            message: `node ${node.name} disabled by ${req.actor.name}`,
+            metadata: { node: node.name },
+        });
+        reconcileSockets();
+        // Healthy peers drop the node from their mesh and ingress configuration on their next work fetch.
+        endpointSnapshot = JSON.stringify(endpointsFor(currentDesiredStates()));
+        wakeAll();
+        res.json({ ok: true, node: node.name });
     });
     api('get', '/api/v1/nodes', null, async (req, res) => {
-        const rows = db.all('SELECT id,name,address,public_address,overlay_ip,ingress,ingress_status_json,status,labels_json,last_seen_at,created_at,updated_at FROM nodes ORDER BY name');
+        const rows = db.all('SELECT id,name,address,public_address,overlay_ip,ingress,ingress_status_json,status,labels_json,last_seen_at,disabled_at,cert_not_after,created_at,updated_at FROM nodes ORDER BY name');
         res.json({
             nodes: rows.map((row) => {
                 const { labels_json, ...rest } = row;
@@ -995,15 +1202,15 @@ export async function createApp(options = {}) {
     api('post', '/api/v1/nodes/:name/rotate-wg-key', null, async (req, res) => {
         if (!MESH)
             return res.status(409).json({ error: 'the WireGuard mesh is not enabled (TUG_MESH=true)' });
-        const node = findNode(req.params.name);
+        const node = findNode(String(req.params.name));
         if (!node)
             return res.status(404).json({ error: 'node not found' });
         res.json({ commandId: queueCommand(node.id, null, 'wg.rotate-key'), nodeId: node.id });
     });
-    // Removes a retired node: its token, WireGuard peer, statuses and queued commands. Refused while a current deployment still places
+    // Removes a retired node: its identity, WireGuard peer, statuses and queued commands. Refused while a current deployment still places
     // something on it (redeploy without it first) or while its agent is still reporting, unless force=true.
     api('delete', '/api/v1/nodes/:name', null, async (req, res) => {
-        const node = db.get('SELECT id,name,last_seen_at FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
+        const node = db.get('SELECT id,name,last_seen_at FROM nodes WHERE name=? OR id=?', String(req.params.name), String(req.params.name));
         if (!node)
             return res.status(404).json({ error: 'node not found' });
         const referenced = [];
@@ -1027,6 +1234,8 @@ export async function createApp(options = {}) {
         db.run('DELETE FROM node_statuses WHERE node_id=?', node.id);
         db.run('DELETE FROM wg_peers WHERE node_id=?', node.id);
         db.run('DELETE FROM nodes WHERE id=?', node.id);
+        db.run('UPDATE enrollment_credentials SET revoked_at=COALESCE(revoked_at,?) WHERE name=? AND node_id IS NULL', now(), node.name);
+        reconcileSockets();
         audit(req.actor, {
             type: 'node.removed',
             message: `node ${node.name} removed`,
@@ -1036,7 +1245,7 @@ export async function createApp(options = {}) {
         res.json({ ok: true, node: node.name });
     });
     api('post', '/api/v1/nodes/:name/ingress', null, async (req, res) => {
-        const node = findNode(req.params.name);
+        const node = findNode(String(req.params.name));
         if (!node)
             return res.status(404).json({ error: 'node not found' });
         const enabled = req.body.enabled !== false;
@@ -1059,12 +1268,12 @@ export async function createApp(options = {}) {
             sendError(res, isUniqueViolation(e) ? httpError(409, 'public key already registered to another node') : e);
         }
     });
-    api('post', '/api/v1/tokens', null, async (req, res) => {
-        const request = parseTokenRequest(req.body);
-        const created = createToken(db, request, req.actor.name);
+    api('post', '/api/v1/credentials', null, async (req, res) => {
+        const request = parseCredentialRequest(req.body);
+        const created = await createCredential(db, pki, advertiseUrl(), request, req.actor.name);
         audit(req.actor, {
-            type: 'token.create',
-            message: `token ${created.name} (${created.role}${created.projects ? `, projects: ${created.projects.join(', ')}` : ''}) created by ${req.actor.name}`,
+            type: 'credential.create',
+            message: `credential ${created.name} (${created.role}${created.projects ? `, projects: ${created.projects.join(', ')}` : ''}) created by ${req.actor.name}`,
             metadata: {
                 name: created.name,
                 role: created.role,
@@ -1072,18 +1281,19 @@ export async function createApp(options = {}) {
                 expiresAt: created.expiresAt,
             },
         });
-        res.json(created);
+        res.json({ ...created, credential: encodeCredential(created.credential) });
     });
-    api('get', '/api/v1/tokens', null, async (_req, res) => res.json({ tokens: listTokens(db) }));
-    api('delete', '/api/v1/tokens/:name', null, async (req, res) => {
-        if (!revokeToken(db, req.params.name))
-            return res.status(404).json({ error: 'token not found' });
+    api('get', '/api/v1/credentials', null, async (_req, res) => res.json({ credentials: listCredentials(db) }));
+    api('delete', '/api/v1/credentials/:name', null, async (req, res) => {
+        if (!revokeCredential(db, String(req.params.name)))
+            return res.status(404).json({ error: 'credential not found' });
+        reconcileSockets();
         audit(req.actor, {
-            type: 'token.revoke',
-            message: `token ${req.params.name} revoked by ${req.actor.name}`,
-            metadata: { name: req.params.name },
+            type: 'credential.revoke',
+            message: `credential ${String(req.params.name)} revoked by ${req.actor.name}`,
+            metadata: { name: String(req.params.name) },
         });
-        res.json({ ok: true, name: req.params.name });
+        res.json({ ok: true, name: String(req.params.name) });
     });
     const backupDestination = () => db.get("SELECT * FROM backup_destination WHERE id='default'");
     const backupEnv = () => {
@@ -1173,13 +1383,13 @@ export async function createApp(options = {}) {
     });
     api('get', '/api/v1/restores', queryProject, async (req, res) => res.json({ restores: restores.list(String(req.query.project || 'site')) }));
     api('post', '/api/v1/restores/:id/release', null, async (req, res) => {
-        const result = restores.releaseInterrupted(req.params.id);
+        const result = restores.releaseInterrupted(String(req.params.id));
         if ('error' in result)
             return res.status(result.status).json({ error: result.error });
         res.json(result);
     });
     api('post', '/api/v1/restores/:id/resume', null, async (req, res) => {
-        const result = restores.resume(req.params.id);
+        const result = restores.resume(String(req.params.id));
         if ('error' in result)
             return res.status(result.status).json({ error: result.error });
         res.json(result);
@@ -1245,7 +1455,7 @@ export async function createApp(options = {}) {
             });
         res.json({ ok: true, removed: removed.changes > 0 });
     });
-    api('get', '/api/v1/backup/key', null, async (req, res) => {
+    api('get', '/api/v1/backup/repository-password', null, async (req, res) => {
         const project = String(req.query.project || '');
         if (!PROJECT_NAME_PATTERN.test(project) || isReservedProject(project))
             return res.status(400).json({ error: 'project is required' });
@@ -1301,14 +1511,14 @@ export async function createApp(options = {}) {
         const project = String(req.query.project || 'site');
         if (isReservedProject(project))
             return res.status(400).json({ error: 'project names starting with _ are reserved' });
-        const removed = db.run('DELETE FROM secrets WHERE project_id=? AND name=?', project, req.params.name);
+        const removed = db.run('DELETE FROM secrets WHERE project_id=? AND name=?', project, String(req.params.name));
         if (removed.changes) {
             wakeAll(project);
             audit(req.actor, {
                 type: 'secret.delete',
                 project,
-                message: `secret ${req.params.name} deleted by ${req.actor.name}`,
-                metadata: { name: req.params.name },
+                message: `secret ${String(req.params.name)} deleted by ${req.actor.name}`,
+                metadata: { name: String(req.params.name) },
             });
         }
         res.json({ ok: true });
@@ -1384,7 +1594,7 @@ export async function createApp(options = {}) {
                 throw httpError(400, 'invalid project or version');
             if (version.startsWith(ROLLBACK_VERSION_PREFIX))
                 throw httpError(400, `version cannot start with "${ROLLBACK_VERSION_PREFIX}" (reserved for rollback deployments)`);
-            const nodeRows = db.all('SELECT name,labels_json,last_seen_at,created_at FROM nodes ORDER BY created_at,name');
+            const nodeRows = db.all('SELECT name,labels_json,last_seen_at,created_at FROM nodes WHERE disabled_at IS NULL ORDER BY created_at,name');
             // Stale nodes (agent not seen recently) are not scheduled onto, so a dead host cannot hold a rollout open.
             const nodeInfos = nodeRows
                 .filter((row) => !isStale(row))
@@ -1397,7 +1607,7 @@ export async function createApp(options = {}) {
             for (const svc of Object.values(compose.services || {})) {
                 const ext = (svc['x-tug'] ||= {});
                 const placement = ext.placement || {};
-                const global = ext.global === true || placement.global === true;
+                const global = ext.global === true;
                 const labels = Array.isArray(placement.labels) ? placement.labels : [];
                 if (!placement.nodes?.length && (labels.length || global)) {
                     const matched = nodeInfos
@@ -1417,12 +1627,17 @@ export async function createApp(options = {}) {
                 }
             }
             const nodes = nodeRows.map((row) => ({ name: row.name }));
+            for (const [name, svc] of Object.entries(compose.services || {}))
+                for (const node of svc['x-tug']?.placement?.nodes || [])
+                    if (!nodes.some((n) => n.name === node))
+                        throw httpError(400, `${name}: placement node ${node} is not an enabled node`);
             try {
                 validateCompose(compose, nodes.map((n) => n.name));
             }
             catch (e) {
                 throw httpError(400, errMessage(e));
             }
+            assertIngressHostsFree(project, compose);
             // The CLI interpolates control secrets as sentinels; a reference to a secret that is gone would deploy an empty value.
             const referenced = sentinelNames(compose);
             if (referenced.includes(REGISTRY_AUTH_SECRET))
@@ -1631,11 +1846,11 @@ export async function createApp(options = {}) {
     }
     get('/agent/v1/images/:buildId', authAgent, async (req, res) => {
         const states = retainedStates();
-        const allowed = states.some((state) => Object.values(state.services).some((service) => service.artifact?.buildId === req.params.buildId &&
+        const allowed = states.some((state) => Object.values(state.services).some((service) => service.artifact?.buildId === String(req.params.buildId) &&
             (!service.placement.nodes?.length || service.placement.nodes.includes(req.node.name))));
         if (!allowed)
             return res.status(404).json({ error: 'image not found' });
-        const row = db.get('SELECT * FROM builds WHERE id=? AND state=?', req.params.buildId, 'complete');
+        const row = db.get('SELECT * FROM builds WHERE id=? AND state=?', String(req.params.buildId), 'complete');
         if (!row?.artifact_path)
             return res.status(404).json({ error: 'image not found' });
         try {
@@ -1653,14 +1868,14 @@ export async function createApp(options = {}) {
                 res.status(404).json({ error: 'image artifact unavailable' });
         }
     });
-    api('get', '/api/v1/deployments/:id', deploymentProject, async (req, res) => res.json(db.get('SELECT * FROM deployments WHERE id=?', req.params.id)));
+    api('get', '/api/v1/deployments/:id', deploymentProject, async (req, res) => res.json(db.get('SELECT * FROM deployments WHERE id=?', String(req.params.id))));
     api('post', '/api/v1/deployments/:id/cancel', deploymentProject, async (req, res) => withPublication(() => {
-        const row = db.get('SELECT * FROM deployments WHERE id=?', req.params.id);
+        const row = db.get('SELECT * FROM deployments WHERE id=?', String(req.params.id));
         if (!row)
             return res.status(404).json({ error: 'deployment not found' });
         if (!ACTIVE_DEPLOYMENT_STATES.includes(row.state))
             return res.status(409).json({ error: `deployment is ${row.state}` });
-        const result = db.run(`UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state IN (${ACTIVE_STATES_SQL})`, 'cancelled', now(), req.params.id);
+        const result = db.run(`UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state IN (${ACTIVE_STATES_SQL})`, 'cancelled', now(), String(req.params.id));
         if (result.changes !== 1)
             return res.status(409).json({ error: 'deployment is no longer active' });
         audit(req.actor, {
@@ -1700,7 +1915,7 @@ export async function createApp(options = {}) {
         let lock;
         try {
             lock = deleteData ? acquireProjectLock(project, 'volume.delete') : null;
-            const volume = db.get('SELECT * FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
+            const volume = db.get('SELECT * FROM volumes WHERE project_id=? AND name=?', project, String(req.params.name));
             if (!volume)
                 return res.status(404).json({ error: 'volume not found' });
             const config = JSON.parse(volume.config_json || '{}');
@@ -1712,7 +1927,7 @@ export async function createApp(options = {}) {
             if (ds) {
                 const state = JSON.parse(ds.state_json);
                 const inUse = Object.entries(state.compose?.services || {})
-                    .filter(([, svc]) => (svc.volumes || []).some((v) => (typeof v === 'string' ? v.split(':')[0] : v.source) === req.params.name))
+                    .filter(([, svc]) => (svc.volumes || []).some((v) => (typeof v === 'string' ? v.split(':')[0] : v.source) === String(req.params.name)))
                     .map(([name]) => name);
                 if (inUse.length)
                     return res
@@ -1723,10 +1938,10 @@ export async function createApp(options = {}) {
             if (deleteData && volume.node_id) {
                 commandId = queueCommand(volume.node_id, project, 'volume.delete', {
                     project,
-                    volume: req.params.name,
+                    volume: String(req.params.name),
                 });
             }
-            db.run('DELETE FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
+            db.run('DELETE FROM volumes WHERE project_id=? AND name=?', project, String(req.params.name));
             res.json({ ok: true, commandId, deletedMetadata: true, deleteData });
         }
         catch (e) {
@@ -1738,25 +1953,42 @@ export async function createApp(options = {}) {
     });
     api('post', '/api/v1/control/backups/run', null, async (req, res) => withWriter(async () => {
         const backupDir = process.env.TUG_CONTROL_BACKUP_DIR || path.join(DATA_DIR, 'control-backups');
-        await ensureDir(backupDir);
+        // The archive holds the master key and the CA key, so nothing here is ever readable beyond the control user.
+        await fs.mkdir(backupDir, { recursive: true, mode: 0o700 });
+        await fs.chmod(backupDir, 0o700);
         const ts = now();
         const backupId = id('cbak');
         const staging = path.join(backupDir, `${backupId}-staging`);
-        await fs.rm(staging, { recursive: true, force: true });
-        await ensureDir(staging);
-        const snapshotPath = path.join(staging, 'tug.db');
-        db.run('VACUUM INTO ?', snapshotPath);
-        await fs.copyFile(masterKeyFile, path.join(staging, 'master.key'));
-        await fs.cp(artifactDir, path.join(staging, 'images'), { recursive: true });
-        await fs.writeFile(path.join(staging, 'metadata.json'), JSON.stringify({ backupId, createdAt: ts, type: 'control' }, null, 2));
         const artifactRef = path.resolve(backupDir, `${backupId}.tar.gz`);
-        await tar.c({ gzip: true, file: artifactRef, cwd: staging }, [
-            'tug.db',
-            'master.key',
-            'metadata.json',
-            'images',
-        ]);
-        await fs.rm(staging, { recursive: true, force: true });
+        try {
+            await fs.rm(staging, { recursive: true, force: true });
+            await fs.mkdir(staging, { mode: 0o700 });
+            const snapshotPath = path.join(staging, 'tug.db');
+            db.run('VACUUM INTO ?', snapshotPath);
+            await fs.chmod(snapshotPath, 0o600);
+            await fs.copyFile(masterKeyFile, path.join(staging, 'master.key'));
+            await fs.chmod(path.join(staging, 'master.key'), 0o600);
+            await fs.cp(path.join(DATA_DIR, 'pki'), path.join(staging, 'pki'), {
+                recursive: true,
+                filter: (src) => path.basename(src) !== 'server.json',
+            });
+            await fs.cp(artifactDir, path.join(staging, 'images'), { recursive: true });
+            await fs.writeFile(path.join(staging, 'metadata.json'), JSON.stringify({ backupId, createdAt: ts, type: 'control' }, null, 2), { mode: 0o600 });
+            await tar.c({ gzip: true, file: artifactRef, cwd: staging, mode: 0o600 }, [
+                'tug.db',
+                'master.key',
+                'metadata.json',
+                'pki',
+                'images',
+            ]);
+        }
+        catch (error) {
+            await fs.rm(artifactRef, { force: true });
+            throw error;
+        }
+        finally {
+            await fs.rm(staging, { recursive: true, force: true });
+        }
         db.run('INSERT INTO backups(id,project_id,target,node_id,state,artifact_ref,metadata_json,created_at,completed_at) VALUES(?,?,?,?,?,?,?,?,?)', backupId, '_control', 'control-plane', null, 'completed', artifactRef, JSON.stringify({ type: 'control' }), ts, now());
         res.json({ backupId, artifactRef, state: 'completed' });
     }));
@@ -1787,8 +2019,9 @@ export async function createApp(options = {}) {
             const integrity = snapshot.get('PRAGMA integrity_check');
             if (integrity?.integrity_check !== 'ok')
                 throw new Error('backup database failed integrity check');
-            if (!snapshot.get("SELECT 1 FROM meta WHERE key='operator_token_hash'"))
-                throw new Error('backup database has no operator identity');
+            await Pki.load(path.join(restoreDir, 'pki'), false).catch((e) => {
+                throw new Error(`backup PKI is not usable: ${errMessage(e)}`, { cause: e });
+            });
             for (const secret of snapshot.all('SELECT * FROM secrets'))
                 decrypt(secret.encrypted_value, key);
             const states = snapshot.all('SELECT * FROM desired_states');
@@ -1820,6 +2053,13 @@ export async function createApp(options = {}) {
             await fs.cp(path.join(restoreDir, 'images'), artifactDir, { recursive: true });
             await fs.copyFile(path.join(restoreDir, 'tug.db'), dbFile);
             await fs.copyFile(path.join(restoreDir, 'master.key'), masterKeyFile);
+            await fs.cp(path.join(restoreDir, 'pki'), path.join(DATA_DIR, 'pki'), {
+                recursive: true,
+                filter: (src) => path.basename(src) !== 'server.json',
+            });
+            // Authorization follows the restored database: older snapshots forget later principals and revive revoked ones.
+            // Close every session, including this one, but only after the response has been delivered.
+            res.once('finish', closeAllSockets);
             res.json({ ok: true, restored: true, restartRequired: true });
         }
         catch (e) {
@@ -1894,19 +2134,19 @@ export async function createApp(options = {}) {
         const cmdId = queueCommand(node.id, project, 'logs.fetch', { project, service, tail });
         res.json({ commandId: cmdId, nodeId: node.id });
     });
-    api('get', '/api/v1/commands/:id', commandProject, async (req, res) => res.json(db.get('SELECT * FROM commands WHERE id=?', req.params.id)));
+    api('get', '/api/v1/commands/:id', commandProject, async (req, res) => res.json(db.get('SELECT * FROM commands WHERE id=?', String(req.params.id))));
     api('get', '/api/v1/status', null, async (req, res) => {
         const staleSeconds = Number(process.env.TUG_STALE_SECONDS || 60);
         const cutoff = new Date(Date.now() - staleSeconds * 1000).toISOString();
-        db.run("UPDATE nodes SET status='stale',updated_at=? WHERE last_seen_at IS NOT NULL AND last_seen_at < ?", now(), cutoff);
+        db.run("UPDATE nodes SET status='stale',updated_at=? WHERE last_seen_at IS NOT NULL AND last_seen_at < ? AND disabled_at IS NULL", now(), cutoff);
         const projects = projectScope(req.actor, 'id');
         const deployments = projectScope(req.actor, 'project_id');
-        const statuses = projectScope(req.actor, 'project_id');
+        const statuses = projectScope(req.actor, 's.project_id');
         res.json({
             projects: db.all(`SELECT * FROM projects WHERE ${projects.where}`, ...projects.args),
             deployments: db.all(`SELECT * FROM deployments WHERE ${deployments.where} ORDER BY created_at DESC LIMIT 20`, ...deployments.args),
             nodes: db.all('SELECT id,name,status,last_seen_at FROM nodes'),
-            statuses: db.all(`SELECT * FROM node_statuses WHERE ${statuses.where}`, ...statuses.args),
+            statuses: db.all(`SELECT s.*,n.name AS node FROM node_statuses s LEFT JOIN nodes n ON n.id=s.node_id WHERE ${statuses.where}`, ...statuses.args),
         });
     });
     api('get', '/api/v1/events', null, async (req, res) => {
@@ -1914,7 +2154,7 @@ export async function createApp(options = {}) {
         if (project)
             assertProject(req.actor, project);
         const limit = Math.min(Number(req.query.limit || 100), 500);
-        const scope = projectScope(req.actor, 'project_id'); // scoped tokens never see node-level or other projects' events
+        const scope = projectScope(req.actor, 'project_id'); // scoped credentials never see node-level or other projects' events
         const events = project
             ? db.all('SELECT * FROM events WHERE project_id=? ORDER BY created_at DESC LIMIT ?', project, limit)
             : db.all(`SELECT * FROM events WHERE ${scope.where} ORDER BY created_at DESC LIMIT ?`, ...scope.args, limit);
@@ -1933,16 +2173,52 @@ export async function createApp(options = {}) {
         await check('sqlite', async () => ({
             value: db.get('SELECT COUNT(*) AS count FROM projects').count,
         }));
-        await check('operator_token', async () => ({
-            configured: Boolean(db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash')),
-        }));
+        await check('certificates', async () => {
+            const nowMs = Date.now();
+            const ca = certificateExpiry(pki.caNotAfter, nowMs, EXPIRY_WARN_DAYS.ca);
+            const saved = JSON.parse(await fs.readFile(path.join(DATA_DIR, 'pki', 'server.json'), 'utf8'));
+            const server = certificateExpiry(new Date(new crypto.X509Certificate(saved.cert).validTo), nowMs, EXPIRY_WARN_DAYS.server);
+            const problems = [
+                ...(ca.state === 'expired'
+                    ? ['the control CA has expired; see the CA-loss recovery runbook']
+                    : []),
+                ...(server.state === 'expired' ? ['the control server certificate has expired'] : []),
+            ];
+            const warnings = [
+                ...(ca.state === 'expiring'
+                    ? [`the control CA expires in ${ca.daysLeft} days; plan a CA rotation`]
+                    : []),
+                ...(server.state === 'expiring'
+                    ? [`the server certificate expires in ${server.daysLeft} days and has not renewed`]
+                    : []),
+            ];
+            return {
+                ok: problems.length === 0,
+                ca,
+                server,
+                ...(problems.length && { error: problems.join('; ') }),
+                ...(warnings.length && { warnings }),
+            };
+        });
+        await check('nodes', async () => {
+            const nowMs = Date.now();
+            const staleSeconds = Number(process.env.TUG_STALE_SECONDS || 60);
+            const nodes = db
+                .all('SELECT * FROM nodes ORDER BY name')
+                .map((n) => diagnoseNode(n, nowMs, staleSeconds));
+            const problems = nodes.flatMap((n) => n.problems.map((p) => `${n.name}: ${p}`));
+            const warnings = nodes.flatMap((n) => n.warnings.map((w) => `${n.name}: ${w}`));
+            return {
+                ok: problems.length === 0,
+                nodes: nodes.map(({ problems: _p, warnings: _w, ...rest }) => rest),
+                ...(problems.length && { error: problems.join('; ') }),
+                ...(warnings.length && { warnings }),
+            };
+        });
         await check('master_key', async () => ({ configured: Boolean(masterKey) }));
         await check('docker', async () => ({
             output: (await runCommand('docker', ['version', '--format', '{{.Server.Version}}'])).out.trim(),
         }));
-        await check('docker_compose', async () => ({
-            output: (await runCommand('docker', ['compose', 'version'])).out.trim(),
-        }));
         if (MESH) {
             await check('mesh', async () => {
                 const nodes = db.all('SELECT n.name, n.overlay_ip, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
@@ -1985,7 +2261,7 @@ export async function createApp(options = {}) {
     }
     function endpointsFor(states) {
         const staleBefore = new Date(Date.now() - Number(process.env.TUG_STALE_SECONDS || 60) * 1000).toISOString();
-        const nodes = db.all('SELECT n.*, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
+        const nodes = db.all('SELECT n.*, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n WHERE n.disabled_at IS NULL');
         const views = nodes.map((n) => ({
             name: n.name,
             overlayIp: MESH && n.has_key ? n.overlay_ip : null,
@@ -2004,7 +2280,7 @@ export async function createApp(options = {}) {
     function networkFor(node) {
         if (!MESH || !node.overlay_ip)
             return undefined;
-        const peers = db.all('SELECT w.*, n.name FROM wg_peers w JOIN nodes n ON n.id=w.node_id WHERE w.node_id != ? ORDER BY n.name', node.id);
+        const peers = db.all('SELECT w.*, n.name FROM wg_peers w JOIN nodes n ON n.id=w.node_id WHERE w.node_id != ? AND n.disabled_at IS NULL ORDER BY n.name', node.id);
         return {
             interface: WG_INTERFACE,
             overlayIp: node.overlay_ip,
@@ -2021,7 +2297,7 @@ export async function createApp(options = {}) {
     }
     /** Names of all nodes and of the ingress nodes, for port binding and upstream decisions. */
     function nodeTopology() {
-        const rows = db.all('SELECT name,ingress FROM nodes ORDER BY name');
+        const rows = db.all('SELECT name,ingress FROM nodes WHERE disabled_at IS NULL ORDER BY name');
         return {
             nodes: rows.map((r) => r.name),
             ingressNodes: rows.filter((r) => r.ingress).map((r) => r.name),
@@ -2108,22 +2384,49 @@ export async function createApp(options = {}) {
     });
     post('/agent/v1/status', authAgent, async (req, res) => {
         const ts = now();
-        const report = req.body;
-        for (const st of report.projects || []) {
-            db.run(`INSERT INTO node_statuses(id,node_id,project_id,observed_version,status_json,health,updated_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(node_id,project_id) DO UPDATE SET observed_version=excluded.observed_version,status_json=excluded.status_json,health=excluded.health,updated_at=excluded.updated_at`, id('ns'), req.node.id, st.project, st.observedVersion, JSON.stringify(st), st.health || 'unknown', ts);
+        let report;
+        try {
+            report = parseStatusReport(req.body);
+        }
+        catch (e) {
+            throw httpError(400, errMessage(e));
+        }
+        for (const st of report.projects) {
+            // A node reports only the services currently placed on it; anything else would feed other nodes' work.
+            const ds = currentDesired(st.project);
+            const desired = ds ? JSON.parse(ds.state_json) : undefined;
+            const placed = Object.entries(desired?.services || {})
+                .filter(([, svc]) => svc.placement?.nodes?.includes(req.node.name))
+                .map(([name]) => name);
+            if (!placed.length)
+                continue;
+            const sanitized = {
+                ...st,
+                ...(st.services && { services: st.services.filter((name) => placed.includes(name)) }),
+                ...(st.serviceStates && {
+                    serviceStates: Object.fromEntries(Object.entries(st.serviceStates).filter(([name]) => placed.includes(name))),
+                }),
+            };
+            db.run(`INSERT INTO node_statuses(id,node_id,project_id,observed_version,status_json,health,updated_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(node_id,project_id) DO UPDATE SET observed_version=excluded.observed_version,status_json=excluded.status_json,health=excluded.health,updated_at=excluded.updated_at`, id('ns'), req.node.id, st.project, st.observedVersion, JSON.stringify(sanitized), st.health, ts);
             updateDeploymentConvergence(st.project);
         }
-        if (report.jobs?.length)
-            backups.onHeartbeat(report.jobs);
-        for (const cr of (report.commandResults || [])) {
-            const result = cr.result || {};
-            if (cr.commandId && cr.state)
-                backups.onResult(cr.commandId, cr.state, result);
-            db.run('UPDATE commands SET state=?,result_json=?,updated_at=? WHERE id=? AND node_id=?', cr.state || 'succeeded', JSON.stringify(result), ts, cr.commandId ?? null, req.node.id);
+        // Commands are resolved under the calling node first: only a running command of this node can be
+        // heartbeated or completed, so foreign and already-finished command ids change nothing.
+        const runningHere = (commandId) => db.get("SELECT id FROM commands WHERE id=? AND node_id=? AND state='running'", commandId, req.node.id);
+        const jobs = (report.jobs || []).filter((job) => runningHere(job.commandId));
+        if (jobs.length)
+            backups.onHeartbeat(jobs);
+        for (const cr of report.commandResults) {
+            if (!runningHere(cr.commandId))
+                continue;
+            backups.onResult(cr.commandId, cr.state, cr.result);
+            db.run("UPDATE commands SET state=?,result_json=?,updated_at=? WHERE id=? AND node_id=? AND state='running'", cr.state, JSON.stringify(cr.result), ts, cr.commandId, req.node.id);
         }
         if (report.ingress)
             db.run('UPDATE nodes SET ingress_status_json=? WHERE id=?', JSON.stringify({ ...report.ingress, reportedAt: ts }), req.node.id);
         db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', ts, 'online', req.node.id);
+        if (report.sentAt)
+            db.run('UPDATE nodes SET clock_skew_ms=?,clock_observed_at=? WHERE id=?', Date.parse(report.sentAt) - Date.parse(ts), ts, req.node.id);
         try {
             restores.advance();
         }
@@ -2164,6 +2467,14 @@ export async function createApp(options = {}) {
             return res.status(409).json({ error: `command is ${command.state}, not running` });
         if (!command.project_id || !backupDestination())
             return res.status(409).json({ error: 'no backup destination is configured' });
+        // A command that outlived the project's placement on this node (a newer version moved it) gets nothing.
+        const desiredRow = currentDesired(command.project_id);
+        const desiredNow = desiredRow
+            ? JSON.parse(desiredRow.state_json)
+            : undefined;
+        const placedHere = Object.values(desiredNow?.services || {}).some((svc) => svc.placement?.nodes?.includes(req.node.name));
+        if (!placedHere)
+            return res.status(403).json({ error: 'project is no longer placed on this node' });
         const response = {
             env: {
                 ...backupEnv(),
@@ -2211,7 +2522,8 @@ export async function createApp(options = {}) {
     });
     return app;
 }
-function attachAgentStream(app, server) {
+/** Attaches the agent and operator WebSocket upgrade handling to `server`; `listen` uses it on the mTLS listener. */
+export function attachAgentStream(app, server) {
     const runtime = streamRuntimes.get(app);
     if (!runtime)
         throw new Error('control app is missing its stream runtime');
@@ -2226,26 +2538,31 @@ function attachAgentStream(app, server) {
             reject(404, 'Not Found');
             return;
         }
-        const raw = String(request.headers.authorization || '').replace(/^Bearer /, '');
         void (async () => {
+            // The same current-state check as every HTTP request.
+            const identity = runtime.isRestoring() ? null : runtime.identify(request);
+            if (!identity) {
+                reject(401, 'Unauthorized');
+                return;
+            }
             if (pathname === '/api/v1/live') {
-                const actor = runtime.authenticateOperator(raw);
-                if (!actor) {
-                    reject(401, 'Unauthorized');
+                if (identity.klass !== 'operator') {
+                    reject(403, 'Forbidden');
                     return;
                 }
-                if (!hasRole(actor, 'deploy')) {
+                if (!hasRole(identity.actor, 'deploy')) {
                     reject(403, 'Forbidden');
                     return;
                 }
+                const actor = identity.actor;
                 wss.handleUpgrade(request, socket, head, (ws) => runtime.live.operator(actor, ws));
             }
             else {
-                const node = runtime.authenticate(raw);
-                if (!node) {
-                    reject(401, 'Unauthorized');
+                if (identity.klass !== 'node') {
+                    reject(403, 'Forbidden');
                     return;
                 }
+                const node = identity.node;
                 wss.handleUpgrade(request, socket, head, (ws) => {
                     runtime.live.agent(node.id, ws);
                     runtime.hub.connect(node.id, ws);
@@ -2255,23 +2572,105 @@ function attachAgentStream(app, server) {
     });
     server.once('close', () => wss.close());
 }
-export async function listen(app, port = PORT) {
-    const certFile = process.env.TUG_TLS_CERT_FILE;
-    const keyFile = process.env.TUG_TLS_KEY_FILE;
-    if (certFile && keyFile) {
-        const [cert, key] = await Promise.all([fs.readFile(certFile), fs.readFile(keyFile)]);
-        const server = https.createServer({ cert, key }, app);
-        attachAgentStream(app, server);
-        return new Promise((resolve) => server.listen(port, () => {
-            console.log(`tug-control listening on ${port} (tls)`);
-            resolve(server);
-        }));
+/** Loads the persisted server credential, or issues one when it is missing, for different names or two thirds expired. */
+async function ensureServerCredential(pki, dir, names, force = false) {
+    const file = path.join(dir, 'pki', 'server.json');
+    try {
+        const saved = JSON.parse(await fs.readFile(file, 'utf8'));
+        const x = new (await import('node:crypto')).X509Certificate(saved.cert);
+        const lifetime = Date.parse(x.validTo) - Date.parse(x.validFrom);
+        const age = Date.now() - Date.parse(x.validFrom);
+        const signedByCa = x.verify(new (await import('node:crypto')).X509Certificate(pki.caPem).publicKey);
+        if (!force && signedByCa && saved.names === namesKey(names) && age < (lifetime * 2) / 3)
+            return { cert: saved.cert, key: saved.key, renewed: false };
     }
-    console.warn('TUG_TLS_CERT_FILE/TUG_TLS_KEY_FILE not set; serving plain HTTP');
-    const server = http.createServer(app);
+    catch {
+        /* Missing or unreadable: issue a new one. */
+    }
+    const issued = await pki.issueServer(names);
+    await writeFileDurable(file, JSON.stringify({ cert: issued.cert, key: issued.key, names: namesKey(names) }));
+    return { cert: issued.cert, key: issued.key, renewed: true };
+}
+/**
+ * Starts the single mTLS listener. Chain validation rejects missing or foreign client certificates during the handshake;
+ * the secureConnection gate then checks current database state, and every request checks it again.
+ */
+export async function listen(app, port = PORT) {
+    const pki = app.locals.pki;
+    const dataDir = app.locals.dataDir;
+    const advertiseUrl = process.env.TUG_ADVERTISE_URL;
+    if (!advertiseUrl)
+        throw new Error('TUG_ADVERTISE_URL is required: the credential-free https origin clients connect to');
+    const url = new URL(advertiseUrl);
+    if (url.protocol !== 'https:' ||
+        url.username ||
+        url.password ||
+        url.search ||
+        url.hash ||
+        url.pathname !== '/')
+        throw new Error('TUG_ADVERTISE_URL must be a credential-free https origin such as https://control.example.com:8443');
+    const names = serverNames(advertiseUrl, (process.env.TUG_EXTRA_SANS || '').split(','));
+    const credential = await ensureServerCredential(pki, dataDir, names);
+    const server = https.createServer(serverTlsOptions({ cert: credential.cert, key: credential.key, ca: pki.caPem }), app);
+    server.requestTimeout = 5 * 60_000;
+    server.headersTimeout = 30_000;
     attachAgentStream(app, server);
+    const runtime = streamRuntimes.get(app);
+    const liveSockets = app.locals.liveSockets;
+    server.on('tlsClientError', (error) => {
+        console.warn(`tls handshake refused: ${errMessage(error)}`.slice(0, 200));
+    });
+    // Connection authorization gate: after a full handshake, refuse reused sessions and identities the database does not authorize.
+    server.on('secureConnection', (socket) => {
+        // After a restore the database is closed until restart: refuse everything, and never let the gate throw.
+        let presented = null;
+        try {
+            if (!runtime.isRestoring() && !socket.isSessionReused())
+                presented = peerIdentity(socket);
+            if (presented && !runtime.resolvePrincipal(presented.uri, presented.keyHash))
+                presented = null;
+        }
+        catch {
+            presented = null;
+        }
+        if (!presented) {
+            socket.destroy();
+            return;
+        }
+        const entry = { socket, ...presented };
+        liveSockets.add(entry);
+        socket.once('close', () => liveSockets.delete(entry));
+    });
+    // Out-of-process changes (host-side root recovery) are noticed within a second; renewal reloads the secure context.
+    const sweep = setInterval(() => {
+        try {
+            app.locals.reconcileSockets();
+        }
+        catch (e) {
+            console.error(`session sweep failed: ${errMessage(e)}`);
+        }
+    }, 1000);
+    const renew = setInterval(() => {
+        ensureServerCredential(pki, dataDir, names)
+            .then((next) => {
+            if (next.renewed) {
+                server.setSecureContext(serverTlsOptions({ cert: next.cert, key: next.key, ca: pki.caPem }));
+                console.log('server certificate renewed');
+            }
+            const left = pki.caNotAfter.getTime() - Date.now();
+            if (left < 90 * 86_400_000)
+                console.warn(`control CA expires in ${Math.floor(left / 86_400_000)} days`);
+        })
+            .catch((e) => console.error(`server certificate renewal failed: ${errMessage(e)}`));
+    }, Math.min(SERVER_LIFETIME_MS / 90, 3_600_000));
+    sweep.unref();
+    renew.unref();
+    server.once('close', () => {
+        clearInterval(sweep);
+        clearInterval(renew);
+    });
     return new Promise((resolve) => server.listen(port, () => {
-        console.log(`tug-control listening on ${port}`);
+        console.log(`tug-control listening on ${port} (mTLS)`);
         resolve(server);
     }));
 }