@wpelevator/tug 0.2.1 → 0.4.0

package/dist/control.js

@@ -9,17 +9,355 @@ import sqlite3 from 'sqlite3';
 import { open } from 'sqlite';
 import { scanArchive } from "./archive.js";
 import https from 'node:https';
+import http from 'node:http';
 import path from 'node:path';
+import { WebSocketServer, WebSocket } from 'ws';
 import * as tar from 'tar';
 import { openDb } from "./db.js";
 import { encrypt, decrypt, loadOrCreateMasterKey } from "./secrets.js";
-import { buildDesiredState, serviceSecretNames, validateCompose } from "./compose.js";
+import { buildDesiredState, loopbackIngressServices, serviceSecretNames, validateCompose, } from "./compose.js";
 import { buildAndPush, unpackContext, run as runCommand } from "./build.js";
+import { sentinelNames } from "./normalize.js";
 import { ensureDir, errMessage, hashToken, id, now, token } from "./util.js";
-import { DEFAULT_MESH_CIDR, DEFAULT_WG_PORT, WG_INTERFACE, WG_KEEPALIVE, allocateOverlayIp, isWgKey, parseCidr, wgEndpoint } from "./mesh.js";
+import { DEFAULT_MESH_CIDR, DEFAULT_WG_PORT, WG_INTERFACE, WG_KEEPALIVE, allocateOverlayIp, isWgKey, parseCidr, wgEndpoint, } from "./mesh.js";
 import { buildIngressWork, resolveServiceEndpoints } from "./ingress.js";
-import { ACTIVE_DEPLOYMENT_STATES } from "./protocol.js";
-import { AUTO_ROLLBACK_ACTOR, ROUTE_ROLES, assertProject, bodyProject, createGuard, createToken, listTokens, parseTokenRequest, queryProject, recordEvent, revokeToken } from "./tokens.js";
+import { ACTIVE_DEPLOYMENT_STATES, LIVE_MAX_FRAME_BYTES, LIVE_MAX_BUFFERED_BYTES, LIVE_MAX_SESSIONS, LIVE_START_TIMEOUT_MS, } from "./protocol.js";
+import { AUTO_ROLLBACK_ACTOR, ROUTE_ROLES, PROJECT_NAME_PATTERN, authenticate, hasRole, assertProject, bodyProject, createGuard, createToken, listTokens, parseTokenRequest, queryProject, recordEvent, revokeToken, } from "./tokens.js";
+class WakeHub {
+    cursors = new Map();
+    projects = new Map();
+    streams = new Map();
+    waiters = new Map();
+    cursor(nodeId) {
+        return this.cursors.get(nodeId) ?? 0;
+    }
+    connect(nodeId, ws) {
+        const previous = this.streams.get(nodeId);
+        if (previous && previous !== ws)
+            previous.close(1000, 'replaced by a newer agent connection');
+        this.streams.set(nodeId, ws);
+        const ready = { type: 'ready', nodeId, cursor: this.cursor(nodeId) };
+        ws.send(JSON.stringify(ready));
+        ws.once('close', () => {
+            if (this.streams.get(nodeId) === ws)
+                this.streams.delete(nodeId);
+        });
+    }
+    wake(nodeId, project) {
+        const cursor = this.cursor(nodeId) + 1;
+        this.cursors.set(nodeId, cursor);
+        this.projects.set(nodeId, project);
+        const ws = this.streams.get(nodeId);
+        if (ws?.readyState === WebSocket.OPEN) {
+            const message = {
+                type: 'work-available',
+                cursor,
+                ...(project ? { project } : {}),
+            };
+            ws.send(JSON.stringify(message));
+        }
+        for (const resolve of this.waiters.get(nodeId) ?? [])
+            resolve();
+    }
+    async wait(nodeId, after, timeoutMs) {
+        const response = () => {
+            const cursor = this.cursor(nodeId);
+            const project = this.projects.get(nodeId);
+            return {
+                type: cursor > after ? 'work-available' : 'timeout',
+                cursor,
+                ...(project ? { project } : {}),
+            };
+        };
+        if (this.cursor(nodeId) > after)
+            return response();
+        await new Promise((resolve) => {
+            const waiters = this.waiters.get(nodeId) ?? new Set();
+            const done = () => {
+                clearTimeout(timer);
+                waiters.delete(done);
+                if (!waiters.size)
+                    this.waiters.delete(nodeId);
+                resolve();
+            };
+            waiters.add(done);
+            this.waiters.set(nodeId, waiters);
+            const timer = setTimeout(done, timeoutMs);
+        });
+        return response();
+    }
+}
+const objectFrame = (value) => !!value && typeof value === 'object' && !Array.isArray(value);
+const frameKeys = (value, keys) => Object.keys(value).every((key) => keys.includes(key));
+const liveDimension = (value) => Number.isInteger(value) && Number(value) >= 1 && Number(value) <= 65535;
+const liveBase64 = (value) => typeof value === 'string' &&
+    /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value);
+function parseLiveRequest(value) {
+    if (!objectFrame(value) ||
+        typeof value.project !== 'string' ||
+        !PROJECT_NAME_PATTERN.test(value.project) ||
+        value.project.length > 128 ||
+        (value.node !== undefined &&
+            (typeof value.node !== 'string' ||
+                !PROJECT_NAME_PATTERN.test(value.node) ||
+                value.node.length > 128)))
+        throw new Error('invalid live request project or node');
+    const common = ['operation', 'project', 'node'];
+    if (value.operation === 'ps' && frameKeys(value, common))
+        return value;
+    if (typeof value.service !== 'string' ||
+        !PROJECT_NAME_PATTERN.test(value.service) ||
+        value.service.length > 128)
+        throw new Error('invalid live request service');
+    if (value.operation === 'inspect' && frameKeys(value, [...common, 'service']))
+        return value;
+    if (value.operation === 'logs' &&
+        frameKeys(value, [...common, 'service', 'follow', 'tail']) &&
+        value.follow === true &&
+        Number.isSafeInteger(value.tail) &&
+        Number(value.tail) >= 0 &&
+        Number(value.tail) <= 10_000)
+        return value;
+    if (value.operation === 'exec' &&
+        frameKeys(value, [...common, 'service', 'command', 'interactive', 'tty', 'columns', 'rows']) &&
+        Array.isArray(value.command) &&
+        value.command.length > 0 &&
+        value.command.length <= 256 &&
+        typeof value.command[0] === 'string' &&
+        value.command[0].length > 0 &&
+        value.command.every((part) => typeof part === 'string' && !part.includes('\0') && part.length <= 32768) &&
+        typeof value.interactive === 'boolean' &&
+        typeof value.tty === 'boolean' &&
+        (value.columns === undefined || liveDimension(value.columns)) &&
+        (value.rows === undefined || liveDimension(value.rows)))
+        return value;
+    throw new Error('invalid or unsupported live request');
+}
+/** Ephemeral sessions are bound to socket identities; reconnecting never replays work. */
+class LiveHub {
+    agents = new Map();
+    sessions = new Map();
+    retired = new WeakMap();
+    startTimeoutMs;
+    resolve;
+    constructor(resolve, startTimeoutMs = LIVE_START_TIMEOUT_MS) {
+        this.resolve = resolve;
+        this.startTimeoutMs = startTimeoutMs;
+    }
+    send(ws, frame) {
+        if (ws.readyState !== WebSocket.OPEN || ws.bufferedAmount > LIVE_MAX_BUFFERED_BYTES)
+            return false;
+        const data = JSON.stringify(frame);
+        if (Buffer.byteLength(data) + ws.bufferedAmount > LIVE_MAX_BUFFERED_BYTES)
+            return false;
+        try {
+            ws.send(data);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    finish(session, frame, cancel = true) {
+        if (!this.sessions.delete(session.id))
+            return;
+        clearTimeout(session.timer);
+        const retired = this.retired.get(session.agent) ?? new Set();
+        retired.add(session.id);
+        if (retired.size > 1024)
+            retired.delete(retired.values().next().value);
+        this.retired.set(session.agent, retired);
+        // If cancellation cannot enter a congested agent socket, disconnect it so its local engine disposes every operation.
+        if (cancel && !this.send(session.agent, { v: 1, type: 'live.cancel', sessionId: session.id }))
+            session.agent.terminate();
+        if (frame)
+            this.send(session.operator, frame);
+        session.operator.close(1000, 'live operation finished');
+    }
+    fail(session, message) {
+        this.finish(session, { v: 1, type: 'live.error', sessionId: session.id, message });
+    }
+    agent(nodeId, ws) {
+        const previous = this.agents.get(nodeId);
+        if (previous && previous !== ws)
+            for (const session of this.sessions.values())
+                if (session.agent === previous)
+                    this.fail(session, 'agent connection replaced');
+        this.agents.set(nodeId, ws);
+        ws.on('error', () => { });
+        ws.on('close', () => {
+            if (this.agents.get(nodeId) === ws)
+                this.agents.delete(nodeId);
+            for (const session of this.sessions.values())
+                if (session.agent === ws)
+                    this.fail(session, 'agent disconnected');
+        });
+        ws.on('message', (data, binary) => {
+            try {
+                if (binary || Buffer.byteLength(data.toString()) > LIVE_MAX_FRAME_BYTES)
+                    throw new Error('invalid frame');
+                const frame = JSON.parse(data.toString());
+                if (!objectFrame(frame) || frame.v !== 1 || typeof frame.sessionId !== 'string')
+                    throw new Error('invalid frame');
+                const session = this.sessions.get(frame.sessionId);
+                if (!session && this.retired.get(ws)?.has(frame.sessionId))
+                    return;
+                if (!session || session.agent !== ws)
+                    throw new Error('foreign or unknown session');
+                const keys = ['v', 'type', 'sessionId'];
+                if (frame.type === 'live.started' && frameKeys(frame, keys) && !session.started) {
+                    session.started = true;
+                    clearTimeout(session.timer);
+                    if (!this.send(session.operator, {
+                        v: 1,
+                        type: 'live.started',
+                        sessionId: session.id,
+                        node: session.node,
+                    }))
+                        this.fail(session, 'operator buffer limit exceeded');
+                }
+                else if (frame.type === 'live.output' &&
+                    frameKeys(frame, [...keys, 'stream', 'data']) &&
+                    (frame.stream === 'stdout' || frame.stream === 'stderr') &&
+                    liveBase64(frame.data)) {
+                    if (!this.send(session.operator, frame))
+                        this.fail(session, 'operator buffer limit exceeded');
+                }
+                else if (frame.type === 'live.result' &&
+                    frameKeys(frame, [...keys, 'result']) &&
+                    Object.hasOwn(frame, 'result') &&
+                    ['ps', 'inspect'].includes(session.request.operation)) {
+                    if (!this.send(session.operator, {
+                        v: 1,
+                        type: 'live.result',
+                        sessionId: session.id,
+                        value: frame.result,
+                    }))
+                        this.fail(session, 'operator buffer limit exceeded');
+                }
+                else if (frame.type === 'live.exit' &&
+                    frameKeys(frame, [...keys, 'code', 'signal', 'cancelled', 'error']) &&
+                    (frame.code === null ||
+                        (Number.isInteger(frame.code) &&
+                            Number(frame.code) >= 0 &&
+                            Number(frame.code) <= 255)) &&
+                    (frame.signal === null || typeof frame.signal === 'string') &&
+                    typeof frame.cancelled === 'boolean' &&
+                    (frame.error === undefined || typeof frame.error === 'string')) {
+                    if (frame.error)
+                        this.fail(session, frame.error);
+                    else
+                        this.finish(session, {
+                            v: 1,
+                            type: 'live.exit',
+                            sessionId: session.id,
+                            code: typeof frame.code === 'number' ? frame.code : frame.cancelled ? 130 : 1,
+                        }, false);
+                }
+                else
+                    throw new Error('unsupported agent frame');
+            }
+            catch {
+                ws.close(1008, 'invalid live session frame');
+            }
+        });
+    }
+    operator(actor, ws) {
+        let opening = false;
+        let session;
+        const timer = setTimeout(() => {
+            if (!session)
+                reject('live.open timeout');
+        }, this.startTimeoutMs);
+        const reject = (message) => {
+            clearTimeout(timer);
+            if (session)
+                this.fail(session, message);
+            else {
+                this.send(ws, { v: 1, type: 'live.error', message });
+                ws.close(1008, message.slice(0, 100));
+            }
+        };
+        ws.on('error', () => { });
+        ws.on('close', () => {
+            clearTimeout(timer);
+            if (session)
+                this.finish(session);
+        });
+        ws.on('message', (data, binary) => {
+            void (async () => {
+                if (binary || Buffer.byteLength(data.toString()) > LIVE_MAX_FRAME_BYTES)
+                    throw new Error('invalid or oversized live frame');
+                const frame = JSON.parse(data.toString());
+                if (!objectFrame(frame) || frame.v !== 1)
+                    throw new Error('invalid live frame');
+                if (!session) {
+                    if (opening || frame.type !== 'live.open' || !frameKeys(frame, ['v', 'type', 'request']))
+                        throw new Error('first frame must be live.open');
+                    opening = true;
+                    const request = parseLiveRequest(frame.request);
+                    const node = await this.resolve(actor, request);
+                    if (ws.readyState !== WebSocket.OPEN)
+                        return;
+                    const agent = this.agents.get(node.id);
+                    if (!agent || agent.readyState !== WebSocket.OPEN)
+                        throw new Error('agent is not connected');
+                    if (this.sessions.size >= LIVE_MAX_SESSIONS)
+                        throw new Error('too many live sessions');
+                    clearTimeout(timer);
+                    const sessionId = id('live');
+                    session = {
+                        id: sessionId,
+                        operator: ws,
+                        agent,
+                        node: node.name,
+                        request,
+                        started: false,
+                        timer: setTimeout(() => {
+                            if (session)
+                                this.fail(session, 'agent live startup timeout');
+                        }, this.startTimeoutMs),
+                    };
+                    this.sessions.set(sessionId, session);
+                    if (!this.send(agent, { v: 1, type: 'live.open', sessionId, request }))
+                        this.fail(session, 'agent buffer limit exceeded');
+                    return;
+                }
+                if (!this.sessions.has(session.id))
+                    return;
+                if (!frameKeys(frame, [
+                    'v',
+                    'type',
+                    ...(frame.type === 'live.stdin'
+                        ? ['data']
+                        : frame.type === 'live.resize'
+                            ? ['columns', 'rows']
+                            : []),
+                ]))
+                    throw new Error('invalid live input');
+                if (frame.type === 'live.cancel') {
+                    this.finish(session, { v: 1, type: 'live.exit', sessionId: session.id, code: 130 });
+                    return;
+                }
+                if (session.request.operation !== 'exec')
+                    throw new Error('operation does not accept input');
+                if (frame.type === 'live.stdin'
+                    ? !session.request.interactive || !liveBase64(frame.data)
+                    : frame.type === 'live.stdin-end'
+                        ? !session.request.interactive
+                        : frame.type === 'live.resize'
+                            ? !session.request.tty ||
+                                !liveDimension(frame.columns) ||
+                                !liveDimension(frame.rows)
+                            : true)
+                    throw new Error('invalid live input');
+                if (!this.send(session.agent, { ...frame, sessionId: session.id }))
+                    this.fail(session, 'agent buffer limit exceeded');
+            })().catch((error) => reject(errMessage(error)));
+        });
+    }
+}
+const streamRuntimes = new WeakMap();
 const httpError = (status, message) => Object.assign(new Error(message), { status });
 const missingProject = (project) => `project ${project} has no desired state`;
 const missingService = (project, service) => `service ${service} not found in project ${project}`;
@@ -30,12 +368,14 @@ function deploymentFailureMessage(rows) {
         const flat = value.replace(/\s+/g, ' ').trim();
         return flat.length > limit ? `${flat.slice(0, limit)}...` : flat;
     };
-    const details = rows.map(row => {
+    const details = rows.map((row) => {
         let status = {};
         try {
             status = JSON.parse(row.status_json);
         }
-        catch { /* a bad report still names the node */ }
+        catch {
+            /* a bad report still names the node */
+        }
         const parts = [];
         if (typeof status.error === 'string' && status.error.trim())
             parts.push(oneLine(status.error));
@@ -50,15 +390,25 @@ function deploymentFailureMessage(rows) {
 }
 /** Versions of rollback deployments start with this; `tug deploy` may not use it, which is how automatic rollback tells them apart. */
 const ROLLBACK_VERSION_PREFIX = 'rollback-';
-const ACTIVE_STATES_SQL = ACTIVE_DEPLOYMENT_STATES.map(s => `'${s}'`).join(',');
+const ACTIVE_STATES_SQL = ACTIVE_DEPLOYMENT_STATES.map((s) => `'${s}'`).join(',');
 const PORT = Number(process.env.PORT || 8080);
 export async function createApp(options = {}) {
-    const images = { build: buildAndPush, save: saveImageArchive, inspect: inspectImage, validate: validateImageArchive, run: runCommand, ...options.images };
+    const images = {
+        build: buildAndPush,
+        save: saveImageArchive,
+        inspect: inspectImage,
+        validate: validateImageArchive,
+        run: runCommand,
+        ...options.images,
+    };
     const DATA_DIR = process.env.TUG_DATA_DIR || './data';
     const MAX_IMAGE_BYTES = Number(process.env.TUG_MAX_IMAGE_BYTES || 5 * 1024 ** 3);
     if (!Number.isSafeInteger(MAX_IMAGE_BYTES) || MAX_IMAGE_BYTES < 1)
         throw new Error('TUG_MAX_IMAGE_BYTES must be a positive safe integer');
-    const upload = multer({ dest: process.env.TUG_UPLOAD_DIR || path.join(DATA_DIR, 'uploads'), limits: { fileSize: MAX_IMAGE_BYTES, files: 64, fields: 1, fieldSize: 1024 ** 2 } });
+    const upload = multer({
+        dest: process.env.TUG_UPLOAD_DIR || path.join(DATA_DIR, 'uploads'),
+        limits: { fileSize: MAX_IMAGE_BYTES, files: 64, fields: 1, fieldSize: 1024 ** 2 },
+    });
     const dbFile = path.join(DATA_DIR, 'tug.db');
     const masterKeyFile = path.join(DATA_DIR, 'master.key');
     const REGISTRY = process.env.TUG_REGISTRY?.trim() || '';
@@ -81,7 +431,7 @@ export async function createApp(options = {}) {
     async function reloadRegistry() {
         if (!REGISTRY_CONTAINER)
             return;
-        await runCommand('docker', ['restart', REGISTRY_CONTAINER]).catch(e => console.error(`registry reload failed: ${e.message}`));
+        await runCommand('docker', ['restart', REGISTRY_CONTAINER]).catch((e) => console.error(`registry reload failed: ${e.message}`));
     }
     async function upsertRegistryCredential(name, password) {
         if (!REGISTRY || !REGISTRY_AUTH_PATH)
@@ -94,7 +444,10 @@ export async function createApp(options = {}) {
                 existing = await fs.readFile(REGISTRY_AUTH_PATH, 'utf8');
             }
             catch { }
-            const lines = existing.split('\n').filter(Boolean).filter(l => !l.startsWith(`${name}:`));
+            const lines = existing
+                .split('\n')
+                .filter(Boolean)
+                .filter((l) => !l.startsWith(`${name}:`));
             lines.push(line);
             await ensureDir(path.dirname(REGISTRY_AUTH_PATH));
             await fs.writeFile(REGISTRY_AUTH_PATH, `${lines.join('\n')}\n`);
@@ -110,13 +463,17 @@ export async function createApp(options = {}) {
                 existing = await fs.readFile(REGISTRY_AUTH_PATH, 'utf8');
             }
             catch { }
-            const lines = existing.split('\n').filter(Boolean).filter(l => !l.startsWith(`${name}:`));
+            const lines = existing
+                .split('\n')
+                .filter(Boolean)
+                .filter((l) => !l.startsWith(`${name}:`));
             await ensureDir(path.dirname(REGISTRY_AUTH_PATH));
             await fs.writeFile(REGISTRY_AUTH_PATH, lines.length ? `${lines.join('\n')}\n` : '');
         });
         await reloadRegistry();
     }
     const db = await (options.openDatabase || openDb)(dbFile);
+    const wakeHub = new WakeHub();
     // Events are an operator-facing log, not a record of truth, so keep them bounded. `0` turns a limit off; anything that is
     // not a whole number falls back to the default.
     const envCount = (name, fallback) => {
@@ -137,32 +494,43 @@ export async function createApp(options = {}) {
         lastEventPrune = Date.now();
         let removed = 0;
         if (EVENT_RETENTION_DAYS > 0)
-            removed += (await db.run('DELETE FROM events WHERE created_at < ?', new Date(Date.now() - EVENT_RETENTION_DAYS * 86_400_000).toISOString())).changes ?? 0;
+            removed +=
+                (await db.run('DELETE FROM events WHERE created_at < ?', new Date(Date.now() - EVENT_RETENTION_DAYS * 86_400_000).toISOString())).changes ?? 0;
         if (EVENT_MAX_ROWS > 0)
-            removed += (await db.run('DELETE FROM events WHERE id IN (SELECT id FROM events ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?)', EVENT_MAX_ROWS)).changes ?? 0;
+            removed +=
+                (await db.run('DELETE FROM events WHERE id IN (SELECT id FROM events ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?)', EVENT_MAX_ROWS)).changes ?? 0;
         if (removed)
             console.log(`removed ${removed} old events (kept ${EVENT_RETENTION_DAYS || 'any'} days, at most ${EVENT_MAX_ROWS || 'unlimited'} rows)`);
         return removed;
     }
-    await pruneEvents().catch(e => console.error(`event pruning failed: ${errMessage(e)}`));
+    await pruneEvents().catch((e) => console.error(`event pruning failed: ${errMessage(e)}`));
     const masterKey = await loadOrCreateMasterKey(masterKeyFile);
     const operator = await db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash');
     if (!operator) {
         const t = process.env.TUG_OPERATOR_TOKEN || token('tug_op');
         await db.run('INSERT INTO meta(key,value) VALUES(?,?)', 'operator_token_hash', hashToken(t));
-        await fs.writeFile(path.join(DATA_DIR, 'operator.token'), `${t}\n`, { mode: 0o600 }).catch(() => { });
+        await fs
+            .writeFile(path.join(DATA_DIR, 'operator.token'), `${t}\n`, { mode: 0o600 })
+            .catch(() => { });
         console.log(`operator token: ${t}`);
     }
     const app = express();
+    const authenticateAgent = async (raw) => raw
+        ? db.get('SELECT * FROM nodes WHERE agent_token_hash=?', hashToken(raw))
+        : undefined;
     let restartRequired = false;
-    app.use((_req, res, next) => restartRequired ? res.status(503).json({ error: 'control restored; restart required', restartRequired: true }) : next());
+    app.use((_req, res, next) => restartRequired
+        ? res.status(503).json({ error: 'control restored; restart required', restartRequired: true })
+        : next());
     app.use(express.json({ limit: '20mb' }));
     // Hold the writer for the operation lifetime, including after an HTTP disconnect.
     let writer = Promise.resolve();
     async function withWriter(operation) {
         const previous = writer;
         let release;
-        writer = new Promise(resolve => { release = resolve; });
+        writer = new Promise((resolve) => {
+            release = resolve;
+        });
         await previous;
         try {
             if (restartRequired)
@@ -177,7 +545,9 @@ export async function createApp(options = {}) {
     async function withPublication(operation) {
         const previous = publication;
         let release;
-        publication = new Promise(resolve => { release = resolve; });
+        publication = new Promise((resolve) => {
+            release = resolve;
+        });
         await previous;
         try {
             return await operation();
@@ -210,18 +580,31 @@ export async function createApp(options = {}) {
         route(method, url, [guard(role, resolve ?? undefined), ...handlers]);
     };
     const deploymentProject = async (req) => (await db.get('SELECT project_id FROM deployments WHERE id=?', req.params.id))?.project_id;
-    const commandProject = async (req) => { const row = await db.get('SELECT project_id FROM commands WHERE id=?', req.params.id); return row ? row.project_id : undefined; };
+    const commandProject = async (req) => {
+        const row = await db.get('SELECT project_id FROM commands WHERE id=?', req.params.id);
+        return row ? row.project_id : undefined;
+    };
     /** Audit trail entry for a control action; bookkeeping failures are logged, never returned to the caller. */
-    const audit = (actor, event) => recordEvent(db, { ...event, actor: actor.name }).catch(e => console.error(`audit event failed: ${errMessage(e)}`));
-    const projectScope = (actor, column) => actor.projects ? { where: `${column} IN (${actor.projects.map(() => '?').join(',') || "''"})`, args: actor.projects } : { where: '1=1', args: [] };
+    const audit = (actor, event) => recordEvent(db, { ...event, actor: actor.name }).catch((e) => console.error(`audit event failed: ${errMessage(e)}`));
+    const projectScope = (actor, column) => actor.projects
+        ? {
+            where: `${column} IN (${actor.projects.map(() => '?').join(',') || "''"})`,
+            args: actor.projects,
+        }
+        : { where: '1=1', args: [] };
     const authAgent = async (req, res, next) => {
         const raw = (req.headers.authorization || '').replace(/^Bearer /, '');
-        const node = raw && await db.get('SELECT * FROM nodes WHERE agent_token_hash=?', hashToken(raw));
+        const node = await authenticateAgent(raw);
         if (!node)
             return res.status(401).json({ error: 'unauthorized' });
         req.node = node;
         next();
     };
+    const wakeAll = async (project) => {
+        for (const node of await db.all('SELECT id FROM nodes'))
+            wakeHub.wake(node.id, project);
+    };
+    let endpointSnapshot = '';
     async function updateDeploymentConvergence(projectId) {
         const project = await db.get('SELECT * FROM projects WHERE id=?', projectId);
         if (!project?.current_deployment_id)
@@ -237,15 +620,22 @@ export async function createApp(options = {}) {
         if (!required.size)
             return;
         const statuses = await db.all('SELECT node_id,observed_version,health,status_json FROM node_statuses WHERE project_id=?', projectId);
-        const byNode = new Map(statuses.map(s => [s.node_id, s]));
-        const reportsRevision = (node) => state.revision ? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision : byNode.get(node)?.observed_version === state.version;
-        const failedNodes = [...required].filter(node => reportsRevision(node) && byNode.get(node)?.health === 'failed').map(node => byNode.get(node));
+        const byNode = new Map(statuses.map((s) => [s.node_id, s]));
+        const reportsRevision = (node) => state.revision
+            ? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision
+            : byNode.get(node)?.observed_version === state.version;
+        const failedNodes = [...required]
+            .filter((node) => reportsRevision(node) && byNode.get(node)?.health === 'failed')
+            .map((node) => byNode.get(node));
         if (failedNodes.length) {
             const marked = await db.run('UPDATE deployments SET state=?,error_message=?,updated_at=? WHERE id=? AND state=?', 'failed', deploymentFailureMessage(failedNodes), now(), project.current_deployment_id, 'rolling_out');
             if (marked.changes === 1)
                 scheduleAutoRollback(projectId, project.current_deployment_id, state);
         }
-        const ok = [...required].every(node => (state.revision ? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision : byNode.get(node)?.observed_version === state.version) && ['serving', 'skipped'].includes(byNode.get(node)?.health ?? ''));
+        const ok = [...required].every((node) => (state.revision
+            ? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision
+            : byNode.get(node)?.observed_version === state.version) &&
+            ['serving', 'skipped'].includes(byNode.get(node)?.health ?? ''));
         if (ok)
             await db.run('UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state=?', 'succeeded', now(), project.current_deployment_id, 'rolling_out');
     }
@@ -280,7 +670,11 @@ export async function createApp(options = {}) {
             await db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, `${ROLLBACK_VERSION_PREFIX}${previousState.version}-${Date.now()}`, 'rolling_out', ts, ts, opts.actor);
             await db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, previousState.version, JSON.stringify(previousState), ts);
             await db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
-            return { deployment: await db.get('SELECT * FROM deployments WHERE id=?', depId), desiredState: previousState };
+            await wakeAll(project);
+            return {
+                deployment: await db.get('SELECT * FROM deployments WHERE id=?', depId),
+                desiredState: previousState,
+            };
         }
         finally {
             await releaseLock(lock);
@@ -302,17 +696,60 @@ export async function createApp(options = {}) {
                 return;
             }
             try {
-                const out = await withWriter(() => createRollback(projectId, { actor: AUTO_ROLLBACK_ACTOR, auto: true, failedDeploymentId: deploymentId }));
-                await recordAutoRollbackEvent(projectId, out.deployment.id, 'deployment.auto_rollback', `deployment ${deploymentId} (${failed?.version}) reported failed health; rolled back to ${out.desiredState.version} as deployment ${out.deployment.id}`, { failedDeploymentId: deploymentId, rollbackDeploymentId: out.deployment.id, version: out.desiredState.version });
+                const out = await withWriter(() => createRollback(projectId, {
+                    actor: AUTO_ROLLBACK_ACTOR,
+                    auto: true,
+                    failedDeploymentId: deploymentId,
+                }));
+                await recordAutoRollbackEvent(projectId, out.deployment.id, 'deployment.auto_rollback', `deployment ${deploymentId} (${failed?.version}) reported failed health; rolled back to ${out.desiredState.version} as deployment ${out.deployment.id}`, {
+                    failedDeploymentId: deploymentId,
+                    rollbackDeploymentId: out.deployment.id,
+                    version: out.desiredState.version,
+                });
             }
             catch (e) {
                 await recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `deployment ${deploymentId} failed but automatic rollback was not possible: ${errMessage(e)}`).catch(() => { });
             }
-        })().catch(e => console.error('auto rollback failed:', errMessage(e)));
+        })().catch((e) => console.error('auto rollback failed:', errMessage(e)));
     }
     async function currentDesired(projectId) {
         return db.get('SELECT ds.* FROM desired_states ds JOIN projects p ON p.current_deployment_id=ds.deployment_id WHERE p.id=?', projectId);
     }
+    const live = new LiveHub(async (actor, request) => {
+        assertProject(actor, request.project);
+        const row = await currentDesired(request.project);
+        if (!row)
+            throw httpError(404, missingProject(request.project));
+        const desired = JSON.parse(row.state_json);
+        const nodes = await db.all('SELECT * FROM nodes');
+        const eligible = new Set();
+        const services = request.operation === 'ps'
+            ? Object.values(desired.services)
+            : [desired.services[request.service]];
+        if (services.some((service) => !service))
+            throw httpError(404, missingService(request.project, 'service' in request ? request.service : ''));
+        for (const service of services)
+            for (const node of nodes)
+                if (!service.placement?.nodes?.length || service.placement.nodes.includes(node.name))
+                    eligible.add(node.name);
+        if (request.node && !eligible.has(request.node))
+            throw httpError(400, 'requested node is not placed for this operation');
+        if (!request.node && eligible.size !== 1)
+            throw httpError(400, 'operation requires --node because placement is ambiguous');
+        const node = nodes.find((node) => node.name === (request.node || [...eligible][0]));
+        if (!node)
+            throw httpError(404, 'no eligible node for this operation');
+        return node;
+    }, options.liveStartTimeoutMs);
+    streamRuntimes.set(app, {
+        hub: wakeHub,
+        authenticate: authenticateAgent,
+        authenticateOperator: (raw) => authenticate(db, raw),
+        live,
+    });
+    api('get', '/api/v1/live', null, (_req, res) => {
+        res.status(426).json({ error: 'WebSocket upgrade required' });
+    });
     async function acquireProjectLock(project, operation, ttlMs = Number(process.env.TUG_LOCK_TTL_MS || 10 * 60 * 1000)) {
         const name = `project:${project}`;
         const ts = now();
@@ -330,7 +767,11 @@ export async function createApp(options = {}) {
             err.lock = existing;
             throw err;
         }
-        const heartbeat = setInterval(() => { void db.run('UPDATE locks SET expires_at=? WHERE name=? AND owner=?', new Date(Date.now() + ttlMs).toISOString(), name, owner).catch(() => { }); }, Math.max(10, Math.floor(ttlMs / 3)));
+        const heartbeat = setInterval(() => {
+            void db
+                .run('UPDATE locks SET expires_at=? WHERE name=? AND owner=?', new Date(Date.now() + ttlMs).toISOString(), name, owner)
+                .catch(() => { });
+        }, Math.max(10, Math.floor(ttlMs / 3)));
         heartbeat.unref();
         return { name, owner, heartbeat };
     }
@@ -338,18 +779,76 @@ export async function createApp(options = {}) {
         if (lock?.heartbeat)
             clearInterval(lock.heartbeat);
         if (lock)
-            await db.run('DELETE FROM locks WHERE name=? AND owner=?', lock.name, lock.owner).catch(() => { });
+            await db
+                .run('DELETE FROM locks WHERE name=? AND owner=?', lock.name, lock.owner)
+                .catch(() => { });
     }
-    function sendError(res, e) { const err = e; res.status(restartRequired ? 503 : err.status || 500).json({ error: err.message, lock: err.lock || undefined }); }
+    function sendError(res, e) {
+        const err = e;
+        res
+            .status(restartRequired ? 503 : err.status || 500)
+            .json({ error: err.message, lock: err.lock || undefined });
+    }
+    api('post', '/api/v1/scale', null, async (req, res) => {
+        const project = req.body?.project;
+        const service = req.body?.service;
+        const replicas = Number(req.body?.replicas);
+        if (!project || !service || !Number.isInteger(replicas) || replicas < 1)
+            throw httpError(400, 'project, service and a positive integer replicas are required');
+        assertProject(req.actor, project);
+        const dsRow = await currentDesired(project);
+        if (!dsRow)
+            throw httpError(404, `project ${project} has no desired state`);
+        const state = JSON.parse(dsRow.state_json);
+        if (!state.services?.[service])
+            throw httpError(404, `service ${service} not found in project ${project}`);
+        const nodeRows = await db.all('SELECT name,labels_json,last_seen_at FROM nodes ORDER BY created_at,name');
+        const nodeInfos = nodeRows
+            .filter((row) => !isStale(row))
+            .map((row) => ({
+            name: row.name,
+            labels: row.labels_json ? JSON.parse(row.labels_json) : [],
+        }));
+        const composeSvc = state.compose?.services?.[service] || {};
+        const ext = composeSvc['x-tug'] || {};
+        const placement = ext.placement || {};
+        const global = ext.global === true || placement.global === true;
+        const labels = Array.isArray(placement.labels) ? placement.labels : [];
+        const matched = nodeInfos.filter((node) => labels.every((label) => node.labels.includes(label)));
+        const pool = labels.length ? matched : nodeInfos;
+        if (!pool.length)
+            throw httpError(400, labels.length
+                ? 'no healthy nodes match the service placement labels'
+                : 'no nodes available for scaling');
+        const selected = global ? pool : pool.slice(0, Math.min(replicas, pool.length));
+        state.services[service].placement = {
+            ...(state.services[service].placement || {}),
+            nodes: selected.map((node) => node.name),
+        };
+        state.services[service].replicas = selected.length;
+        state.revision = id('rev');
+        await db.run('UPDATE desired_states SET state_json=? WHERE id=?', JSON.stringify(state), dsRow.id);
+        await wakeAll(project);
+        res.json({
+            ok: true,
+            project,
+            service,
+            replicas: selected.length,
+            nodes: selected.map((node) => node.name),
+        });
+    });
     get('/healthz', (req, res) => res.json({ ok: true }));
-    api('post', '/api/v1/projects', req => String(req.body?.name || 'site'), async (req, res) => {
+    api('post', '/api/v1/projects', (req) => String(req.body?.name || 'site'), async (req, res) => {
         const name = req.body.name || 'site';
         const ts = now();
         await db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', name, name, ts, ts);
         res.json(await db.get('SELECT * FROM projects WHERE id=?', name));
     });
-    api('get', '/api/v1/projects/:project', req => req.params.project, async (req, res) => res.json(await db.get('SELECT * FROM projects WHERE id=?', req.params.project)));
-    const NODE_INFO_SQL = 'SELECT id,name,address,public_address,overlay_ip,status,created_at,updated_at FROM nodes WHERE id=?';
+    api('get', '/api/v1/projects/:project', (req) => req.params.project, async (req, res) => res.json(await db.get('SELECT * FROM projects WHERE id=?', req.params.project)));
+    /** A node whose agent has been seen before but not within TUG_STALE_SECONDS; never-seen (just registered) nodes stay schedulable. */
+    const isStale = (row) => !!row.last_seen_at &&
+        Date.parse(row.last_seen_at) < Date.now() - Number(process.env.TUG_STALE_SECONDS || 60) * 1000;
+    const NODE_INFO_SQL = 'SELECT id,name,address,public_address,overlay_ip,labels_json,status,created_at,updated_at FROM nodes WHERE id=?';
     /** Allocates the node's stable overlay IP on first registration (retrying if a concurrent register took it). */
     async function ensureOverlayIp(nodeId) {
         if (!MESH)
@@ -358,7 +857,7 @@ export async function createApp(options = {}) {
             const row = await db.get('SELECT overlay_ip FROM nodes WHERE id=?', nodeId);
             if (row?.overlay_ip)
                 return;
-            const used = (await db.all('SELECT overlay_ip FROM nodes WHERE overlay_ip IS NOT NULL')).map(r => r.overlay_ip);
+            const used = (await db.all('SELECT overlay_ip FROM nodes WHERE overlay_ip IS NOT NULL')).map((r) => r.overlay_ip);
             try {
                 await db.run('UPDATE nodes SET overlay_ip=? WHERE id=? AND overlay_ip IS NULL', allocateOverlayIp(MESH_CIDR, used), nodeId);
                 return;
@@ -382,18 +881,32 @@ export async function createApp(options = {}) {
         await db.run(`INSERT INTO wg_peers(node_id,public_key,overlay_ip,endpoint,allowed_ips,updated_at) VALUES(?,?,?,?,?,?)
       ON CONFLICT(node_id) DO UPDATE SET public_key=excluded.public_key,overlay_ip=excluded.overlay_ip,endpoint=excluded.endpoint,allowed_ips=excluded.allowed_ips,updated_at=excluded.updated_at`, nodeId, key, node.overlay_ip, wgEndpoint(node.public_address, WG_PORT) ?? null, `${node.overlay_ip}/32`, now());
     }
-    async function registerNode({ name, address, publicAddress, wgPublicKey }) {
+    async function registerNode({ name, address, publicAddress, wgPublicKey, labels, }) {
         if (wgPublicKey !== undefined && !isWgKey(wgPublicKey))
             throw httpError(400, 'wgPublicKey must be a base64 WireGuard public key');
         const agentToken = token('tug_agent');
         const registryToken = token('tug_reg');
         const ts = now();
-        await db.run(`INSERT INTO nodes(id,name,address,public_address,status,agent_token_hash,registry_token_hash,created_at,updated_at)
-      VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET name=excluded.name,address=excluded.address,public_address=excluded.public_address,agent_token_hash=excluded.agent_token_hash,registry_token_hash=excluded.registry_token_hash,updated_at=excluded.updated_at`, name, name, address || null, publicAddress || null, 'registered', hashToken(agentToken), hashToken(registryToken), ts, ts);
+        await db.run(`INSERT INTO nodes(id,name,address,public_address,status,labels_json,agent_token_hash,registry_token_hash,created_at,updated_at)
+      VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET name=excluded.name,address=excluded.address,public_address=excluded.public_address,labels_json=excluded.labels_json,agent_token_hash=excluded.agent_token_hash,registry_token_hash=excluded.registry_token_hash,updated_at=excluded.updated_at`, name, name, address || null, publicAddress || null, 'registered', JSON.stringify(Array.isArray(labels) ? labels : []), hashToken(agentToken), hashToken(registryToken), ts, ts);
         await ensureOverlayIp(name);
         await syncWgPeer(name, wgPublicKey);
         await upsertRegistryCredential(name, registryToken);
-        const node = (await db.get(NODE_INFO_SQL, name));
+        await wakeAll();
+        const row = await db.get(NODE_INFO_SQL, name);
+        if (!row)
+            throw httpError(500, 'node not found after registration');
+        const node = {
+            id: row.id,
+            name: row.name,
+            address: row.address,
+            public_address: row.public_address,
+            overlay_ip: row.overlay_ip,
+            labels: row.labels_json ? JSON.parse(row.labels_json) : [],
+            status: row.status,
+            created_at: row.created_at,
+            updated_at: row.updated_at,
+        };
         return { node, agentToken, registryToken };
     }
     api('post', '/api/v1/nodes/register', null, async (req, res) => {
@@ -401,7 +914,13 @@ export async function createApp(options = {}) {
         if (!name)
             return res.status(400).json({ error: 'name required' });
         try {
-            res.json(await registerNode({ name, address: req.body.address, publicAddress: req.body.publicAddress, wgPublicKey: req.body.wgPublicKey }));
+            res.json(await registerNode({
+                name,
+                address: req.body.address,
+                publicAddress: req.body.publicAddress,
+                wgPublicKey: req.body.wgPublicKey,
+                labels: req.body.labels,
+            }));
         }
         catch (e) {
             sendError(res, e);
@@ -431,10 +950,18 @@ export async function createApp(options = {}) {
         }
         const name = row.name || body.name;
         if (!name)
-            return res.status(400).json({ error: 'node name required (bind one to the enroll token or send name)' });
+            return res
+                .status(400)
+                .json({ error: 'node name required (bind one to the enroll token or send name)' });
         let result;
         try {
-            result = await registerNode({ name, address: body.address, publicAddress: body.publicAddress, wgPublicKey: body.wgPublicKey });
+            result = await registerNode({
+                name,
+                address: body.address,
+                publicAddress: body.publicAddress,
+                wgPublicKey: body.wgPublicKey,
+                labels: body.labels,
+            });
         }
         catch (e) {
             return sendError(res, e);
@@ -447,7 +974,15 @@ export async function createApp(options = {}) {
         const enrolled = { ...result, registry: REGISTRY };
         res.json(enrolled);
     });
-    api('get', '/api/v1/nodes', null, async (req, res) => res.json({ nodes: await db.all('SELECT id,name,address,public_address,overlay_ip,ingress,ingress_status_json,status,last_seen_at,created_at,updated_at FROM nodes ORDER BY name') }));
+    api('get', '/api/v1/nodes', null, async (req, res) => {
+        const rows = await db.all('SELECT id,name,address,public_address,overlay_ip,ingress,ingress_status_json,status,labels_json,last_seen_at,created_at,updated_at FROM nodes ORDER BY name');
+        res.json({
+            nodes: rows.map((row) => {
+                const { labels_json, ...rest } = row;
+                return { ...rest, labels: labels_json ? JSON.parse(labels_json) : [] };
+            }),
+        });
+    });
     api('post', '/api/v1/nodes/:name/rotate-registry-token', null, async (req, res) => {
         const node = await db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
         if (!node)
@@ -465,11 +1000,15 @@ export async function createApp(options = {}) {
         await removeRegistryCredential(node.name);
         res.json({ ok: true, node: node.name, revoked: true });
     });
-    async function findNode(nameOrId) { return db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', nameOrId, nameOrId); }
-    function queueCommand(nodeId, projectId, type, payload = {}) {
+    async function findNode(nameOrId) {
+        return db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', nameOrId, nameOrId);
+    }
+    async function queueCommand(nodeId, projectId, type, payload = {}) {
         const commandId = id('cmd');
         const ts = now();
-        return db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', commandId, nodeId, projectId, type, 'pending', JSON.stringify(payload), 0, ts, null, ts, ts).then(() => commandId);
+        await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', commandId, nodeId, projectId, type, 'pending', JSON.stringify(payload), 0, ts, null, ts, ts);
+        wakeHub.wake(nodeId, projectId ?? undefined);
+        return commandId;
     }
     // The private key lives only on the node, so rotation is a command the agent executes: it
     // generates a new keypair, reports the public key and peers pick it up from their next work response.
@@ -481,12 +1020,48 @@ export async function createApp(options = {}) {
             return res.status(404).json({ error: 'node not found' });
         res.json({ commandId: await queueCommand(node.id, null, 'wg.rotate-key'), nodeId: node.id });
     });
+    // Removes a retired node: its token, WireGuard peer, statuses and queued commands. Refused while a current deployment still places
+    // something on it (redeploy without it first) or while its agent is still reporting, unless force=true.
+    api('delete', '/api/v1/nodes/:name', null, async (req, res) => {
+        const node = await db.get('SELECT id,name,last_seen_at FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
+        if (!node)
+            return res.status(404).json({ error: 'node not found' });
+        const referenced = [];
+        for (const state of await currentDesiredStates()) {
+            for (const [name, svc] of Object.entries(state.services || {}))
+                if (svc.placement?.nodes?.includes(node.name))
+                    referenced.push(`${state.project}/${name}`);
+            for (const [name, vol] of Object.entries(state.compose?.volumes || {}))
+                if (vol?.['x-tug']?.placement?.nodes?.includes(node.name))
+                    referenced.push(`${state.project}/volume ${name}`);
+        }
+        if (referenced.length)
+            return res.status(409).json({
+                error: `node ${node.name} is still placed in current deployments: ${referenced.join(', ')}; deploy without it first`,
+            });
+        if (node.last_seen_at && !isStale(node) && String(req.query.force) !== 'true')
+            return res.status(409).json({
+                error: `node ${node.name} is still reporting; stop its agent first or pass force=true`,
+            });
+        await db.run('DELETE FROM commands WHERE node_id=?', node.id);
+        await db.run('DELETE FROM node_statuses WHERE node_id=?', node.id);
+        await db.run('DELETE FROM wg_peers WHERE node_id=?', node.id);
+        await db.run('DELETE FROM nodes WHERE id=?', node.id);
+        await audit(req.actor, {
+            type: 'node.removed',
+            message: `node ${node.name} removed`,
+            metadata: { node: node.name },
+        });
+        await wakeAll();
+        res.json({ ok: true, node: node.name });
+    });
     api('post', '/api/v1/nodes/:name/ingress', null, async (req, res) => {
         const node = await findNode(req.params.name);
         if (!node)
             return res.status(404).json({ error: 'node not found' });
         const enabled = req.body.enabled !== false;
         await db.run('UPDATE nodes SET ingress=?,updated_at=? WHERE id=?', enabled ? 1 : 0, now(), node.id);
+        wakeHub.wake(node.id);
         res.json({ ok: true, node: node.name, ingress: enabled });
     });
     post('/agent/v1/wireguard/key', authAgent, async (req, res) => {
@@ -497,16 +1072,21 @@ export async function createApp(options = {}) {
             return res.status(400).json({ error: 'publicKey must be a base64 WireGuard public key' });
         try {
             await syncWgPeer(req.node.id, publicKey);
+            await wakeAll();
             res.json({ ok: true });
         }
         catch (e) {
-            sendError(res, /UNIQUE constraint/.test(errMessage(e)) ? httpError(409, 'public key already registered to another node') : e);
+            sendError(res, /UNIQUE constraint/.test(errMessage(e))
+                ? httpError(409, 'public key already registered to another node')
+                : e);
         }
     });
     post('/api/v1/registry/auth', async (req, res) => {
         const username = req.body.username;
         const password = req.body.password;
-        const node = username && password ? await db.get('SELECT id,name FROM nodes WHERE (name=? OR id=?) AND registry_token_hash IS NOT NULL AND registry_token_hash=?', username, username, hashToken(password)) : null;
+        const node = username && password
+            ? await db.get('SELECT id,name FROM nodes WHERE (name=? OR id=?) AND registry_token_hash IS NOT NULL AND registry_token_hash=?', username, username, hashToken(password))
+            : null;
         if (!node)
             return res.status(401).json({ ok: false, error: 'invalid registry credentials' });
         res.json({ ok: true, node: node.name });
@@ -514,14 +1094,27 @@ export async function createApp(options = {}) {
     api('post', '/api/v1/tokens', null, async (req, res) => {
         const request = parseTokenRequest(req.body);
         const created = await createToken(db, request, req.actor.name);
-        await audit(req.actor, { type: 'token.create', message: `token ${created.name} (${created.role}${created.projects ? `, projects: ${created.projects.join(', ')}` : ''}) created by ${req.actor.name}`, metadata: { name: created.name, role: created.role, projects: created.projects, expiresAt: created.expiresAt } });
+        await audit(req.actor, {
+            type: 'token.create',
+            message: `token ${created.name} (${created.role}${created.projects ? `, projects: ${created.projects.join(', ')}` : ''}) created by ${req.actor.name}`,
+            metadata: {
+                name: created.name,
+                role: created.role,
+                projects: created.projects,
+                expiresAt: created.expiresAt,
+            },
+        });
         res.json(created);
     });
     api('get', '/api/v1/tokens', null, async (_req, res) => res.json({ tokens: await listTokens(db) }));
     api('delete', '/api/v1/tokens/:name', null, async (req, res) => {
-        if (!await revokeToken(db, req.params.name))
+        if (!(await revokeToken(db, req.params.name)))
             return res.status(404).json({ error: 'token not found' });
-        await audit(req.actor, { type: 'token.revoke', message: `token ${req.params.name} revoked by ${req.actor.name}`, metadata: { name: req.params.name } });
+        await audit(req.actor, {
+            type: 'token.revoke',
+            message: `token ${req.params.name} revoked by ${req.actor.name}`,
+            metadata: { name: req.params.name },
+        });
         res.json({ ok: true, name: req.params.name });
     });
     api('post', '/api/v1/secrets', bodyProject, async (req, res) => {
@@ -535,7 +1128,13 @@ export async function createApp(options = {}) {
             const ts = now();
             await db.run(`INSERT INTO secrets(id,project_id,name,encrypted_value,version,created_at,updated_at) VALUES(?,?,?,?,?,?,?)
         ON CONFLICT(project_id,name) DO UPDATE SET encrypted_value=excluded.encrypted_value,version=excluded.version,updated_at=excluded.updated_at`, id('sec'), project, name, encrypt(value, masterKey), (old?.version || 0) + 1, ts, ts);
-            await audit(req.actor, { type: 'secret.set', project, message: `secret ${name} set to version ${(old?.version || 0) + 1} by ${req.actor.name}`, metadata: { name, version: (old?.version || 0) + 1 } });
+            await wakeAll(project);
+            await audit(req.actor, {
+                type: 'secret.set',
+                project,
+                message: `secret ${name} set to version ${(old?.version || 0) + 1} by ${req.actor.name}`,
+                metadata: { name, version: (old?.version || 0) + 1 },
+            });
             res.json({ ok: true, name, version: (old?.version || 0) + 1 });
         }
         catch (e) {
@@ -545,24 +1144,33 @@ export async function createApp(options = {}) {
             await releaseLock(lock);
         }
     });
-    api('get', '/api/v1/secrets', queryProject, async (req, res) => res.json({ secrets: await db.all('SELECT project_id,name,version,updated_at FROM secrets WHERE project_id=? ORDER BY name', req.query.project || 'site') }));
+    api('get', '/api/v1/secrets', queryProject, async (req, res) => res.json({
+        secrets: await db.all('SELECT project_id,name,version,updated_at FROM secrets WHERE project_id=? ORDER BY name', req.query.project || 'site'),
+    }));
     api('delete', '/api/v1/secrets/:name', queryProject, async (req, res) => {
         const project = String(req.query.project || 'site');
         const removed = await db.run('DELETE FROM secrets WHERE project_id=? AND name=?', project, req.params.name);
-        if (removed.changes)
-            await audit(req.actor, { type: 'secret.delete', project, message: `secret ${req.params.name} deleted by ${req.actor.name}`, metadata: { name: req.params.name } });
+        if (removed.changes) {
+            await wakeAll(project);
+            await audit(req.actor, {
+                type: 'secret.delete',
+                project,
+                message: `secret ${req.params.name} deleted by ${req.actor.name}`,
+                metadata: { name: req.params.name },
+            });
+        }
         res.json({ ok: true });
     });
     const artifactDir = path.join(DATA_DIR, 'images');
     await ensureDir(artifactDir);
     async function retainedStates() {
         // All published history is retained. This conservative superset protects failed/lagging versions too.
-        return (await db.all('SELECT * FROM desired_states')).map(row => JSON.parse(row.state_json));
+        return (await db.all('SELECT * FROM desired_states')).map((row) => JSON.parse(row.state_json));
     }
     async function collectArtifacts() {
         const states = await retainedStates();
-        const refs = new Set(states.flatMap(state => Object.values(state.services).flatMap(service => service.artifact ? [service.artifact.buildId] : [])));
-        const imageRefs = new Set(states.flatMap(state => Object.values(state.services).map(service => service.image)));
+        const refs = new Set(states.flatMap((state) => Object.values(state.services).flatMap((service) => service.artifact ? [service.artifact.buildId] : [])));
+        const imageRefs = new Set(states.flatMap((state) => Object.values(state.services).map((service) => service.image)));
         const rows = await db.all('SELECT * FROM builds');
         for (const row of rows) {
             if (refs.has(row.id) || imageRefs.has(row.image_ref))
@@ -575,7 +1183,7 @@ export async function createApp(options = {}) {
             await db.run('DELETE FROM builds WHERE id=?', row.id);
         }
         const remaining = await db.all('SELECT id FROM builds');
-        const owned = new Set(remaining.flatMap(row => [`${row.id}.tar.gz`, `${row.id}.tar.gz.tmp`]));
+        const owned = new Set(remaining.flatMap((row) => [`${row.id}.tar.gz`, `${row.id}.tar.gz.tmp`]));
         for (const name of await fs.readdir(artifactDir))
             if (!owned.has(name))
                 await fs.rm(path.join(artifactDir, name), { force: true });
@@ -585,10 +1193,10 @@ export async function createApp(options = {}) {
     await collectArtifacts();
     const receiveDeployment = upload.any();
     api('post', '/api/v1/deployments', null, (req, res) => {
-        receiveDeployment(req, res, error => {
+        receiveDeployment(req, res, (error) => {
             void withWriter(() => deployRequest(req, res, error)).catch(async (failure) => {
                 const files = Array.isArray(req.files) ? req.files : [];
-                await Promise.all(files.map(file => fs.rm(file.path, { force: true }).catch(() => { })));
+                await Promise.all(files.map((file) => fs.rm(file.path, { force: true }).catch(() => { })));
                 sendError(res, failure);
             });
         });
@@ -616,31 +1224,97 @@ export async function createApp(options = {}) {
             const compose = body.compose;
             const version = body.version || `git-${Date.now()}`;
             assertProject(req.actor, project);
-            if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(project) || typeof version !== 'string' || !version.length)
+            if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(project) ||
+                typeof version !== 'string' ||
+                !version.length)
                 throw httpError(400, 'invalid project or version');
             if (version.startsWith(ROLLBACK_VERSION_PREFIX))
                 throw httpError(400, `version cannot start with "${ROLLBACK_VERSION_PREFIX}" (reserved for rollback deployments)`);
-            const nodes = await db.all('SELECT name FROM nodes ORDER BY created_at,name');
+            const nodeRows = await db.all('SELECT name,labels_json,last_seen_at,created_at FROM nodes ORDER BY created_at,name');
+            // Stale nodes (agent not seen recently) are not scheduled onto, so a dead host cannot hold a rollout open.
+            const nodeInfos = nodeRows
+                .filter((row) => !isStale(row))
+                .map((row) => ({
+                name: row.name,
+                labels: row.labels_json ? JSON.parse(row.labels_json) : [],
+                created_at: row.created_at,
+            }));
+            // Resolve label-based and global placement into concrete node lists before validation.
+            for (const svc of Object.values(compose.services || {})) {
+                const ext = (svc['x-tug'] ||= {});
+                const placement = ext.placement || {};
+                const global = ext.global === true || placement.global === true;
+                const labels = Array.isArray(placement.labels) ? placement.labels : [];
+                if (!placement.nodes?.length && (labels.length || global)) {
+                    const matched = nodeInfos
+                        .filter((node) => labels.every((label) => node.labels.includes(label)))
+                        .sort((a, b) => a.name.localeCompare(b.name));
+                    if (!matched.length)
+                        throw httpError(400, 'no nodes match placement labels');
+                    let selected = matched;
+                    if (!global) {
+                        const replicas = ext.replicas || matched.length;
+                        if (!Number.isInteger(replicas) || replicas < 1)
+                            throw httpError(400, 'replicas must be a positive integer');
+                        selected = matched.slice(0, Math.min(replicas, matched.length));
+                    }
+                    ext.placement = { ...placement, nodes: selected.map((node) => node.name) };
+                    ext.replicas = selected.length;
+                }
+            }
+            const nodes = nodeRows.map((row) => ({ name: row.name }));
+            for (const vol of Object.values(compose.volumes || {})) {
+                const xe = vol?.['x-tug'];
+                if (!xe ||
+                    (xe.type !== 'nfs' && xe.driver !== 'nfs') ||
+                    xe.address ||
+                    !xe.placement?.nodes?.[0])
+                    continue;
+                const exporter = await db.get('SELECT address,overlay_ip FROM nodes WHERE name=?', xe.placement.nodes[0]);
+                const addr = (MESH && exporter?.overlay_ip) || exporter?.address;
+                if (addr)
+                    xe.address = addr;
+            }
             try {
-                validateCompose(compose, nodes.map(n => n.name));
+                validateCompose(compose, nodes.map((n) => n.name));
             }
             catch (e) {
                 throw httpError(400, errMessage(e));
             }
-            const builtServices = Object.entries(compose.services).filter(([, service]) => service.build).map(([name]) => name);
+            // The CLI interpolates control secrets as sentinels; a reference to a secret that is gone would deploy an empty value.
+            const referenced = sentinelNames(compose);
+            if (referenced.length) {
+                const known = new Set((await db.all('SELECT name FROM secrets WHERE project_id=?', project)).map((row) => row.name));
+                const missing = referenced.filter((name) => !known.has(name));
+                if (missing.length)
+                    throw httpError(400, `secrets not found in project ${project}: ${missing.join(', ')}; create them with tug secret-set`);
+            }
+            const builtServices = Object.entries(compose.services)
+                .filter(([, service]) => service.build)
+                .map(([name]) => name);
             const local = body.localImages;
-            const context = files.find(file => file.fieldname === 'context');
+            const context = files.find((file) => file.fieldname === 'context');
             if (local !== undefined && (!Array.isArray(local) || context))
                 throw httpError(400, 'localImages must be an array and cannot accompany a source context');
-            if (files.filter(file => file.fieldname === 'context').length > 1)
+            if (files.filter((file) => file.fieldname === 'context').length > 1)
                 throw httpError(400, 'duplicate source context');
             if (local) {
-                if (local.some(item => !item || typeof item !== 'object'))
+                if (local.some((item) => !item || typeof item !== 'object'))
                     throw httpError(400, 'invalid local image manifest');
-                if (local.length !== builtServices.length || new Set(local.map(item => item.service)).size !== local.length || new Set(local.map(item => item.field)).size !== local.length)
+                if (local.length !== builtServices.length ||
+                    new Set(local.map((item) => item.service)).size !== local.length ||
+                    new Set(local.map((item) => item.field)).size !== local.length)
                     throw httpError(400, 'localImages must contain exactly one image for each build service');
                 for (const item of local) {
-                    if (!/^tug-build\/[a-f0-9-]{36}\/image-\d+:local$/.test(item.imageRef) || item.imageRef.split('/').at(-1) !== `${item.field}:local` || !builtServices.includes(item.service) || !/^image-\d+$/.test(item.field) || files.filter(file => file.fieldname === item.field).length !== 1 || !/^[a-f0-9]{64}$/.test(item.sha256) || !Number.isSafeInteger(item.sizeBytes) || item.sizeBytes <= 0 || item.sizeBytes > MAX_IMAGE_BYTES)
+                    if (!/^tug-build\/[a-f0-9-]{36}\/image-\d+:local$/.test(item.imageRef) ||
+                        item.imageRef.split('/').at(-1) !== `${item.field}:local` ||
+                        !builtServices.includes(item.service) ||
+                        !/^image-\d+$/.test(item.field) ||
+                        files.filter((file) => file.fieldname === item.field).length !== 1 ||
+                        !/^[a-f0-9]{64}$/.test(item.sha256) ||
+                        !Number.isSafeInteger(item.sizeBytes) ||
+                        item.sizeBytes <= 0 ||
+                        item.sizeBytes > MAX_IMAGE_BYTES)
                         throw httpError(400, 'invalid local image manifest or file');
                 }
             }
@@ -653,7 +1327,7 @@ export async function createApp(options = {}) {
                 if (ing?.tls !== 'custom')
                     continue;
                 for (const secret of [ing.cert, ing.key])
-                    if (!await db.get('SELECT 1 FROM secrets WHERE project_id=? AND name=?', project, secret))
+                    if (!(await db.get('SELECT 1 FROM secrets WHERE project_id=? AND name=?', project, secret)))
                         throw httpError(400, `${svcName}: custom TLS secret ${secret} not found in project ${project}; set it with secret-set first`);
             }
             lock = await acquireProjectLock(project, 'deploy');
@@ -664,7 +1338,14 @@ export async function createApp(options = {}) {
             const ts = now();
             await db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', project, project, ts, ts);
             await db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, version, 'building', ts, ts, req.actor.name);
-            const desired = buildDesiredState({ project, version, revision: depId, compose, registry: REGISTRY, defaultNode: nodes[0]?.name || null });
+            const desired = buildDesiredState({
+                project,
+                version,
+                revision: depId,
+                compose,
+                registry: REGISTRY,
+                defaultNode: nodes[0]?.name || null,
+            });
             if (context) {
                 contextDir = path.resolve(DATA_DIR, 'build-contexts', depId);
                 await unpackContext(context.path, contextDir);
@@ -685,15 +1366,18 @@ export async function createApp(options = {}) {
                 const temporary = `${final}.tmp`;
                 try {
                     if (local) {
-                        const item = local.find(item => item.service === service);
-                        const file = files.find(file => file.fieldname === item.field);
+                        const item = local.find((item) => item.service === service);
+                        const file = files.find((file) => file.fieldname === item.field);
                         try {
                             info = await images.validate(file.path, item.imageRef);
                         }
                         catch (error) {
                             throw httpError(400, `${service}: invalid image archive: ${errMessage(error)}`);
                         }
-                        if (info.sha256 !== item.sha256 || info.sizeBytes !== item.sizeBytes || info.imageId !== item.imageId || info.platform !== item.platform)
+                        if (info.sha256 !== item.sha256 ||
+                            info.sizeBytes !== item.sizeBytes ||
+                            info.imageId !== item.imageId ||
+                            info.platform !== item.platform)
                             throw httpError(400, `${service}: local image metadata/checksum mismatch`);
                         importedClientRef = item.imageRef;
                         await images.run('docker', ['load', '-i', file.path]);
@@ -703,7 +1387,13 @@ export async function createApp(options = {}) {
                         await images.run('docker', ['tag', item.imageRef, imageRef]);
                     }
                     else
-                        await images.build({ contextDir: contextDir, service, buildSpec: compose.services[service].build, imageRef, push: false });
+                        await images.build({
+                            contextDir: contextDir,
+                            service,
+                            buildSpec: compose.services[service].build,
+                            imageRef,
+                            push: false,
+                        });
                     if (REGISTRY) {
                         if (process.env.TUG_PUSH !== 'false')
                             await images.run('docker', ['push', imageRef]);
@@ -733,13 +1423,16 @@ export async function createApp(options = {}) {
                 await db.exec('BEGIN IMMEDIATE');
                 try {
                     const published = await db.run("UPDATE deployments SET state='rolling_out',updated_at=? WHERE id=? AND state='building'", now(), depId);
-                    if (published.changes !== 1 || !await db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now()))
+                    if (published.changes !== 1 ||
+                        !(await db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now())))
                         throw httpError(409, 'deployment was cancelled or its build lease was lost');
                     await db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, version, JSON.stringify(desired), ts);
                     for (const [volumeName, volumeSpec] of Object.entries(compose.volumes || {})) {
                         const lp = volumeSpec?.['x-tug'] || {};
                         const nodeName = lp.placement?.nodes?.[0] || null;
-                        const node = nodeName ? await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName) : null;
+                        const node = nodeName
+                            ? await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName)
+                            : null;
                         await db.run(`INSERT INTO volumes(id,project_id,name,type,driver,node_id,config_json,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(project_id,name) DO UPDATE SET type=excluded.type,driver=excluded.driver,node_id=excluded.node_id,config_json=excluded.config_json,updated_at=excluded.updated_at`, id('vol'), project, volumeName, lp.type || 'local', lp.driver || null, node?.id || null, JSON.stringify(lp), ts, ts);
                     }
                     await db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
@@ -750,20 +1443,32 @@ export async function createApp(options = {}) {
                     throw e;
                 }
             });
-            await audit(req.actor, { type: 'deploy', project, deploymentId: depId, message: `deployment ${depId} (${version}) started by ${req.actor.name}`, metadata: { version } });
-            responseBody = { deployment: { id: depId, project, version, state: 'rolling_out', actor: req.actor.name }, desiredState: desired };
+            await audit(req.actor, {
+                type: 'deploy',
+                project,
+                deploymentId: depId,
+                message: `deployment ${depId} (${version}) started by ${req.actor.name}`,
+                metadata: { version },
+            });
+            await wakeAll(project);
+            responseBody = {
+                deployment: { id: depId, project, version, state: 'rolling_out', actor: req.actor.name },
+                desiredState: desired,
+            };
         }
         catch (e) {
             if (depId)
-                await db.run("UPDATE deployments SET state='build_failed',error_message=?,updated_at=? WHERE id=? AND state='building'", errMessage(e), now(), depId).catch(() => { });
+                await db
+                    .run("UPDATE deployments SET state='build_failed',error_message=?,updated_at=? WHERE id=? AND state='building'", errMessage(e), now(), depId)
+                    .catch(() => { });
             responseError = e;
         }
         finally {
             await releaseLock(lock);
-            await Promise.all(files.map(file => fs.rm(file.path, { force: true }).catch(() => { })));
+            await Promise.all(files.map((file) => fs.rm(file.path, { force: true }).catch(() => { })));
             if (contextDir)
                 await fs.rm(contextDir, { recursive: true, force: true });
-            await collectArtifacts().catch(e => console.error('image cleanup failed:', errMessage(e)));
+            await collectArtifacts().catch((e) => console.error('image cleanup failed:', errMessage(e)));
         }
         if (responseError)
             sendError(res, responseError);
@@ -772,7 +1477,8 @@ export async function createApp(options = {}) {
     }
     get('/agent/v1/images/:buildId', authAgent, async (req, res) => {
         const states = await retainedStates();
-        const allowed = states.some(state => Object.values(state.services).some(service => service.artifact?.buildId === req.params.buildId && (!service.placement.nodes?.length || service.placement.nodes.includes(req.node.name))));
+        const allowed = states.some((state) => Object.values(state.services).some((service) => service.artifact?.buildId === req.params.buildId &&
+            (!service.placement.nodes?.length || service.placement.nodes.includes(req.node.name))));
         if (!allowed)
             return res.status(404).json({ error: 'image not found' });
         const row = await db.get('SELECT * FROM builds WHERE id=? AND state=?', req.params.buildId, 'complete');
@@ -780,7 +1486,12 @@ export async function createApp(options = {}) {
             return res.status(404).json({ error: 'image not found' });
         try {
             await fs.access(path.join(DATA_DIR, row.artifact_path));
-            res.set({ 'Content-Type': 'application/gzip', 'Content-Length': String(row.size_bytes), 'X-Tug-Sha256': row.sha256, 'Cache-Control': 'private, no-store' });
+            res.set({
+                'Content-Type': 'application/gzip',
+                'Content-Length': String(row.size_bytes),
+                'X-Tug-Sha256': row.sha256,
+                'Cache-Control': 'private, no-store',
+            });
             await pipeline(createReadStream(path.join(DATA_DIR, row.artifact_path)), res);
         }
         catch {
@@ -798,21 +1509,37 @@ export async function createApp(options = {}) {
         const result = await db.run(`UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state IN (${ACTIVE_STATES_SQL})`, 'cancelled', now(), req.params.id);
         if (result.changes !== 1)
             return res.status(409).json({ error: 'deployment is no longer active' });
-        await audit(req.actor, { type: 'cancel', project: row.project_id, deploymentId: row.id, message: `deployment ${row.id} cancelled by ${req.actor.name}` });
+        await audit(req.actor, {
+            type: 'cancel',
+            project: row.project_id,
+            deploymentId: row.id,
+            message: `deployment ${row.id} cancelled by ${req.actor.name}`,
+        });
         res.json({ ok: true });
     }));
     api('post', '/api/v1/rollback', bodyProject, async (req, res) => withWriter(async () => {
         const project = req.body.project || 'site';
         try {
-            const out = await createRollback(project, { actor: req.actor.name, force: Boolean(req.body.force) });
-            await audit(req.actor, { type: 'rollback', project, deploymentId: out.deployment.id, message: `rollback to ${out.desiredState.version} (deployment ${out.deployment.id}) started by ${req.actor.name}`, metadata: { version: out.desiredState.version, force: Boolean(req.body.force) } });
+            const out = await createRollback(project, {
+                actor: req.actor.name,
+                force: Boolean(req.body.force),
+            });
+            await audit(req.actor, {
+                type: 'rollback',
+                project,
+                deploymentId: out.deployment.id,
+                message: `rollback to ${out.desiredState.version} (deployment ${out.deployment.id}) started by ${req.actor.name}`,
+                metadata: { version: out.desiredState.version, force: Boolean(req.body.force) },
+            });
             res.json(out);
         }
         catch (e) {
             sendError(res, e);
         }
     }));
-    api('get', '/api/v1/volumes', queryProject, async (req, res) => res.json({ volumes: await db.all('SELECT * FROM volumes WHERE project_id=? ORDER BY name', req.query.project || 'site') }));
+    api('get', '/api/v1/volumes', queryProject, async (req, res) => res.json({
+        volumes: await db.all('SELECT * FROM volumes WHERE project_id=? ORDER BY name', req.query.project || 'site'),
+    }));
     api('delete', '/api/v1/volumes/:name', queryProject, async (req, res) => {
         const project = String(req.query.project || 'site');
         const deleteData = String(req.query.deleteData || 'false') === 'true';
@@ -824,19 +1551,26 @@ export async function createApp(options = {}) {
                 return res.status(404).json({ error: 'volume not found' });
             const config = JSON.parse(volume.config_json || '{}');
             if ((config.persistent || volume.type !== 'ephemeral') && !deleteData)
-                return res.status(409).json({ error: 'refusing to delete persistent volume without deleteData=true' });
+                return res
+                    .status(409)
+                    .json({ error: 'refusing to delete persistent volume without deleteData=true' });
             const ds = await currentDesired(project);
             if (ds) {
                 const state = JSON.parse(ds.state_json);
-                const inUse = Object.entries(state.compose?.services || {}).filter(([, svc]) => (svc.volumes || []).some(v => (typeof v === 'string' ? v.split(':')[0] : v.source) === req.params.name)).map(([name]) => name);
+                const inUse = Object.entries(state.compose?.services || {})
+                    .filter(([, svc]) => (svc.volumes || []).some((v) => (typeof v === 'string' ? v.split(':')[0] : v.source) === req.params.name))
+                    .map(([name]) => name);
                 if (inUse.length)
-                    return res.status(409).json({ error: `volume is still referenced by services: ${inUse.join(', ')}` });
+                    return res
+                        .status(409)
+                        .json({ error: `volume is still referenced by services: ${inUse.join(', ')}` });
             }
             let commandId = null;
             if (deleteData && volume.node_id) {
-                commandId = id('cmd');
-                const ts = now();
-                await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', commandId, volume.node_id, project, 'volume.delete', 'pending', JSON.stringify({ project, volume: req.params.name }), 0, ts, null, ts, ts);
+                commandId = await queueCommand(volume.node_id, project, 'volume.delete', {
+                    project,
+                    volume: req.params.name,
+                });
             }
             await db.run('DELETE FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
             res.json({ ok: true, commandId, deletedMetadata: true, deleteData });
@@ -862,14 +1596,21 @@ export async function createApp(options = {}) {
         await fs.cp(artifactDir, path.join(staging, 'images'), { recursive: true });
         await fs.writeFile(path.join(staging, 'metadata.json'), JSON.stringify({ backupId, createdAt: ts, type: 'control' }, null, 2));
         const artifactRef = path.resolve(backupDir, `${backupId}.tar.gz`);
-        await tar.c({ gzip: true, file: artifactRef, cwd: staging }, ['tug.db', 'master.key', 'metadata.json', 'images']);
+        await tar.c({ gzip: true, file: artifactRef, cwd: staging }, [
+            'tug.db',
+            'master.key',
+            'metadata.json',
+            'images',
+        ]);
         await fs.rm(staging, { recursive: true, force: true });
         await db.run('INSERT INTO backups(id,project_id,target,node_id,state,artifact_ref,metadata_json,created_at,completed_at) VALUES(?,?,?,?,?,?,?,?,?)', backupId, '_control', 'control-plane', null, 'completed', artifactRef, JSON.stringify({ type: 'control' }), ts, now());
         res.json({ backupId, artifactRef, state: 'completed' });
     }));
     api('post', '/api/v1/control/restore', null, async (req, res) => withWriter(async () => {
         if (!req.body.confirm)
-            return res.status(400).json({ error: 'confirm=true is required; restore overwrites control data then requires restart' });
+            return res.status(400).json({
+                error: 'confirm=true is required; restore overwrites control data then requires restart',
+            });
         const artifactRef = req.body.artifactRef;
         if (typeof artifactRef !== 'string' || !artifactRef)
             return res.status(400).json({ error: 'artifactRef required' });
@@ -878,19 +1619,25 @@ export async function createApp(options = {}) {
         try {
             await fs.rm(restoreDir, { recursive: true, force: true });
             await ensureDir(restoreDir);
-            await scanArchive(artifactRef, entry => {
-                if (path.isAbsolute(entry.path) || entry.path.split('/').includes('..') || !['File', 'Directory'].includes(entry.type))
+            await scanArchive(artifactRef, (entry) => {
+                if (path.isAbsolute(entry.path) ||
+                    entry.path.split('/').includes('..') ||
+                    !['File', 'Directory'].includes(entry.type))
                     throw new Error('unsafe control backup entry');
             });
             await tar.x({ file: artifactRef, cwd: restoreDir, strict: true });
             const key = (await fs.readFile(path.join(restoreDir, 'master.key'), 'utf8')).trim();
             if (!/^[A-Za-z0-9+/]{43}=$/.test(key))
                 throw new Error('invalid backup master key');
-            snapshot = await open({ filename: path.join(restoreDir, 'tug.db'), driver: sqlite3.Database, mode: sqlite3.OPEN_READONLY });
+            snapshot = await open({
+                filename: path.join(restoreDir, 'tug.db'),
+                driver: sqlite3.Database,
+                mode: sqlite3.OPEN_READONLY,
+            });
             const integrity = await snapshot.get('PRAGMA integrity_check');
             if (integrity?.integrity_check !== 'ok')
                 throw new Error('backup database failed integrity check');
-            if (!await snapshot.get("SELECT 1 FROM meta WHERE key='operator_token_hash'"))
+            if (!(await snapshot.get("SELECT 1 FROM meta WHERE key='operator_token_hash'")))
                 throw new Error('backup database has no operator identity');
             for (const secret of await snapshot.all('SELECT * FROM secrets'))
                 decrypt(secret.encrypted_value, key);
@@ -901,10 +1648,14 @@ export async function createApp(options = {}) {
                     if (!service.artifact)
                         continue;
                     const build = await snapshot.get('SELECT * FROM builds WHERE id=? AND state=?', service.artifact.buildId, 'complete');
-                    if (!build?.artifact_path || !/^images\/build_[a-zA-Z0-9_-]+\.tar\.gz$/.test(build.artifact_path))
+                    if (!build?.artifact_path ||
+                        !/^images\/build_[a-zA-Z0-9_-]+\.tar\.gz$/.test(build.artifact_path))
                         throw new Error('backup contains an unsafe or missing image reference');
                     const actual = await hashArchive(path.join(restoreDir, build.artifact_path));
-                    if (actual.sha256 !== build.sha256 || actual.sizeBytes !== build.size_bytes || actual.sha256 !== service.artifact.sha256 || actual.sizeBytes !== service.artifact.sizeBytes)
+                    if (actual.sha256 !== build.sha256 ||
+                        actual.sizeBytes !== build.size_bytes ||
+                        actual.sha256 !== service.artifact.sha256 ||
+                        actual.sizeBytes !== service.artifact.sizeBytes)
                         throw new Error('backup image archive checksum mismatch');
                 }
             }
@@ -927,7 +1678,7 @@ export async function createApp(options = {}) {
             res.status(restartRequired ? 500 : 400).json({ error: errMessage(e), restartRequired });
         }
     }));
-    async function commandForService({ project, service, type, payload }) {
+    async function commandForService({ project, service, type, payload, }) {
         if (!service)
             throw httpError(400, 'service required');
         const ds = await currentDesired(project);
@@ -943,16 +1694,24 @@ export async function createApp(options = {}) {
         const node = await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
         if (!node)
             throw httpError(404, 'node not found');
-        const cmdId = id('cmd');
-        const ts = now();
-        await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', cmdId, node.id, project, type, 'pending', JSON.stringify({ project, service, ...payload }), 0, ts, null, ts, ts);
+        const cmdId = await queueCommand(node.id, project, type, { project, service, ...payload });
         return { commandId: cmdId, nodeId: node.id };
     }
     api('post', '/api/v1/restart', bodyProject, async (req, res) => {
         try {
             const project = req.body.project || 'site';
-            const queued = await commandForService({ project, service: req.body.service, type: 'service.restart', payload: {} });
-            await audit(req.actor, { type: 'restart', project, message: `service ${req.body.service} restart requested by ${req.actor.name}`, metadata: { service: req.body.service, commandId: queued.commandId } });
+            const queued = await commandForService({
+                project,
+                service: req.body.service,
+                type: 'service.restart',
+                payload: {},
+            });
+            await audit(req.actor, {
+                type: 'restart',
+                project,
+                message: `service ${req.body.service} restart requested by ${req.actor.name}`,
+                metadata: { service: req.body.service, commandId: queued.commandId },
+            });
             res.json(queued);
         }
         catch (e) {
@@ -978,9 +1737,7 @@ export async function createApp(options = {}) {
         const node = await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
         if (!node)
             return res.status(404).json({ error: 'node not found' });
-        const cmdId = id('cmd');
-        const ts = now();
-        await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', cmdId, node.id, project, 'logs.fetch', 'pending', JSON.stringify({ project, service, tail }), 0, ts, null, ts, ts);
+        const cmdId = await queueCommand(node.id, project, 'logs.fetch', { project, service, tail });
         res.json({ commandId: cmdId, nodeId: node.id });
     });
     api('get', '/api/v1/commands/:id', commandProject, async (req, res) => res.json(await db.get('SELECT * FROM commands WHERE id=?', req.params.id)));
@@ -991,7 +1748,12 @@ export async function createApp(options = {}) {
         const projects = projectScope(req.actor, 'id');
         const deployments = projectScope(req.actor, 'project_id');
         const statuses = projectScope(req.actor, 'project_id');
-        res.json({ projects: await db.all(`SELECT * FROM projects WHERE ${projects.where}`, ...projects.args), deployments: await db.all(`SELECT * FROM deployments WHERE ${deployments.where} ORDER BY created_at DESC LIMIT 20`, ...deployments.args), nodes: await db.all('SELECT id,name,status,last_seen_at FROM nodes'), statuses: await db.all(`SELECT * FROM node_statuses WHERE ${statuses.where}`, ...statuses.args) });
+        res.json({
+            projects: await db.all(`SELECT * FROM projects WHERE ${projects.where}`, ...projects.args),
+            deployments: await db.all(`SELECT * FROM deployments WHERE ${deployments.where} ORDER BY created_at DESC LIMIT 20`, ...deployments.args),
+            nodes: await db.all('SELECT id,name,status,last_seen_at FROM nodes'),
+            statuses: await db.all(`SELECT * FROM node_statuses WHERE ${statuses.where}`, ...statuses.args),
+        });
     });
     api('get', '/api/v1/events', null, async (req, res) => {
         const project = req.query.project ? String(req.query.project) : null;
@@ -1014,15 +1776,23 @@ export async function createApp(options = {}) {
                 checks.push({ name, ok: false, error: errMessage(e) });
             }
         }
-        await check('sqlite', async () => ({ value: (await db.get('SELECT COUNT(*) AS count FROM projects')).count }));
-        await check('operator_token', async () => ({ configured: Boolean(await db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash')) }));
+        await check('sqlite', async () => ({
+            value: (await db.get('SELECT COUNT(*) AS count FROM projects')).count,
+        }));
+        await check('operator_token', async () => ({
+            configured: Boolean(await db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash')),
+        }));
         await check('master_key', async () => ({ configured: Boolean(masterKey) }));
-        await check('docker', async () => ({ output: (await runCommand('docker', ['version', '--format', '{{.Server.Version}}'])).out.trim() }));
-        await check('docker_compose', async () => ({ output: (await runCommand('docker', ['compose', 'version'])).out.trim() }));
+        await check('docker', async () => ({
+            output: (await runCommand('docker', ['version', '--format', '{{.Server.Version}}'])).out.trim(),
+        }));
+        await check('docker_compose', async () => ({
+            output: (await runCommand('docker', ['compose', 'version'])).out.trim(),
+        }));
         if (MESH) {
             await check('mesh', async () => {
                 const nodes = await db.all('SELECT n.name, n.overlay_ip, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
-                const missing = nodes.filter(n => !n.has_key).map(n => n.name);
+                const missing = nodes.filter((n) => !n.has_key).map((n) => n.name);
                 if (missing.length)
                     throw new Error(`nodes without a reported WireGuard key (is TUG_MESH=true on their agents?): ${missing.join(', ')}`);
                 return { cidr: MESH_CIDR, port: WG_PORT, nodes: nodes.length };
@@ -1031,23 +1801,26 @@ export async function createApp(options = {}) {
         const ingressNodes = await db.all('SELECT name,ingress_status_json FROM nodes WHERE ingress=1');
         if (ingressNodes.length) {
             await check('ingress', async () => {
-                const bad = ingressNodes.filter(n => (n.ingress_status_json ? JSON.parse(n.ingress_status_json).state : 'unreported') !== 'serving').map(n => n.name);
+                const bad = ingressNodes
+                    .filter((n) => (n.ingress_status_json ? JSON.parse(n.ingress_status_json).state : 'unreported') !==
+                    'serving')
+                    .map((n) => n.name);
                 if (bad.length)
                     throw new Error(`ingress nodes not serving: ${bad.join(', ')}`);
-                return { nodes: ingressNodes.map(n => n.name) };
+                return { nodes: ingressNodes.map((n) => n.name) };
             });
         }
-        res.json({ ok: checks.every(c => c.ok), registry: REGISTRY, checks });
+        res.json({ ok: checks.every((c) => c.ok), registry: REGISTRY, checks });
     });
     api('post', '/api/v1/diagnostics/nodes', null, async (req, res) => {
-        const nodes = req.body.node ? await db.all('SELECT id FROM nodes WHERE id=? OR name=?', req.body.node, req.body.node) : await db.all('SELECT id FROM nodes');
+        const nodes = req.body.node
+            ? await db.all('SELECT id FROM nodes WHERE id=? OR name=?', req.body.node, req.body.node)
+            : await db.all('SELECT id FROM nodes');
         if (req.body.node && !nodes.length)
             return res.status(404).json({ error: `node ${req.body.node} not found` });
-        const ts = now();
         const commands = [];
         for (const node of nodes) {
-            const cmdId = id('cmd');
-            await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', cmdId, node.id, null, 'diagnostics.run', 'pending', '{}', 0, ts, null, ts, ts);
+            const cmdId = await queueCommand(node.id, null, 'diagnostics.run');
             commands.push({ commandId: cmdId, nodeId: node.id });
         }
         res.json({ commands });
@@ -1059,9 +1832,17 @@ export async function createApp(options = {}) {
     async function endpointsFor(states) {
         const staleBefore = new Date(Date.now() - Number(process.env.TUG_STALE_SECONDS || 60) * 1000).toISOString();
         const nodes = await db.all('SELECT n.*, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
-        const views = nodes.map(n => ({ name: n.name, overlayIp: MESH && n.has_key ? n.overlay_ip : null, address: n.address, lastSeenAt: n.last_seen_at }));
-        const statuses = (await db.all('SELECT node_id,project_id,status_json FROM node_statuses'))
-            .map(r => ({ node: nodes.find(n => n.id === r.node_id)?.name ?? r.node_id, project: r.project_id, status: JSON.parse(r.status_json) }));
+        const views = nodes.map((n) => ({
+            name: n.name,
+            overlayIp: MESH && n.has_key ? n.overlay_ip : null,
+            address: n.address,
+            lastSeenAt: n.last_seen_at,
+        }));
+        const statuses = (await db.all('SELECT node_id,project_id,status_json FROM node_statuses')).map((r) => ({
+            node: nodes.find((n) => n.id === r.node_id)?.name ?? r.node_id,
+            project: r.project_id,
+            status: JSON.parse(r.status_json),
+        }));
         return resolveServiceEndpoints(states, views, statuses, staleBefore);
     }
     async function networkFor(node) {
@@ -1069,10 +1850,39 @@ export async function createApp(options = {}) {
             return undefined;
         const peers = await db.all('SELECT w.*, n.name FROM wg_peers w JOIN nodes n ON n.id=w.node_id WHERE w.node_id != ? ORDER BY n.name', node.id);
         return {
-            interface: WG_INTERFACE, overlayIp: node.overlay_ip, cidr: MESH_CIDR, listenPort: WG_PORT,
-            peers: peers.map(p => ({ node: p.name, publicKey: p.public_key, endpoint: p.endpoint ?? undefined, allowedIps: p.allowed_ips.split(','), persistentKeepalive: WG_KEEPALIVE }))
+            interface: WG_INTERFACE,
+            overlayIp: node.overlay_ip,
+            cidr: MESH_CIDR,
+            listenPort: WG_PORT,
+            peers: peers.map((p) => ({
+                node: p.name,
+                publicKey: p.public_key,
+                endpoint: p.endpoint ?? undefined,
+                allowedIps: p.allowed_ips.split(','),
+                persistentKeepalive: WG_KEEPALIVE,
+            })),
         };
     }
+    /** Names of all nodes and of the ingress nodes, for port binding and upstream decisions. */
+    async function nodeTopology() {
+        const rows = await db.all('SELECT name,ingress FROM nodes ORDER BY name');
+        return {
+            nodes: rows.map((r) => r.name),
+            ingressNodes: rows.filter((r) => r.ingress).map((r) => r.name),
+        };
+    }
+    /** Where this node's published ports listen: the address other nodes and the ingress reach it on, or loopback. */
+    async function portBindingFor(node, states) {
+        const hasKey = MESH && node.overlay_ip && (await db.get('SELECT 1 FROM wg_peers WHERE node_id=?', node.id));
+        const { nodes, ingressNodes } = await nodeTopology();
+        const loopback = {};
+        for (const state of states) {
+            const services = loopbackIngressServices(state, node.name, ingressNodes, nodes);
+            if (services.length)
+                loopback[state.project] = services;
+        }
+        return { address: (hasKey ? node.overlay_ip : node.address) || null, loopback };
+    }
     async function ingressFor(node, states) {
         const secrets = new Map();
         for (const state of states) {
@@ -1087,24 +1897,49 @@ export async function createApp(options = {}) {
                 }
             }
         }
-        return buildIngressWork({ states, endpoints: await endpointsFor(states), node: node.name, secrets, acmeEmail: ACME_EMAIL });
+        return buildIngressWork({
+            states,
+            endpoints: await endpointsFor(states),
+            node: node.name,
+            ...(await nodeTopology()),
+            secrets,
+            acmeEmail: ACME_EMAIL,
+        });
     }
+    get('/agent/v1/wake', authAgent, async (req, res) => {
+        const after = Math.max(0, Number.parseInt(String(req.query.after ?? '0'), 10) || 0);
+        const timeoutMs = Math.min(30_000, Math.max(0, Number.parseInt(String(req.query.timeout ?? '25000'), 10) || 0));
+        await db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
+        res.json(await wakeHub.wait(req.node.id, after, timeoutMs));
+    });
     get('/agent/v1/work', authAgent, async (req, res) => {
         await db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
         const node = (await db.get('SELECT * FROM nodes WHERE id=?', req.node.id));
         const allStates = await currentDesiredStates();
-        const desiredStates = allStates.filter(st => Object.values(st.services).some(svc => (svc.placement?.nodes || []).includes(node.name) || !(svc.placement?.nodes?.length)));
+        const desiredStates = allStates.filter((st) => Object.values(st.services).some((svc) => (svc.placement?.nodes || []).includes(node.name) || !svc.placement?.nodes?.length));
         const commands = await db.all('SELECT * FROM commands WHERE node_id=? AND state=?', node.id, 'pending');
         if (commands.length)
             await db.run(`UPDATE commands SET state='running',attempts=attempts+1,updated_at=? WHERE node_id=? AND state='pending'`, now(), node.id);
-        const retainedImages = [...new Set((await retainedStates()).flatMap(state => Object.values(state.services).filter(service => service.artifact && (!service.placement.nodes?.length || service.placement.nodes.includes(node.name))).map(service => service.image)))];
-        const work = { serverTime: now(), nodeId: node.id, desiredStates, commands, retainedImages };
+        const retainedImages = [
+            ...new Set((await retainedStates()).flatMap((state) => Object.values(state.services)
+                .filter((service) => service.artifact &&
+                (!service.placement.nodes?.length || service.placement.nodes.includes(node.name)))
+                .map((service) => service.image))),
+        ];
+        const work = {
+            serverTime: now(),
+            nodeId: node.id,
+            desiredStates,
+            commands,
+            retainedImages,
+        };
         const network = await networkFor(node);
         if (network)
             work.network = network;
+        work.ports = await portBindingFor(node, desiredStates);
         if (SERVICE_DNS) {
             const all = await endpointsFor(allStates);
-            work.serviceEndpoints = Object.fromEntries(desiredStates.map(st => [st.project, all[st.project] || {}]));
+            work.serviceEndpoints = Object.fromEntries(desiredStates.map((st) => [st.project, all[st.project] || {}]));
         }
         if (node.ingress)
             work.ingress = await ingressFor(node, allStates);
@@ -1124,6 +1959,11 @@ export async function createApp(options = {}) {
         if (report.ingress)
             await db.run('UPDATE nodes SET ingress_status_json=? WHERE id=?', JSON.stringify({ ...report.ingress, reportedAt: ts }), req.node.id);
         await db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', ts, 'online', req.node.id);
+        const nextEndpointSnapshot = JSON.stringify(await endpointsFor(await currentDesiredStates()));
+        if (nextEndpointSnapshot !== endpointSnapshot) {
+            endpointSnapshot = nextEndpointSnapshot;
+            await wakeAll();
+        }
         res.json({ ok: true });
     });
     post('/agent/v1/events', authAgent, async (req, res) => {
@@ -1131,7 +1971,7 @@ export async function createApp(options = {}) {
         for (const ev of evs)
             await db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at) VALUES(?,?,?,?,?,?,?,?,?)', id('evt'), ev.projectId || null, ev.deploymentId || null, req.node.id, ev.level || 'info', ev.type || 'event', ev.message || '', JSON.stringify(ev.metadata || {}), now());
         if (Date.now() - lastEventPrune >= eventPruneIntervalMs)
-            await pruneEvents().catch(e => console.error(`event pruning failed: ${errMessage(e)}`));
+            await pruneEvents().catch((e) => console.error(`event pruning failed: ${errMessage(e)}`));
         res.json({ ok: true });
     });
     get('/agent/v1/secrets', authAgent, async (req, res) => {
@@ -1139,11 +1979,19 @@ export async function createApp(options = {}) {
         let rows = await db.all('SELECT name,encrypted_value,version FROM secrets WHERE project_id=?', project);
         if (req.query.services) {
             const ds = await currentDesired(project);
-            const names = String(req.query.services).split(',').map(s => s.trim()).filter(Boolean);
-            const allowed = ds ? serviceSecretNames(JSON.parse(ds.state_json).compose, names) : new Set();
-            rows = rows.filter(r => allowed.has(r.name));
+            const names = String(req.query.services)
+                .split(',')
+                .map((s) => s.trim())
+                .filter(Boolean);
+            const allowed = ds
+                ? serviceSecretNames(JSON.parse(ds.state_json).compose, names)
+                : new Set();
+            rows = rows.filter((r) => allowed.has(r.name));
         }
-        const secrets = { secrets: Object.fromEntries(rows.map(r => [r.name, decrypt(r.encrypted_value, masterKey)])), versions: Object.fromEntries(rows.map(r => [r.name, r.version])) };
+        const secrets = {
+            secrets: Object.fromEntries(rows.map((r) => [r.name, decrypt(r.encrypted_value, masterKey)])),
+            versions: Object.fromEntries(rows.map((r) => [r.name, r.version])),
+        };
         res.json(secrets);
     });
     app.use((error, _req, res, next) => {
@@ -1153,16 +2001,69 @@ export async function createApp(options = {}) {
     });
     return app;
 }
+function attachAgentStream(app, server) {
+    const runtime = streamRuntimes.get(app);
+    if (!runtime)
+        throw new Error('control app is missing its stream runtime');
+    const wss = new WebSocketServer({ noServer: true, maxPayload: LIVE_MAX_FRAME_BYTES });
+    server.on('upgrade', (request, socket, head) => {
+        const reject = (status, reason) => {
+            socket.write(`HTTP/1.1 ${status} ${reason}\r\nConnection: close\r\n\r\n`);
+            socket.destroy();
+        };
+        const pathname = new URL(request.url || '/', 'http://localhost').pathname;
+        if (!['/agent/v1/stream', '/api/v1/live'].includes(pathname)) {
+            reject(404, 'Not Found');
+            return;
+        }
+        const raw = String(request.headers.authorization || '').replace(/^Bearer /, '');
+        void (async () => {
+            if (pathname === '/api/v1/live') {
+                const actor = await runtime.authenticateOperator(raw);
+                if (!actor) {
+                    reject(401, 'Unauthorized');
+                    return;
+                }
+                if (!hasRole(actor, 'deploy')) {
+                    reject(403, 'Forbidden');
+                    return;
+                }
+                wss.handleUpgrade(request, socket, head, (ws) => runtime.live.operator(actor, ws));
+            }
+            else {
+                const node = await runtime.authenticate(raw);
+                if (!node) {
+                    reject(401, 'Unauthorized');
+                    return;
+                }
+                wss.handleUpgrade(request, socket, head, (ws) => {
+                    runtime.live.agent(node.id, ws);
+                    runtime.hub.connect(node.id, ws);
+                });
+            }
+        })().catch(() => reject(500, 'Internal Server Error'));
+    });
+    server.once('close', () => wss.close());
+}
 export async function listen(app, port = PORT) {
     const certFile = process.env.TUG_TLS_CERT_FILE;
     const keyFile = process.env.TUG_TLS_KEY_FILE;
     if (certFile && keyFile) {
         const [cert, key] = await Promise.all([fs.readFile(certFile), fs.readFile(keyFile)]);
         const server = https.createServer({ cert, key }, app);
-        return new Promise(resolve => server.listen(port, () => { console.log(`tug-control listening on ${port} (tls)`); resolve(server); }));
+        attachAgentStream(app, server);
+        return new Promise((resolve) => server.listen(port, () => {
+            console.log(`tug-control listening on ${port} (tls)`);
+            resolve(server);
+        }));
     }
     console.warn('TUG_TLS_CERT_FILE/TUG_TLS_KEY_FILE not set; serving plain HTTP');
-    return new Promise(resolve => { const server = app.listen(port, () => { console.log(`tug-control listening on ${port}`); resolve(server); }); });
+    const server = http.createServer(app);
+    attachAgentStream(app, server);
+    return new Promise((resolve) => server.listen(port, () => {
+        console.log(`tug-control listening on ${port}`);
+        resolve(server);
+    }));
 }
 if (import.meta.url === `file://${process.argv[1]}`)
-    createApp().then(app => listen(app));
+    createApp().then((app) => listen(app));