@wpelevator/tug 0.0.0 → 0.2.0

package/dist/control.js

@@ -0,0 +1,1105 @@
+#!/usr/bin/env node
+import express from 'express';
+import multer from 'multer';
+import fs from 'node:fs/promises';
+import { createReadStream } from 'node:fs';
+import { pipeline } from 'node:stream/promises';
+import { saveImageArchive, inspectImage, validateImageArchive, hashArchive } from "./artifacts.js";
+import sqlite3 from 'sqlite3';
+import { open } from 'sqlite';
+import { scanArchive } from "./archive.js";
+import https from 'node:https';
+import path from 'node:path';
+import * as tar from 'tar';
+import { openDb } from "./db.js";
+import { encrypt, decrypt, loadOrCreateMasterKey } from "./secrets.js";
+import { buildDesiredState, serviceSecretNames, validateCompose } from "./compose.js";
+import { buildAndPush, unpackContext, run as runCommand } from "./build.js";
+import { ensureDir, errMessage, hashToken, id, now, token } from "./util.js";
+import { DEFAULT_MESH_CIDR, DEFAULT_WG_PORT, WG_INTERFACE, WG_KEEPALIVE, allocateOverlayIp, isWgKey, parseCidr, wgEndpoint } from "./mesh.js";
+import { buildIngressWork, resolveServiceEndpoints } from "./ingress.js";
+import { ACTIVE_DEPLOYMENT_STATES } from "./protocol.js";
+import { AUTO_ROLLBACK_ACTOR, ROUTE_ROLES, assertProject, bodyProject, createGuard, createToken, listTokens, parseTokenRequest, queryProject, recordEvent, revokeToken } from "./tokens.js";
+const httpError = (status, message) => Object.assign(new Error(message), { status });
+/** Versions of rollback deployments start with this; `tug deploy` may not use it, which is how automatic rollback tells them apart. */
+const ROLLBACK_VERSION_PREFIX = 'rollback-';
+const ACTIVE_STATES_SQL = ACTIVE_DEPLOYMENT_STATES.map(s => `'${s}'`).join(',');
+const PORT = Number(process.env.PORT || 8080);
+export async function createApp(options = {}) {
+    const images = { build: buildAndPush, save: saveImageArchive, inspect: inspectImage, validate: validateImageArchive, run: runCommand, ...options.images };
+    const DATA_DIR = process.env.TUG_DATA_DIR || './data';
+    const MAX_IMAGE_BYTES = Number(process.env.TUG_MAX_IMAGE_BYTES || 5 * 1024 ** 3);
+    if (!Number.isSafeInteger(MAX_IMAGE_BYTES) || MAX_IMAGE_BYTES < 1)
+        throw new Error('TUG_MAX_IMAGE_BYTES must be a positive safe integer');
+    const upload = multer({ dest: process.env.TUG_UPLOAD_DIR || path.join(DATA_DIR, 'uploads'), limits: { fileSize: MAX_IMAGE_BYTES, files: 64, fields: 1, fieldSize: 1024 ** 2 } });
+    const dbFile = path.join(DATA_DIR, 'tug.db');
+    const masterKeyFile = path.join(DATA_DIR, 'master.key');
+    const REGISTRY = process.env.TUG_REGISTRY?.trim() || '';
+    const REGISTRY_AUTH_PATH = process.env.TUG_REGISTRY_AUTH_PATH || null;
+    const REGISTRY_CONTAINER = process.env.TUG_REGISTRY_CONTAINER || null;
+    // Networking layers are opt-in; with all of them off the work response is unchanged.
+    const MESH = process.env.TUG_MESH === 'true';
+    const MESH_CIDR = process.env.TUG_MESH_CIDR || DEFAULT_MESH_CIDR;
+    const WG_PORT = Number(process.env.TUG_WG_PORT || DEFAULT_WG_PORT);
+    const SERVICE_DNS = (process.env.TUG_SERVICE_DNS ?? String(MESH)) === 'true';
+    const ACME_EMAIL = process.env.TUG_ACME_EMAIL || undefined;
+    if (MESH)
+        parseCidr(MESH_CIDR);
+    let htpasswdQueue = Promise.resolve();
+    function withHtpasswdLock(fn) {
+        const next = htpasswdQueue.catch(() => { }).then(fn);
+        htpasswdQueue = next.then(() => { }, () => { });
+        return next;
+    }
+    async function reloadRegistry() {
+        if (!REGISTRY_CONTAINER)
+            return;
+        await runCommand('docker', ['restart', REGISTRY_CONTAINER]).catch(e => console.error(`registry reload failed: ${e.message}`));
+    }
+    async function upsertRegistryCredential(name, password) {
+        if (!REGISTRY || !REGISTRY_AUTH_PATH)
+            return;
+        await withHtpasswdLock(async () => {
+            const { out } = await runCommand('htpasswd', ['-nbB', name, password]);
+            const line = out.trim();
+            let existing = '';
+            try {
+                existing = await fs.readFile(REGISTRY_AUTH_PATH, 'utf8');
+            }
+            catch { }
+            const lines = existing.split('\n').filter(Boolean).filter(l => !l.startsWith(`${name}:`));
+            lines.push(line);
+            await ensureDir(path.dirname(REGISTRY_AUTH_PATH));
+            await fs.writeFile(REGISTRY_AUTH_PATH, `${lines.join('\n')}\n`);
+        });
+        await reloadRegistry();
+    }
+    async function removeRegistryCredential(name) {
+        if (!REGISTRY_AUTH_PATH)
+            return;
+        await withHtpasswdLock(async () => {
+            let existing = '';
+            try {
+                existing = await fs.readFile(REGISTRY_AUTH_PATH, 'utf8');
+            }
+            catch { }
+            const lines = existing.split('\n').filter(Boolean).filter(l => !l.startsWith(`${name}:`));
+            await ensureDir(path.dirname(REGISTRY_AUTH_PATH));
+            await fs.writeFile(REGISTRY_AUTH_PATH, lines.length ? `${lines.join('\n')}\n` : '');
+        });
+        await reloadRegistry();
+    }
+    const db = await (options.openDatabase || openDb)(dbFile);
+    const masterKey = await loadOrCreateMasterKey(masterKeyFile);
+    const operator = await db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash');
+    if (!operator) {
+        const t = process.env.TUG_OPERATOR_TOKEN || token('tug_op');
+        await db.run('INSERT INTO meta(key,value) VALUES(?,?)', 'operator_token_hash', hashToken(t));
+        await fs.writeFile(path.join(DATA_DIR, 'operator.token'), `${t}\n`, { mode: 0o600 }).catch(() => { });
+        console.log(`operator token: ${t}`);
+    }
+    const app = express();
+    let restartRequired = false;
+    app.use((_req, res, next) => restartRequired ? res.status(503).json({ error: 'control restored; restart required', restartRequired: true }) : next());
+    app.use(express.json({ limit: '20mb' }));
+    // Hold the writer for the operation lifetime, including after an HTTP disconnect.
+    let writer = Promise.resolve();
+    async function withWriter(operation) {
+        const previous = writer;
+        let release;
+        writer = new Promise(resolve => { release = resolve; });
+        await previous;
+        try {
+            if (restartRequired)
+                throw httpError(503, 'control restored; restart required');
+            return await operation();
+        }
+        finally {
+            release();
+        }
+    }
+    let publication = Promise.resolve();
+    async function withPublication(operation) {
+        const previous = publication;
+        let release;
+        publication = new Promise(resolve => { release = resolve; });
+        await previous;
+        try {
+            return await operation();
+        }
+        finally {
+            release();
+        }
+    }
+    const safe = (handler) => (req, res, next) => {
+        try {
+            void Promise.resolve(handler(req, res, next)).catch(next);
+        }
+        catch (error) {
+            next(error);
+        }
+    };
+    const route = (method, url, handlers) => app[method](url, ...handlers.map(safe));
+    const get = (url, ...handlers) => route('get', url, handlers);
+    const post = (url, ...handlers) => route('post', url, handlers);
+    // Operator-facing routes go through api(): the role table in tokens.ts is the only place a route's access is declared.
+    const guard = createGuard(db);
+    const apiRoutes = new Map();
+    app.locals.apiRoutes = apiRoutes;
+    const api = (method, url, resolve, ...handlers) => {
+        const key = `${method.toUpperCase()} ${url}`;
+        const role = ROUTE_ROLES[key];
+        if (!role || role === 'public')
+            throw new Error(`API route ${key} has no declared role in ROUTE_ROLES`);
+        apiRoutes.set(key, role);
+        route(method, url, [guard(role, resolve ?? undefined), ...handlers]);
+    };
+    const deploymentProject = async (req) => (await db.get('SELECT project_id FROM deployments WHERE id=?', req.params.id))?.project_id;
+    const commandProject = async (req) => { const row = await db.get('SELECT project_id FROM commands WHERE id=?', req.params.id); return row ? row.project_id : undefined; };
+    /** Audit trail entry for a control action; bookkeeping failures are logged, never returned to the caller. */
+    const audit = (actor, event) => recordEvent(db, { ...event, actor: actor.name }).catch(e => console.error(`audit event failed: ${errMessage(e)}`));
+    const projectScope = (actor, column) => actor.projects ? { where: `${column} IN (${actor.projects.map(() => '?').join(',') || "''"})`, args: actor.projects } : { where: '1=1', args: [] };
+    const authAgent = async (req, res, next) => {
+        const raw = (req.headers.authorization || '').replace(/^Bearer /, '');
+        const node = raw && await db.get('SELECT * FROM nodes WHERE agent_token_hash=?', hashToken(raw));
+        if (!node)
+            return res.status(401).json({ error: 'unauthorized' });
+        req.node = node;
+        next();
+    };
+    async function updateDeploymentConvergence(projectId) {
+        const project = await db.get('SELECT * FROM projects WHERE id=?', projectId);
+        if (!project?.current_deployment_id)
+            return;
+        const ds = await db.get('SELECT * FROM desired_states WHERE deployment_id=?', project.current_deployment_id);
+        if (!ds)
+            return;
+        const state = JSON.parse(ds.state_json);
+        const required = new Set();
+        for (const svc of Object.values(state.services || {}))
+            for (const node of svc.placement?.nodes || [])
+                required.add(node);
+        if (!required.size)
+            return;
+        const statuses = await db.all('SELECT node_id,observed_version,health,status_json FROM node_statuses WHERE project_id=?', projectId);
+        const byNode = new Map(statuses.map(s => [s.node_id, s]));
+        const reportsRevision = (node) => state.revision ? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision : byNode.get(node)?.observed_version === state.version;
+        const failed = [...required].some(node => reportsRevision(node) && byNode.get(node)?.health === 'failed');
+        if (failed) {
+            const marked = await db.run('UPDATE deployments SET state=?,error_message=?,updated_at=? WHERE id=? AND state=?', 'failed', 'one or more assigned nodes reported failed health', now(), project.current_deployment_id, 'rolling_out');
+            if (marked.changes === 1)
+                scheduleAutoRollback(projectId, project.current_deployment_id, state);
+        }
+        const ok = [...required].every(node => (state.revision ? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision : byNode.get(node)?.observed_version === state.version) && ['serving', 'skipped'].includes(byNode.get(node)?.health ?? ''));
+        if (ok)
+            await db.run('UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state=?', 'succeeded', now(), project.current_deployment_id, 'rolling_out');
+    }
+    /** Events control writes when it rolls back on its own; they are attributed to a reserved system actor. */
+    async function recordAutoRollbackEvent(projectId, deploymentId, type, message, metadata = {}) {
+        await db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at,actor) VALUES(?,?,?,?,?,?,?,?,?,?)', id('evt'), projectId, deploymentId, null, 'warn', type, message, JSON.stringify(metadata), now(), AUTO_ROLLBACK_ACTOR);
+    }
+    /**
+     * Creates a rollback deployment that copies the most recent earlier deployment's desired state under a
+     * new revision. Shared by `POST /api/v1/rollback` and automatic rollback; the caller holds the writer queue.
+     * `auto` only considers `succeeded` deployments and requires `failedDeploymentId` to still be current.
+     */
+    async function createRollback(project, opts) {
+        let lock;
+        try {
+            lock = await acquireProjectLock(project, opts.auto ? 'auto-rollback' : 'rollback');
+            const active = await db.get(`SELECT id,state FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) ORDER BY created_at DESC LIMIT 1`, project);
+            if (active && !opts.force)
+                throw httpError(409, `deployment ${active.id} is already ${active.state}; retry with force to rollback anyway`);
+            const current = await db.get('SELECT current_deployment_id FROM projects WHERE id=?', project);
+            if (opts.failedDeploymentId && current?.current_deployment_id !== opts.failedDeploymentId)
+                throw httpError(409, `deployment ${opts.failedDeploymentId} is no longer the current deployment`);
+            const states = opts.auto ? `'succeeded'` : `'succeeded','rolling_out'`;
+            const previousDs = await db.get(`SELECT ds.*, d.version AS deployment_version FROM desired_states ds JOIN deployments d ON d.id=ds.deployment_id
+        WHERE ds.project_id=? AND ds.deployment_id != ? AND d.state IN (${states}) ORDER BY ds.created_at DESC LIMIT 1`, project, current?.current_deployment_id || '');
+            if (!previousDs)
+                throw httpError(404, 'no previous deployment found');
+            const ts = now();
+            const depId = id('dep');
+            const previousState = JSON.parse(previousDs.state_json);
+            previousState.revision = depId;
+            await db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, `${ROLLBACK_VERSION_PREFIX}${previousState.version}-${Date.now()}`, 'rolling_out', ts, ts, opts.actor);
+            await db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, previousState.version, JSON.stringify(previousState), ts);
+            await db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
+            return { deployment: await db.get('SELECT * FROM deployments WHERE id=?', depId), desiredState: previousState };
+        }
+        finally {
+            await releaseLock(lock);
+        }
+    }
+    /**
+     * With `x-tug.rollback: auto`, a deployment that just turned `failed` is rolled back to the previous
+     * `succeeded` one. Rollback deployments are never rolled back (their versions carry a reserved prefix
+     * that `tug deploy` cannot use), so a bad rollback target cannot loop. Runs in the background so a
+     * busy writer queue never blocks agent status reports; the outcome is recorded as an event either way.
+     */
+    function scheduleAutoRollback(projectId, deploymentId, state) {
+        if (state.compose['x-tug']?.rollback !== 'auto')
+            return;
+        void (async () => {
+            const failed = await db.get('SELECT * FROM deployments WHERE id=?', deploymentId);
+            if (failed?.version.startsWith(ROLLBACK_VERSION_PREFIX)) {
+                await recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `rollback deployment ${failed.version} failed; not rolling back a rollback`);
+                return;
+            }
+            try {
+                const out = await withWriter(() => createRollback(projectId, { actor: AUTO_ROLLBACK_ACTOR, auto: true, failedDeploymentId: deploymentId }));
+                await recordAutoRollbackEvent(projectId, out.deployment.id, 'deployment.auto_rollback', `deployment ${deploymentId} (${failed?.version}) reported failed health; rolled back to ${out.desiredState.version} as deployment ${out.deployment.id}`, { failedDeploymentId: deploymentId, rollbackDeploymentId: out.deployment.id, version: out.desiredState.version });
+            }
+            catch (e) {
+                await recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `deployment ${deploymentId} failed but automatic rollback was not possible: ${errMessage(e)}`).catch(() => { });
+            }
+        })().catch(e => console.error('auto rollback failed:', errMessage(e)));
+    }
+    async function currentDesired(projectId) {
+        return db.get('SELECT ds.* FROM desired_states ds JOIN projects p ON p.current_deployment_id=ds.deployment_id WHERE p.id=?', projectId);
+    }
+    async function acquireProjectLock(project, operation, ttlMs = Number(process.env.TUG_LOCK_TTL_MS || 10 * 60 * 1000)) {
+        const name = `project:${project}`;
+        const ts = now();
+        await db.run('DELETE FROM locks WHERE expires_at < ?', ts);
+        const owner = `${operation}:${id('lock')}`;
+        try {
+            await db.run('INSERT INTO locks(name,owner,expires_at,created_at) VALUES(?,?,?,?)', name, owner, new Date(Date.now() + ttlMs).toISOString(), ts);
+        }
+        catch (e) {
+            if (!/UNIQUE constraint/.test(errMessage(e)))
+                throw e;
+            const existing = await db.get('SELECT * FROM locks WHERE name=?', name);
+            const err = new Error(`project ${project} is locked by ${existing?.owner} until ${existing?.expires_at}`);
+            err.status = 409;
+            err.lock = existing;
+            throw err;
+        }
+        const heartbeat = setInterval(() => { void db.run('UPDATE locks SET expires_at=? WHERE name=? AND owner=?', new Date(Date.now() + ttlMs).toISOString(), name, owner).catch(() => { }); }, Math.max(10, Math.floor(ttlMs / 3)));
+        heartbeat.unref();
+        return { name, owner, heartbeat };
+    }
+    async function releaseLock(lock) {
+        if (lock?.heartbeat)
+            clearInterval(lock.heartbeat);
+        if (lock)
+            await db.run('DELETE FROM locks WHERE name=? AND owner=?', lock.name, lock.owner).catch(() => { });
+    }
+    function sendError(res, e) { const err = e; res.status(restartRequired ? 503 : err.status || 500).json({ error: err.message, lock: err.lock || undefined }); }
+    get('/healthz', (req, res) => res.json({ ok: true }));
+    api('post', '/api/v1/projects', req => String(req.body?.name || 'site'), async (req, res) => {
+        const name = req.body.name || 'site';
+        const ts = now();
+        await db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', name, name, ts, ts);
+        res.json(await db.get('SELECT * FROM projects WHERE id=?', name));
+    });
+    api('get', '/api/v1/projects/:project', req => req.params.project, async (req, res) => res.json(await db.get('SELECT * FROM projects WHERE id=?', req.params.project)));
+    const NODE_INFO_SQL = 'SELECT id,name,address,public_address,overlay_ip,status,created_at,updated_at FROM nodes WHERE id=?';
+    /** Allocates the node's stable overlay IP on first registration (retrying if a concurrent register took it). */
+    async function ensureOverlayIp(nodeId) {
+        if (!MESH)
+            return;
+        for (let attempt = 0; attempt < 5; attempt++) {
+            const row = await db.get('SELECT overlay_ip FROM nodes WHERE id=?', nodeId);
+            if (row?.overlay_ip)
+                return;
+            const used = (await db.all('SELECT overlay_ip FROM nodes WHERE overlay_ip IS NOT NULL')).map(r => r.overlay_ip);
+            try {
+                await db.run('UPDATE nodes SET overlay_ip=? WHERE id=? AND overlay_ip IS NULL', allocateOverlayIp(MESH_CIDR, used), nodeId);
+                return;
+            }
+            catch (e) {
+                if (!/UNIQUE constraint/.test(errMessage(e)))
+                    throw e;
+            }
+        }
+        throw new Error('could not allocate an overlay address');
+    }
+    /** Stores (or refreshes) the node's WireGuard peer entry; the private key never reaches control. */
+    async function syncWgPeer(nodeId, publicKey) {
+        if (!MESH)
+            return;
+        const node = await db.get('SELECT * FROM nodes WHERE id=?', nodeId);
+        const existing = await db.get('SELECT * FROM wg_peers WHERE node_id=?', nodeId);
+        const key = publicKey ?? existing?.public_key;
+        if (!node?.overlay_ip || !key)
+            return;
+        await db.run(`INSERT INTO wg_peers(node_id,public_key,overlay_ip,endpoint,allowed_ips,updated_at) VALUES(?,?,?,?,?,?)
+      ON CONFLICT(node_id) DO UPDATE SET public_key=excluded.public_key,overlay_ip=excluded.overlay_ip,endpoint=excluded.endpoint,allowed_ips=excluded.allowed_ips,updated_at=excluded.updated_at`, nodeId, key, node.overlay_ip, wgEndpoint(node.public_address, WG_PORT) ?? null, `${node.overlay_ip}/32`, now());
+    }
+    async function registerNode({ name, address, publicAddress, wgPublicKey }) {
+        if (wgPublicKey !== undefined && !isWgKey(wgPublicKey))
+            throw httpError(400, 'wgPublicKey must be a base64 WireGuard public key');
+        const agentToken = token('tug_agent');
+        const registryToken = token('tug_reg');
+        const ts = now();
+        await db.run(`INSERT INTO nodes(id,name,address,public_address,status,agent_token_hash,registry_token_hash,created_at,updated_at)
+      VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET name=excluded.name,address=excluded.address,public_address=excluded.public_address,agent_token_hash=excluded.agent_token_hash,registry_token_hash=excluded.registry_token_hash,updated_at=excluded.updated_at`, name, name, address || null, publicAddress || null, 'registered', hashToken(agentToken), hashToken(registryToken), ts, ts);
+        await ensureOverlayIp(name);
+        await syncWgPeer(name, wgPublicKey);
+        await upsertRegistryCredential(name, registryToken);
+        const node = (await db.get(NODE_INFO_SQL, name));
+        return { node, agentToken, registryToken };
+    }
+    api('post', '/api/v1/nodes/register', null, async (req, res) => {
+        const name = req.body.name;
+        if (!name)
+            return res.status(400).json({ error: 'name required' });
+        try {
+            res.json(await registerNode({ name, address: req.body.address, publicAddress: req.body.publicAddress, wgPublicKey: req.body.wgPublicKey }));
+        }
+        catch (e) {
+            sendError(res, e);
+        }
+    });
+    // Self-service enrollment: operator mints a short-lived, limited-use key; the agent
+    // exchanges it for durable credentials on first boot (Portainer Edge-style join).
+    api('post', '/api/v1/nodes/enroll-token', null, async (req, res) => {
+        const name = req.body.name || null;
+        const ttlSeconds = Math.max(1, Number(req.body.ttlSeconds ?? 3600));
+        const uses = Math.max(1, Number(req.body.uses ?? 1));
+        const enrollToken = token('tug_enroll');
+        const expiresAt = new Date(Date.now() + ttlSeconds * 1000).toISOString();
+        await db.run('INSERT INTO enroll_tokens(token_hash,name,uses_remaining,expires_at,created_at) VALUES(?,?,?,?,?)', hashToken(enrollToken), name, uses, expiresAt, now());
+        res.json({ enrollToken, name, usesRemaining: uses, expiresAt, registry: REGISTRY });
+    });
+    post('/agent/v1/enroll', async (req, res) => {
+        const body = req.body;
+        const raw = String(body.enrollToken || (req.headers.authorization || '').replace(/^Bearer /, '')).trim();
+        if (!raw)
+            return res.status(400).json({ error: 'enrollToken required' });
+        const row = await db.get('SELECT * FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
+        if (!row || row.uses_remaining < 1 || row.expires_at < now()) {
+            if (row)
+                await db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
+            return res.status(401).json({ error: 'invalid or expired enroll token' });
+        }
+        const name = row.name || body.name;
+        if (!name)
+            return res.status(400).json({ error: 'node name required (bind one to the enroll token or send name)' });
+        let result;
+        try {
+            result = await registerNode({ name, address: body.address, publicAddress: body.publicAddress, wgPublicKey: body.wgPublicKey });
+        }
+        catch (e) {
+            return sendError(res, e);
+        }
+        const remaining = row.uses_remaining - 1;
+        if (remaining < 1)
+            await db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
+        else
+            await db.run('UPDATE enroll_tokens SET uses_remaining=? WHERE token_hash=?', remaining, hashToken(raw));
+        const enrolled = { ...result, registry: REGISTRY };
+        res.json(enrolled);
+    });
+    api('get', '/api/v1/nodes', null, async (req, res) => res.json({ nodes: await db.all('SELECT id,name,address,public_address,overlay_ip,ingress,ingress_status_json,status,last_seen_at,created_at,updated_at FROM nodes ORDER BY name') }));
+    api('post', '/api/v1/nodes/:name/rotate-registry-token', null, async (req, res) => {
+        const node = await db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
+        if (!node)
+            return res.status(404).json({ error: 'node not found' });
+        const registryToken = token('tug_reg');
+        await db.run('UPDATE nodes SET registry_token_hash=?,updated_at=? WHERE id=?', hashToken(registryToken), now(), node.id);
+        await upsertRegistryCredential(node.name, registryToken);
+        res.json({ ok: true, node: node.name, registryToken });
+    });
+    api('post', '/api/v1/nodes/:name/revoke-registry-token', null, async (req, res) => {
+        const node = await db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
+        if (!node)
+            return res.status(404).json({ error: 'node not found' });
+        await db.run('UPDATE nodes SET registry_token_hash=NULL,updated_at=? WHERE id=?', now(), node.id);
+        await removeRegistryCredential(node.name);
+        res.json({ ok: true, node: node.name, revoked: true });
+    });
+    async function findNode(nameOrId) { return db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', nameOrId, nameOrId); }
+    function queueCommand(nodeId, projectId, type, payload = {}) {
+        const commandId = id('cmd');
+        const ts = now();
+        return db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', commandId, nodeId, projectId, type, 'pending', JSON.stringify(payload), 0, ts, null, ts, ts).then(() => commandId);
+    }
+    // The private key lives only on the node, so rotation is a command the agent executes: it
+    // generates a new keypair, reports the public key and peers pick it up from their next work response.
+    api('post', '/api/v1/nodes/:name/rotate-wg-key', null, async (req, res) => {
+        if (!MESH)
+            return res.status(409).json({ error: 'the WireGuard mesh is not enabled (TUG_MESH=true)' });
+        const node = await findNode(req.params.name);
+        if (!node)
+            return res.status(404).json({ error: 'node not found' });
+        res.json({ commandId: await queueCommand(node.id, null, 'wg.rotate-key'), nodeId: node.id });
+    });
+    api('post', '/api/v1/nodes/:name/ingress', null, async (req, res) => {
+        const node = await findNode(req.params.name);
+        if (!node)
+            return res.status(404).json({ error: 'node not found' });
+        const enabled = req.body.enabled !== false;
+        await db.run('UPDATE nodes SET ingress=?,updated_at=? WHERE id=?', enabled ? 1 : 0, now(), node.id);
+        res.json({ ok: true, node: node.name, ingress: enabled });
+    });
+    post('/agent/v1/wireguard/key', authAgent, async (req, res) => {
+        if (!MESH)
+            return res.status(409).json({ error: 'the WireGuard mesh is not enabled' });
+        const { publicKey } = req.body;
+        if (!isWgKey(publicKey))
+            return res.status(400).json({ error: 'publicKey must be a base64 WireGuard public key' });
+        try {
+            await syncWgPeer(req.node.id, publicKey);
+            res.json({ ok: true });
+        }
+        catch (e) {
+            sendError(res, /UNIQUE constraint/.test(errMessage(e)) ? httpError(409, 'public key already registered to another node') : e);
+        }
+    });
+    post('/api/v1/registry/auth', async (req, res) => {
+        const username = req.body.username;
+        const password = req.body.password;
+        const node = username && password ? await db.get('SELECT id,name FROM nodes WHERE (name=? OR id=?) AND registry_token_hash IS NOT NULL AND registry_token_hash=?', username, username, hashToken(password)) : null;
+        if (!node)
+            return res.status(401).json({ ok: false, error: 'invalid registry credentials' });
+        res.json({ ok: true, node: node.name });
+    });
+    api('post', '/api/v1/tokens', null, async (req, res) => {
+        const request = parseTokenRequest(req.body);
+        const created = await createToken(db, request, req.actor.name);
+        await audit(req.actor, { type: 'token.create', message: `token ${created.name} (${created.role}${created.projects ? `, projects: ${created.projects.join(', ')}` : ''}) created by ${req.actor.name}`, metadata: { name: created.name, role: created.role, projects: created.projects, expiresAt: created.expiresAt } });
+        res.json(created);
+    });
+    api('get', '/api/v1/tokens', null, async (_req, res) => res.json({ tokens: await listTokens(db) }));
+    api('delete', '/api/v1/tokens/:name', null, async (req, res) => {
+        if (!await revokeToken(db, req.params.name))
+            return res.status(404).json({ error: 'token not found' });
+        await audit(req.actor, { type: 'token.revoke', message: `token ${req.params.name} revoked by ${req.actor.name}`, metadata: { name: req.params.name } });
+        res.json({ ok: true, name: req.params.name });
+    });
+    api('post', '/api/v1/secrets', bodyProject, async (req, res) => {
+        const { project = 'site', name, value } = req.body;
+        if (!name || value == null)
+            return res.status(400).json({ error: 'name and value required' });
+        let lock;
+        try {
+            lock = await acquireProjectLock(project, 'secret.set');
+            const old = await db.get('SELECT version FROM secrets WHERE project_id=? AND name=?', project, name);
+            const ts = now();
+            await db.run(`INSERT INTO secrets(id,project_id,name,encrypted_value,version,created_at,updated_at) VALUES(?,?,?,?,?,?,?)
+        ON CONFLICT(project_id,name) DO UPDATE SET encrypted_value=excluded.encrypted_value,version=excluded.version,updated_at=excluded.updated_at`, id('sec'), project, name, encrypt(value, masterKey), (old?.version || 0) + 1, ts, ts);
+            await audit(req.actor, { type: 'secret.set', project, message: `secret ${name} set to version ${(old?.version || 0) + 1} by ${req.actor.name}`, metadata: { name, version: (old?.version || 0) + 1 } });
+            res.json({ ok: true, name, version: (old?.version || 0) + 1 });
+        }
+        catch (e) {
+            sendError(res, e);
+        }
+        finally {
+            await releaseLock(lock);
+        }
+    });
+    api('get', '/api/v1/secrets', queryProject, async (req, res) => res.json({ secrets: await db.all('SELECT project_id,name,version,updated_at FROM secrets WHERE project_id=? ORDER BY name', req.query.project || 'site') }));
+    api('delete', '/api/v1/secrets/:name', queryProject, async (req, res) => {
+        const project = String(req.query.project || 'site');
+        const removed = await db.run('DELETE FROM secrets WHERE project_id=? AND name=?', project, req.params.name);
+        if (removed.changes)
+            await audit(req.actor, { type: 'secret.delete', project, message: `secret ${req.params.name} deleted by ${req.actor.name}`, metadata: { name: req.params.name } });
+        res.json({ ok: true });
+    });
+    const artifactDir = path.join(DATA_DIR, 'images');
+    await ensureDir(artifactDir);
+    async function retainedStates() {
+        // All published history is retained. This conservative superset protects failed/lagging versions too.
+        return (await db.all('SELECT * FROM desired_states')).map(row => JSON.parse(row.state_json));
+    }
+    async function collectArtifacts() {
+        const states = await retainedStates();
+        const refs = new Set(states.flatMap(state => Object.values(state.services).flatMap(service => service.artifact ? [service.artifact.buildId] : [])));
+        const imageRefs = new Set(states.flatMap(state => Object.values(state.services).map(service => service.image)));
+        const rows = await db.all('SELECT * FROM builds');
+        for (const row of rows) {
+            if (refs.has(row.id) || imageRefs.has(row.image_ref))
+                continue;
+            const active = await db.get("SELECT 1 FROM deployments WHERE id=? AND state IN ('uploading','building')", row.deployment_id);
+            if (active)
+                continue;
+            await fs.rm(path.join(DATA_DIR, row.artifact_path || path.join('images', `${row.id}.tar.gz`)), { force: true });
+            await images.run('docker', ['image', 'rm', row.image_ref]).catch(() => { });
+            await db.run('DELETE FROM builds WHERE id=?', row.id);
+        }
+        const remaining = await db.all('SELECT id FROM builds');
+        const owned = new Set(remaining.flatMap(row => [`${row.id}.tar.gz`, `${row.id}.tar.gz.tmp`]));
+        for (const name of await fs.readdir(artifactDir))
+            if (!owned.has(name))
+                await fs.rm(path.join(artifactDir, name), { force: true });
+    }
+    // Previous-process builds cannot finish after a restart; clear their unpublished files.
+    await db.run("UPDATE deployments SET state='build_failed',error_message='control restarted during build' WHERE state IN ('uploading','building')");
+    await collectArtifacts();
+    const receiveDeployment = upload.any();
+    api('post', '/api/v1/deployments', null, (req, res) => {
+        receiveDeployment(req, res, error => {
+            void withWriter(() => deployRequest(req, res, error)).catch(async (failure) => {
+                const files = Array.isArray(req.files) ? req.files : [];
+                await Promise.all(files.map(file => fs.rm(file.path, { force: true }).catch(() => { })));
+                sendError(res, failure);
+            });
+        });
+    });
+    async function deployRequest(req, res, uploadError) {
+        const files = (Array.isArray(req.files) ? req.files : []);
+        let responseBody;
+        let responseError;
+        let lock;
+        let depId;
+        let contextDir;
+        try {
+            if (uploadError)
+                throw httpError(400, errMessage(uploadError));
+            let body;
+            try {
+                body = req.is('multipart/form-data') ? JSON.parse(req.body.payload) : req.body;
+            }
+            catch {
+                throw httpError(400, 'invalid deployment payload JSON');
+            }
+            if (!body || typeof body !== 'object')
+                throw httpError(400, 'deployment payload is required');
+            const project = body.project || 'site';
+            const compose = body.compose;
+            const version = body.version || `git-${Date.now()}`;
+            assertProject(req.actor, project);
+            if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(project) || typeof version !== 'string' || !version.length)
+                throw httpError(400, 'invalid project or version');
+            if (version.startsWith(ROLLBACK_VERSION_PREFIX))
+                throw httpError(400, `version cannot start with "${ROLLBACK_VERSION_PREFIX}" (reserved for rollback deployments)`);
+            const nodes = await db.all('SELECT name FROM nodes ORDER BY created_at,name');
+            try {
+                validateCompose(compose, nodes.map(n => n.name));
+            }
+            catch (e) {
+                throw httpError(400, errMessage(e));
+            }
+            const builtServices = Object.entries(compose.services).filter(([, service]) => service.build).map(([name]) => name);
+            const local = body.localImages;
+            const context = files.find(file => file.fieldname === 'context');
+            if (local !== undefined && (!Array.isArray(local) || context))
+                throw httpError(400, 'localImages must be an array and cannot accompany a source context');
+            if (files.filter(file => file.fieldname === 'context').length > 1)
+                throw httpError(400, 'duplicate source context');
+            if (local) {
+                if (local.some(item => !item || typeof item !== 'object'))
+                    throw httpError(400, 'invalid local image manifest');
+                if (local.length !== builtServices.length || new Set(local.map(item => item.service)).size !== local.length || new Set(local.map(item => item.field)).size !== local.length)
+                    throw httpError(400, 'localImages must contain exactly one image for each build service');
+                for (const item of local) {
+                    if (!/^tug-build\/[a-f0-9-]{36}\/image-\d+:local$/.test(item.imageRef) || item.imageRef.split('/').at(-1) !== `${item.field}:local` || !builtServices.includes(item.service) || !/^image-\d+$/.test(item.field) || files.filter(file => file.fieldname === item.field).length !== 1 || !/^[a-f0-9]{64}$/.test(item.sha256) || !Number.isSafeInteger(item.sizeBytes) || item.sizeBytes <= 0 || item.sizeBytes > MAX_IMAGE_BYTES)
+                        throw httpError(400, 'invalid local image manifest or file');
+                }
+            }
+            if (files.length !== (local ? local.length : context ? 1 : 0))
+                throw httpError(400, 'unexpected deployment upload');
+            if (!REGISTRY && builtServices.length && !context && !local)
+                throw httpError(400, 'build services require a source context or localImages upload without a registry');
+            for (const [svcName, svc] of Object.entries(compose.services)) {
+                const ing = svc['x-tug']?.ingress;
+                if (ing?.tls !== 'custom')
+                    continue;
+                for (const secret of [ing.cert, ing.key])
+                    if (!await db.get('SELECT 1 FROM secrets WHERE project_id=? AND name=?', project, secret))
+                        throw httpError(400, `${svcName}: custom TLS secret ${secret} not found in project ${project}; set it with secret-set first`);
+            }
+            lock = await acquireProjectLock(project, 'deploy');
+            const active = await db.get(`SELECT * FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) LIMIT 1`, project);
+            if (active)
+                throw httpError(409, `deployment ${active.id} is already ${active.state}`);
+            depId = id('dep');
+            const ts = now();
+            await db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', project, project, ts, ts);
+            await db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, version, 'building', ts, ts, req.actor.name);
+            const desired = buildDesiredState({ project, version, revision: depId, compose, registry: REGISTRY, defaultNode: nodes[0]?.name || null });
+            if (context) {
+                contextDir = path.resolve(DATA_DIR, 'build-contexts', depId);
+                await unpackContext(context.path, contextDir);
+            }
+            for (const service of builtServices) {
+                if (!contextDir && !local) {
+                    desired.services[service].image = `${REGISTRY}/${project}/${service}:${version}`;
+                    continue;
+                } // Explicit registry/no-build-upload means an existing pushed image.
+                const spec = desired.services[service];
+                const buildId = id('build');
+                const imageRef = spec.image;
+                await db.run('INSERT INTO builds(id,project_id,deployment_id,service,image_ref,state,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)', buildId, project, depId, service, imageRef, 'building', ts, ts);
+                let info;
+                let importedClientRef;
+                const relative = path.join('images', `${buildId}.tar.gz`);
+                const final = path.join(DATA_DIR, relative);
+                const temporary = `${final}.tmp`;
+                try {
+                    if (local) {
+                        const item = local.find(item => item.service === service);
+                        const file = files.find(file => file.fieldname === item.field);
+                        try {
+                            info = await images.validate(file.path, item.imageRef);
+                        }
+                        catch (error) {
+                            throw httpError(400, `${service}: invalid image archive: ${errMessage(error)}`);
+                        }
+                        if (info.sha256 !== item.sha256 || info.sizeBytes !== item.sizeBytes || info.imageId !== item.imageId || info.platform !== item.platform)
+                            throw httpError(400, `${service}: local image metadata/checksum mismatch`);
+                        importedClientRef = item.imageRef;
+                        await images.run('docker', ['load', '-i', file.path]);
+                        const imported = await images.inspect(item.imageRef);
+                        if (imported.platform !== info.platform)
+                            throw httpError(400, 'imported image identity mismatch');
+                        await images.run('docker', ['tag', item.imageRef, imageRef]);
+                    }
+                    else
+                        await images.build({ contextDir: contextDir, service, buildSpec: compose.services[service].build, imageRef, push: false });
+                    if (REGISTRY) {
+                        if (process.env.TUG_PUSH !== 'false')
+                            await images.run('docker', ['push', imageRef]);
+                        const inspected = await images.inspect(imageRef);
+                        await db.run('UPDATE builds SET state=?,image_id=?,platform=?,updated_at=? WHERE id=?', 'complete', inspected.imageId, inspected.platform, now(), buildId);
+                    }
+                    else {
+                        info = await images.save(imageRef, temporary);
+                        await fs.rename(temporary, final);
+                        const artifact = { buildId, imageRef, ...info };
+                        spec.artifact = artifact;
+                        await db.run('UPDATE builds SET state=?,image_id=?,platform=?,sha256=?,size_bytes=?,artifact_path=?,updated_at=? WHERE id=?', 'complete', info.imageId, info.platform, info.sha256, info.sizeBytes, relative, now(), buildId);
+                    }
+                }
+                finally {
+                    if (importedClientRef)
+                        await images.run('docker', ['image', 'rm', importedClientRef]).catch(() => { });
+                    await fs.rm(temporary, { force: true });
+                }
+            }
+            const buildLock = lock;
+            await withPublication(async () => {
+                const owned = await db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now());
+                const pending = await db.get('SELECT 1 FROM deployments WHERE id=? AND state=?', depId, 'building');
+                if (!owned || !pending)
+                    throw httpError(409, 'deployment was cancelled or its build lease was lost');
+                await db.exec('BEGIN IMMEDIATE');
+                try {
+                    const published = await db.run("UPDATE deployments SET state='rolling_out',updated_at=? WHERE id=? AND state='building'", now(), depId);
+                    if (published.changes !== 1 || !await db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now()))
+                        throw httpError(409, 'deployment was cancelled or its build lease was lost');
+                    await db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, version, JSON.stringify(desired), ts);
+                    for (const [volumeName, volumeSpec] of Object.entries(compose.volumes || {})) {
+                        const lp = volumeSpec?.['x-tug'] || {};
+                        const nodeName = lp.placement?.nodes?.[0] || null;
+                        const node = nodeName ? await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName) : null;
+                        await db.run(`INSERT INTO volumes(id,project_id,name,type,driver,node_id,config_json,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(project_id,name) DO UPDATE SET type=excluded.type,driver=excluded.driver,node_id=excluded.node_id,config_json=excluded.config_json,updated_at=excluded.updated_at`, id('vol'), project, volumeName, lp.type || 'local', lp.driver || null, node?.id || null, JSON.stringify(lp), ts, ts);
+                    }
+                    await db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
+                    await db.exec('COMMIT');
+                }
+                catch (e) {
+                    await db.exec('ROLLBACK');
+                    throw e;
+                }
+            });
+            await audit(req.actor, { type: 'deploy', project, deploymentId: depId, message: `deployment ${depId} (${version}) started by ${req.actor.name}`, metadata: { version } });
+            responseBody = { deployment: { id: depId, project, version, state: 'rolling_out', actor: req.actor.name }, desiredState: desired };
+        }
+        catch (e) {
+            if (depId)
+                await db.run("UPDATE deployments SET state='build_failed',error_message=?,updated_at=? WHERE id=? AND state='building'", errMessage(e), now(), depId).catch(() => { });
+            responseError = e;
+        }
+        finally {
+            await releaseLock(lock);
+            await Promise.all(files.map(file => fs.rm(file.path, { force: true }).catch(() => { })));
+            if (contextDir)
+                await fs.rm(contextDir, { recursive: true, force: true });
+            await collectArtifacts().catch(e => console.error('image cleanup failed:', errMessage(e)));
+        }
+        if (responseError)
+            sendError(res, responseError);
+        else
+            res.json(responseBody);
+    }
+    get('/agent/v1/images/:buildId', authAgent, async (req, res) => {
+        const states = await retainedStates();
+        const allowed = states.some(state => Object.values(state.services).some(service => service.artifact?.buildId === req.params.buildId && (!service.placement.nodes?.length || service.placement.nodes.includes(req.node.name))));
+        if (!allowed)
+            return res.status(404).json({ error: 'image not found' });
+        const row = await db.get('SELECT * FROM builds WHERE id=? AND state=?', req.params.buildId, 'complete');
+        if (!row?.artifact_path)
+            return res.status(404).json({ error: 'image not found' });
+        try {
+            await fs.access(path.join(DATA_DIR, row.artifact_path));
+            res.set({ 'Content-Type': 'application/gzip', 'Content-Length': String(row.size_bytes), 'X-Tug-Sha256': row.sha256, 'Cache-Control': 'private, no-store' });
+            await pipeline(createReadStream(path.join(DATA_DIR, row.artifact_path)), res);
+        }
+        catch {
+            if (!res.headersSent)
+                res.status(404).json({ error: 'image artifact unavailable' });
+        }
+    });
+    api('get', '/api/v1/deployments/:id', deploymentProject, async (req, res) => res.json(await db.get('SELECT * FROM deployments WHERE id=?', req.params.id)));
+    api('post', '/api/v1/deployments/:id/cancel', deploymentProject, async (req, res) => withPublication(async () => {
+        const row = await db.get('SELECT * FROM deployments WHERE id=?', req.params.id);
+        if (!row)
+            return res.status(404).json({ error: 'deployment not found' });
+        if (!ACTIVE_DEPLOYMENT_STATES.includes(row.state))
+            return res.status(409).json({ error: `deployment is ${row.state}` });
+        const result = await db.run(`UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state IN (${ACTIVE_STATES_SQL})`, 'cancelled', now(), req.params.id);
+        if (result.changes !== 1)
+            return res.status(409).json({ error: 'deployment is no longer active' });
+        await audit(req.actor, { type: 'cancel', project: row.project_id, deploymentId: row.id, message: `deployment ${row.id} cancelled by ${req.actor.name}` });
+        res.json({ ok: true });
+    }));
+    api('post', '/api/v1/rollback', bodyProject, async (req, res) => withWriter(async () => {
+        const project = req.body.project || 'site';
+        try {
+            const out = await createRollback(project, { actor: req.actor.name, force: Boolean(req.body.force) });
+            await audit(req.actor, { type: 'rollback', project, deploymentId: out.deployment.id, message: `rollback to ${out.desiredState.version} (deployment ${out.deployment.id}) started by ${req.actor.name}`, metadata: { version: out.desiredState.version, force: Boolean(req.body.force) } });
+            res.json(out);
+        }
+        catch (e) {
+            sendError(res, e);
+        }
+    }));
+    api('get', '/api/v1/volumes', queryProject, async (req, res) => res.json({ volumes: await db.all('SELECT * FROM volumes WHERE project_id=? ORDER BY name', req.query.project || 'site') }));
+    api('delete', '/api/v1/volumes/:name', queryProject, async (req, res) => {
+        const project = String(req.query.project || 'site');
+        const deleteData = String(req.query.deleteData || 'false') === 'true';
+        let lock;
+        try {
+            lock = deleteData ? await acquireProjectLock(project, 'volume.delete') : null;
+            const volume = await db.get('SELECT * FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
+            if (!volume)
+                return res.status(404).json({ error: 'volume not found' });
+            const config = JSON.parse(volume.config_json || '{}');
+            if ((config.persistent || volume.type !== 'ephemeral') && !deleteData)
+                return res.status(409).json({ error: 'refusing to delete persistent volume without deleteData=true' });
+            const ds = await currentDesired(project);
+            if (ds) {
+                const state = JSON.parse(ds.state_json);
+                const inUse = Object.entries(state.compose?.services || {}).filter(([, svc]) => (svc.volumes || []).some(v => (typeof v === 'string' ? v.split(':')[0] : v.source) === req.params.name)).map(([name]) => name);
+                if (inUse.length)
+                    return res.status(409).json({ error: `volume is still referenced by services: ${inUse.join(', ')}` });
+            }
+            let commandId = null;
+            if (deleteData && volume.node_id) {
+                commandId = id('cmd');
+                const ts = now();
+                await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', commandId, volume.node_id, project, 'volume.delete', 'pending', JSON.stringify({ project, volume: req.params.name }), 0, ts, null, ts, ts);
+            }
+            await db.run('DELETE FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
+            res.json({ ok: true, commandId, deletedMetadata: true, deleteData });
+        }
+        catch (e) {
+            sendError(res, e);
+        }
+        finally {
+            await releaseLock(lock);
+        }
+    });
+    api('post', '/api/v1/control/backups/run', null, async (req, res) => withWriter(async () => {
+        const backupDir = process.env.TUG_CONTROL_BACKUP_DIR || path.join(DATA_DIR, 'control-backups');
+        await ensureDir(backupDir);
+        const ts = now();
+        const backupId = id('cbak');
+        const staging = path.join(backupDir, `${backupId}-staging`);
+        await fs.rm(staging, { recursive: true, force: true });
+        await ensureDir(staging);
+        const snapshotPath = path.join(staging, 'tug.db').replaceAll("'", "''");
+        await db.exec(`VACUUM INTO '${snapshotPath}'`);
+        await fs.copyFile(masterKeyFile, path.join(staging, 'master.key'));
+        await fs.cp(artifactDir, path.join(staging, 'images'), { recursive: true });
+        await fs.writeFile(path.join(staging, 'metadata.json'), JSON.stringify({ backupId, createdAt: ts, type: 'control' }, null, 2));
+        const artifactRef = path.resolve(backupDir, `${backupId}.tar.gz`);
+        await tar.c({ gzip: true, file: artifactRef, cwd: staging }, ['tug.db', 'master.key', 'metadata.json', 'images']);
+        await fs.rm(staging, { recursive: true, force: true });
+        await db.run('INSERT INTO backups(id,project_id,target,node_id,state,artifact_ref,metadata_json,created_at,completed_at) VALUES(?,?,?,?,?,?,?,?,?)', backupId, '_control', 'control-plane', null, 'completed', artifactRef, JSON.stringify({ type: 'control' }), ts, now());
+        res.json({ backupId, artifactRef, state: 'completed' });
+    }));
+    api('post', '/api/v1/control/restore', null, async (req, res) => withWriter(async () => {
+        if (!req.body.confirm)
+            return res.status(400).json({ error: 'confirm=true is required; restore overwrites control data then requires restart' });
+        const artifactRef = req.body.artifactRef;
+        if (typeof artifactRef !== 'string' || !artifactRef)
+            return res.status(400).json({ error: 'artifactRef required' });
+        const restoreDir = process.env.TUG_CONTROL_RESTORE_DIR || path.join(DATA_DIR, 'control-restore');
+        let snapshot;
+        try {
+            await fs.rm(restoreDir, { recursive: true, force: true });
+            await ensureDir(restoreDir);
+            await scanArchive(artifactRef, entry => {
+                if (path.isAbsolute(entry.path) || entry.path.split('/').includes('..') || !['File', 'Directory'].includes(entry.type))
+                    throw new Error('unsafe control backup entry');
+            });
+            await tar.x({ file: artifactRef, cwd: restoreDir, strict: true });
+            const key = (await fs.readFile(path.join(restoreDir, 'master.key'), 'utf8')).trim();
+            if (!/^[A-Za-z0-9+/]{43}=$/.test(key))
+                throw new Error('invalid backup master key');
+            snapshot = await open({ filename: path.join(restoreDir, 'tug.db'), driver: sqlite3.Database, mode: sqlite3.OPEN_READONLY });
+            const integrity = await snapshot.get('PRAGMA integrity_check');
+            if (integrity?.integrity_check !== 'ok')
+                throw new Error('backup database failed integrity check');
+            if (!await snapshot.get("SELECT 1 FROM meta WHERE key='operator_token_hash'"))
+                throw new Error('backup database has no operator identity');
+            for (const secret of await snapshot.all('SELECT * FROM secrets'))
+                decrypt(secret.encrypted_value, key);
+            const states = await snapshot.all('SELECT * FROM desired_states');
+            for (const stateRow of states) {
+                const state = JSON.parse(stateRow.state_json);
+                for (const service of Object.values(state.services)) {
+                    if (!service.artifact)
+                        continue;
+                    const build = await snapshot.get('SELECT * FROM builds WHERE id=? AND state=?', service.artifact.buildId, 'complete');
+                    if (!build?.artifact_path || !/^images\/build_[a-zA-Z0-9_-]+\.tar\.gz$/.test(build.artifact_path))
+                        throw new Error('backup contains an unsafe or missing image reference');
+                    const actual = await hashArchive(path.join(restoreDir, build.artifact_path));
+                    if (actual.sha256 !== build.sha256 || actual.sizeBytes !== build.size_bytes || actual.sha256 !== service.artifact.sha256 || actual.sizeBytes !== service.artifact.sizeBytes)
+                        throw new Error('backup image archive checksum mismatch');
+                }
+            }
+            await snapshot.close();
+            snapshot = undefined;
+            await ensureDir(path.join(restoreDir, 'images'));
+            // Validation completes before changing any live file. Stop DB-dependent work until restart.
+            restartRequired = true;
+            await db.close();
+            await fs.rm(`${dbFile}-wal`, { force: true });
+            await fs.rm(`${dbFile}-shm`, { force: true });
+            await fs.cp(path.join(restoreDir, 'images'), artifactDir, { recursive: true });
+            await fs.copyFile(path.join(restoreDir, 'tug.db'), dbFile);
+            await fs.copyFile(path.join(restoreDir, 'master.key'), masterKeyFile);
+            res.json({ ok: true, restored: true, restartRequired: true });
+        }
+        catch (e) {
+            if (snapshot)
+                await snapshot.close().catch(() => { });
+            res.status(restartRequired ? 500 : 400).json({ error: errMessage(e), restartRequired });
+        }
+    }));
+    async function commandForService({ project, service, type, payload }) {
+        if (!service)
+            throw httpError(400, 'service required');
+        const ds = await currentDesired(project);
+        if (!ds)
+            throw httpError(404, 'project has no desired state');
+        const state = JSON.parse(ds.state_json);
+        const svc = state.services?.[service];
+        if (!svc)
+            throw httpError(404, 'service not found');
+        const nodeName = svc.placement?.nodes?.[0];
+        if (!nodeName)
+            throw httpError(400, 'service has no explicit node placement');
+        const node = await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
+        if (!node)
+            throw httpError(404, 'node not found');
+        const cmdId = id('cmd');
+        const ts = now();
+        await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', cmdId, node.id, project, type, 'pending', JSON.stringify({ project, service, ...payload }), 0, ts, null, ts, ts);
+        return { commandId: cmdId, nodeId: node.id };
+    }
+    api('post', '/api/v1/restart', bodyProject, async (req, res) => {
+        try {
+            const project = req.body.project || 'site';
+            const queued = await commandForService({ project, service: req.body.service, type: 'service.restart', payload: {} });
+            await audit(req.actor, { type: 'restart', project, message: `service ${req.body.service} restart requested by ${req.actor.name}`, metadata: { service: req.body.service, commandId: queued.commandId } });
+            res.json(queued);
+        }
+        catch (e) {
+            res.status(e.status || 500).json({ error: errMessage(e) });
+        }
+    });
+    api('post', '/api/v1/logs', bodyProject, async (req, res) => {
+        const project = req.body.project || 'site';
+        const service = req.body.service;
+        const tail = Number(req.body.tail || 200);
+        if (!service)
+            return res.status(400).json({ error: 'service required' });
+        const ds = await currentDesired(project);
+        if (!ds)
+            return res.status(404).json({ error: 'project has no desired state' });
+        const state = JSON.parse(ds.state_json);
+        const svc = state.services?.[service];
+        if (!svc)
+            return res.status(404).json({ error: 'service not found' });
+        const nodeName = svc.placement?.nodes?.[0];
+        if (!nodeName)
+            return res.status(400).json({ error: 'service has no explicit node placement' });
+        const node = await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
+        if (!node)
+            return res.status(404).json({ error: 'node not found' });
+        const cmdId = id('cmd');
+        const ts = now();
+        await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', cmdId, node.id, project, 'logs.fetch', 'pending', JSON.stringify({ project, service, tail }), 0, ts, null, ts, ts);
+        res.json({ commandId: cmdId, nodeId: node.id });
+    });
+    api('get', '/api/v1/commands/:id', commandProject, async (req, res) => res.json(await db.get('SELECT * FROM commands WHERE id=?', req.params.id)));
+    api('get', '/api/v1/status', null, async (req, res) => {
+        const staleSeconds = Number(process.env.TUG_STALE_SECONDS || 60);
+        const cutoff = new Date(Date.now() - staleSeconds * 1000).toISOString();
+        await db.run("UPDATE nodes SET status='stale',updated_at=? WHERE last_seen_at IS NOT NULL AND last_seen_at < ?", now(), cutoff);
+        const projects = projectScope(req.actor, 'id');
+        const deployments = projectScope(req.actor, 'project_id');
+        const statuses = projectScope(req.actor, 'project_id');
+        res.json({ projects: await db.all(`SELECT * FROM projects WHERE ${projects.where}`, ...projects.args), deployments: await db.all(`SELECT * FROM deployments WHERE ${deployments.where} ORDER BY created_at DESC LIMIT 20`, ...deployments.args), nodes: await db.all('SELECT id,name,status,last_seen_at FROM nodes'), statuses: await db.all(`SELECT * FROM node_statuses WHERE ${statuses.where}`, ...statuses.args) });
+    });
+    api('get', '/api/v1/events', null, async (req, res) => {
+        const project = req.query.project ? String(req.query.project) : null;
+        if (project)
+            assertProject(req.actor, project);
+        const limit = Math.min(Number(req.query.limit || 100), 500);
+        const scope = projectScope(req.actor, 'project_id'); // scoped tokens never see node-level or other projects' events
+        const events = project
+            ? await db.all('SELECT * FROM events WHERE project_id=? ORDER BY created_at DESC LIMIT ?', project, limit)
+            : await db.all(`SELECT * FROM events WHERE ${scope.where} ORDER BY created_at DESC LIMIT ?`, ...scope.args, limit);
+        res.json({ events });
+    });
+    api('get', '/api/v1/diagnostics', null, async (req, res) => {
+        const checks = [];
+        async function check(name, fn) {
+            try {
+                checks.push({ name, ok: true, ...(await fn()) });
+            }
+            catch (e) {
+                checks.push({ name, ok: false, error: errMessage(e) });
+            }
+        }
+        await check('sqlite', async () => ({ value: (await db.get('SELECT COUNT(*) AS count FROM projects')).count }));
+        await check('operator_token', async () => ({ configured: Boolean(await db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash')) }));
+        await check('master_key', async () => ({ configured: Boolean(masterKey) }));
+        await check('docker', async () => ({ output: (await runCommand('docker', ['version', '--format', '{{.Server.Version}}'])).out.trim() }));
+        await check('docker_compose', async () => ({ output: (await runCommand('docker', ['compose', 'version'])).out.trim() }));
+        if (MESH) {
+            await check('mesh', async () => {
+                const nodes = await db.all('SELECT n.name, n.overlay_ip, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
+                const missing = nodes.filter(n => !n.has_key).map(n => n.name);
+                if (missing.length)
+                    throw new Error(`nodes without a reported WireGuard key (is TUG_MESH=true on their agents?): ${missing.join(', ')}`);
+                return { cidr: MESH_CIDR, port: WG_PORT, nodes: nodes.length };
+            });
+        }
+        const ingressNodes = await db.all('SELECT name,ingress_status_json FROM nodes WHERE ingress=1');
+        if (ingressNodes.length) {
+            await check('ingress', async () => {
+                const bad = ingressNodes.filter(n => (n.ingress_status_json ? JSON.parse(n.ingress_status_json).state : 'unreported') !== 'serving').map(n => n.name);
+                if (bad.length)
+                    throw new Error(`ingress nodes not serving: ${bad.join(', ')}`);
+                return { nodes: ingressNodes.map(n => n.name) };
+            });
+        }
+        res.json({ ok: checks.every(c => c.ok), registry: REGISTRY, checks });
+    });
+    api('post', '/api/v1/diagnostics/nodes', null, async (req, res) => {
+        const nodes = req.body.node ? await db.all('SELECT id FROM nodes WHERE id=? OR name=?', req.body.node, req.body.node) : await db.all('SELECT id FROM nodes');
+        const ts = now();
+        const commands = [];
+        for (const node of nodes) {
+            const cmdId = id('cmd');
+            await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', cmdId, node.id, null, 'diagnostics.run', 'pending', '{}', 0, ts, null, ts, ts);
+            commands.push({ commandId: cmdId, nodeId: node.id });
+        }
+        res.json({ commands });
+    });
+    async function currentDesiredStates() {
+        const rows = await db.all(`SELECT ds.* FROM desired_states ds JOIN projects p ON p.current_deployment_id=ds.deployment_id ORDER BY ds.created_at DESC`);
+        return rows.map((s) => JSON.parse(s.state_json));
+    }
+    async function endpointsFor(states) {
+        const staleBefore = new Date(Date.now() - Number(process.env.TUG_STALE_SECONDS || 60) * 1000).toISOString();
+        const nodes = await db.all('SELECT n.*, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
+        const views = nodes.map(n => ({ name: n.name, overlayIp: MESH && n.has_key ? n.overlay_ip : null, address: n.address, lastSeenAt: n.last_seen_at }));
+        const statuses = (await db.all('SELECT node_id,project_id,status_json FROM node_statuses'))
+            .map(r => ({ node: nodes.find(n => n.id === r.node_id)?.name ?? r.node_id, project: r.project_id, status: JSON.parse(r.status_json) }));
+        return resolveServiceEndpoints(states, views, statuses, staleBefore);
+    }
+    async function networkFor(node) {
+        if (!MESH || !node.overlay_ip)
+            return undefined;
+        const peers = await db.all('SELECT w.*, n.name FROM wg_peers w JOIN nodes n ON n.id=w.node_id WHERE w.node_id != ? ORDER BY n.name', node.id);
+        return {
+            interface: WG_INTERFACE, overlayIp: node.overlay_ip, cidr: MESH_CIDR, listenPort: WG_PORT,
+            peers: peers.map(p => ({ node: p.name, publicKey: p.public_key, endpoint: p.endpoint ?? undefined, allowedIps: p.allowed_ips.split(','), persistentKeepalive: WG_KEEPALIVE }))
+        };
+    }
+    async function ingressFor(node, states) {
+        const secrets = new Map();
+        for (const state of states) {
+            for (const svc of Object.values(state.compose.services || {})) {
+                const ing = svc['x-tug']?.ingress;
+                if (ing?.tls !== 'custom')
+                    continue;
+                for (const name of [ing.cert, ing.key]) {
+                    const row = await db.get('SELECT encrypted_value FROM secrets WHERE project_id=? AND name=?', state.project, name);
+                    if (row)
+                        secrets.set(`${state.project}/${name}`, decrypt(row.encrypted_value, masterKey));
+                }
+            }
+        }
+        return buildIngressWork({ states, endpoints: await endpointsFor(states), node: node.name, secrets, acmeEmail: ACME_EMAIL });
+    }
+    get('/agent/v1/work', authAgent, async (req, res) => {
+        await db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
+        const node = (await db.get('SELECT * FROM nodes WHERE id=?', req.node.id));
+        const allStates = await currentDesiredStates();
+        const desiredStates = allStates.filter(st => Object.values(st.services).some(svc => (svc.placement?.nodes || []).includes(node.name) || !(svc.placement?.nodes?.length)));
+        const commands = await db.all('SELECT * FROM commands WHERE node_id=? AND state=?', node.id, 'pending');
+        if (commands.length)
+            await db.run(`UPDATE commands SET state='running',attempts=attempts+1,updated_at=? WHERE node_id=? AND state='pending'`, now(), node.id);
+        const retainedImages = [...new Set((await retainedStates()).flatMap(state => Object.values(state.services).filter(service => service.artifact && (!service.placement.nodes?.length || service.placement.nodes.includes(node.name))).map(service => service.image)))];
+        const work = { serverTime: now(), nodeId: node.id, desiredStates, commands, retainedImages };
+        const network = await networkFor(node);
+        if (network)
+            work.network = network;
+        if (SERVICE_DNS) {
+            const all = await endpointsFor(allStates);
+            work.serviceEndpoints = Object.fromEntries(desiredStates.map(st => [st.project, all[st.project] || {}]));
+        }
+        if (node.ingress)
+            work.ingress = await ingressFor(node, allStates);
+        res.json(work);
+    });
+    post('/agent/v1/status', authAgent, async (req, res) => {
+        const ts = now();
+        const report = req.body;
+        for (const st of report.projects || []) {
+            await db.run(`INSERT INTO node_statuses(id,node_id,project_id,observed_version,status_json,health,updated_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(node_id,project_id) DO UPDATE SET observed_version=excluded.observed_version,status_json=excluded.status_json,health=excluded.health,updated_at=excluded.updated_at`, id('ns'), req.node.id, st.project, st.observedVersion, JSON.stringify(st), st.health || 'unknown', ts);
+            await updateDeploymentConvergence(st.project);
+        }
+        for (const cr of (report.commandResults || [])) {
+            const result = cr.result || {};
+            await db.run('UPDATE commands SET state=?,result_json=?,updated_at=? WHERE id=? AND node_id=?', cr.state || 'succeeded', JSON.stringify(result), ts, cr.commandId, req.node.id);
+        }
+        if (report.ingress)
+            await db.run('UPDATE nodes SET ingress_status_json=? WHERE id=?', JSON.stringify({ ...report.ingress, reportedAt: ts }), req.node.id);
+        await db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', ts, 'online', req.node.id);
+        res.json({ ok: true });
+    });
+    post('/agent/v1/events', authAgent, async (req, res) => { const evs = req.body.events || [req.body]; for (const ev of evs)
+        await db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at) VALUES(?,?,?,?,?,?,?,?,?)', id('evt'), ev.projectId || null, ev.deploymentId || null, req.node.id, ev.level || 'info', ev.type || 'event', ev.message || '', JSON.stringify(ev.metadata || {}), now()); res.json({ ok: true }); });
+    get('/agent/v1/secrets', authAgent, async (req, res) => {
+        const project = String(req.query.project || 'site');
+        let rows = await db.all('SELECT name,encrypted_value,version FROM secrets WHERE project_id=?', project);
+        if (req.query.services) {
+            const ds = await currentDesired(project);
+            const names = String(req.query.services).split(',').map(s => s.trim()).filter(Boolean);
+            const allowed = ds ? serviceSecretNames(JSON.parse(ds.state_json).compose, names) : new Set();
+            rows = rows.filter(r => allowed.has(r.name));
+        }
+        const secrets = { secrets: Object.fromEntries(rows.map(r => [r.name, decrypt(r.encrypted_value, masterKey)])), versions: Object.fromEntries(rows.map(r => [r.name, r.version])) };
+        res.json(secrets);
+    });
+    app.use((error, _req, res, next) => {
+        if (res.headersSent)
+            return next(error);
+        sendError(res, error);
+    });
+    return app;
+}
+export async function listen(app, port = PORT) {
+    const certFile = process.env.TUG_TLS_CERT_FILE;
+    const keyFile = process.env.TUG_TLS_KEY_FILE;
+    if (certFile && keyFile) {
+        const [cert, key] = await Promise.all([fs.readFile(certFile), fs.readFile(keyFile)]);
+        const server = https.createServer({ cert, key }, app);
+        return new Promise(resolve => server.listen(port, () => { console.log(`tug-control listening on ${port} (tls)`); resolve(server); }));
+    }
+    console.warn('TUG_TLS_CERT_FILE/TUG_TLS_KEY_FILE not set; serving plain HTTP');
+    return new Promise(resolve => { const server = app.listen(port, () => { console.log(`tug-control listening on ${port}`); resolve(server); }); });
+}
+if (import.meta.url === `file://${process.argv[1]}`)
+    createApp().then(app => listen(app));