@wpelevator/tug 0.1.0 → 0.2.1

package/dist/control.js

@@ -2,6 +2,12 @@
 import express from 'express';
 import multer from 'multer';
 import fs from 'node:fs/promises';
+import { createReadStream } from 'node:fs';
+import { pipeline } from 'node:stream/promises';
+import { saveImageArchive, inspectImage, validateImageArchive, hashArchive } from "./artifacts.js";
+import sqlite3 from 'sqlite3';
+import { open } from 'sqlite';
+import { scanArchive } from "./archive.js";
 import https from 'node:https';
 import path from 'node:path';
 import * as tar from 'tar';
@@ -13,15 +19,49 @@ import { ensureDir, errMessage, hashToken, id, now, token } from "./util.js";
 import { DEFAULT_MESH_CIDR, DEFAULT_WG_PORT, WG_INTERFACE, WG_KEEPALIVE, allocateOverlayIp, isWgKey, parseCidr, wgEndpoint } from "./mesh.js";
 import { buildIngressWork, resolveServiceEndpoints } from "./ingress.js";
 import { ACTIVE_DEPLOYMENT_STATES } from "./protocol.js";
+import { AUTO_ROLLBACK_ACTOR, ROUTE_ROLES, assertProject, bodyProject, createGuard, createToken, listTokens, parseTokenRequest, queryProject, recordEvent, revokeToken } from "./tokens.js";
 const httpError = (status, message) => Object.assign(new Error(message), { status });
+const missingProject = (project) => `project ${project} has no desired state`;
+const missingService = (project, service) => `service ${service} not found in project ${project}`;
+/** Collapse a node report into the deployment error `tug deploy` prints. Compose failures are multi-line; keep the reason, not the progress log. */
+function deploymentFailureMessage(rows) {
+    const limit = 500;
+    const oneLine = (value) => {
+        const flat = value.replace(/\s+/g, ' ').trim();
+        return flat.length > limit ? `${flat.slice(0, limit)}...` : flat;
+    };
+    const details = rows.map(row => {
+        let status = {};
+        try {
+            status = JSON.parse(row.status_json);
+        }
+        catch { /* a bad report still names the node */ }
+        const parts = [];
+        if (typeof status.error === 'string' && status.error.trim())
+            parts.push(oneLine(status.error));
+        for (const [service, state] of Object.entries(status.serviceStates || {})) {
+            if (typeof state?.rolloutError === 'string' && state.rolloutError.trim())
+                parts.push(`${service}: ${oneLine(state.rolloutError)}`);
+        }
+        const detail = parts.join('; ');
+        return detail ? `${row.node_id}: ${detail}` : row.node_id;
+    });
+    return `one or more assigned nodes reported failed health: ${details.join('; ')}`;
+}
+/** Versions of rollback deployments start with this; `tug deploy` may not use it, which is how automatic rollback tells them apart. */
+const ROLLBACK_VERSION_PREFIX = 'rollback-';
 const ACTIVE_STATES_SQL = ACTIVE_DEPLOYMENT_STATES.map(s => `'${s}'`).join(',');
 const PORT = Number(process.env.PORT || 8080);
-export async function createApp() {
+export async function createApp(options = {}) {
+    const images = { build: buildAndPush, save: saveImageArchive, inspect: inspectImage, validate: validateImageArchive, run: runCommand, ...options.images };
     const DATA_DIR = process.env.TUG_DATA_DIR || './data';
-    const upload = multer({ dest: process.env.TUG_UPLOAD_DIR || path.join(DATA_DIR, 'uploads') });
+    const MAX_IMAGE_BYTES = Number(process.env.TUG_MAX_IMAGE_BYTES || 5 * 1024 ** 3);
+    if (!Number.isSafeInteger(MAX_IMAGE_BYTES) || MAX_IMAGE_BYTES < 1)
+        throw new Error('TUG_MAX_IMAGE_BYTES must be a positive safe integer');
+    const upload = multer({ dest: process.env.TUG_UPLOAD_DIR || path.join(DATA_DIR, 'uploads'), limits: { fileSize: MAX_IMAGE_BYTES, files: 64, fields: 1, fieldSize: 1024 ** 2 } });
     const dbFile = path.join(DATA_DIR, 'tug.db');
     const masterKeyFile = path.join(DATA_DIR, 'master.key');
-    const REGISTRY = process.env.TUG_REGISTRY || 'localhost:5000';
+    const REGISTRY = process.env.TUG_REGISTRY?.trim() || '';
     const REGISTRY_AUTH_PATH = process.env.TUG_REGISTRY_AUTH_PATH || null;
     const REGISTRY_CONTAINER = process.env.TUG_REGISTRY_CONTAINER || null;
     // Networking layers are opt-in; with all of them off the work response is unchanged.
@@ -44,7 +84,7 @@ export async function createApp() {
         await runCommand('docker', ['restart', REGISTRY_CONTAINER]).catch(e => console.error(`registry reload failed: ${e.message}`));
     }
     async function upsertRegistryCredential(name, password) {
-        if (!REGISTRY_AUTH_PATH)
+        if (!REGISTRY || !REGISTRY_AUTH_PATH)
             return;
         await withHtpasswdLock(async () => {
             const { out } = await runCommand('htpasswd', ['-nbB', name, password]);
@@ -76,7 +116,35 @@ export async function createApp() {
         });
         await reloadRegistry();
     }
-    const db = await openDb(dbFile);
+    const db = await (options.openDatabase || openDb)(dbFile);
+    // Events are an operator-facing log, not a record of truth, so keep them bounded. `0` turns a limit off; anything that is
+    // not a whole number falls back to the default.
+    const envCount = (name, fallback) => {
+        const raw = process.env[name];
+        if (raw === undefined || raw === '')
+            return fallback;
+        const value = Number(raw);
+        if (Number.isInteger(value) && value >= 0)
+            return value;
+        console.warn(`${name}=${JSON.stringify(raw)} is not a whole number; using ${fallback}`);
+        return fallback;
+    };
+    const EVENT_RETENTION_DAYS = envCount('TUG_EVENT_RETENTION_DAYS', 30);
+    const EVENT_MAX_ROWS = envCount('TUG_EVENT_MAX_ROWS', 100_000);
+    const eventPruneIntervalMs = options.eventPruneIntervalMs ?? 60 * 60 * 1000;
+    let lastEventPrune = Date.now();
+    async function pruneEvents() {
+        lastEventPrune = Date.now();
+        let removed = 0;
+        if (EVENT_RETENTION_DAYS > 0)
+            removed += (await db.run('DELETE FROM events WHERE created_at < ?', new Date(Date.now() - EVENT_RETENTION_DAYS * 86_400_000).toISOString())).changes ?? 0;
+        if (EVENT_MAX_ROWS > 0)
+            removed += (await db.run('DELETE FROM events WHERE id IN (SELECT id FROM events ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?)', EVENT_MAX_ROWS)).changes ?? 0;
+        if (removed)
+            console.log(`removed ${removed} old events (kept ${EVENT_RETENTION_DAYS || 'any'} days, at most ${EVENT_MAX_ROWS || 'unlimited'} rows)`);
+        return removed;
+    }
+    await pruneEvents().catch(e => console.error(`event pruning failed: ${errMessage(e)}`));
     const masterKey = await loadOrCreateMasterKey(masterKeyFile);
     const operator = await db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash');
     if (!operator) {
@@ -86,14 +154,66 @@ export async function createApp() {
         console.log(`operator token: ${t}`);
     }
     const app = express();
+    let restartRequired = false;
+    app.use((_req, res, next) => restartRequired ? res.status(503).json({ error: 'control restored; restart required', restartRequired: true }) : next());
     app.use(express.json({ limit: '20mb' }));
-    const authOperator = async (req, res, next) => {
-        const raw = (req.headers.authorization || '').replace(/^Bearer /, '');
-        const row = await db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash');
-        if (!raw || hashToken(raw) !== row?.value)
-            return res.status(401).json({ error: 'unauthorized' });
-        next();
+    // Hold the writer for the operation lifetime, including after an HTTP disconnect.
+    let writer = Promise.resolve();
+    async function withWriter(operation) {
+        const previous = writer;
+        let release;
+        writer = new Promise(resolve => { release = resolve; });
+        await previous;
+        try {
+            if (restartRequired)
+                throw httpError(503, 'control restored; restart required');
+            return await operation();
+        }
+        finally {
+            release();
+        }
+    }
+    let publication = Promise.resolve();
+    async function withPublication(operation) {
+        const previous = publication;
+        let release;
+        publication = new Promise(resolve => { release = resolve; });
+        await previous;
+        try {
+            return await operation();
+        }
+        finally {
+            release();
+        }
+    }
+    const safe = (handler) => (req, res, next) => {
+        try {
+            void Promise.resolve(handler(req, res, next)).catch(next);
+        }
+        catch (error) {
+            next(error);
+        }
+    };
+    const route = (method, url, handlers) => app[method](url, ...handlers.map(safe));
+    const get = (url, ...handlers) => route('get', url, handlers);
+    const post = (url, ...handlers) => route('post', url, handlers);
+    // Operator-facing routes go through api(): the role table in tokens.ts is the only place a route's access is declared.
+    const guard = createGuard(db);
+    const apiRoutes = new Map();
+    app.locals.apiRoutes = apiRoutes;
+    const api = (method, url, resolve, ...handlers) => {
+        const key = `${method.toUpperCase()} ${url}`;
+        const role = ROUTE_ROLES[key];
+        if (!role || role === 'public')
+            throw new Error(`API route ${key} has no declared role in ROUTE_ROLES`);
+        apiRoutes.set(key, role);
+        route(method, url, [guard(role, resolve ?? undefined), ...handlers]);
     };
+    const deploymentProject = async (req) => (await db.get('SELECT project_id FROM deployments WHERE id=?', req.params.id))?.project_id;
+    const commandProject = async (req) => { const row = await db.get('SELECT project_id FROM commands WHERE id=?', req.params.id); return row ? row.project_id : undefined; };
+    /** Audit trail entry for a control action; bookkeeping failures are logged, never returned to the caller. */
+    const audit = (actor, event) => recordEvent(db, { ...event, actor: actor.name }).catch(e => console.error(`audit event failed: ${errMessage(e)}`));
+    const projectScope = (actor, column) => actor.projects ? { where: `${column} IN (${actor.projects.map(() => '?').join(',') || "''"})`, args: actor.projects } : { where: '1=1', args: [] };
     const authAgent = async (req, res, next) => {
         const raw = (req.headers.authorization || '').replace(/^Bearer /, '');
         const node = raw && await db.get('SELECT * FROM nodes WHERE agent_token_hash=?', hashToken(raw));
@@ -116,15 +236,80 @@ export async function createApp() {
                 required.add(node);
         if (!required.size)
             return;
-        const statuses = await db.all('SELECT node_id,observed_version,health FROM node_statuses WHERE project_id=?', projectId);
+        const statuses = await db.all('SELECT node_id,observed_version,health,status_json FROM node_statuses WHERE project_id=?', projectId);
         const byNode = new Map(statuses.map(s => [s.node_id, s]));
-        const failed = [...required].some(node => byNode.get(node)?.observed_version === state.version && byNode.get(node)?.health === 'failed');
-        if (failed)
-            await db.run('UPDATE deployments SET state=?,error_message=?,updated_at=? WHERE id=? AND state=?', 'failed', 'one or more assigned nodes reported failed health', now(), project.current_deployment_id, 'rolling_out');
-        const ok = [...required].every(node => byNode.get(node)?.observed_version === state.version && ['serving', 'skipped'].includes(byNode.get(node)?.health ?? ''));
+        const reportsRevision = (node) => state.revision ? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision : byNode.get(node)?.observed_version === state.version;
+        const failedNodes = [...required].filter(node => reportsRevision(node) && byNode.get(node)?.health === 'failed').map(node => byNode.get(node));
+        if (failedNodes.length) {
+            const marked = await db.run('UPDATE deployments SET state=?,error_message=?,updated_at=? WHERE id=? AND state=?', 'failed', deploymentFailureMessage(failedNodes), now(), project.current_deployment_id, 'rolling_out');
+            if (marked.changes === 1)
+                scheduleAutoRollback(projectId, project.current_deployment_id, state);
+        }
+        const ok = [...required].every(node => (state.revision ? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision : byNode.get(node)?.observed_version === state.version) && ['serving', 'skipped'].includes(byNode.get(node)?.health ?? ''));
         if (ok)
             await db.run('UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state=?', 'succeeded', now(), project.current_deployment_id, 'rolling_out');
     }
+    /** Events control writes when it rolls back on its own; they are attributed to a reserved system actor. */
+    async function recordAutoRollbackEvent(projectId, deploymentId, type, message, metadata = {}) {
+        await db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at,actor) VALUES(?,?,?,?,?,?,?,?,?,?)', id('evt'), projectId, deploymentId, null, 'warn', type, message, JSON.stringify(metadata), now(), AUTO_ROLLBACK_ACTOR);
+    }
+    /**
+     * Creates a rollback deployment that copies the most recent earlier deployment's desired state under a
+     * new revision. Shared by `POST /api/v1/rollback` and automatic rollback; the caller holds the writer queue.
+     * `auto` only considers `succeeded` deployments and requires `failedDeploymentId` to still be current.
+     */
+    async function createRollback(project, opts) {
+        let lock;
+        try {
+            lock = await acquireProjectLock(project, opts.auto ? 'auto-rollback' : 'rollback');
+            const active = await db.get(`SELECT id,state FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) ORDER BY created_at DESC LIMIT 1`, project);
+            if (active && !opts.force)
+                throw httpError(409, `deployment ${active.id} is already ${active.state}; retry with force to rollback anyway`);
+            const current = await db.get('SELECT current_deployment_id FROM projects WHERE id=?', project);
+            if (opts.failedDeploymentId && current?.current_deployment_id !== opts.failedDeploymentId)
+                throw httpError(409, `deployment ${opts.failedDeploymentId} is no longer the current deployment`);
+            const states = opts.auto ? `'succeeded'` : `'succeeded','rolling_out'`;
+            const previousDs = await db.get(`SELECT ds.*, d.version AS deployment_version FROM desired_states ds JOIN deployments d ON d.id=ds.deployment_id
+        WHERE ds.project_id=? AND ds.deployment_id != ? AND d.state IN (${states}) ORDER BY ds.created_at DESC LIMIT 1`, project, current?.current_deployment_id || '');
+            if (!previousDs)
+                throw httpError(404, 'no previous deployment found');
+            const ts = now();
+            const depId = id('dep');
+            const previousState = JSON.parse(previousDs.state_json);
+            previousState.revision = depId;
+            await db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, `${ROLLBACK_VERSION_PREFIX}${previousState.version}-${Date.now()}`, 'rolling_out', ts, ts, opts.actor);
+            await db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, previousState.version, JSON.stringify(previousState), ts);
+            await db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
+            return { deployment: await db.get('SELECT * FROM deployments WHERE id=?', depId), desiredState: previousState };
+        }
+        finally {
+            await releaseLock(lock);
+        }
+    }
+    /**
+     * With `x-tug.rollback: auto`, a deployment that just turned `failed` is rolled back to the previous
+     * `succeeded` one. Rollback deployments are never rolled back (their versions carry a reserved prefix
+     * that `tug deploy` cannot use), so a bad rollback target cannot loop. Runs in the background so a
+     * busy writer queue never blocks agent status reports; the outcome is recorded as an event either way.
+     */
+    function scheduleAutoRollback(projectId, deploymentId, state) {
+        if (state.compose['x-tug']?.rollback !== 'auto')
+            return;
+        void (async () => {
+            const failed = await db.get('SELECT * FROM deployments WHERE id=?', deploymentId);
+            if (failed?.version.startsWith(ROLLBACK_VERSION_PREFIX)) {
+                await recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `rollback deployment ${failed.version} failed; not rolling back a rollback`);
+                return;
+            }
+            try {
+                const out = await withWriter(() => createRollback(projectId, { actor: AUTO_ROLLBACK_ACTOR, auto: true, failedDeploymentId: deploymentId }));
+                await recordAutoRollbackEvent(projectId, out.deployment.id, 'deployment.auto_rollback', `deployment ${deploymentId} (${failed?.version}) reported failed health; rolled back to ${out.desiredState.version} as deployment ${out.deployment.id}`, { failedDeploymentId: deploymentId, rollbackDeploymentId: out.deployment.id, version: out.desiredState.version });
+            }
+            catch (e) {
+                await recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `deployment ${deploymentId} failed but automatic rollback was not possible: ${errMessage(e)}`).catch(() => { });
+            }
+        })().catch(e => console.error('auto rollback failed:', errMessage(e)));
+    }
     async function currentDesired(projectId) {
         return db.get('SELECT ds.* FROM desired_states ds JOIN projects p ON p.current_deployment_id=ds.deployment_id WHERE p.id=?', projectId);
     }
@@ -145,21 +330,25 @@ export async function createApp() {
             err.lock = existing;
             throw err;
         }
-        return { name, owner };
+        const heartbeat = setInterval(() => { void db.run('UPDATE locks SET expires_at=? WHERE name=? AND owner=?', new Date(Date.now() + ttlMs).toISOString(), name, owner).catch(() => { }); }, Math.max(10, Math.floor(ttlMs / 3)));
+        heartbeat.unref();
+        return { name, owner, heartbeat };
     }
     async function releaseLock(lock) {
+        if (lock?.heartbeat)
+            clearInterval(lock.heartbeat);
         if (lock)
             await db.run('DELETE FROM locks WHERE name=? AND owner=?', lock.name, lock.owner).catch(() => { });
     }
-    function sendError(res, e) { const err = e; res.status(err.status || 500).json({ error: err.message, lock: err.lock || undefined }); }
-    app.get('/healthz', (req, res) => res.json({ ok: true }));
-    app.post('/api/v1/projects', authOperator, async (req, res) => {
+    function sendError(res, e) { const err = e; res.status(restartRequired ? 503 : err.status || 500).json({ error: err.message, lock: err.lock || undefined }); }
+    get('/healthz', (req, res) => res.json({ ok: true }));
+    api('post', '/api/v1/projects', req => String(req.body?.name || 'site'), async (req, res) => {
         const name = req.body.name || 'site';
         const ts = now();
         await db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', name, name, ts, ts);
         res.json(await db.get('SELECT * FROM projects WHERE id=?', name));
     });
-    app.get('/api/v1/projects/:project', authOperator, async (req, res) => res.json(await db.get('SELECT * FROM projects WHERE id=?', req.params.project)));
+    api('get', '/api/v1/projects/:project', req => req.params.project, async (req, res) => res.json(await db.get('SELECT * FROM projects WHERE id=?', req.params.project)));
     const NODE_INFO_SQL = 'SELECT id,name,address,public_address,overlay_ip,status,created_at,updated_at FROM nodes WHERE id=?';
     /** Allocates the node's stable overlay IP on first registration (retrying if a concurrent register took it). */
     async function ensureOverlayIp(nodeId) {
@@ -207,7 +396,7 @@ export async function createApp() {
         const node = (await db.get(NODE_INFO_SQL, name));
         return { node, agentToken, registryToken };
     }
-    app.post('/api/v1/nodes/register', authOperator, async (req, res) => {
+    api('post', '/api/v1/nodes/register', null, async (req, res) => {
         const name = req.body.name;
         if (!name)
             return res.status(400).json({ error: 'name required' });
@@ -220,7 +409,7 @@ export async function createApp() {
     });
     // Self-service enrollment: operator mints a short-lived, limited-use key; the agent
     // exchanges it for durable credentials on first boot (Portainer Edge-style join).
-    app.post('/api/v1/nodes/enroll-token', authOperator, async (req, res) => {
+    api('post', '/api/v1/nodes/enroll-token', null, async (req, res) => {
         const name = req.body.name || null;
         const ttlSeconds = Math.max(1, Number(req.body.ttlSeconds ?? 3600));
         const uses = Math.max(1, Number(req.body.uses ?? 1));
@@ -229,7 +418,7 @@ export async function createApp() {
         await db.run('INSERT INTO enroll_tokens(token_hash,name,uses_remaining,expires_at,created_at) VALUES(?,?,?,?,?)', hashToken(enrollToken), name, uses, expiresAt, now());
         res.json({ enrollToken, name, usesRemaining: uses, expiresAt, registry: REGISTRY });
     });
-    app.post('/agent/v1/enroll', async (req, res) => {
+    post('/agent/v1/enroll', async (req, res) => {
         const body = req.body;
         const raw = String(body.enrollToken || (req.headers.authorization || '').replace(/^Bearer /, '')).trim();
         if (!raw)
@@ -258,8 +447,8 @@ export async function createApp() {
         const enrolled = { ...result, registry: REGISTRY };
         res.json(enrolled);
     });
-    app.get('/api/v1/nodes', authOperator, async (req, res) => res.json({ nodes: await db.all('SELECT id,name,address,public_address,overlay_ip,ingress,ingress_status_json,status,last_seen_at,created_at,updated_at FROM nodes ORDER BY name') }));
-    app.post('/api/v1/nodes/:name/rotate-registry-token', authOperator, async (req, res) => {
+    api('get', '/api/v1/nodes', null, async (req, res) => res.json({ nodes: await db.all('SELECT id,name,address,public_address,overlay_ip,ingress,ingress_status_json,status,last_seen_at,created_at,updated_at FROM nodes ORDER BY name') }));
+    api('post', '/api/v1/nodes/:name/rotate-registry-token', null, async (req, res) => {
         const node = await db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
         if (!node)
             return res.status(404).json({ error: 'node not found' });
@@ -268,7 +457,7 @@ export async function createApp() {
         await upsertRegistryCredential(node.name, registryToken);
         res.json({ ok: true, node: node.name, registryToken });
     });
-    app.post('/api/v1/nodes/:name/revoke-registry-token', authOperator, async (req, res) => {
+    api('post', '/api/v1/nodes/:name/revoke-registry-token', null, async (req, res) => {
         const node = await db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
         if (!node)
             return res.status(404).json({ error: 'node not found' });
@@ -284,7 +473,7 @@ export async function createApp() {
     }
     // The private key lives only on the node, so rotation is a command the agent executes: it
     // generates a new keypair, reports the public key and peers pick it up from their next work response.
-    app.post('/api/v1/nodes/:name/rotate-wg-key', authOperator, async (req, res) => {
+    api('post', '/api/v1/nodes/:name/rotate-wg-key', null, async (req, res) => {
         if (!MESH)
             return res.status(409).json({ error: 'the WireGuard mesh is not enabled (TUG_MESH=true)' });
         const node = await findNode(req.params.name);
@@ -292,7 +481,7 @@ export async function createApp() {
             return res.status(404).json({ error: 'node not found' });
         res.json({ commandId: await queueCommand(node.id, null, 'wg.rotate-key'), nodeId: node.id });
     });
-    app.post('/api/v1/nodes/:name/ingress', authOperator, async (req, res) => {
+    api('post', '/api/v1/nodes/:name/ingress', null, async (req, res) => {
         const node = await findNode(req.params.name);
         if (!node)
             return res.status(404).json({ error: 'node not found' });
@@ -300,7 +489,7 @@ export async function createApp() {
         await db.run('UPDATE nodes SET ingress=?,updated_at=? WHERE id=?', enabled ? 1 : 0, now(), node.id);
         res.json({ ok: true, node: node.name, ingress: enabled });
     });
-    app.post('/agent/v1/wireguard/key', authAgent, async (req, res) => {
+    post('/agent/v1/wireguard/key', authAgent, async (req, res) => {
         if (!MESH)
             return res.status(409).json({ error: 'the WireGuard mesh is not enabled' });
         const { publicKey } = req.body;
@@ -314,7 +503,7 @@ export async function createApp() {
             sendError(res, /UNIQUE constraint/.test(errMessage(e)) ? httpError(409, 'public key already registered to another node') : e);
         }
     });
-    app.post('/api/v1/registry/auth', async (req, res) => {
+    post('/api/v1/registry/auth', async (req, res) => {
         const username = req.body.username;
         const password = req.body.password;
         const node = username && password ? await db.get('SELECT id,name FROM nodes WHERE (name=? OR id=?) AND registry_token_hash IS NOT NULL AND registry_token_hash=?', username, username, hashToken(password)) : null;
@@ -322,7 +511,20 @@ export async function createApp() {
             return res.status(401).json({ ok: false, error: 'invalid registry credentials' });
         res.json({ ok: true, node: node.name });
     });
-    app.post('/api/v1/secrets', authOperator, async (req, res) => {
+    api('post', '/api/v1/tokens', null, async (req, res) => {
+        const request = parseTokenRequest(req.body);
+        const created = await createToken(db, request, req.actor.name);
+        await audit(req.actor, { type: 'token.create', message: `token ${created.name} (${created.role}${created.projects ? `, projects: ${created.projects.join(', ')}` : ''}) created by ${req.actor.name}`, metadata: { name: created.name, role: created.role, projects: created.projects, expiresAt: created.expiresAt } });
+        res.json(created);
+    });
+    api('get', '/api/v1/tokens', null, async (_req, res) => res.json({ tokens: await listTokens(db) }));
+    api('delete', '/api/v1/tokens/:name', null, async (req, res) => {
+        if (!await revokeToken(db, req.params.name))
+            return res.status(404).json({ error: 'token not found' });
+        await audit(req.actor, { type: 'token.revoke', message: `token ${req.params.name} revoked by ${req.actor.name}`, metadata: { name: req.params.name } });
+        res.json({ ok: true, name: req.params.name });
+    });
+    api('post', '/api/v1/secrets', bodyProject, async (req, res) => {
         const { project = 'site', name, value } = req.body;
         if (!name || value == null)
             return res.status(400).json({ error: 'name and value required' });
@@ -333,6 +535,7 @@ export async function createApp() {
             const ts = now();
             await db.run(`INSERT INTO secrets(id,project_id,name,encrypted_value,version,created_at,updated_at) VALUES(?,?,?,?,?,?,?)
         ON CONFLICT(project_id,name) DO UPDATE SET encrypted_value=excluded.encrypted_value,version=excluded.version,updated_at=excluded.updated_at`, id('sec'), project, name, encrypt(value, masterKey), (old?.version || 0) + 1, ts, ts);
+            await audit(req.actor, { type: 'secret.set', project, message: `secret ${name} set to version ${(old?.version || 0) + 1} by ${req.actor.name}`, metadata: { name, version: (old?.version || 0) + 1 } });
             res.json({ ok: true, name, version: (old?.version || 0) + 1 });
         }
         catch (e) {
@@ -342,109 +545,275 @@ export async function createApp() {
             await releaseLock(lock);
         }
     });
-    app.get('/api/v1/secrets', authOperator, async (req, res) => res.json({ secrets: await db.all('SELECT project_id,name,version,updated_at FROM secrets WHERE project_id=? ORDER BY name', req.query.project || 'site') }));
-    app.delete('/api/v1/secrets/:name', authOperator, async (req, res) => { await db.run('DELETE FROM secrets WHERE project_id=? AND name=?', req.query.project || 'site', req.params.name); res.json({ ok: true }); });
-    app.post('/api/v1/deployments', authOperator, upload.single('context'), async (req, res) => {
-        const body = req.file ? JSON.parse(req.body.payload) : req.body;
-        const project = body.project || 'site';
-        const compose = body.compose;
-        const version = body.version || `git-${Date.now()}`;
-        const nodes = await db.all('SELECT name FROM nodes');
-        try {
-            validateCompose(compose, nodes.map(n => n.name));
-        }
-        catch (e) {
-            return res.status(400).json({ error: errMessage(e) });
-        }
-        for (const [svcName, svc] of Object.entries(compose.services)) {
-            const ing = svc['x-tug']?.ingress;
-            if (ing?.tls !== 'custom')
+    api('get', '/api/v1/secrets', queryProject, async (req, res) => res.json({ secrets: await db.all('SELECT project_id,name,version,updated_at FROM secrets WHERE project_id=? ORDER BY name', req.query.project || 'site') }));
+    api('delete', '/api/v1/secrets/:name', queryProject, async (req, res) => {
+        const project = String(req.query.project || 'site');
+        const removed = await db.run('DELETE FROM secrets WHERE project_id=? AND name=?', project, req.params.name);
+        if (removed.changes)
+            await audit(req.actor, { type: 'secret.delete', project, message: `secret ${req.params.name} deleted by ${req.actor.name}`, metadata: { name: req.params.name } });
+        res.json({ ok: true });
+    });
+    const artifactDir = path.join(DATA_DIR, 'images');
+    await ensureDir(artifactDir);
+    async function retainedStates() {
+        // All published history is retained. This conservative superset protects failed/lagging versions too.
+        return (await db.all('SELECT * FROM desired_states')).map(row => JSON.parse(row.state_json));
+    }
+    async function collectArtifacts() {
+        const states = await retainedStates();
+        const refs = new Set(states.flatMap(state => Object.values(state.services).flatMap(service => service.artifact ? [service.artifact.buildId] : [])));
+        const imageRefs = new Set(states.flatMap(state => Object.values(state.services).map(service => service.image)));
+        const rows = await db.all('SELECT * FROM builds');
+        for (const row of rows) {
+            if (refs.has(row.id) || imageRefs.has(row.image_ref))
                 continue;
-            for (const secret of [ing.cert, ing.key]) {
-                if (!await db.get('SELECT 1 FROM secrets WHERE project_id=? AND name=?', project, secret))
-                    return res.status(400).json({ error: `${svcName}: custom TLS secret ${secret} not found in project ${project}; set it with secret:set first` });
-            }
+            const active = await db.get("SELECT 1 FROM deployments WHERE id=? AND state IN ('uploading','building')", row.deployment_id);
+            if (active)
+                continue;
+            await fs.rm(path.join(DATA_DIR, row.artifact_path || path.join('images', `${row.id}.tar.gz`)), { force: true });
+            await images.run('docker', ['image', 'rm', row.image_ref]).catch(() => { });
+            await db.run('DELETE FROM builds WHERE id=?', row.id);
         }
-        const active = await db.get(`SELECT id,state FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) ORDER BY created_at DESC LIMIT 1`, project);
-        if (active)
-            return res.status(409).json({ error: `deployment ${active.id} is already ${active.state}` });
-        const depId = id('dep');
-        const ts = now();
+        const remaining = await db.all('SELECT id FROM builds');
+        const owned = new Set(remaining.flatMap(row => [`${row.id}.tar.gz`, `${row.id}.tar.gz.tmp`]));
+        for (const name of await fs.readdir(artifactDir))
+            if (!owned.has(name))
+                await fs.rm(path.join(artifactDir, name), { force: true });
+    }
+    // Previous-process builds cannot finish after a restart; clear their unpublished files.
+    await db.run("UPDATE deployments SET state='build_failed',error_message='control restarted during build' WHERE state IN ('uploading','building')");
+    await collectArtifacts();
+    const receiveDeployment = upload.any();
+    api('post', '/api/v1/deployments', null, (req, res) => {
+        receiveDeployment(req, res, error => {
+            void withWriter(() => deployRequest(req, res, error)).catch(async (failure) => {
+                const files = Array.isArray(req.files) ? req.files : [];
+                await Promise.all(files.map(file => fs.rm(file.path, { force: true }).catch(() => { })));
+                sendError(res, failure);
+            });
+        });
+    });
+    async function deployRequest(req, res, uploadError) {
+        const files = (Array.isArray(req.files) ? req.files : []);
+        let responseBody;
+        let responseError;
         let lock;
+        let depId;
+        let contextDir;
         try {
+            if (uploadError)
+                throw httpError(400, errMessage(uploadError));
+            let body;
+            try {
+                body = req.is('multipart/form-data') ? JSON.parse(req.body.payload) : req.body;
+            }
+            catch {
+                throw httpError(400, 'invalid deployment payload JSON');
+            }
+            if (!body || typeof body !== 'object')
+                throw httpError(400, 'deployment payload is required');
+            const project = body.project || 'site';
+            const compose = body.compose;
+            const version = body.version || `git-${Date.now()}`;
+            assertProject(req.actor, project);
+            if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(project) || typeof version !== 'string' || !version.length)
+                throw httpError(400, 'invalid project or version');
+            if (version.startsWith(ROLLBACK_VERSION_PREFIX))
+                throw httpError(400, `version cannot start with "${ROLLBACK_VERSION_PREFIX}" (reserved for rollback deployments)`);
+            const nodes = await db.all('SELECT name FROM nodes ORDER BY created_at,name');
+            try {
+                validateCompose(compose, nodes.map(n => n.name));
+            }
+            catch (e) {
+                throw httpError(400, errMessage(e));
+            }
+            const builtServices = Object.entries(compose.services).filter(([, service]) => service.build).map(([name]) => name);
+            const local = body.localImages;
+            const context = files.find(file => file.fieldname === 'context');
+            if (local !== undefined && (!Array.isArray(local) || context))
+                throw httpError(400, 'localImages must be an array and cannot accompany a source context');
+            if (files.filter(file => file.fieldname === 'context').length > 1)
+                throw httpError(400, 'duplicate source context');
+            if (local) {
+                if (local.some(item => !item || typeof item !== 'object'))
+                    throw httpError(400, 'invalid local image manifest');
+                if (local.length !== builtServices.length || new Set(local.map(item => item.service)).size !== local.length || new Set(local.map(item => item.field)).size !== local.length)
+                    throw httpError(400, 'localImages must contain exactly one image for each build service');
+                for (const item of local) {
+                    if (!/^tug-build\/[a-f0-9-]{36}\/image-\d+:local$/.test(item.imageRef) || item.imageRef.split('/').at(-1) !== `${item.field}:local` || !builtServices.includes(item.service) || !/^image-\d+$/.test(item.field) || files.filter(file => file.fieldname === item.field).length !== 1 || !/^[a-f0-9]{64}$/.test(item.sha256) || !Number.isSafeInteger(item.sizeBytes) || item.sizeBytes <= 0 || item.sizeBytes > MAX_IMAGE_BYTES)
+                        throw httpError(400, 'invalid local image manifest or file');
+                }
+            }
+            if (files.length !== (local ? local.length : context ? 1 : 0))
+                throw httpError(400, 'unexpected deployment upload');
+            if (!REGISTRY && builtServices.length && !context && !local)
+                throw httpError(400, 'build services require a source context or localImages upload without a registry');
+            for (const [svcName, svc] of Object.entries(compose.services)) {
+                const ing = svc['x-tug']?.ingress;
+                if (ing?.tls !== 'custom')
+                    continue;
+                for (const secret of [ing.cert, ing.key])
+                    if (!await db.get('SELECT 1 FROM secrets WHERE project_id=? AND name=?', project, secret))
+                        throw httpError(400, `${svcName}: custom TLS secret ${secret} not found in project ${project}; set it with secret-set first`);
+            }
             lock = await acquireProjectLock(project, 'deploy');
+            const active = await db.get(`SELECT * FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) LIMIT 1`, project);
+            if (active)
+                throw httpError(409, `deployment ${active.id} is already ${active.state}`);
+            depId = id('dep');
+            const ts = now();
             await db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', project, project, ts, ts);
-            await db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at) VALUES(?,?,?,?,?,?)', depId, project, version, 'building', ts, ts);
-            const desired = buildDesiredState({ project, version, compose, registry: REGISTRY, defaultNode: nodes[0]?.name || null });
-            if (req.file) {
-                const contextDir = path.resolve(DATA_DIR, 'build-contexts', depId);
-                await unpackContext(req.file.path, contextDir);
-                for (const [svc, spec] of Object.entries(compose.services))
-                    if (spec.build)
-                        await buildAndPush({ contextDir, service: svc, buildSpec: spec.build, imageRef: desired.services[svc].image, push: process.env.TUG_PUSH !== 'false' });
+            await db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, version, 'building', ts, ts, req.actor.name);
+            const desired = buildDesiredState({ project, version, revision: depId, compose, registry: REGISTRY, defaultNode: nodes[0]?.name || null });
+            if (context) {
+                contextDir = path.resolve(DATA_DIR, 'build-contexts', depId);
+                await unpackContext(context.path, contextDir);
             }
-            await db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, version, JSON.stringify(desired), ts);
-            for (const [volumeName, volumeSpec] of Object.entries(compose.volumes || {})) {
-                const lp = volumeSpec?.['x-tug'] || {};
-                const nodeName = lp.placement?.nodes?.[0] || null;
-                const node = nodeName ? await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName) : null;
-                await db.run(`INSERT INTO volumes(id,project_id,name,type,driver,node_id,config_json,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?)
-          ON CONFLICT(project_id,name) DO UPDATE SET type=excluded.type,driver=excluded.driver,node_id=excluded.node_id,config_json=excluded.config_json,updated_at=excluded.updated_at`, id('vol'), project, volumeName, lp.type || 'local', lp.driver || null, node?.id || null, JSON.stringify(lp), ts, ts);
+            for (const service of builtServices) {
+                if (!contextDir && !local) {
+                    desired.services[service].image = `${REGISTRY}/${project}/${service}:${version}`;
+                    continue;
+                } // Explicit registry/no-build-upload means an existing pushed image.
+                const spec = desired.services[service];
+                const buildId = id('build');
+                const imageRef = spec.image;
+                await db.run('INSERT INTO builds(id,project_id,deployment_id,service,image_ref,state,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)', buildId, project, depId, service, imageRef, 'building', ts, ts);
+                let info;
+                let importedClientRef;
+                const relative = path.join('images', `${buildId}.tar.gz`);
+                const final = path.join(DATA_DIR, relative);
+                const temporary = `${final}.tmp`;
+                try {
+                    if (local) {
+                        const item = local.find(item => item.service === service);
+                        const file = files.find(file => file.fieldname === item.field);
+                        try {
+                            info = await images.validate(file.path, item.imageRef);
+                        }
+                        catch (error) {
+                            throw httpError(400, `${service}: invalid image archive: ${errMessage(error)}`);
+                        }
+                        if (info.sha256 !== item.sha256 || info.sizeBytes !== item.sizeBytes || info.imageId !== item.imageId || info.platform !== item.platform)
+                            throw httpError(400, `${service}: local image metadata/checksum mismatch`);
+                        importedClientRef = item.imageRef;
+                        await images.run('docker', ['load', '-i', file.path]);
+                        const imported = await images.inspect(item.imageRef);
+                        if (imported.platform !== info.platform)
+                            throw httpError(400, 'imported image identity mismatch');
+                        await images.run('docker', ['tag', item.imageRef, imageRef]);
+                    }
+                    else
+                        await images.build({ contextDir: contextDir, service, buildSpec: compose.services[service].build, imageRef, push: false });
+                    if (REGISTRY) {
+                        if (process.env.TUG_PUSH !== 'false')
+                            await images.run('docker', ['push', imageRef]);
+                        const inspected = await images.inspect(imageRef);
+                        await db.run('UPDATE builds SET state=?,image_id=?,platform=?,updated_at=? WHERE id=?', 'complete', inspected.imageId, inspected.platform, now(), buildId);
+                    }
+                    else {
+                        info = await images.save(imageRef, temporary);
+                        await fs.rename(temporary, final);
+                        const artifact = { buildId, imageRef, ...info };
+                        spec.artifact = artifact;
+                        await db.run('UPDATE builds SET state=?,image_id=?,platform=?,sha256=?,size_bytes=?,artifact_path=?,updated_at=? WHERE id=?', 'complete', info.imageId, info.platform, info.sha256, info.sizeBytes, relative, now(), buildId);
+                    }
+                }
+                finally {
+                    if (importedClientRef)
+                        await images.run('docker', ['image', 'rm', importedClientRef]).catch(() => { });
+                    await fs.rm(temporary, { force: true });
+                }
             }
-            await db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
-            await db.run('UPDATE deployments SET state=?,updated_at=? WHERE id=?', 'rolling_out', now(), depId);
-            res.json({ deployment: { id: depId, project, version, state: 'rolling_out' }, desiredState: desired });
+            const buildLock = lock;
+            await withPublication(async () => {
+                const owned = await db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now());
+                const pending = await db.get('SELECT 1 FROM deployments WHERE id=? AND state=?', depId, 'building');
+                if (!owned || !pending)
+                    throw httpError(409, 'deployment was cancelled or its build lease was lost');
+                await db.exec('BEGIN IMMEDIATE');
+                try {
+                    const published = await db.run("UPDATE deployments SET state='rolling_out',updated_at=? WHERE id=? AND state='building'", now(), depId);
+                    if (published.changes !== 1 || !await db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now()))
+                        throw httpError(409, 'deployment was cancelled or its build lease was lost');
+                    await db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, version, JSON.stringify(desired), ts);
+                    for (const [volumeName, volumeSpec] of Object.entries(compose.volumes || {})) {
+                        const lp = volumeSpec?.['x-tug'] || {};
+                        const nodeName = lp.placement?.nodes?.[0] || null;
+                        const node = nodeName ? await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName) : null;
+                        await db.run(`INSERT INTO volumes(id,project_id,name,type,driver,node_id,config_json,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(project_id,name) DO UPDATE SET type=excluded.type,driver=excluded.driver,node_id=excluded.node_id,config_json=excluded.config_json,updated_at=excluded.updated_at`, id('vol'), project, volumeName, lp.type || 'local', lp.driver || null, node?.id || null, JSON.stringify(lp), ts, ts);
+                    }
+                    await db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
+                    await db.exec('COMMIT');
+                }
+                catch (e) {
+                    await db.exec('ROLLBACK');
+                    throw e;
+                }
+            });
+            await audit(req.actor, { type: 'deploy', project, deploymentId: depId, message: `deployment ${depId} (${version}) started by ${req.actor.name}`, metadata: { version } });
+            responseBody = { deployment: { id: depId, project, version, state: 'rolling_out', actor: req.actor.name }, desiredState: desired };
         }
         catch (e) {
-            await db.run('UPDATE deployments SET state=?,error_message=?,updated_at=? WHERE id=?', 'build_failed', errMessage(e), now(), depId).catch(() => { });
-            sendError(res, e);
+            if (depId)
+                await db.run("UPDATE deployments SET state='build_failed',error_message=?,updated_at=? WHERE id=? AND state='building'", errMessage(e), now(), depId).catch(() => { });
+            responseError = e;
         }
         finally {
             await releaseLock(lock);
-            if (req.file)
-                await fs.rm(req.file.path, { force: true }).catch(() => { });
+            await Promise.all(files.map(file => fs.rm(file.path, { force: true }).catch(() => { })));
+            if (contextDir)
+                await fs.rm(contextDir, { recursive: true, force: true });
+            await collectArtifacts().catch(e => console.error('image cleanup failed:', errMessage(e)));
+        }
+        if (responseError)
+            sendError(res, responseError);
+        else
+            res.json(responseBody);
+    }
+    get('/agent/v1/images/:buildId', authAgent, async (req, res) => {
+        const states = await retainedStates();
+        const allowed = states.some(state => Object.values(state.services).some(service => service.artifact?.buildId === req.params.buildId && (!service.placement.nodes?.length || service.placement.nodes.includes(req.node.name))));
+        if (!allowed)
+            return res.status(404).json({ error: 'image not found' });
+        const row = await db.get('SELECT * FROM builds WHERE id=? AND state=?', req.params.buildId, 'complete');
+        if (!row?.artifact_path)
+            return res.status(404).json({ error: 'image not found' });
+        try {
+            await fs.access(path.join(DATA_DIR, row.artifact_path));
+            res.set({ 'Content-Type': 'application/gzip', 'Content-Length': String(row.size_bytes), 'X-Tug-Sha256': row.sha256, 'Cache-Control': 'private, no-store' });
+            await pipeline(createReadStream(path.join(DATA_DIR, row.artifact_path)), res);
+        }
+        catch {
+            if (!res.headersSent)
+                res.status(404).json({ error: 'image artifact unavailable' });
         }
     });
-    app.get('/api/v1/deployments/:id', authOperator, async (req, res) => res.json(await db.get('SELECT * FROM deployments WHERE id=?', req.params.id)));
-    app.post('/api/v1/deployments/:id/cancel', authOperator, async (req, res) => {
+    api('get', '/api/v1/deployments/:id', deploymentProject, async (req, res) => res.json(await db.get('SELECT * FROM deployments WHERE id=?', req.params.id)));
+    api('post', '/api/v1/deployments/:id/cancel', deploymentProject, async (req, res) => withPublication(async () => {
         const row = await db.get('SELECT * FROM deployments WHERE id=?', req.params.id);
         if (!row)
             return res.status(404).json({ error: 'deployment not found' });
         if (!ACTIVE_DEPLOYMENT_STATES.includes(row.state))
             return res.status(409).json({ error: `deployment is ${row.state}` });
-        await db.run('UPDATE deployments SET state=?,updated_at=? WHERE id=?', 'cancelled', now(), req.params.id);
+        const result = await db.run(`UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state IN (${ACTIVE_STATES_SQL})`, 'cancelled', now(), req.params.id);
+        if (result.changes !== 1)
+            return res.status(409).json({ error: 'deployment is no longer active' });
+        await audit(req.actor, { type: 'cancel', project: row.project_id, deploymentId: row.id, message: `deployment ${row.id} cancelled by ${req.actor.name}` });
         res.json({ ok: true });
-    });
-    app.post('/api/v1/rollback', authOperator, async (req, res) => {
+    }));
+    api('post', '/api/v1/rollback', bodyProject, async (req, res) => withWriter(async () => {
         const project = req.body.project || 'site';
-        let lock;
         try {
-            lock = await acquireProjectLock(project, 'rollback');
-            const active = await db.get(`SELECT id,state FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) ORDER BY created_at DESC LIMIT 1`, project);
-            if (active && !req.body.force)
-                return res.status(409).json({ error: `deployment ${active.id} is already ${active.state}; retry with force to rollback anyway` });
-            const current = await db.get('SELECT current_deployment_id FROM projects WHERE id=?', project);
-            const previousDs = await db.get(`SELECT ds.*, d.version AS deployment_version FROM desired_states ds JOIN deployments d ON d.id=ds.deployment_id
-        WHERE ds.project_id=? AND ds.deployment_id != ? AND d.state IN ('succeeded','rolling_out') ORDER BY ds.created_at DESC LIMIT 1`, project, current?.current_deployment_id || '');
-            if (!previousDs)
-                return res.status(404).json({ error: 'no previous deployment found' });
-            const ts = now();
-            const depId = id('dep');
-            const previousState = JSON.parse(previousDs.state_json);
-            await db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at) VALUES(?,?,?,?,?,?)', depId, project, `rollback-${previousState.version}-${Date.now()}`, 'rolling_out', ts, ts);
-            await db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, previousState.version, JSON.stringify(previousState), ts);
-            await db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
-            res.json({ deployment: await db.get('SELECT * FROM deployments WHERE id=?', depId), desiredState: previousState });
+            const out = await createRollback(project, { actor: req.actor.name, force: Boolean(req.body.force) });
+            await audit(req.actor, { type: 'rollback', project, deploymentId: out.deployment.id, message: `rollback to ${out.desiredState.version} (deployment ${out.deployment.id}) started by ${req.actor.name}`, metadata: { version: out.desiredState.version, force: Boolean(req.body.force) } });
+            res.json(out);
         }
         catch (e) {
             sendError(res, e);
         }
-        finally {
-            await releaseLock(lock);
-        }
-    });
-    app.get('/api/v1/volumes', authOperator, async (req, res) => res.json({ volumes: await db.all('SELECT * FROM volumes WHERE project_id=? ORDER BY name', req.query.project || 'site') }));
-    app.delete('/api/v1/volumes/:name', authOperator, async (req, res) => {
+    }));
+    api('get', '/api/v1/volumes', queryProject, async (req, res) => res.json({ volumes: await db.all('SELECT * FROM volumes WHERE project_id=? ORDER BY name', req.query.project || 'site') }));
+    api('delete', '/api/v1/volumes/:name', queryProject, async (req, res) => {
         const project = String(req.query.project || 'site');
         const deleteData = String(req.query.deleteData || 'false') === 'true';
         let lock;
@@ -479,7 +848,7 @@ export async function createApp() {
             await releaseLock(lock);
         }
     });
-    app.post('/api/v1/control/backups/run', authOperator, async (req, res) => {
+    api('post', '/api/v1/control/backups/run', null, async (req, res) => withWriter(async () => {
         const backupDir = process.env.TUG_CONTROL_BACKUP_DIR || path.join(DATA_DIR, 'control-backups');
         await ensureDir(backupDir);
         const ts = now();
@@ -487,40 +856,87 @@ export async function createApp() {
         const staging = path.join(backupDir, `${backupId}-staging`);
         await fs.rm(staging, { recursive: true, force: true });
         await ensureDir(staging);
-        await db.exec('PRAGMA wal_checkpoint(FULL);');
-        await fs.copyFile(dbFile, path.join(staging, 'tug.db'));
+        const snapshotPath = path.join(staging, 'tug.db').replaceAll("'", "''");
+        await db.exec(`VACUUM INTO '${snapshotPath}'`);
         await fs.copyFile(masterKeyFile, path.join(staging, 'master.key'));
+        await fs.cp(artifactDir, path.join(staging, 'images'), { recursive: true });
         await fs.writeFile(path.join(staging, 'metadata.json'), JSON.stringify({ backupId, createdAt: ts, type: 'control' }, null, 2));
         const artifactRef = path.resolve(backupDir, `${backupId}.tar.gz`);
-        await tar.c({ gzip: true, file: artifactRef, cwd: staging }, ['tug.db', 'master.key', 'metadata.json']);
+        await tar.c({ gzip: true, file: artifactRef, cwd: staging }, ['tug.db', 'master.key', 'metadata.json', 'images']);
         await fs.rm(staging, { recursive: true, force: true });
         await db.run('INSERT INTO backups(id,project_id,target,node_id,state,artifact_ref,metadata_json,created_at,completed_at) VALUES(?,?,?,?,?,?,?,?,?)', backupId, '_control', 'control-plane', null, 'completed', artifactRef, JSON.stringify({ type: 'control' }), ts, now());
         res.json({ backupId, artifactRef, state: 'completed' });
-    });
-    app.post('/api/v1/control/restore', authOperator, async (req, res) => {
+    }));
+    api('post', '/api/v1/control/restore', null, async (req, res) => withWriter(async () => {
         if (!req.body.confirm)
-            return res.status(400).json({ error: 'confirm=true is required; restore overwrites control database and master key, then requires process restart' });
+            return res.status(400).json({ error: 'confirm=true is required; restore overwrites control data then requires restart' });
         const artifactRef = req.body.artifactRef;
-        if (!artifactRef)
+        if (typeof artifactRef !== 'string' || !artifactRef)
             return res.status(400).json({ error: 'artifactRef required' });
         const restoreDir = process.env.TUG_CONTROL_RESTORE_DIR || path.join(DATA_DIR, 'control-restore');
-        await fs.rm(restoreDir, { recursive: true, force: true });
-        await ensureDir(restoreDir);
-        await tar.x({ file: artifactRef, cwd: restoreDir });
-        await fs.copyFile(path.join(restoreDir, 'tug.db'), dbFile);
-        await fs.copyFile(path.join(restoreDir, 'master.key'), masterKeyFile);
-        res.json({ ok: true, restored: true, restartRequired: true });
-    });
+        let snapshot;
+        try {
+            await fs.rm(restoreDir, { recursive: true, force: true });
+            await ensureDir(restoreDir);
+            await scanArchive(artifactRef, entry => {
+                if (path.isAbsolute(entry.path) || entry.path.split('/').includes('..') || !['File', 'Directory'].includes(entry.type))
+                    throw new Error('unsafe control backup entry');
+            });
+            await tar.x({ file: artifactRef, cwd: restoreDir, strict: true });
+            const key = (await fs.readFile(path.join(restoreDir, 'master.key'), 'utf8')).trim();
+            if (!/^[A-Za-z0-9+/]{43}=$/.test(key))
+                throw new Error('invalid backup master key');
+            snapshot = await open({ filename: path.join(restoreDir, 'tug.db'), driver: sqlite3.Database, mode: sqlite3.OPEN_READONLY });
+            const integrity = await snapshot.get('PRAGMA integrity_check');
+            if (integrity?.integrity_check !== 'ok')
+                throw new Error('backup database failed integrity check');
+            if (!await snapshot.get("SELECT 1 FROM meta WHERE key='operator_token_hash'"))
+                throw new Error('backup database has no operator identity');
+            for (const secret of await snapshot.all('SELECT * FROM secrets'))
+                decrypt(secret.encrypted_value, key);
+            const states = await snapshot.all('SELECT * FROM desired_states');
+            for (const stateRow of states) {
+                const state = JSON.parse(stateRow.state_json);
+                for (const service of Object.values(state.services)) {
+                    if (!service.artifact)
+                        continue;
+                    const build = await snapshot.get('SELECT * FROM builds WHERE id=? AND state=?', service.artifact.buildId, 'complete');
+                    if (!build?.artifact_path || !/^images\/build_[a-zA-Z0-9_-]+\.tar\.gz$/.test(build.artifact_path))
+                        throw new Error('backup contains an unsafe or missing image reference');
+                    const actual = await hashArchive(path.join(restoreDir, build.artifact_path));
+                    if (actual.sha256 !== build.sha256 || actual.sizeBytes !== build.size_bytes || actual.sha256 !== service.artifact.sha256 || actual.sizeBytes !== service.artifact.sizeBytes)
+                        throw new Error('backup image archive checksum mismatch');
+                }
+            }
+            await snapshot.close();
+            snapshot = undefined;
+            await ensureDir(path.join(restoreDir, 'images'));
+            // Validation completes before changing any live file. Stop DB-dependent work until restart.
+            restartRequired = true;
+            await db.close();
+            await fs.rm(`${dbFile}-wal`, { force: true });
+            await fs.rm(`${dbFile}-shm`, { force: true });
+            await fs.cp(path.join(restoreDir, 'images'), artifactDir, { recursive: true });
+            await fs.copyFile(path.join(restoreDir, 'tug.db'), dbFile);
+            await fs.copyFile(path.join(restoreDir, 'master.key'), masterKeyFile);
+            res.json({ ok: true, restored: true, restartRequired: true });
+        }
+        catch (e) {
+            if (snapshot)
+                await snapshot.close().catch(() => { });
+            res.status(restartRequired ? 500 : 400).json({ error: errMessage(e), restartRequired });
+        }
+    }));
     async function commandForService({ project, service, type, payload }) {
         if (!service)
             throw httpError(400, 'service required');
         const ds = await currentDesired(project);
         if (!ds)
-            throw httpError(404, 'project has no desired state');
+            throw httpError(404, missingProject(project));
         const state = JSON.parse(ds.state_json);
         const svc = state.services?.[service];
         if (!svc)
-            throw httpError(404, 'service not found');
+            throw httpError(404, missingService(project, service));
         const nodeName = svc.placement?.nodes?.[0];
         if (!nodeName)
             throw httpError(400, 'service has no explicit node placement');
@@ -532,15 +948,18 @@ export async function createApp() {
         await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', cmdId, node.id, project, type, 'pending', JSON.stringify({ project, service, ...payload }), 0, ts, null, ts, ts);
         return { commandId: cmdId, nodeId: node.id };
     }
-    app.post('/api/v1/restart', authOperator, async (req, res) => {
+    api('post', '/api/v1/restart', bodyProject, async (req, res) => {
         try {
-            res.json(await commandForService({ project: req.body.project || 'site', service: req.body.service, type: 'service.restart', payload: {} }));
+            const project = req.body.project || 'site';
+            const queued = await commandForService({ project, service: req.body.service, type: 'service.restart', payload: {} });
+            await audit(req.actor, { type: 'restart', project, message: `service ${req.body.service} restart requested by ${req.actor.name}`, metadata: { service: req.body.service, commandId: queued.commandId } });
+            res.json(queued);
         }
         catch (e) {
             res.status(e.status || 500).json({ error: errMessage(e) });
         }
     });
-    app.post('/api/v1/logs', authOperator, async (req, res) => {
+    api('post', '/api/v1/logs', bodyProject, async (req, res) => {
         const project = req.body.project || 'site';
         const service = req.body.service;
         const tail = Number(req.body.tail || 200);
@@ -548,11 +967,11 @@ export async function createApp() {
             return res.status(400).json({ error: 'service required' });
         const ds = await currentDesired(project);
         if (!ds)
-            return res.status(404).json({ error: 'project has no desired state' });
+            return res.status(404).json({ error: missingProject(project) });
         const state = JSON.parse(ds.state_json);
         const svc = state.services?.[service];
         if (!svc)
-            return res.status(404).json({ error: 'service not found' });
+            return res.status(404).json({ error: missingService(project, service) });
         const nodeName = svc.placement?.nodes?.[0];
         if (!nodeName)
             return res.status(400).json({ error: 'service has no explicit node placement' });
@@ -564,22 +983,28 @@ export async function createApp() {
         await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', cmdId, node.id, project, 'logs.fetch', 'pending', JSON.stringify({ project, service, tail }), 0, ts, null, ts, ts);
         res.json({ commandId: cmdId, nodeId: node.id });
     });
-    app.get('/api/v1/commands/:id', authOperator, async (req, res) => res.json(await db.get('SELECT * FROM commands WHERE id=?', req.params.id)));
-    app.get('/api/v1/status', authOperator, async (req, res) => {
+    api('get', '/api/v1/commands/:id', commandProject, async (req, res) => res.json(await db.get('SELECT * FROM commands WHERE id=?', req.params.id)));
+    api('get', '/api/v1/status', null, async (req, res) => {
         const staleSeconds = Number(process.env.TUG_STALE_SECONDS || 60);
         const cutoff = new Date(Date.now() - staleSeconds * 1000).toISOString();
         await db.run("UPDATE nodes SET status='stale',updated_at=? WHERE last_seen_at IS NOT NULL AND last_seen_at < ?", now(), cutoff);
-        res.json({ projects: await db.all('SELECT * FROM projects'), deployments: await db.all('SELECT * FROM deployments ORDER BY created_at DESC LIMIT 20'), nodes: await db.all('SELECT id,name,status,last_seen_at FROM nodes'), statuses: await db.all('SELECT * FROM node_statuses') });
+        const projects = projectScope(req.actor, 'id');
+        const deployments = projectScope(req.actor, 'project_id');
+        const statuses = projectScope(req.actor, 'project_id');
+        res.json({ projects: await db.all(`SELECT * FROM projects WHERE ${projects.where}`, ...projects.args), deployments: await db.all(`SELECT * FROM deployments WHERE ${deployments.where} ORDER BY created_at DESC LIMIT 20`, ...deployments.args), nodes: await db.all('SELECT id,name,status,last_seen_at FROM nodes'), statuses: await db.all(`SELECT * FROM node_statuses WHERE ${statuses.where}`, ...statuses.args) });
     });
-    app.get('/api/v1/events', authOperator, async (req, res) => {
-        const project = req.query.project || null;
+    api('get', '/api/v1/events', null, async (req, res) => {
+        const project = req.query.project ? String(req.query.project) : null;
+        if (project)
+            assertProject(req.actor, project);
         const limit = Math.min(Number(req.query.limit || 100), 500);
+        const scope = projectScope(req.actor, 'project_id'); // scoped tokens never see node-level or other projects' events
         const events = project
             ? await db.all('SELECT * FROM events WHERE project_id=? ORDER BY created_at DESC LIMIT ?', project, limit)
-            : await db.all('SELECT * FROM events ORDER BY created_at DESC LIMIT ?', limit);
+            : await db.all(`SELECT * FROM events WHERE ${scope.where} ORDER BY created_at DESC LIMIT ?`, ...scope.args, limit);
         res.json({ events });
     });
-    app.get('/api/v1/diagnostics', authOperator, async (req, res) => {
+    api('get', '/api/v1/diagnostics', null, async (req, res) => {
         const checks = [];
         async function check(name, fn) {
             try {
@@ -614,8 +1039,10 @@ export async function createApp() {
         }
         res.json({ ok: checks.every(c => c.ok), registry: REGISTRY, checks });
     });
-    app.post('/api/v1/diagnostics/nodes', authOperator, async (req, res) => {
+    api('post', '/api/v1/diagnostics/nodes', null, async (req, res) => {
         const nodes = req.body.node ? await db.all('SELECT id FROM nodes WHERE id=? OR name=?', req.body.node, req.body.node) : await db.all('SELECT id FROM nodes');
+        if (req.body.node && !nodes.length)
+            return res.status(404).json({ error: `node ${req.body.node} not found` });
         const ts = now();
         const commands = [];
         for (const node of nodes) {
@@ -662,7 +1089,7 @@ export async function createApp() {
         }
         return buildIngressWork({ states, endpoints: await endpointsFor(states), node: node.name, secrets, acmeEmail: ACME_EMAIL });
     }
-    app.get('/agent/v1/work', authAgent, async (req, res) => {
+    get('/agent/v1/work', authAgent, async (req, res) => {
         await db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
         const node = (await db.get('SELECT * FROM nodes WHERE id=?', req.node.id));
         const allStates = await currentDesiredStates();
@@ -670,7 +1097,8 @@ export async function createApp() {
         const commands = await db.all('SELECT * FROM commands WHERE node_id=? AND state=?', node.id, 'pending');
         if (commands.length)
             await db.run(`UPDATE commands SET state='running',attempts=attempts+1,updated_at=? WHERE node_id=? AND state='pending'`, now(), node.id);
-        const work = { serverTime: now(), nodeId: node.id, desiredStates, commands };
+        const retainedImages = [...new Set((await retainedStates()).flatMap(state => Object.values(state.services).filter(service => service.artifact && (!service.placement.nodes?.length || service.placement.nodes.includes(node.name))).map(service => service.image)))];
+        const work = { serverTime: now(), nodeId: node.id, desiredStates, commands, retainedImages };
         const network = await networkFor(node);
         if (network)
             work.network = network;
@@ -682,7 +1110,7 @@ export async function createApp() {
             work.ingress = await ingressFor(node, allStates);
         res.json(work);
     });
-    app.post('/agent/v1/status', authAgent, async (req, res) => {
+    post('/agent/v1/status', authAgent, async (req, res) => {
         const ts = now();
         const report = req.body;
         for (const st of report.projects || []) {
@@ -698,9 +1126,15 @@ export async function createApp() {
         await db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', ts, 'online', req.node.id);
         res.json({ ok: true });
     });
-    app.post('/agent/v1/events', authAgent, async (req, res) => { const evs = req.body.events || [req.body]; for (const ev of evs)
-        await db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at) VALUES(?,?,?,?,?,?,?,?,?)', id('evt'), ev.projectId || null, ev.deploymentId || null, req.node.id, ev.level || 'info', ev.type || 'event', ev.message || '', JSON.stringify(ev.metadata || {}), now()); res.json({ ok: true }); });
-    app.get('/agent/v1/secrets', authAgent, async (req, res) => {
+    post('/agent/v1/events', authAgent, async (req, res) => {
+        const evs = req.body.events || [req.body];
+        for (const ev of evs)
+            await db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at) VALUES(?,?,?,?,?,?,?,?,?)', id('evt'), ev.projectId || null, ev.deploymentId || null, req.node.id, ev.level || 'info', ev.type || 'event', ev.message || '', JSON.stringify(ev.metadata || {}), now());
+        if (Date.now() - lastEventPrune >= eventPruneIntervalMs)
+            await pruneEvents().catch(e => console.error(`event pruning failed: ${errMessage(e)}`));
+        res.json({ ok: true });
+    });
+    get('/agent/v1/secrets', authAgent, async (req, res) => {
         const project = String(req.query.project || 'site');
         let rows = await db.all('SELECT name,encrypted_value,version FROM secrets WHERE project_id=?', project);
         if (req.query.services) {
@@ -712,6 +1146,11 @@ export async function createApp() {
         const secrets = { secrets: Object.fromEntries(rows.map(r => [r.name, decrypt(r.encrypted_value, masterKey)])), versions: Object.fromEntries(rows.map(r => [r.name, r.version])) };
         res.json(secrets);
     });
+    app.use((error, _req, res, next) => {
+        if (res.headersSent)
+            return next(error);
+        sendError(res, error);
+    });
     return app;
 }
 export async function listen(app, port = PORT) {