@wpelevator/tug 0.4.0 → 0.6.0

package/dist/control.js

@@ -1,27 +1,30 @@
 #!/usr/bin/env node
 import express from 'express';
+import { resolveImage, parseRegistryCredentials, imageLocation, REGISTRY_AUTH_SECRET, } from "./registry.js";
 import multer from 'multer';
 import fs from 'node:fs/promises';
 import { createReadStream } from 'node:fs';
 import { pipeline } from 'node:stream/promises';
 import { saveImageArchive, inspectImage, validateImageArchive, hashArchive } from "./artifacts.js";
-import sqlite3 from 'sqlite3';
-import { open } from 'sqlite';
 import { scanArchive } from "./archive.js";
 import https from 'node:https';
 import http from 'node:http';
 import path from 'node:path';
 import { WebSocketServer, WebSocket } from 'ws';
 import * as tar from 'tar';
-import { openDb } from "./db.js";
+import { openDb, openSnapshotDb } from "./db.js";
+import { isUniqueViolation } from "./db-driver.js";
 import { encrypt, decrypt, loadOrCreateMasterKey } from "./secrets.js";
 import { buildDesiredState, loopbackIngressServices, serviceSecretNames, validateCompose, } from "./compose.js";
-import { buildAndPush, unpackContext, run as runCommand } from "./build.js";
+import { buildImage, unpackContext, run as runCommand } from "./build.js";
 import { sentinelNames } from "./normalize.js";
 import { ensureDir, errMessage, hashToken, id, now, token } from "./util.js";
 import { DEFAULT_MESH_CIDR, DEFAULT_WG_PORT, WG_INTERFACE, WG_KEEPALIVE, allocateOverlayIp, isWgKey, parseCidr, wgEndpoint, } from "./mesh.js";
 import { buildIngressWork, resolveServiceEndpoints } from "./ingress.js";
-import { ACTIVE_DEPLOYMENT_STATES, LIVE_MAX_FRAME_BYTES, LIVE_MAX_BUFFERED_BYTES, LIVE_MAX_SESSIONS, LIVE_START_TIMEOUT_MS, } from "./protocol.js";
+import { ACTIVE_DEPLOYMENT_STATES, LIVE_MAX_FRAME_BYTES, LIVE_MAX_BUFFERED_BYTES, LIVE_MAX_SESSIONS, LIVE_START_TIMEOUT_MS, BACKUP_COMMAND_TYPES, } from "./protocol.js";
+import { BACKUP_SECRET_PROJECT, isReservedProject, parseBackupDestination, projectRepository, repositoryPassword, redactSecrets, } from "./backup-destination.js";
+import { createBackupService } from "./backup-scheduler.js";
+import { createRestoreService } from "./backup-restore.js";
 import { AUTO_ROLLBACK_ACTOR, ROUTE_ROLES, PROJECT_NAME_PATTERN, authenticate, hasRole, assertProject, bodyProject, createGuard, createToken, listTokens, parseTokenRequest, queryProject, recordEvent, revokeToken, } from "./tokens.js";
 class WakeHub {
     cursors = new Map();
@@ -119,7 +122,18 @@ function parseLiveRequest(value) {
         Number(value.tail) <= 10_000)
         return value;
     if (value.operation === 'exec' &&
-        frameKeys(value, [...common, 'service', 'command', 'interactive', 'tty', 'columns', 'rows']) &&
+        frameKeys(value, [
+            ...common,
+            'service',
+            'command',
+            'interactive',
+            'tty',
+            'columns',
+            'rows',
+            'oneOff',
+        ]) &&
+        (value.oneOff === undefined || value.oneOff === true) &&
+        (value.oneOff !== true || (value.interactive === false && value.tty === false)) &&
         Array.isArray(value.command) &&
         value.command.length > 0 &&
         value.command.length <= 256 &&
@@ -394,7 +408,8 @@ const ACTIVE_STATES_SQL = ACTIVE_DEPLOYMENT_STATES.map((s) => `'${s}'`).join(','
 const PORT = Number(process.env.PORT || 8080);
 export async function createApp(options = {}) {
     const images = {
-        build: buildAndPush,
+        resolve: resolveImage,
+        build: buildImage,
         save: saveImageArchive,
         inspect: inspectImage,
         validate: validateImageArchive,
@@ -411,9 +426,6 @@ export async function createApp(options = {}) {
     });
     const dbFile = path.join(DATA_DIR, 'tug.db');
     const masterKeyFile = path.join(DATA_DIR, 'master.key');
-    const REGISTRY = process.env.TUG_REGISTRY?.trim() || '';
-    const REGISTRY_AUTH_PATH = process.env.TUG_REGISTRY_AUTH_PATH || null;
-    const REGISTRY_CONTAINER = process.env.TUG_REGISTRY_CONTAINER || null;
     // Networking layers are opt-in; with all of them off the work response is unchanged.
     const MESH = process.env.TUG_MESH === 'true';
     const MESH_CIDR = process.env.TUG_MESH_CIDR || DEFAULT_MESH_CIDR;
@@ -422,56 +434,6 @@ export async function createApp(options = {}) {
     const ACME_EMAIL = process.env.TUG_ACME_EMAIL || undefined;
     if (MESH)
         parseCidr(MESH_CIDR);
-    let htpasswdQueue = Promise.resolve();
-    function withHtpasswdLock(fn) {
-        const next = htpasswdQueue.catch(() => { }).then(fn);
-        htpasswdQueue = next.then(() => { }, () => { });
-        return next;
-    }
-    async function reloadRegistry() {
-        if (!REGISTRY_CONTAINER)
-            return;
-        await runCommand('docker', ['restart', REGISTRY_CONTAINER]).catch((e) => console.error(`registry reload failed: ${e.message}`));
-    }
-    async function upsertRegistryCredential(name, password) {
-        if (!REGISTRY || !REGISTRY_AUTH_PATH)
-            return;
-        await withHtpasswdLock(async () => {
-            const { out } = await runCommand('htpasswd', ['-nbB', name, password]);
-            const line = out.trim();
-            let existing = '';
-            try {
-                existing = await fs.readFile(REGISTRY_AUTH_PATH, 'utf8');
-            }
-            catch { }
-            const lines = existing
-                .split('\n')
-                .filter(Boolean)
-                .filter((l) => !l.startsWith(`${name}:`));
-            lines.push(line);
-            await ensureDir(path.dirname(REGISTRY_AUTH_PATH));
-            await fs.writeFile(REGISTRY_AUTH_PATH, `${lines.join('\n')}\n`);
-        });
-        await reloadRegistry();
-    }
-    async function removeRegistryCredential(name) {
-        if (!REGISTRY_AUTH_PATH)
-            return;
-        await withHtpasswdLock(async () => {
-            let existing = '';
-            try {
-                existing = await fs.readFile(REGISTRY_AUTH_PATH, 'utf8');
-            }
-            catch { }
-            const lines = existing
-                .split('\n')
-                .filter(Boolean)
-                .filter((l) => !l.startsWith(`${name}:`));
-            await ensureDir(path.dirname(REGISTRY_AUTH_PATH));
-            await fs.writeFile(REGISTRY_AUTH_PATH, lines.length ? `${lines.join('\n')}\n` : '');
-        });
-        await reloadRegistry();
-    }
     const db = await (options.openDatabase || openDb)(dbFile);
     const wakeHub = new WakeHub();
     // Events are an operator-facing log, not a record of truth, so keep them bounded. `0` turns a limit off; anything that is
@@ -490,32 +452,37 @@ export async function createApp(options = {}) {
     const EVENT_MAX_ROWS = envCount('TUG_EVENT_MAX_ROWS', 100_000);
     const eventPruneIntervalMs = options.eventPruneIntervalMs ?? 60 * 60 * 1000;
     let lastEventPrune = Date.now();
-    async function pruneEvents() {
+    function pruneEvents() {
         lastEventPrune = Date.now();
         let removed = 0;
         if (EVENT_RETENTION_DAYS > 0)
             removed +=
-                (await db.run('DELETE FROM events WHERE created_at < ?', new Date(Date.now() - EVENT_RETENTION_DAYS * 86_400_000).toISOString())).changes ?? 0;
+                db.run('DELETE FROM events WHERE created_at < ?', new Date(Date.now() - EVENT_RETENTION_DAYS * 86_400_000).toISOString()).changes ?? 0;
         if (EVENT_MAX_ROWS > 0)
             removed +=
-                (await db.run('DELETE FROM events WHERE id IN (SELECT id FROM events ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?)', EVENT_MAX_ROWS)).changes ?? 0;
+                db.run('DELETE FROM events WHERE id IN (SELECT id FROM events ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?)', EVENT_MAX_ROWS).changes ?? 0;
         if (removed)
             console.log(`removed ${removed} old events (kept ${EVENT_RETENTION_DAYS || 'any'} days, at most ${EVENT_MAX_ROWS || 'unlimited'} rows)`);
         return removed;
     }
-    await pruneEvents().catch((e) => console.error(`event pruning failed: ${errMessage(e)}`));
+    try {
+        pruneEvents();
+    }
+    catch (e) {
+        console.error(`event pruning failed: ${errMessage(e)}`);
+    }
     const masterKey = await loadOrCreateMasterKey(masterKeyFile);
-    const operator = await db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash');
+    const operator = db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash');
     if (!operator) {
         const t = process.env.TUG_OPERATOR_TOKEN || token('tug_op');
-        await db.run('INSERT INTO meta(key,value) VALUES(?,?)', 'operator_token_hash', hashToken(t));
+        db.run('INSERT INTO meta(key,value) VALUES(?,?)', 'operator_token_hash', hashToken(t));
         await fs
             .writeFile(path.join(DATA_DIR, 'operator.token'), `${t}\n`, { mode: 0o600 })
             .catch(() => { });
         console.log(`operator token: ${t}`);
     }
     const app = express();
-    const authenticateAgent = async (raw) => raw
+    const authenticateAgent = (raw) => raw
         ? db.get('SELECT * FROM nodes WHERE agent_token_hash=?', hashToken(raw))
         : undefined;
     let restartRequired = false;
@@ -579,13 +546,21 @@ export async function createApp(options = {}) {
         apiRoutes.set(key, role);
         route(method, url, [guard(role, resolve ?? undefined), ...handlers]);
     };
-    const deploymentProject = async (req) => (await db.get('SELECT project_id FROM deployments WHERE id=?', req.params.id))?.project_id;
-    const commandProject = async (req) => {
-        const row = await db.get('SELECT project_id FROM commands WHERE id=?', req.params.id);
+    const deploymentProject = (req) => db.get('SELECT project_id FROM deployments WHERE id=?', req.params.id)
+        ?.project_id;
+    const commandProject = (req) => {
+        const row = db.get('SELECT project_id FROM commands WHERE id=?', req.params.id);
         return row ? row.project_id : undefined;
     };
     /** Audit trail entry for a control action; bookkeeping failures are logged, never returned to the caller. */
-    const audit = (actor, event) => recordEvent(db, { ...event, actor: actor.name }).catch((e) => console.error(`audit event failed: ${errMessage(e)}`));
+    const audit = (actor, event) => {
+        try {
+            recordEvent(db, { ...event, actor: actor.name });
+        }
+        catch (e) {
+            console.error(`audit event failed: ${errMessage(e)}`);
+        }
+    };
     const projectScope = (actor, column) => actor.projects
         ? {
             where: `${column} IN (${actor.projects.map(() => '?').join(',') || "''"})`,
@@ -594,22 +569,22 @@ export async function createApp(options = {}) {
         : { where: '1=1', args: [] };
     const authAgent = async (req, res, next) => {
         const raw = (req.headers.authorization || '').replace(/^Bearer /, '');
-        const node = await authenticateAgent(raw);
+        const node = authenticateAgent(raw);
         if (!node)
             return res.status(401).json({ error: 'unauthorized' });
         req.node = node;
         next();
     };
-    const wakeAll = async (project) => {
-        for (const node of await db.all('SELECT id FROM nodes'))
+    const wakeAll = (project) => {
+        for (const node of db.all('SELECT id FROM nodes'))
             wakeHub.wake(node.id, project);
     };
     let endpointSnapshot = '';
-    async function updateDeploymentConvergence(projectId) {
-        const project = await db.get('SELECT * FROM projects WHERE id=?', projectId);
+    function updateDeploymentConvergence(projectId) {
+        const project = db.get('SELECT * FROM projects WHERE id=?', projectId);
         if (!project?.current_deployment_id)
             return;
-        const ds = await db.get('SELECT * FROM desired_states WHERE deployment_id=?', project.current_deployment_id);
+        const ds = db.get('SELECT * FROM desired_states WHERE deployment_id=?', project.current_deployment_id);
         if (!ds)
             return;
         const state = JSON.parse(ds.state_json);
@@ -619,7 +594,7 @@ export async function createApp(options = {}) {
                 required.add(node);
         if (!required.size)
             return;
-        const statuses = await db.all('SELECT node_id,observed_version,health,status_json FROM node_statuses WHERE project_id=?', projectId);
+        const statuses = db.all('SELECT node_id,observed_version,health,status_json FROM node_statuses WHERE project_id=?', projectId);
         const byNode = new Map(statuses.map((s) => [s.node_id, s]));
         const reportsRevision = (node) => state.revision
             ? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision
@@ -628,7 +603,7 @@ export async function createApp(options = {}) {
             .filter((node) => reportsRevision(node) && byNode.get(node)?.health === 'failed')
             .map((node) => byNode.get(node));
         if (failedNodes.length) {
-            const marked = await db.run('UPDATE deployments SET state=?,error_message=?,updated_at=? WHERE id=? AND state=?', 'failed', deploymentFailureMessage(failedNodes), now(), project.current_deployment_id, 'rolling_out');
+            const marked = db.run('UPDATE deployments SET state=?,error_message=?,updated_at=? WHERE id=? AND state=?', 'failed', deploymentFailureMessage(failedNodes), now(), project.current_deployment_id, 'rolling_out');
             if (marked.changes === 1)
                 scheduleAutoRollback(projectId, project.current_deployment_id, state);
         }
@@ -637,29 +612,29 @@ export async function createApp(options = {}) {
             : byNode.get(node)?.observed_version === state.version) &&
             ['serving', 'skipped'].includes(byNode.get(node)?.health ?? ''));
         if (ok)
-            await db.run('UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state=?', 'succeeded', now(), project.current_deployment_id, 'rolling_out');
+            db.run('UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state=?', 'succeeded', now(), project.current_deployment_id, 'rolling_out');
     }
     /** Events control writes when it rolls back on its own; they are attributed to a reserved system actor. */
-    async function recordAutoRollbackEvent(projectId, deploymentId, type, message, metadata = {}) {
-        await db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at,actor) VALUES(?,?,?,?,?,?,?,?,?,?)', id('evt'), projectId, deploymentId, null, 'warn', type, message, JSON.stringify(metadata), now(), AUTO_ROLLBACK_ACTOR);
+    function recordAutoRollbackEvent(projectId, deploymentId, type, message, metadata = {}) {
+        db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at,actor) VALUES(?,?,?,?,?,?,?,?,?,?)', id('evt'), projectId, deploymentId, null, 'warn', type, message, JSON.stringify(metadata), now(), AUTO_ROLLBACK_ACTOR);
     }
     /**
      * Creates a rollback deployment that copies the most recent earlier deployment's desired state under a
      * new revision. Shared by `POST /api/v1/rollback` and automatic rollback; the caller holds the writer queue.
      * `auto` only considers `succeeded` deployments and requires `failedDeploymentId` to still be current.
      */
-    async function createRollback(project, opts) {
+    function createRollback(project, opts) {
         let lock;
         try {
-            lock = await acquireProjectLock(project, opts.auto ? 'auto-rollback' : 'rollback');
-            const active = await db.get(`SELECT id,state FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) ORDER BY created_at DESC LIMIT 1`, project);
+            lock = acquireProjectLock(project, opts.auto ? 'auto-rollback' : 'rollback');
+            const active = db.get(`SELECT id,state FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) ORDER BY created_at DESC LIMIT 1`, project);
             if (active && !opts.force)
                 throw httpError(409, `deployment ${active.id} is already ${active.state}; retry with force to rollback anyway`);
-            const current = await db.get('SELECT current_deployment_id FROM projects WHERE id=?', project);
+            const current = db.get('SELECT current_deployment_id FROM projects WHERE id=?', project);
             if (opts.failedDeploymentId && current?.current_deployment_id !== opts.failedDeploymentId)
                 throw httpError(409, `deployment ${opts.failedDeploymentId} is no longer the current deployment`);
             const states = opts.auto ? `'succeeded'` : `'succeeded','rolling_out'`;
-            const previousDs = await db.get(`SELECT ds.*, d.version AS deployment_version FROM desired_states ds JOIN deployments d ON d.id=ds.deployment_id
+            const previousDs = db.get(`SELECT ds.*, d.version AS deployment_version FROM desired_states ds JOIN deployments d ON d.id=ds.deployment_id
         WHERE ds.project_id=? AND ds.deployment_id != ? AND d.state IN (${states}) ORDER BY ds.created_at DESC LIMIT 1`, project, current?.current_deployment_id || '');
             if (!previousDs)
                 throw httpError(404, 'no previous deployment found');
@@ -667,17 +642,17 @@ export async function createApp(options = {}) {
             const depId = id('dep');
             const previousState = JSON.parse(previousDs.state_json);
             previousState.revision = depId;
-            await db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, `${ROLLBACK_VERSION_PREFIX}${previousState.version}-${Date.now()}`, 'rolling_out', ts, ts, opts.actor);
-            await db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, previousState.version, JSON.stringify(previousState), ts);
-            await db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
-            await wakeAll(project);
+            db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, `${ROLLBACK_VERSION_PREFIX}${previousState.version}-${Date.now()}`, 'rolling_out', ts, ts, opts.actor);
+            db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, previousState.version, JSON.stringify(previousState), ts);
+            db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
+            wakeAll(project);
             return {
-                deployment: await db.get('SELECT * FROM deployments WHERE id=?', depId),
+                deployment: db.get('SELECT * FROM deployments WHERE id=?', depId),
                 desiredState: previousState,
             };
         }
         finally {
-            await releaseLock(lock);
+            releaseLock(lock);
         }
     }
     /**
@@ -690,9 +665,9 @@ export async function createApp(options = {}) {
         if (state.compose['x-tug']?.rollback !== 'auto')
             return;
         void (async () => {
-            const failed = await db.get('SELECT * FROM deployments WHERE id=?', deploymentId);
+            const failed = db.get('SELECT * FROM deployments WHERE id=?', deploymentId);
             if (failed?.version.startsWith(ROLLBACK_VERSION_PREFIX)) {
-                await recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `rollback deployment ${failed.version} failed; not rolling back a rollback`);
+                recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `rollback deployment ${failed.version} failed; not rolling back a rollback`);
                 return;
             }
             try {
@@ -701,27 +676,32 @@ export async function createApp(options = {}) {
                     auto: true,
                     failedDeploymentId: deploymentId,
                 }));
-                await recordAutoRollbackEvent(projectId, out.deployment.id, 'deployment.auto_rollback', `deployment ${deploymentId} (${failed?.version}) reported failed health; rolled back to ${out.desiredState.version} as deployment ${out.deployment.id}`, {
+                recordAutoRollbackEvent(projectId, out.deployment.id, 'deployment.auto_rollback', `deployment ${deploymentId} (${failed?.version}) reported failed health; rolled back to ${out.desiredState.version} as deployment ${out.deployment.id}`, {
                     failedDeploymentId: deploymentId,
                     rollbackDeploymentId: out.deployment.id,
                     version: out.desiredState.version,
                 });
             }
             catch (e) {
-                await recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `deployment ${deploymentId} failed but automatic rollback was not possible: ${errMessage(e)}`).catch(() => { });
+                try {
+                    recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `deployment ${deploymentId} failed but automatic rollback was not possible: ${errMessage(e)}`);
+                }
+                catch {
+                    /* Best-effort audit event. */
+                }
             }
         })().catch((e) => console.error('auto rollback failed:', errMessage(e)));
     }
-    async function currentDesired(projectId) {
+    function currentDesired(projectId) {
         return db.get('SELECT ds.* FROM desired_states ds JOIN projects p ON p.current_deployment_id=ds.deployment_id WHERE p.id=?', projectId);
     }
     const live = new LiveHub(async (actor, request) => {
         assertProject(actor, request.project);
-        const row = await currentDesired(request.project);
+        const row = currentDesired(request.project);
         if (!row)
             throw httpError(404, missingProject(request.project));
         const desired = JSON.parse(row.state_json);
-        const nodes = await db.all('SELECT * FROM nodes');
+        const nodes = db.all('SELECT * FROM nodes');
         const eligible = new Set();
         const services = request.operation === 'ps'
             ? Object.values(desired.services)
@@ -739,6 +719,17 @@ export async function createApp(options = {}) {
         const node = nodes.find((node) => node.name === (request.node || [...eligible][0]));
         if (!node)
             throw httpError(404, 'no eligible node for this operation');
+        if (request.operation === 'exec' && request.oneOff)
+            audit(actor, {
+                type: 'job.run',
+                project: request.project,
+                message: `${actor.name} ran a one-off ${request.service} container on ${node.name}: ${request.command.join(' ').slice(0, 200)}`,
+                metadata: {
+                    service: request.service,
+                    node: node.name,
+                    command: request.command.slice(0, 64),
+                },
+            });
         return node;
     }, options.liveStartTimeoutMs);
     streamRuntimes.set(app, {
@@ -750,38 +741,44 @@ export async function createApp(options = {}) {
     api('get', '/api/v1/live', null, (_req, res) => {
         res.status(426).json({ error: 'WebSocket upgrade required' });
     });
-    async function acquireProjectLock(project, operation, ttlMs = Number(process.env.TUG_LOCK_TTL_MS || 10 * 60 * 1000)) {
+    function acquireProjectLock(project, operation, ttlMs = Number(process.env.TUG_LOCK_TTL_MS || 10 * 60 * 1000)) {
         const name = `project:${project}`;
         const ts = now();
-        await db.run('DELETE FROM locks WHERE expires_at < ?', ts);
+        db.run('DELETE FROM locks WHERE expires_at < ?', ts);
         const owner = `${operation}:${id('lock')}`;
         try {
-            await db.run('INSERT INTO locks(name,owner,expires_at,created_at) VALUES(?,?,?,?)', name, owner, new Date(Date.now() + ttlMs).toISOString(), ts);
+            db.run('INSERT INTO locks(name,owner,expires_at,created_at) VALUES(?,?,?,?)', name, owner, new Date(Date.now() + ttlMs).toISOString(), ts);
         }
         catch (e) {
-            if (!/UNIQUE constraint/.test(errMessage(e)))
+            if (!isUniqueViolation(e))
                 throw e;
-            const existing = await db.get('SELECT * FROM locks WHERE name=?', name);
+            const existing = db.get('SELECT * FROM locks WHERE name=?', name);
             const err = new Error(`project ${project} is locked by ${existing?.owner} until ${existing?.expires_at}`);
             err.status = 409;
             err.lock = existing;
             throw err;
         }
         const heartbeat = setInterval(() => {
-            void db
-                .run('UPDATE locks SET expires_at=? WHERE name=? AND owner=?', new Date(Date.now() + ttlMs).toISOString(), name, owner)
-                .catch(() => { });
+            try {
+                db.run('UPDATE locks SET expires_at=? WHERE name=? AND owner=?', new Date(Date.now() + ttlMs).toISOString(), name, owner);
+            }
+            catch {
+                /* Best-effort bookkeeping. */
+            }
         }, Math.max(10, Math.floor(ttlMs / 3)));
         heartbeat.unref();
         return { name, owner, heartbeat };
     }
-    async function releaseLock(lock) {
+    function releaseLock(lock) {
         if (lock?.heartbeat)
             clearInterval(lock.heartbeat);
         if (lock)
-            await db
-                .run('DELETE FROM locks WHERE name=? AND owner=?', lock.name, lock.owner)
-                .catch(() => { });
+            try {
+                db.run('DELETE FROM locks WHERE name=? AND owner=?', lock.name, lock.owner);
+            }
+            catch {
+                /* Best-effort bookkeeping. */
+            }
     }
     function sendError(res, e) {
         const err = e;
@@ -796,13 +793,13 @@ export async function createApp(options = {}) {
         if (!project || !service || !Number.isInteger(replicas) || replicas < 1)
             throw httpError(400, 'project, service and a positive integer replicas are required');
         assertProject(req.actor, project);
-        const dsRow = await currentDesired(project);
+        const dsRow = currentDesired(project);
         if (!dsRow)
             throw httpError(404, `project ${project} has no desired state`);
         const state = JSON.parse(dsRow.state_json);
         if (!state.services?.[service])
             throw httpError(404, `service ${service} not found in project ${project}`);
-        const nodeRows = await db.all('SELECT name,labels_json,last_seen_at FROM nodes ORDER BY created_at,name');
+        const nodeRows = db.all('SELECT name,labels_json,last_seen_at FROM nodes ORDER BY created_at,name');
         const nodeInfos = nodeRows
             .filter((row) => !isStale(row))
             .map((row) => ({
@@ -827,8 +824,8 @@ export async function createApp(options = {}) {
         };
         state.services[service].replicas = selected.length;
         state.revision = id('rev');
-        await db.run('UPDATE desired_states SET state_json=? WHERE id=?', JSON.stringify(state), dsRow.id);
-        await wakeAll(project);
+        db.run('UPDATE desired_states SET state_json=? WHERE id=?', JSON.stringify(state), dsRow.id);
+        wakeAll(project);
         res.json({
             ok: true,
             project,
@@ -841,59 +838,59 @@ export async function createApp(options = {}) {
     api('post', '/api/v1/projects', (req) => String(req.body?.name || 'site'), async (req, res) => {
         const name = req.body.name || 'site';
         const ts = now();
-        await db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', name, name, ts, ts);
-        res.json(await db.get('SELECT * FROM projects WHERE id=?', name));
+        db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', name, name, ts, ts);
+        res.json(db.get('SELECT * FROM projects WHERE id=?', name));
     });
-    api('get', '/api/v1/projects/:project', (req) => req.params.project, async (req, res) => res.json(await db.get('SELECT * FROM projects WHERE id=?', req.params.project)));
+    api('get', '/api/v1/projects/:project', (req) => req.params.project, async (req, res) => res.json(db.get('SELECT * FROM projects WHERE id=?', req.params.project)));
     /** A node whose agent has been seen before but not within TUG_STALE_SECONDS; never-seen (just registered) nodes stay schedulable. */
     const isStale = (row) => !!row.last_seen_at &&
         Date.parse(row.last_seen_at) < Date.now() - Number(process.env.TUG_STALE_SECONDS || 60) * 1000;
     const NODE_INFO_SQL = 'SELECT id,name,address,public_address,overlay_ip,labels_json,status,created_at,updated_at FROM nodes WHERE id=?';
     /** Allocates the node's stable overlay IP on first registration (retrying if a concurrent register took it). */
-    async function ensureOverlayIp(nodeId) {
+    function ensureOverlayIp(nodeId) {
         if (!MESH)
             return;
         for (let attempt = 0; attempt < 5; attempt++) {
-            const row = await db.get('SELECT overlay_ip FROM nodes WHERE id=?', nodeId);
+            const row = db.get('SELECT overlay_ip FROM nodes WHERE id=?', nodeId);
             if (row?.overlay_ip)
                 return;
-            const used = (await db.all('SELECT overlay_ip FROM nodes WHERE overlay_ip IS NOT NULL')).map((r) => r.overlay_ip);
+            const used = db
+                .all('SELECT overlay_ip FROM nodes WHERE overlay_ip IS NOT NULL')
+                .map((r) => r.overlay_ip);
             try {
-                await db.run('UPDATE nodes SET overlay_ip=? WHERE id=? AND overlay_ip IS NULL', allocateOverlayIp(MESH_CIDR, used), nodeId);
+                db.run('UPDATE nodes SET overlay_ip=? WHERE id=? AND overlay_ip IS NULL', allocateOverlayIp(MESH_CIDR, used), nodeId);
                 return;
             }
             catch (e) {
-                if (!/UNIQUE constraint/.test(errMessage(e)))
+                if (!isUniqueViolation(e))
                     throw e;
             }
         }
         throw new Error('could not allocate an overlay address');
     }
     /** Stores (or refreshes) the node's WireGuard peer entry; the private key never reaches control. */
-    async function syncWgPeer(nodeId, publicKey) {
+    function syncWgPeer(nodeId, publicKey) {
         if (!MESH)
             return;
-        const node = await db.get('SELECT * FROM nodes WHERE id=?', nodeId);
-        const existing = await db.get('SELECT * FROM wg_peers WHERE node_id=?', nodeId);
+        const node = db.get('SELECT * FROM nodes WHERE id=?', nodeId);
+        const existing = db.get('SELECT * FROM wg_peers WHERE node_id=?', nodeId);
         const key = publicKey ?? existing?.public_key;
         if (!node?.overlay_ip || !key)
             return;
-        await db.run(`INSERT INTO wg_peers(node_id,public_key,overlay_ip,endpoint,allowed_ips,updated_at) VALUES(?,?,?,?,?,?)
+        db.run(`INSERT INTO wg_peers(node_id,public_key,overlay_ip,endpoint,allowed_ips,updated_at) VALUES(?,?,?,?,?,?)
       ON CONFLICT(node_id) DO UPDATE SET public_key=excluded.public_key,overlay_ip=excluded.overlay_ip,endpoint=excluded.endpoint,allowed_ips=excluded.allowed_ips,updated_at=excluded.updated_at`, nodeId, key, node.overlay_ip, wgEndpoint(node.public_address, WG_PORT) ?? null, `${node.overlay_ip}/32`, now());
     }
     async function registerNode({ name, address, publicAddress, wgPublicKey, labels, }) {
         if (wgPublicKey !== undefined && !isWgKey(wgPublicKey))
             throw httpError(400, 'wgPublicKey must be a base64 WireGuard public key');
         const agentToken = token('tug_agent');
-        const registryToken = token('tug_reg');
         const ts = now();
-        await db.run(`INSERT INTO nodes(id,name,address,public_address,status,labels_json,agent_token_hash,registry_token_hash,created_at,updated_at)
-      VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET name=excluded.name,address=excluded.address,public_address=excluded.public_address,labels_json=excluded.labels_json,agent_token_hash=excluded.agent_token_hash,registry_token_hash=excluded.registry_token_hash,updated_at=excluded.updated_at`, name, name, address || null, publicAddress || null, 'registered', JSON.stringify(Array.isArray(labels) ? labels : []), hashToken(agentToken), hashToken(registryToken), ts, ts);
-        await ensureOverlayIp(name);
-        await syncWgPeer(name, wgPublicKey);
-        await upsertRegistryCredential(name, registryToken);
-        await wakeAll();
-        const row = await db.get(NODE_INFO_SQL, name);
+        db.run(`INSERT INTO nodes(id,name,address,public_address,status,labels_json,agent_token_hash,created_at,updated_at)
+      VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET name=excluded.name,address=excluded.address,public_address=excluded.public_address,labels_json=excluded.labels_json,agent_token_hash=excluded.agent_token_hash,updated_at=excluded.updated_at`, name, name, address || null, publicAddress || null, 'registered', JSON.stringify(Array.isArray(labels) ? labels : []), hashToken(agentToken), ts, ts);
+        ensureOverlayIp(name);
+        syncWgPeer(name, wgPublicKey);
+        wakeAll();
+        const row = db.get(NODE_INFO_SQL, name);
         if (!row)
             throw httpError(500, 'node not found after registration');
         const node = {
@@ -907,7 +904,7 @@ export async function createApp(options = {}) {
             created_at: row.created_at,
             updated_at: row.updated_at,
         };
-        return { node, agentToken, registryToken };
+        return { node, agentToken };
     }
     api('post', '/api/v1/nodes/register', null, async (req, res) => {
         const name = req.body.name;
@@ -934,18 +931,18 @@ export async function createApp(options = {}) {
         const uses = Math.max(1, Number(req.body.uses ?? 1));
         const enrollToken = token('tug_enroll');
         const expiresAt = new Date(Date.now() + ttlSeconds * 1000).toISOString();
-        await db.run('INSERT INTO enroll_tokens(token_hash,name,uses_remaining,expires_at,created_at) VALUES(?,?,?,?,?)', hashToken(enrollToken), name, uses, expiresAt, now());
-        res.json({ enrollToken, name, usesRemaining: uses, expiresAt, registry: REGISTRY });
+        db.run('INSERT INTO enroll_tokens(token_hash,name,uses_remaining,expires_at,created_at) VALUES(?,?,?,?,?)', hashToken(enrollToken), name, uses, expiresAt, now());
+        res.json({ enrollToken, name, usesRemaining: uses, expiresAt });
     });
     post('/agent/v1/enroll', async (req, res) => {
         const body = req.body;
         const raw = String(body.enrollToken || (req.headers.authorization || '').replace(/^Bearer /, '')).trim();
         if (!raw)
             return res.status(400).json({ error: 'enrollToken required' });
-        const row = await db.get('SELECT * FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
+        const row = db.get('SELECT * FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
         if (!row || row.uses_remaining < 1 || row.expires_at < now()) {
             if (row)
-                await db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
+                db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
             return res.status(401).json({ error: 'invalid or expired enroll token' });
         }
         const name = row.name || body.name;
@@ -968,14 +965,14 @@ export async function createApp(options = {}) {
         }
         const remaining = row.uses_remaining - 1;
         if (remaining < 1)
-            await db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
+            db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
         else
-            await db.run('UPDATE enroll_tokens SET uses_remaining=? WHERE token_hash=?', remaining, hashToken(raw));
-        const enrolled = { ...result, registry: REGISTRY };
+            db.run('UPDATE enroll_tokens SET uses_remaining=? WHERE token_hash=?', remaining, hashToken(raw));
+        const enrolled = { ...result };
         res.json(enrolled);
     });
     api('get', '/api/v1/nodes', null, async (req, res) => {
-        const rows = await db.all('SELECT id,name,address,public_address,overlay_ip,ingress,ingress_status_json,status,labels_json,last_seen_at,created_at,updated_at FROM nodes ORDER BY name');
+        const rows = db.all('SELECT id,name,address,public_address,overlay_ip,ingress,ingress_status_json,status,labels_json,last_seen_at,created_at,updated_at FROM nodes ORDER BY name');
         res.json({
             nodes: rows.map((row) => {
                 const { labels_json, ...rest } = row;
@@ -983,30 +980,13 @@ export async function createApp(options = {}) {
             }),
         });
     });
-    api('post', '/api/v1/nodes/:name/rotate-registry-token', null, async (req, res) => {
-        const node = await db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
-        if (!node)
-            return res.status(404).json({ error: 'node not found' });
-        const registryToken = token('tug_reg');
-        await db.run('UPDATE nodes SET registry_token_hash=?,updated_at=? WHERE id=?', hashToken(registryToken), now(), node.id);
-        await upsertRegistryCredential(node.name, registryToken);
-        res.json({ ok: true, node: node.name, registryToken });
-    });
-    api('post', '/api/v1/nodes/:name/revoke-registry-token', null, async (req, res) => {
-        const node = await db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
-        if (!node)
-            return res.status(404).json({ error: 'node not found' });
-        await db.run('UPDATE nodes SET registry_token_hash=NULL,updated_at=? WHERE id=?', now(), node.id);
-        await removeRegistryCredential(node.name);
-        res.json({ ok: true, node: node.name, revoked: true });
-    });
-    async function findNode(nameOrId) {
+    function findNode(nameOrId) {
         return db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', nameOrId, nameOrId);
     }
-    async function queueCommand(nodeId, projectId, type, payload = {}) {
+    function queueCommand(nodeId, projectId, type, payload = {}, deadline = null) {
         const commandId = id('cmd');
         const ts = now();
-        await db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', commandId, nodeId, projectId, type, 'pending', JSON.stringify(payload), 0, ts, null, ts, ts);
+        db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', commandId, nodeId, projectId, type, 'pending', JSON.stringify(payload), 0, ts, deadline, ts, ts);
         wakeHub.wake(nodeId, projectId ?? undefined);
         return commandId;
     }
@@ -1015,19 +995,19 @@ export async function createApp(options = {}) {
     api('post', '/api/v1/nodes/:name/rotate-wg-key', null, async (req, res) => {
         if (!MESH)
             return res.status(409).json({ error: 'the WireGuard mesh is not enabled (TUG_MESH=true)' });
-        const node = await findNode(req.params.name);
+        const node = findNode(req.params.name);
         if (!node)
             return res.status(404).json({ error: 'node not found' });
-        res.json({ commandId: await queueCommand(node.id, null, 'wg.rotate-key'), nodeId: node.id });
+        res.json({ commandId: queueCommand(node.id, null, 'wg.rotate-key'), nodeId: node.id });
     });
     // Removes a retired node: its token, WireGuard peer, statuses and queued commands. Refused while a current deployment still places
     // something on it (redeploy without it first) or while its agent is still reporting, unless force=true.
     api('delete', '/api/v1/nodes/:name', null, async (req, res) => {
-        const node = await db.get('SELECT id,name,last_seen_at FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
+        const node = db.get('SELECT id,name,last_seen_at FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
         if (!node)
             return res.status(404).json({ error: 'node not found' });
         const referenced = [];
-        for (const state of await currentDesiredStates()) {
+        for (const state of currentDesiredStates()) {
             for (const [name, svc] of Object.entries(state.services || {}))
                 if (svc.placement?.nodes?.includes(node.name))
                     referenced.push(`${state.project}/${name}`);
@@ -1043,24 +1023,24 @@ export async function createApp(options = {}) {
             return res.status(409).json({
                 error: `node ${node.name} is still reporting; stop its agent first or pass force=true`,
             });
-        await db.run('DELETE FROM commands WHERE node_id=?', node.id);
-        await db.run('DELETE FROM node_statuses WHERE node_id=?', node.id);
-        await db.run('DELETE FROM wg_peers WHERE node_id=?', node.id);
-        await db.run('DELETE FROM nodes WHERE id=?', node.id);
-        await audit(req.actor, {
+        db.run('DELETE FROM commands WHERE node_id=?', node.id);
+        db.run('DELETE FROM node_statuses WHERE node_id=?', node.id);
+        db.run('DELETE FROM wg_peers WHERE node_id=?', node.id);
+        db.run('DELETE FROM nodes WHERE id=?', node.id);
+        audit(req.actor, {
             type: 'node.removed',
             message: `node ${node.name} removed`,
             metadata: { node: node.name },
         });
-        await wakeAll();
+        wakeAll();
         res.json({ ok: true, node: node.name });
     });
     api('post', '/api/v1/nodes/:name/ingress', null, async (req, res) => {
-        const node = await findNode(req.params.name);
+        const node = findNode(req.params.name);
         if (!node)
             return res.status(404).json({ error: 'node not found' });
         const enabled = req.body.enabled !== false;
-        await db.run('UPDATE nodes SET ingress=?,updated_at=? WHERE id=?', enabled ? 1 : 0, now(), node.id);
+        db.run('UPDATE nodes SET ingress=?,updated_at=? WHERE id=?', enabled ? 1 : 0, now(), node.id);
         wakeHub.wake(node.id);
         res.json({ ok: true, node: node.name, ingress: enabled });
     });
@@ -1071,30 +1051,18 @@ export async function createApp(options = {}) {
         if (!isWgKey(publicKey))
             return res.status(400).json({ error: 'publicKey must be a base64 WireGuard public key' });
         try {
-            await syncWgPeer(req.node.id, publicKey);
-            await wakeAll();
+            syncWgPeer(req.node.id, publicKey);
+            wakeAll();
             res.json({ ok: true });
         }
         catch (e) {
-            sendError(res, /UNIQUE constraint/.test(errMessage(e))
-                ? httpError(409, 'public key already registered to another node')
-                : e);
+            sendError(res, isUniqueViolation(e) ? httpError(409, 'public key already registered to another node') : e);
         }
     });
-    post('/api/v1/registry/auth', async (req, res) => {
-        const username = req.body.username;
-        const password = req.body.password;
-        const node = username && password
-            ? await db.get('SELECT id,name FROM nodes WHERE (name=? OR id=?) AND registry_token_hash IS NOT NULL AND registry_token_hash=?', username, username, hashToken(password))
-            : null;
-        if (!node)
-            return res.status(401).json({ ok: false, error: 'invalid registry credentials' });
-        res.json({ ok: true, node: node.name });
-    });
     api('post', '/api/v1/tokens', null, async (req, res) => {
         const request = parseTokenRequest(req.body);
-        const created = await createToken(db, request, req.actor.name);
-        await audit(req.actor, {
+        const created = createToken(db, request, req.actor.name);
+        audit(req.actor, {
             type: 'token.create',
             message: `token ${created.name} (${created.role}${created.projects ? `, projects: ${created.projects.join(', ')}` : ''}) created by ${req.actor.name}`,
             metadata: {
@@ -1106,30 +1074,210 @@ export async function createApp(options = {}) {
         });
         res.json(created);
     });
-    api('get', '/api/v1/tokens', null, async (_req, res) => res.json({ tokens: await listTokens(db) }));
+    api('get', '/api/v1/tokens', null, async (_req, res) => res.json({ tokens: listTokens(db) }));
     api('delete', '/api/v1/tokens/:name', null, async (req, res) => {
-        if (!(await revokeToken(db, req.params.name)))
+        if (!revokeToken(db, req.params.name))
             return res.status(404).json({ error: 'token not found' });
-        await audit(req.actor, {
+        audit(req.actor, {
             type: 'token.revoke',
             message: `token ${req.params.name} revoked by ${req.actor.name}`,
             metadata: { name: req.params.name },
         });
         res.json({ ok: true, name: req.params.name });
     });
+    const backupDestination = () => db.get("SELECT * FROM backup_destination WHERE id='default'");
+    const backupEnv = () => {
+        const row = backupDestination();
+        if (!row)
+            return {};
+        const env = {};
+        for (const name of JSON.parse(row.env_names_json)) {
+            const secret = db.get('SELECT encrypted_value FROM secrets WHERE project_id=? AND name=?', BACKUP_SECRET_PROJECT, name);
+            if (secret)
+                env[name] = decrypt(secret.encrypted_value, masterKey);
+        }
+        return env;
+    };
+    const backups = createBackupService({
+        db,
+        queueCommand,
+        resticImage: process.env.TUG_RESTIC_IMAGE,
+        // Defensive: the agent already redacts, but a stored error must never carry a destination credential.
+        redact: (text) => redactSecrets(text, Object.values(backupEnv())),
+    });
+    app.locals.backups = backups;
+    const restores = createRestoreService({ db, backups, wake: (project) => wakeAll(project) });
+    app.locals.restores = restores;
+    const backupTickMs = Number(process.env.TUG_BACKUP_TICK_MS) || 30_000;
+    const backupTimer = setInterval(() => {
+        try {
+            backups.tick();
+            restores.advance();
+        }
+        catch (e) {
+            console.error(`backup scheduler failed: ${errMessage(e)}`);
+        }
+    }, backupTickMs);
+    backupTimer.unref();
+    try {
+        backups.tick(); // catch up on slots missed while control was down
+        restores.advance(); // re-adopt unfinished restores and keep their projects locked
+    }
+    catch (e) {
+        console.error(`backup scheduler failed: ${errMessage(e)}`);
+    }
+    api('post', '/api/v1/backups/run', bodyProject, async (req, res) => {
+        const project = String(req.body.project || 'site');
+        const targets = req.body.targets;
+        if (targets !== undefined &&
+            (!Array.isArray(targets) || targets.some((t) => typeof t !== 'string')))
+            return res.status(400).json({ error: 'targets must be a list of service or volume names' });
+        if (!db.get('SELECT 1 FROM projects WHERE id=?', project))
+            return res.status(404).json({ error: `project ${project} not found` });
+        const result = backups.runNow(project, targets, req.actor.name);
+        if (!result.runs.length)
+            return res.status(result.errors.length ? 409 : 404).json({
+                error: result.errors.join('; ') || `project ${project} has no backup targets`,
+            });
+        res.json(result);
+    });
+    api('post', '/api/v1/restores', bodyProject, async (req, res) => {
+        const body = req.body ?? {};
+        const project = String(body.project || 'site');
+        if (typeof body.target !== 'string' || !body.target)
+            return res
+                .status(400)
+                .json({ error: 'target is required, such as service:db or volume:uploads' });
+        const target = body.target.includes(':')
+            ? body.target
+            : (db.get('SELECT target FROM backup_schedules WHERE project_id=? AND enabled=1 AND target LIKE ?', project, `%:${body.target}`)?.target ?? body.target);
+        const request = {
+            project,
+            target,
+            from: typeof body.from === 'string' ? body.from : undefined,
+            scratch: body.scratch === true,
+            safetyBackup: body.safetyBackup !== false,
+            writers: Array.isArray(body.writers) ? body.writers.map(String) : undefined,
+            noWriters: body.noWriters === true,
+            actor: req.actor.name,
+        };
+        const plan = restores.plan(request);
+        if ('error' in plan)
+            return res.status(plan.status).json({ error: plan.error });
+        if (body.confirm !== true)
+            return res.json({ confirmed: false, plan });
+        const started = restores.start(request, plan);
+        if ('error' in started)
+            return res.status(started.status).json({ error: started.error });
+        res.json({ confirmed: true, plan, ...started });
+    });
+    api('get', '/api/v1/restores', queryProject, async (req, res) => res.json({ restores: restores.list(String(req.query.project || 'site')) }));
+    api('post', '/api/v1/restores/:id/release', null, async (req, res) => {
+        const result = restores.releaseInterrupted(req.params.id);
+        if ('error' in result)
+            return res.status(result.status).json({ error: result.error });
+        res.json(result);
+    });
+    api('post', '/api/v1/restores/:id/resume', null, async (req, res) => {
+        const result = restores.resume(req.params.id);
+        if ('error' in result)
+            return res.status(result.status).json({ error: result.error });
+        res.json(result);
+    });
+    api('get', '/api/v1/backups', queryProject, async (req, res) => {
+        const project = String(req.query.project || 'site');
+        const limit = Math.min(Math.max(Number(req.query.limit) || 50, 1), 500);
+        res.json({
+            project,
+            destinationConfigured: Boolean(backupDestination()),
+            ...backups.list(project, limit),
+        });
+    });
+    api('get', '/api/v1/backup/destination', null, async (_req, res) => {
+        const row = backupDestination();
+        res.json(row
+            ? {
+                configured: true,
+                url: row.url,
+                envNames: JSON.parse(row.env_names_json),
+                checkedAt: row.checked_at,
+                checkResult: row.check_result_json ? JSON.parse(row.check_result_json) : null,
+                updatedAt: row.updated_at,
+            }
+            : { configured: false });
+    });
+    api('post', '/api/v1/backup/destination', null, async (req, res) => {
+        let parsed;
+        try {
+            parsed = parseBackupDestination(req.body);
+        }
+        catch (e) {
+            return res.status(400).json({ error: errMessage(e) });
+        }
+        const ts = now();
+        db.exec('BEGIN IMMEDIATE');
+        try {
+            db.run('DELETE FROM secrets WHERE project_id=?', BACKUP_SECRET_PROJECT);
+            for (const [name, value] of Object.entries(parsed.env))
+                db.run('INSERT INTO secrets(id,project_id,name,encrypted_value,version,created_at,updated_at) VALUES(?,?,?,?,?,?,?)', id('sec'), BACKUP_SECRET_PROJECT, name, encrypt(value, masterKey), 1, ts, ts);
+            db.run(`INSERT INTO backup_destination(id,url,env_names_json,checked_at,check_result_json,updated_at) VALUES('default',?,?,NULL,NULL,?)
+        ON CONFLICT(id) DO UPDATE SET url=excluded.url,env_names_json=excluded.env_names_json,checked_at=NULL,check_result_json=NULL,updated_at=excluded.updated_at`, parsed.url, JSON.stringify(Object.keys(parsed.env).sort()), ts);
+            db.exec('COMMIT');
+        }
+        catch (e) {
+            db.exec('ROLLBACK');
+            throw e;
+        }
+        audit(req.actor, {
+            type: 'backup.destination_set',
+            message: `backup destination set to ${parsed.url} by ${req.actor.name}`,
+            metadata: { url: parsed.url, envNames: Object.keys(parsed.env).sort() },
+        });
+        res.json({ ok: true, url: parsed.url, envNames: Object.keys(parsed.env).sort() });
+    });
+    api('delete', '/api/v1/backup/destination', null, async (req, res) => {
+        db.run('DELETE FROM secrets WHERE project_id=?', BACKUP_SECRET_PROJECT);
+        const removed = db.run("DELETE FROM backup_destination WHERE id='default'");
+        if (removed.changes)
+            audit(req.actor, {
+                type: 'backup.destination_removed',
+                message: `backup destination removed by ${req.actor.name}`,
+            });
+        res.json({ ok: true, removed: removed.changes > 0 });
+    });
+    api('get', '/api/v1/backup/key', null, async (req, res) => {
+        const project = String(req.query.project || '');
+        if (!PROJECT_NAME_PATTERN.test(project) || isReservedProject(project))
+            return res.status(400).json({ error: 'project is required' });
+        if (!db.get('SELECT 1 FROM projects WHERE id=?', project))
+            return res.status(404).json({ error: `project ${project} not found` });
+        const destination = backupDestination();
+        audit(req.actor, {
+            type: 'backup.key_read',
+            project,
+            message: `repository password for ${project} read by ${req.actor.name}`,
+        });
+        res.json({
+            project,
+            repository: destination ? projectRepository(destination.url, project) : null,
+            password: repositoryPassword(masterKey, project),
+        });
+    });
     api('post', '/api/v1/secrets', bodyProject, async (req, res) => {
         const { project = 'site', name, value } = req.body;
+        if (isReservedProject(String(project)))
+            return res.status(400).json({ error: `project names starting with _ are reserved` });
         if (!name || value == null)
             return res.status(400).json({ error: 'name and value required' });
         let lock;
         try {
-            lock = await acquireProjectLock(project, 'secret.set');
-            const old = await db.get('SELECT version FROM secrets WHERE project_id=? AND name=?', project, name);
+            lock = acquireProjectLock(project, 'secret.set');
+            const old = db.get('SELECT version FROM secrets WHERE project_id=? AND name=?', project, name);
             const ts = now();
-            await db.run(`INSERT INTO secrets(id,project_id,name,encrypted_value,version,created_at,updated_at) VALUES(?,?,?,?,?,?,?)
+            db.run(`INSERT INTO secrets(id,project_id,name,encrypted_value,version,created_at,updated_at) VALUES(?,?,?,?,?,?,?)
         ON CONFLICT(project_id,name) DO UPDATE SET encrypted_value=excluded.encrypted_value,version=excluded.version,updated_at=excluded.updated_at`, id('sec'), project, name, encrypt(value, masterKey), (old?.version || 0) + 1, ts, ts);
-            await wakeAll(project);
-            await audit(req.actor, {
+            wakeAll(project);
+            audit(req.actor, {
                 type: 'secret.set',
                 project,
                 message: `secret ${name} set to version ${(old?.version || 0) + 1} by ${req.actor.name}`,
@@ -1141,18 +1289,22 @@ export async function createApp(options = {}) {
             sendError(res, e);
         }
         finally {
-            await releaseLock(lock);
+            releaseLock(lock);
         }
     });
-    api('get', '/api/v1/secrets', queryProject, async (req, res) => res.json({
-        secrets: await db.all('SELECT project_id,name,version,updated_at FROM secrets WHERE project_id=? ORDER BY name', req.query.project || 'site'),
-    }));
+    api('get', '/api/v1/secrets', queryProject, async (req, res) => isReservedProject(String(req.query.project || 'site'))
+        ? res.status(400).json({ error: 'project names starting with _ are reserved' })
+        : res.json({
+            secrets: db.all('SELECT project_id,name,version,updated_at FROM secrets WHERE project_id=? ORDER BY name', String(req.query.project || 'site')),
+        }));
     api('delete', '/api/v1/secrets/:name', queryProject, async (req, res) => {
         const project = String(req.query.project || 'site');
-        const removed = await db.run('DELETE FROM secrets WHERE project_id=? AND name=?', project, req.params.name);
+        if (isReservedProject(project))
+            return res.status(400).json({ error: 'project names starting with _ are reserved' });
+        const removed = db.run('DELETE FROM secrets WHERE project_id=? AND name=?', project, req.params.name);
         if (removed.changes) {
-            await wakeAll(project);
-            await audit(req.actor, {
+            wakeAll(project);
+            audit(req.actor, {
                 type: 'secret.delete',
                 project,
                 message: `secret ${req.params.name} deleted by ${req.actor.name}`,
@@ -1163,33 +1315,35 @@ export async function createApp(options = {}) {
     });
     const artifactDir = path.join(DATA_DIR, 'images');
     await ensureDir(artifactDir);
-    async function retainedStates() {
+    function retainedStates() {
         // All published history is retained. This conservative superset protects failed/lagging versions too.
-        return (await db.all('SELECT * FROM desired_states')).map((row) => JSON.parse(row.state_json));
+        return db
+            .all('SELECT * FROM desired_states')
+            .map((row) => JSON.parse(row.state_json));
     }
     async function collectArtifacts() {
-        const states = await retainedStates();
+        const states = retainedStates();
         const refs = new Set(states.flatMap((state) => Object.values(state.services).flatMap((service) => service.artifact ? [service.artifact.buildId] : [])));
         const imageRefs = new Set(states.flatMap((state) => Object.values(state.services).map((service) => service.image)));
-        const rows = await db.all('SELECT * FROM builds');
+        const rows = db.all('SELECT * FROM builds');
         for (const row of rows) {
             if (refs.has(row.id) || imageRefs.has(row.image_ref))
                 continue;
-            const active = await db.get("SELECT 1 FROM deployments WHERE id=? AND state IN ('uploading','building')", row.deployment_id);
+            const active = db.get("SELECT 1 FROM deployments WHERE id=? AND state IN ('uploading','building')", row.deployment_id);
             if (active)
                 continue;
             await fs.rm(path.join(DATA_DIR, row.artifact_path || path.join('images', `${row.id}.tar.gz`)), { force: true });
             await images.run('docker', ['image', 'rm', row.image_ref]).catch(() => { });
-            await db.run('DELETE FROM builds WHERE id=?', row.id);
+            db.run('DELETE FROM builds WHERE id=?', row.id);
         }
-        const remaining = await db.all('SELECT id FROM builds');
+        const remaining = db.all('SELECT id FROM builds');
         const owned = new Set(remaining.flatMap((row) => [`${row.id}.tar.gz`, `${row.id}.tar.gz.tmp`]));
         for (const name of await fs.readdir(artifactDir))
             if (!owned.has(name))
                 await fs.rm(path.join(artifactDir, name), { force: true });
     }
     // Previous-process builds cannot finish after a restart; clear their unpublished files.
-    await db.run("UPDATE deployments SET state='build_failed',error_message='control restarted during build' WHERE state IN ('uploading','building')");
+    db.run("UPDATE deployments SET state='build_failed',error_message='control restarted during build' WHERE state IN ('uploading','building')");
     await collectArtifacts();
     const receiveDeployment = upload.any();
     api('post', '/api/v1/deployments', null, (req, res) => {
@@ -1230,7 +1384,7 @@ export async function createApp(options = {}) {
                 throw httpError(400, 'invalid project or version');
             if (version.startsWith(ROLLBACK_VERSION_PREFIX))
                 throw httpError(400, `version cannot start with "${ROLLBACK_VERSION_PREFIX}" (reserved for rollback deployments)`);
-            const nodeRows = await db.all('SELECT name,labels_json,last_seen_at,created_at FROM nodes ORDER BY created_at,name');
+            const nodeRows = db.all('SELECT name,labels_json,last_seen_at,created_at FROM nodes ORDER BY created_at,name');
             // Stale nodes (agent not seen recently) are not scheduled onto, so a dead host cannot hold a rollout open.
             const nodeInfos = nodeRows
                 .filter((row) => !isStale(row))
@@ -1263,18 +1417,6 @@ export async function createApp(options = {}) {
                 }
             }
             const nodes = nodeRows.map((row) => ({ name: row.name }));
-            for (const vol of Object.values(compose.volumes || {})) {
-                const xe = vol?.['x-tug'];
-                if (!xe ||
-                    (xe.type !== 'nfs' && xe.driver !== 'nfs') ||
-                    xe.address ||
-                    !xe.placement?.nodes?.[0])
-                    continue;
-                const exporter = await db.get('SELECT address,overlay_ip FROM nodes WHERE name=?', xe.placement.nodes[0]);
-                const addr = (MESH && exporter?.overlay_ip) || exporter?.address;
-                if (addr)
-                    xe.address = addr;
-            }
             try {
                 validateCompose(compose, nodes.map((n) => n.name));
             }
@@ -1283,8 +1425,13 @@ export async function createApp(options = {}) {
             }
             // The CLI interpolates control secrets as sentinels; a reference to a secret that is gone would deploy an empty value.
             const referenced = sentinelNames(compose);
+            if (referenced.includes(REGISTRY_AUTH_SECRET))
+                throw httpError(400, REGISTRY_AUTH_SECRET +
+                    ' is reserved for registry authentication and cannot be used in service definitions');
             if (referenced.length) {
-                const known = new Set((await db.all('SELECT name FROM secrets WHERE project_id=?', project)).map((row) => row.name));
+                const known = new Set(db
+                    .all('SELECT name FROM secrets WHERE project_id=?', project)
+                    .map((row) => row.name));
                 const missing = referenced.filter((name) => !known.has(name));
                 if (missing.length)
                     throw httpError(400, `secrets not found in project ${project}: ${missing.join(', ')}; create them with tug secret-set`);
@@ -1320,30 +1467,29 @@ export async function createApp(options = {}) {
             }
             if (files.length !== (local ? local.length : context ? 1 : 0))
                 throw httpError(400, 'unexpected deployment upload');
-            if (!REGISTRY && builtServices.length && !context && !local)
-                throw httpError(400, 'build services require a source context or localImages upload without a registry');
+            if (builtServices.length && !context && !local)
+                throw httpError(400, 'build services require a source context or localImages upload');
             for (const [svcName, svc] of Object.entries(compose.services)) {
                 const ing = svc['x-tug']?.ingress;
                 if (ing?.tls !== 'custom')
                     continue;
                 for (const secret of [ing.cert, ing.key])
-                    if (!(await db.get('SELECT 1 FROM secrets WHERE project_id=? AND name=?', project, secret)))
+                    if (!db.get('SELECT 1 FROM secrets WHERE project_id=? AND name=?', project, secret ?? null))
                         throw httpError(400, `${svcName}: custom TLS secret ${secret} not found in project ${project}; set it with secret-set first`);
             }
-            lock = await acquireProjectLock(project, 'deploy');
-            const active = await db.get(`SELECT * FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) LIMIT 1`, project);
+            lock = acquireProjectLock(project, 'deploy');
+            const active = db.get(`SELECT * FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) LIMIT 1`, project);
             if (active)
                 throw httpError(409, `deployment ${active.id} is already ${active.state}`);
             depId = id('dep');
             const ts = now();
-            await db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', project, project, ts, ts);
-            await db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, version, 'building', ts, ts, req.actor.name);
+            db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', project, project, ts, ts);
+            db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, version, 'building', ts, ts, req.actor.name);
             const desired = buildDesiredState({
                 project,
                 version,
                 revision: depId,
                 compose,
-                registry: REGISTRY,
                 defaultNode: nodes[0]?.name || null,
             });
             if (context) {
@@ -1351,14 +1497,10 @@ export async function createApp(options = {}) {
                 await unpackContext(context.path, contextDir);
             }
             for (const service of builtServices) {
-                if (!contextDir && !local) {
-                    desired.services[service].image = `${REGISTRY}/${project}/${service}:${version}`;
-                    continue;
-                } // Explicit registry/no-build-upload means an existing pushed image.
                 const spec = desired.services[service];
                 const buildId = id('build');
                 const imageRef = spec.image;
-                await db.run('INSERT INTO builds(id,project_id,deployment_id,service,image_ref,state,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)', buildId, project, depId, service, imageRef, 'building', ts, ts);
+                db.run('INSERT INTO builds(id,project_id,deployment_id,service,image_ref,state,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)', buildId, project, depId, service, imageRef, 'building', ts, ts);
                 let info;
                 let importedClientRef;
                 const relative = path.join('images', `${buildId}.tar.gz`);
@@ -1392,21 +1534,12 @@ export async function createApp(options = {}) {
                             service,
                             buildSpec: compose.services[service].build,
                             imageRef,
-                            push: false,
                         });
-                    if (REGISTRY) {
-                        if (process.env.TUG_PUSH !== 'false')
-                            await images.run('docker', ['push', imageRef]);
-                        const inspected = await images.inspect(imageRef);
-                        await db.run('UPDATE builds SET state=?,image_id=?,platform=?,updated_at=? WHERE id=?', 'complete', inspected.imageId, inspected.platform, now(), buildId);
-                    }
-                    else {
-                        info = await images.save(imageRef, temporary);
-                        await fs.rename(temporary, final);
-                        const artifact = { buildId, imageRef, ...info };
-                        spec.artifact = artifact;
-                        await db.run('UPDATE builds SET state=?,image_id=?,platform=?,sha256=?,size_bytes=?,artifact_path=?,updated_at=? WHERE id=?', 'complete', info.imageId, info.platform, info.sha256, info.sizeBytes, relative, now(), buildId);
-                    }
+                    info = await images.save(imageRef, temporary);
+                    await fs.rename(temporary, final);
+                    const artifact = { buildId, imageRef, ...info };
+                    spec.artifact = artifact;
+                    db.run('UPDATE builds SET state=?,image_id=?,platform=?,sha256=?,size_bytes=?,artifact_path=?,updated_at=? WHERE id=?', 'complete', info.imageId, info.platform, info.sha256, info.sizeBytes, relative, now(), buildId);
                 }
                 finally {
                     if (importedClientRef)
@@ -1414,43 +1547,61 @@ export async function createApp(options = {}) {
                     await fs.rm(temporary, { force: true });
                 }
             }
+            const registrySecret = db.get('SELECT * FROM secrets WHERE project_id=? AND name=?', project, REGISTRY_AUTH_SECRET);
+            const credentials = parseRegistryCredentials(registrySecret ? decrypt(registrySecret.encrypted_value, masterKey) : undefined);
+            const resolved = new Map();
+            for (const [name, spec] of Object.entries(desired.services)) {
+                if (spec.artifact)
+                    continue;
+                const image = spec.image;
+                if (!resolved.has(image))
+                    resolved.set(image, await images.resolve(image, credentials));
+                spec.image = resolved.get(image);
+                desired.compose.services[name].image = spec.image;
+            }
             const buildLock = lock;
-            await withPublication(async () => {
-                const owned = await db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now());
-                const pending = await db.get('SELECT 1 FROM deployments WHERE id=? AND state=?', depId, 'building');
+            const publishingId = depId;
+            await withPublication(() => {
+                const owned = db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now());
+                const pending = db.get('SELECT 1 FROM deployments WHERE id=? AND state=?', publishingId, 'building');
                 if (!owned || !pending)
                     throw httpError(409, 'deployment was cancelled or its build lease was lost');
-                await db.exec('BEGIN IMMEDIATE');
+                db.exec('BEGIN IMMEDIATE');
                 try {
-                    const published = await db.run("UPDATE deployments SET state='rolling_out',updated_at=? WHERE id=? AND state='building'", now(), depId);
+                    const published = db.run("UPDATE deployments SET state='rolling_out',updated_at=? WHERE id=? AND state='building'", now(), publishingId);
                     if (published.changes !== 1 ||
-                        !(await db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now())))
+                        !db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now()))
                         throw httpError(409, 'deployment was cancelled or its build lease was lost');
-                    await db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, version, JSON.stringify(desired), ts);
+                    db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, publishingId, version, JSON.stringify(desired), ts);
                     for (const [volumeName, volumeSpec] of Object.entries(compose.volumes || {})) {
                         const lp = volumeSpec?.['x-tug'] || {};
                         const nodeName = lp.placement?.nodes?.[0] || null;
                         const node = nodeName
-                            ? await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName)
+                            ? db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName)
                             : null;
-                        await db.run(`INSERT INTO volumes(id,project_id,name,type,driver,node_id,config_json,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(project_id,name) DO UPDATE SET type=excluded.type,driver=excluded.driver,node_id=excluded.node_id,config_json=excluded.config_json,updated_at=excluded.updated_at`, id('vol'), project, volumeName, lp.type || 'local', lp.driver || null, node?.id || null, JSON.stringify(lp), ts, ts);
+                        db.run(`INSERT INTO volumes(id,project_id,name,type,driver,node_id,config_json,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(project_id,name) DO UPDATE SET type=excluded.type,driver=excluded.driver,node_id=excluded.node_id,config_json=excluded.config_json,updated_at=excluded.updated_at`, id('vol'), project, volumeName, lp.type || 'local', lp.driver || null, node?.id || null, JSON.stringify(lp), ts, ts);
                     }
-                    await db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
-                    await db.exec('COMMIT');
+                    db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', publishingId, now(), project);
+                    db.exec('COMMIT');
                 }
                 catch (e) {
-                    await db.exec('ROLLBACK');
+                    try {
+                        db.exec('ROLLBACK');
+                    }
+                    catch {
+                        /* Preserve the publication error. */
+                    }
                     throw e;
                 }
             });
-            await audit(req.actor, {
+            audit(req.actor, {
                 type: 'deploy',
                 project,
                 deploymentId: depId,
                 message: `deployment ${depId} (${version}) started by ${req.actor.name}`,
                 metadata: { version },
             });
-            await wakeAll(project);
+            wakeAll(project);
             responseBody = {
                 deployment: { id: depId, project, version, state: 'rolling_out', actor: req.actor.name },
                 desiredState: desired,
@@ -1458,13 +1609,16 @@ export async function createApp(options = {}) {
         }
         catch (e) {
             if (depId)
-                await db
-                    .run("UPDATE deployments SET state='build_failed',error_message=?,updated_at=? WHERE id=? AND state='building'", errMessage(e), now(), depId)
-                    .catch(() => { });
+                try {
+                    db.run("UPDATE deployments SET state='build_failed',error_message=?,updated_at=? WHERE id=? AND state='building'", errMessage(e), now(), depId);
+                }
+                catch {
+                    /* Best-effort bookkeeping. */
+                }
             responseError = e;
         }
         finally {
-            await releaseLock(lock);
+            releaseLock(lock);
             await Promise.all(files.map((file) => fs.rm(file.path, { force: true }).catch(() => { })));
             if (contextDir)
                 await fs.rm(contextDir, { recursive: true, force: true });
@@ -1476,12 +1630,12 @@ export async function createApp(options = {}) {
             res.json(responseBody);
     }
     get('/agent/v1/images/:buildId', authAgent, async (req, res) => {
-        const states = await retainedStates();
+        const states = retainedStates();
         const allowed = states.some((state) => Object.values(state.services).some((service) => service.artifact?.buildId === req.params.buildId &&
             (!service.placement.nodes?.length || service.placement.nodes.includes(req.node.name))));
         if (!allowed)
             return res.status(404).json({ error: 'image not found' });
-        const row = await db.get('SELECT * FROM builds WHERE id=? AND state=?', req.params.buildId, 'complete');
+        const row = db.get('SELECT * FROM builds WHERE id=? AND state=?', req.params.buildId, 'complete');
         if (!row?.artifact_path)
             return res.status(404).json({ error: 'image not found' });
         try {
@@ -1499,17 +1653,17 @@ export async function createApp(options = {}) {
                 res.status(404).json({ error: 'image artifact unavailable' });
         }
     });
-    api('get', '/api/v1/deployments/:id', deploymentProject, async (req, res) => res.json(await db.get('SELECT * FROM deployments WHERE id=?', req.params.id)));
-    api('post', '/api/v1/deployments/:id/cancel', deploymentProject, async (req, res) => withPublication(async () => {
-        const row = await db.get('SELECT * FROM deployments WHERE id=?', req.params.id);
+    api('get', '/api/v1/deployments/:id', deploymentProject, async (req, res) => res.json(db.get('SELECT * FROM deployments WHERE id=?', req.params.id)));
+    api('post', '/api/v1/deployments/:id/cancel', deploymentProject, async (req, res) => withPublication(() => {
+        const row = db.get('SELECT * FROM deployments WHERE id=?', req.params.id);
         if (!row)
             return res.status(404).json({ error: 'deployment not found' });
         if (!ACTIVE_DEPLOYMENT_STATES.includes(row.state))
             return res.status(409).json({ error: `deployment is ${row.state}` });
-        const result = await db.run(`UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state IN (${ACTIVE_STATES_SQL})`, 'cancelled', now(), req.params.id);
+        const result = db.run(`UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state IN (${ACTIVE_STATES_SQL})`, 'cancelled', now(), req.params.id);
         if (result.changes !== 1)
             return res.status(409).json({ error: 'deployment is no longer active' });
-        await audit(req.actor, {
+        audit(req.actor, {
             type: 'cancel',
             project: row.project_id,
             deploymentId: row.id,
@@ -1520,11 +1674,11 @@ export async function createApp(options = {}) {
     api('post', '/api/v1/rollback', bodyProject, async (req, res) => withWriter(async () => {
         const project = req.body.project || 'site';
         try {
-            const out = await createRollback(project, {
+            const out = createRollback(project, {
                 actor: req.actor.name,
                 force: Boolean(req.body.force),
             });
-            await audit(req.actor, {
+            audit(req.actor, {
                 type: 'rollback',
                 project,
                 deploymentId: out.deployment.id,
@@ -1538,15 +1692,15 @@ export async function createApp(options = {}) {
         }
     }));
     api('get', '/api/v1/volumes', queryProject, async (req, res) => res.json({
-        volumes: await db.all('SELECT * FROM volumes WHERE project_id=? ORDER BY name', req.query.project || 'site'),
+        volumes: db.all('SELECT * FROM volumes WHERE project_id=? ORDER BY name', String(req.query.project || 'site')),
     }));
     api('delete', '/api/v1/volumes/:name', queryProject, async (req, res) => {
         const project = String(req.query.project || 'site');
         const deleteData = String(req.query.deleteData || 'false') === 'true';
         let lock;
         try {
-            lock = deleteData ? await acquireProjectLock(project, 'volume.delete') : null;
-            const volume = await db.get('SELECT * FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
+            lock = deleteData ? acquireProjectLock(project, 'volume.delete') : null;
+            const volume = db.get('SELECT * FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
             if (!volume)
                 return res.status(404).json({ error: 'volume not found' });
             const config = JSON.parse(volume.config_json || '{}');
@@ -1554,7 +1708,7 @@ export async function createApp(options = {}) {
                 return res
                     .status(409)
                     .json({ error: 'refusing to delete persistent volume without deleteData=true' });
-            const ds = await currentDesired(project);
+            const ds = currentDesired(project);
             if (ds) {
                 const state = JSON.parse(ds.state_json);
                 const inUse = Object.entries(state.compose?.services || {})
@@ -1567,19 +1721,19 @@ export async function createApp(options = {}) {
             }
             let commandId = null;
             if (deleteData && volume.node_id) {
-                commandId = await queueCommand(volume.node_id, project, 'volume.delete', {
+                commandId = queueCommand(volume.node_id, project, 'volume.delete', {
                     project,
                     volume: req.params.name,
                 });
             }
-            await db.run('DELETE FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
+            db.run('DELETE FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
             res.json({ ok: true, commandId, deletedMetadata: true, deleteData });
         }
         catch (e) {
             sendError(res, e);
         }
         finally {
-            await releaseLock(lock);
+            releaseLock(lock);
         }
     });
     api('post', '/api/v1/control/backups/run', null, async (req, res) => withWriter(async () => {
@@ -1590,8 +1744,8 @@ export async function createApp(options = {}) {
         const staging = path.join(backupDir, `${backupId}-staging`);
         await fs.rm(staging, { recursive: true, force: true });
         await ensureDir(staging);
-        const snapshotPath = path.join(staging, 'tug.db').replaceAll("'", "''");
-        await db.exec(`VACUUM INTO '${snapshotPath}'`);
+        const snapshotPath = path.join(staging, 'tug.db');
+        db.run('VACUUM INTO ?', snapshotPath);
         await fs.copyFile(masterKeyFile, path.join(staging, 'master.key'));
         await fs.cp(artifactDir, path.join(staging, 'images'), { recursive: true });
         await fs.writeFile(path.join(staging, 'metadata.json'), JSON.stringify({ backupId, createdAt: ts, type: 'control' }, null, 2));
@@ -1603,7 +1757,7 @@ export async function createApp(options = {}) {
             'images',
         ]);
         await fs.rm(staging, { recursive: true, force: true });
-        await db.run('INSERT INTO backups(id,project_id,target,node_id,state,artifact_ref,metadata_json,created_at,completed_at) VALUES(?,?,?,?,?,?,?,?,?)', backupId, '_control', 'control-plane', null, 'completed', artifactRef, JSON.stringify({ type: 'control' }), ts, now());
+        db.run('INSERT INTO backups(id,project_id,target,node_id,state,artifact_ref,metadata_json,created_at,completed_at) VALUES(?,?,?,?,?,?,?,?,?)', backupId, '_control', 'control-plane', null, 'completed', artifactRef, JSON.stringify({ type: 'control' }), ts, now());
         res.json({ backupId, artifactRef, state: 'completed' });
     }));
     api('post', '/api/v1/control/restore', null, async (req, res) => withWriter(async () => {
@@ -1629,25 +1783,21 @@ export async function createApp(options = {}) {
             const key = (await fs.readFile(path.join(restoreDir, 'master.key'), 'utf8')).trim();
             if (!/^[A-Za-z0-9+/]{43}=$/.test(key))
                 throw new Error('invalid backup master key');
-            snapshot = await open({
-                filename: path.join(restoreDir, 'tug.db'),
-                driver: sqlite3.Database,
-                mode: sqlite3.OPEN_READONLY,
-            });
-            const integrity = await snapshot.get('PRAGMA integrity_check');
+            snapshot = openSnapshotDb(path.join(restoreDir, 'tug.db'));
+            const integrity = snapshot.get('PRAGMA integrity_check');
             if (integrity?.integrity_check !== 'ok')
                 throw new Error('backup database failed integrity check');
-            if (!(await snapshot.get("SELECT 1 FROM meta WHERE key='operator_token_hash'")))
+            if (!snapshot.get("SELECT 1 FROM meta WHERE key='operator_token_hash'"))
                 throw new Error('backup database has no operator identity');
-            for (const secret of await snapshot.all('SELECT * FROM secrets'))
+            for (const secret of snapshot.all('SELECT * FROM secrets'))
                 decrypt(secret.encrypted_value, key);
-            const states = await snapshot.all('SELECT * FROM desired_states');
+            const states = snapshot.all('SELECT * FROM desired_states');
             for (const stateRow of states) {
                 const state = JSON.parse(stateRow.state_json);
                 for (const service of Object.values(state.services)) {
                     if (!service.artifact)
                         continue;
-                    const build = await snapshot.get('SELECT * FROM builds WHERE id=? AND state=?', service.artifact.buildId, 'complete');
+                    const build = snapshot.get('SELECT * FROM builds WHERE id=? AND state=?', service.artifact.buildId, 'complete');
                     if (!build?.artifact_path ||
                         !/^images\/build_[a-zA-Z0-9_-]+\.tar\.gz$/.test(build.artifact_path))
                         throw new Error('backup contains an unsafe or missing image reference');
@@ -1659,12 +1809,12 @@ export async function createApp(options = {}) {
                         throw new Error('backup image archive checksum mismatch');
                 }
             }
-            await snapshot.close();
+            snapshot.close();
             snapshot = undefined;
             await ensureDir(path.join(restoreDir, 'images'));
             // Validation completes before changing any live file. Stop DB-dependent work until restart.
             restartRequired = true;
-            await db.close();
+            db.close();
             await fs.rm(`${dbFile}-wal`, { force: true });
             await fs.rm(`${dbFile}-shm`, { force: true });
             await fs.cp(path.join(restoreDir, 'images'), artifactDir, { recursive: true });
@@ -1673,15 +1823,19 @@ export async function createApp(options = {}) {
             res.json({ ok: true, restored: true, restartRequired: true });
         }
         catch (e) {
-            if (snapshot)
-                await snapshot.close().catch(() => { });
+            try {
+                snapshot?.close();
+            }
+            catch {
+                /* Preserve the restore error. */
+            }
             res.status(restartRequired ? 500 : 400).json({ error: errMessage(e), restartRequired });
         }
     }));
-    async function commandForService({ project, service, type, payload, }) {
+    function commandForService({ project, service, type, payload, }) {
         if (!service)
             throw httpError(400, 'service required');
-        const ds = await currentDesired(project);
+        const ds = currentDesired(project);
         if (!ds)
             throw httpError(404, missingProject(project));
         const state = JSON.parse(ds.state_json);
@@ -1691,22 +1845,22 @@ export async function createApp(options = {}) {
         const nodeName = svc.placement?.nodes?.[0];
         if (!nodeName)
             throw httpError(400, 'service has no explicit node placement');
-        const node = await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
+        const node = db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
         if (!node)
             throw httpError(404, 'node not found');
-        const cmdId = await queueCommand(node.id, project, type, { project, service, ...payload });
+        const cmdId = queueCommand(node.id, project, type, { project, service, ...payload });
         return { commandId: cmdId, nodeId: node.id };
     }
     api('post', '/api/v1/restart', bodyProject, async (req, res) => {
         try {
             const project = req.body.project || 'site';
-            const queued = await commandForService({
+            const queued = commandForService({
                 project,
                 service: req.body.service,
                 type: 'service.restart',
                 payload: {},
             });
-            await audit(req.actor, {
+            audit(req.actor, {
                 type: 'restart',
                 project,
                 message: `service ${req.body.service} restart requested by ${req.actor.name}`,
@@ -1724,7 +1878,7 @@ export async function createApp(options = {}) {
         const tail = Number(req.body.tail || 200);
         if (!service)
             return res.status(400).json({ error: 'service required' });
-        const ds = await currentDesired(project);
+        const ds = currentDesired(project);
         if (!ds)
             return res.status(404).json({ error: missingProject(project) });
         const state = JSON.parse(ds.state_json);
@@ -1734,25 +1888,25 @@ export async function createApp(options = {}) {
         const nodeName = svc.placement?.nodes?.[0];
         if (!nodeName)
             return res.status(400).json({ error: 'service has no explicit node placement' });
-        const node = await db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
+        const node = db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
         if (!node)
             return res.status(404).json({ error: 'node not found' });
-        const cmdId = await queueCommand(node.id, project, 'logs.fetch', { project, service, tail });
+        const cmdId = queueCommand(node.id, project, 'logs.fetch', { project, service, tail });
         res.json({ commandId: cmdId, nodeId: node.id });
     });
-    api('get', '/api/v1/commands/:id', commandProject, async (req, res) => res.json(await db.get('SELECT * FROM commands WHERE id=?', req.params.id)));
+    api('get', '/api/v1/commands/:id', commandProject, async (req, res) => res.json(db.get('SELECT * FROM commands WHERE id=?', req.params.id)));
     api('get', '/api/v1/status', null, async (req, res) => {
         const staleSeconds = Number(process.env.TUG_STALE_SECONDS || 60);
         const cutoff = new Date(Date.now() - staleSeconds * 1000).toISOString();
-        await db.run("UPDATE nodes SET status='stale',updated_at=? WHERE last_seen_at IS NOT NULL AND last_seen_at < ?", now(), cutoff);
+        db.run("UPDATE nodes SET status='stale',updated_at=? WHERE last_seen_at IS NOT NULL AND last_seen_at < ?", now(), cutoff);
         const projects = projectScope(req.actor, 'id');
         const deployments = projectScope(req.actor, 'project_id');
         const statuses = projectScope(req.actor, 'project_id');
         res.json({
-            projects: await db.all(`SELECT * FROM projects WHERE ${projects.where}`, ...projects.args),
-            deployments: await db.all(`SELECT * FROM deployments WHERE ${deployments.where} ORDER BY created_at DESC LIMIT 20`, ...deployments.args),
-            nodes: await db.all('SELECT id,name,status,last_seen_at FROM nodes'),
-            statuses: await db.all(`SELECT * FROM node_statuses WHERE ${statuses.where}`, ...statuses.args),
+            projects: db.all(`SELECT * FROM projects WHERE ${projects.where}`, ...projects.args),
+            deployments: db.all(`SELECT * FROM deployments WHERE ${deployments.where} ORDER BY created_at DESC LIMIT 20`, ...deployments.args),
+            nodes: db.all('SELECT id,name,status,last_seen_at FROM nodes'),
+            statuses: db.all(`SELECT * FROM node_statuses WHERE ${statuses.where}`, ...statuses.args),
         });
     });
     api('get', '/api/v1/events', null, async (req, res) => {
@@ -1762,8 +1916,8 @@ export async function createApp(options = {}) {
         const limit = Math.min(Number(req.query.limit || 100), 500);
         const scope = projectScope(req.actor, 'project_id'); // scoped tokens never see node-level or other projects' events
         const events = project
-            ? await db.all('SELECT * FROM events WHERE project_id=? ORDER BY created_at DESC LIMIT ?', project, limit)
-            : await db.all(`SELECT * FROM events WHERE ${scope.where} ORDER BY created_at DESC LIMIT ?`, ...scope.args, limit);
+            ? db.all('SELECT * FROM events WHERE project_id=? ORDER BY created_at DESC LIMIT ?', project, limit)
+            : db.all(`SELECT * FROM events WHERE ${scope.where} ORDER BY created_at DESC LIMIT ?`, ...scope.args, limit);
         res.json({ events });
     });
     api('get', '/api/v1/diagnostics', null, async (req, res) => {
@@ -1777,10 +1931,10 @@ export async function createApp(options = {}) {
             }
         }
         await check('sqlite', async () => ({
-            value: (await db.get('SELECT COUNT(*) AS count FROM projects')).count,
+            value: db.get('SELECT COUNT(*) AS count FROM projects').count,
         }));
         await check('operator_token', async () => ({
-            configured: Boolean(await db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash')),
+            configured: Boolean(db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash')),
         }));
         await check('master_key', async () => ({ configured: Boolean(masterKey) }));
         await check('docker', async () => ({
@@ -1791,14 +1945,14 @@ export async function createApp(options = {}) {
         }));
         if (MESH) {
             await check('mesh', async () => {
-                const nodes = await db.all('SELECT n.name, n.overlay_ip, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
+                const nodes = db.all('SELECT n.name, n.overlay_ip, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
                 const missing = nodes.filter((n) => !n.has_key).map((n) => n.name);
                 if (missing.length)
                     throw new Error(`nodes without a reported WireGuard key (is TUG_MESH=true on their agents?): ${missing.join(', ')}`);
                 return { cidr: MESH_CIDR, port: WG_PORT, nodes: nodes.length };
             });
         }
-        const ingressNodes = await db.all('SELECT name,ingress_status_json FROM nodes WHERE ingress=1');
+        const ingressNodes = db.all('SELECT name,ingress_status_json FROM nodes WHERE ingress=1');
         if (ingressNodes.length) {
             await check('ingress', async () => {
                 const bad = ingressNodes
@@ -1810,45 +1964,47 @@ export async function createApp(options = {}) {
                 return { nodes: ingressNodes.map((n) => n.name) };
             });
         }
-        res.json({ ok: checks.every((c) => c.ok), registry: REGISTRY, checks });
+        res.json({ ok: checks.every((c) => c.ok), checks });
     });
     api('post', '/api/v1/diagnostics/nodes', null, async (req, res) => {
         const nodes = req.body.node
-            ? await db.all('SELECT id FROM nodes WHERE id=? OR name=?', req.body.node, req.body.node)
-            : await db.all('SELECT id FROM nodes');
+            ? db.all('SELECT id FROM nodes WHERE id=? OR name=?', req.body.node, req.body.node)
+            : db.all('SELECT id FROM nodes');
         if (req.body.node && !nodes.length)
             return res.status(404).json({ error: `node ${req.body.node} not found` });
         const commands = [];
         for (const node of nodes) {
-            const cmdId = await queueCommand(node.id, null, 'diagnostics.run');
+            const cmdId = queueCommand(node.id, null, 'diagnostics.run');
             commands.push({ commandId: cmdId, nodeId: node.id });
         }
         res.json({ commands });
     });
-    async function currentDesiredStates() {
-        const rows = await db.all(`SELECT ds.* FROM desired_states ds JOIN projects p ON p.current_deployment_id=ds.deployment_id ORDER BY ds.created_at DESC`);
+    function currentDesiredStates() {
+        const rows = db.all(`SELECT ds.* FROM desired_states ds JOIN projects p ON p.current_deployment_id=ds.deployment_id ORDER BY ds.created_at DESC`);
         return rows.map((s) => JSON.parse(s.state_json));
     }
-    async function endpointsFor(states) {
+    function endpointsFor(states) {
         const staleBefore = new Date(Date.now() - Number(process.env.TUG_STALE_SECONDS || 60) * 1000).toISOString();
-        const nodes = await db.all('SELECT n.*, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
+        const nodes = db.all('SELECT n.*, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
         const views = nodes.map((n) => ({
             name: n.name,
             overlayIp: MESH && n.has_key ? n.overlay_ip : null,
             address: n.address,
             lastSeenAt: n.last_seen_at,
         }));
-        const statuses = (await db.all('SELECT node_id,project_id,status_json FROM node_statuses')).map((r) => ({
+        const statuses = db
+            .all('SELECT node_id,project_id,status_json FROM node_statuses')
+            .map((r) => ({
             node: nodes.find((n) => n.id === r.node_id)?.name ?? r.node_id,
             project: r.project_id,
             status: JSON.parse(r.status_json),
         }));
         return resolveServiceEndpoints(states, views, statuses, staleBefore);
     }
-    async function networkFor(node) {
+    function networkFor(node) {
         if (!MESH || !node.overlay_ip)
             return undefined;
-        const peers = await db.all('SELECT w.*, n.name FROM wg_peers w JOIN nodes n ON n.id=w.node_id WHERE w.node_id != ? ORDER BY n.name', node.id);
+        const peers = db.all('SELECT w.*, n.name FROM wg_peers w JOIN nodes n ON n.id=w.node_id WHERE w.node_id != ? ORDER BY n.name', node.id);
         return {
             interface: WG_INTERFACE,
             overlayIp: node.overlay_ip,
@@ -1864,17 +2020,17 @@ export async function createApp(options = {}) {
         };
     }
     /** Names of all nodes and of the ingress nodes, for port binding and upstream decisions. */
-    async function nodeTopology() {
-        const rows = await db.all('SELECT name,ingress FROM nodes ORDER BY name');
+    function nodeTopology() {
+        const rows = db.all('SELECT name,ingress FROM nodes ORDER BY name');
         return {
             nodes: rows.map((r) => r.name),
             ingressNodes: rows.filter((r) => r.ingress).map((r) => r.name),
         };
     }
     /** Where this node's published ports listen: the address other nodes and the ingress reach it on, or loopback. */
-    async function portBindingFor(node, states) {
-        const hasKey = MESH && node.overlay_ip && (await db.get('SELECT 1 FROM wg_peers WHERE node_id=?', node.id));
-        const { nodes, ingressNodes } = await nodeTopology();
+    function portBindingFor(node, states) {
+        const hasKey = MESH && node.overlay_ip && db.get('SELECT 1 FROM wg_peers WHERE node_id=?', node.id);
+        const { nodes, ingressNodes } = nodeTopology();
         const loopback = {};
         for (const state of states) {
             const services = loopbackIngressServices(state, node.name, ingressNodes, nodes);
@@ -1883,7 +2039,7 @@ export async function createApp(options = {}) {
         }
         return { address: (hasKey ? node.overlay_ip : node.address) || null, loopback };
     }
-    async function ingressFor(node, states) {
+    function ingressFor(node, states) {
         const secrets = new Map();
         for (const state of states) {
             for (const svc of Object.values(state.compose.services || {})) {
@@ -1891,7 +2047,7 @@ export async function createApp(options = {}) {
                 if (ing?.tls !== 'custom')
                     continue;
                 for (const name of [ing.cert, ing.key]) {
-                    const row = await db.get('SELECT encrypted_value FROM secrets WHERE project_id=? AND name=?', state.project, name);
+                    const row = db.get('SELECT encrypted_value FROM secrets WHERE project_id=? AND name=?', state.project, name ?? null);
                     if (row)
                         secrets.set(`${state.project}/${name}`, decrypt(row.encrypted_value, masterKey));
                 }
@@ -1899,9 +2055,9 @@ export async function createApp(options = {}) {
         }
         return buildIngressWork({
             states,
-            endpoints: await endpointsFor(states),
+            endpoints: endpointsFor(states),
             node: node.name,
-            ...(await nodeTopology()),
+            ...nodeTopology(),
             secrets,
             acmeEmail: ACME_EMAIL,
         });
@@ -1909,19 +2065,21 @@ export async function createApp(options = {}) {
     get('/agent/v1/wake', authAgent, async (req, res) => {
         const after = Math.max(0, Number.parseInt(String(req.query.after ?? '0'), 10) || 0);
         const timeoutMs = Math.min(30_000, Math.max(0, Number.parseInt(String(req.query.timeout ?? '25000'), 10) || 0));
-        await db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
+        db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
         res.json(await wakeHub.wait(req.node.id, after, timeoutMs));
     });
     get('/agent/v1/work', authAgent, async (req, res) => {
-        await db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
-        const node = (await db.get('SELECT * FROM nodes WHERE id=?', req.node.id));
-        const allStates = await currentDesiredStates();
+        db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
+        const node = db.get('SELECT * FROM nodes WHERE id=?', req.node.id);
+        const allStates = currentDesiredStates();
         const desiredStates = allStates.filter((st) => Object.values(st.services).some((svc) => (svc.placement?.nodes || []).includes(node.name) || !svc.placement?.nodes?.length));
-        const commands = await db.all('SELECT * FROM commands WHERE node_id=? AND state=?', node.id, 'pending');
+        // A command past its deadline is never delivered late; the backup scheduler turns it into a missed run.
+        db.run("UPDATE commands SET state='expired',updated_at=? WHERE node_id=? AND state='pending' AND deadline IS NOT NULL AND deadline<?", now(), node.id, now());
+        const commands = db.all('SELECT * FROM commands WHERE node_id=? AND state=?', node.id, 'pending');
         if (commands.length)
-            await db.run(`UPDATE commands SET state='running',attempts=attempts+1,updated_at=? WHERE node_id=? AND state='pending'`, now(), node.id);
+            db.run(`UPDATE commands SET state='running',attempts=attempts+1,updated_at=? WHERE node_id=? AND state='pending'`, now(), node.id);
         const retainedImages = [
-            ...new Set((await retainedStates()).flatMap((state) => Object.values(state.services)
+            ...new Set(retainedStates().flatMap((state) => Object.values(state.services)
                 .filter((service) => service.artifact &&
                 (!service.placement.nodes?.length || service.placement.nodes.includes(node.name)))
                 .map((service) => service.image))),
@@ -1933,64 +2091,116 @@ export async function createApp(options = {}) {
             commands,
             retainedImages,
         };
-        const network = await networkFor(node);
+        const network = networkFor(node);
         if (network)
             work.network = network;
-        work.ports = await portBindingFor(node, desiredStates);
+        work.ports = portBindingFor(node, desiredStates);
+        const holds = restores.holds();
+        if (Object.keys(holds).length)
+            work.holds = holds;
         if (SERVICE_DNS) {
-            const all = await endpointsFor(allStates);
+            const all = endpointsFor(allStates);
             work.serviceEndpoints = Object.fromEntries(desiredStates.map((st) => [st.project, all[st.project] || {}]));
         }
         if (node.ingress)
-            work.ingress = await ingressFor(node, allStates);
+            work.ingress = ingressFor(node, allStates);
         res.json(work);
     });
     post('/agent/v1/status', authAgent, async (req, res) => {
         const ts = now();
         const report = req.body;
         for (const st of report.projects || []) {
-            await db.run(`INSERT INTO node_statuses(id,node_id,project_id,observed_version,status_json,health,updated_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(node_id,project_id) DO UPDATE SET observed_version=excluded.observed_version,status_json=excluded.status_json,health=excluded.health,updated_at=excluded.updated_at`, id('ns'), req.node.id, st.project, st.observedVersion, JSON.stringify(st), st.health || 'unknown', ts);
-            await updateDeploymentConvergence(st.project);
+            db.run(`INSERT INTO node_statuses(id,node_id,project_id,observed_version,status_json,health,updated_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(node_id,project_id) DO UPDATE SET observed_version=excluded.observed_version,status_json=excluded.status_json,health=excluded.health,updated_at=excluded.updated_at`, id('ns'), req.node.id, st.project, st.observedVersion, JSON.stringify(st), st.health || 'unknown', ts);
+            updateDeploymentConvergence(st.project);
         }
+        if (report.jobs?.length)
+            backups.onHeartbeat(report.jobs);
         for (const cr of (report.commandResults || [])) {
             const result = cr.result || {};
-            await db.run('UPDATE commands SET state=?,result_json=?,updated_at=? WHERE id=? AND node_id=?', cr.state || 'succeeded', JSON.stringify(result), ts, cr.commandId, req.node.id);
+            if (cr.commandId && cr.state)
+                backups.onResult(cr.commandId, cr.state, result);
+            db.run('UPDATE commands SET state=?,result_json=?,updated_at=? WHERE id=? AND node_id=?', cr.state || 'succeeded', JSON.stringify(result), ts, cr.commandId ?? null, req.node.id);
         }
         if (report.ingress)
-            await db.run('UPDATE nodes SET ingress_status_json=? WHERE id=?', JSON.stringify({ ...report.ingress, reportedAt: ts }), req.node.id);
-        await db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', ts, 'online', req.node.id);
-        const nextEndpointSnapshot = JSON.stringify(await endpointsFor(await currentDesiredStates()));
+            db.run('UPDATE nodes SET ingress_status_json=? WHERE id=?', JSON.stringify({ ...report.ingress, reportedAt: ts }), req.node.id);
+        db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', ts, 'online', req.node.id);
+        try {
+            restores.advance();
+        }
+        catch (e) {
+            console.error(`restore failed to advance: ${errMessage(e)}`);
+        }
+        const nextEndpointSnapshot = JSON.stringify(endpointsFor(currentDesiredStates()));
         if (nextEndpointSnapshot !== endpointSnapshot) {
             endpointSnapshot = nextEndpointSnapshot;
-            await wakeAll();
+            wakeAll();
         }
         res.json({ ok: true });
     });
     post('/agent/v1/events', authAgent, async (req, res) => {
         const evs = req.body.events || [req.body];
         for (const ev of evs)
-            await db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at) VALUES(?,?,?,?,?,?,?,?,?)', id('evt'), ev.projectId || null, ev.deploymentId || null, req.node.id, ev.level || 'info', ev.type || 'event', ev.message || '', JSON.stringify(ev.metadata || {}), now());
+            db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at) VALUES(?,?,?,?,?,?,?,?,?)', id('evt'), ev.projectId || null, ev.deploymentId || null, req.node.id, ev.level || 'info', ev.type || 'event', ev.message || '', JSON.stringify(ev.metadata || {}), now());
         if (Date.now() - lastEventPrune >= eventPruneIntervalMs)
-            await pruneEvents().catch((e) => console.error(`event pruning failed: ${errMessage(e)}`));
+            try {
+                pruneEvents();
+            }
+            catch (e) {
+                console.error(`event pruning failed: ${errMessage(e)}`);
+            }
         res.json({ ok: true });
     });
+    // Credentials for a backup helper are released only while its command is running on the calling node, and
+    // never ride in the command payload, events or results.
+    get('/agent/v1/backup/credentials', authAgent, async (req, res) => {
+        const command = db.get('SELECT node_id,project_id,type,state FROM commands WHERE id=?', String(req.query.command || ''));
+        if (!command)
+            return res.status(404).json({ error: 'command not found' });
+        if (command.node_id !== req.node.id)
+            return res.status(403).json({ error: 'command belongs to another node' });
+        if (!BACKUP_COMMAND_TYPES.includes(command.type))
+            return res.status(403).json({ error: 'command does not use backup credentials' });
+        if (command.state !== 'running')
+            return res.status(409).json({ error: `command is ${command.state}, not running` });
+        if (!command.project_id || !backupDestination())
+            return res.status(409).json({ error: 'no backup destination is configured' });
+        const response = {
+            env: {
+                ...backupEnv(),
+                RESTIC_PASSWORD: repositoryPassword(masterKey, command.project_id),
+            },
+        };
+        res.json(response);
+    });
     get('/agent/v1/secrets', authAgent, async (req, res) => {
         const project = String(req.query.project || 'site');
-        let rows = await db.all('SELECT name,encrypted_value,version FROM secrets WHERE project_id=?', project);
-        if (req.query.services) {
-            const ds = await currentDesired(project);
-            const names = String(req.query.services)
+        let rows = db.all('SELECT name,encrypted_value,version FROM secrets WHERE project_id=?', project);
+        const ds = currentDesired(project);
+        const desired = ds ? JSON.parse(ds.state_json) : undefined;
+        const requested = req.query.services
+            ? String(req.query.services)
                 .split(',')
                 .map((s) => s.trim())
-                .filter(Boolean);
-            const allowed = ds
-                ? serviceSecretNames(JSON.parse(ds.state_json).compose, names)
-                : new Set();
-            rows = rows.filter((r) => allowed.has(r.name));
+                .filter(Boolean)
+            : Object.keys(desired?.services || {});
+        const names = requested.filter((name) => desired?.services[name]?.placement.nodes?.includes(req.node.name));
+        const allowed = serviceSecretNames(desired?.compose, names);
+        const registrySecret = rows.find((row) => row.name === REGISTRY_AUTH_SECRET);
+        const registryAuth = {};
+        const credentials = parseRegistryCredentials(registrySecret ? decrypt(registrySecret.encrypted_value, masterKey) : undefined);
+        for (const name of names) {
+            const spec = desired.services[name];
+            if (!spec.artifact && spec.image) {
+                const host = imageLocation(spec.image).host;
+                if (credentials[host])
+                    registryAuth[host] = credentials[host];
+            }
         }
+        rows = rows.filter((row) => allowed.has(row.name) && row.name !== REGISTRY_AUTH_SECRET);
         const secrets = {
             secrets: Object.fromEntries(rows.map((r) => [r.name, decrypt(r.encrypted_value, masterKey)])),
             versions: Object.fromEntries(rows.map((r) => [r.name, r.version])),
+            ...(Object.keys(registryAuth).length ? { registryAuth } : {}),
         };
         res.json(secrets);
     });
@@ -2019,7 +2229,7 @@ function attachAgentStream(app, server) {
         const raw = String(request.headers.authorization || '').replace(/^Bearer /, '');
         void (async () => {
             if (pathname === '/api/v1/live') {
-                const actor = await runtime.authenticateOperator(raw);
+                const actor = runtime.authenticateOperator(raw);
                 if (!actor) {
                     reject(401, 'Unauthorized');
                     return;
@@ -2031,7 +2241,7 @@ function attachAgentStream(app, server) {
                 wss.handleUpgrade(request, socket, head, (ws) => runtime.live.operator(actor, ws));
             }
             else {
-                const node = await runtime.authenticate(raw);
+                const node = runtime.authenticate(raw);
                 if (!node) {
                     reject(401, 'Unauthorized');
                     return;