@wpelevator/tug 0.10.0 → 0.12.0

package/dist/commands/observe.js

@@ -0,0 +1,71 @@
+import { errMessage } from "../util.js";
+import { formatStatus } from "../cli-status.js";
+import { loadCfg, print, request, runCapture, waitForCommand } from "./context.js";
+export function registerObserve(program) {
+    program
+        .command('status')
+        .description('summarize projects, deployments and nodes')
+        .option('--json', 'print the raw status as JSON for scripts')
+        .action(async (o) => {
+        const status = await request('GET', '/api/v1/status');
+        console.log(o.json ? JSON.stringify(status, null, 2) : formatStatus(status));
+    });
+    program
+        .command('events')
+        .description('list recent events with the credential that caused each control action; defaults to the resolved project')
+        .option('--limit <n>', 'number of events', '100')
+        .option('--all', 'list events across all projects (limited to your projects for scoped credentials)')
+        .action(async (o) => {
+        if (o.all && o.project)
+            throw new Error('--all and --project cannot be combined');
+        const cfg = await loadCfg();
+        const project = o.all ? undefined : o.project || cfg.project;
+        const query = `limit=${encodeURIComponent(o.limit)}${project ? `&project=${encodeURIComponent(project)}` : ''}`;
+        print(await request('GET', `/api/v1/events?${query}`));
+    });
+    program
+        .command('doctor')
+        .description('check this machine and the control plane: certificate expiry, node connectivity and clock skew')
+        .option('--nodes', 'also run host diagnostics on every node through its agent and wait')
+        .option('--node <name>', 'run host diagnostics on one node through its agent and wait')
+        .option('--timeout <seconds>', 'wait timeout for node diagnostics', '60')
+        .action(async (o) => {
+        const local = [];
+        for (const [name, cmd, args] of [
+            ['node', 'node', ['--version']],
+            ['docker', 'docker', ['version', '--format', '{{.Server.Version}}']],
+            ['docker_compose', 'docker', ['compose', 'version']],
+        ]) {
+            try {
+                local.push({ name, ok: true, output: runCapture(cmd, args).trim() });
+            }
+            catch (e) {
+                local.push({ name, ok: false, error: errMessage(e) });
+            }
+        }
+        const controlDiagnostics = await request('GET', '/api/v1/diagnostics');
+        const warnings = (controlDiagnostics.checks ?? []).flatMap((check) => (check.warnings ?? []).map((warning) => `${check.name}: ${warning}`));
+        const output = {
+            ok: local.every((c) => c.ok) && controlDiagnostics.ok,
+            ...(warnings.length && { warnings }),
+            local,
+            control: controlDiagnostics,
+        };
+        if (o.nodes || o.node) {
+            const queued = await request('POST', '/api/v1/diagnostics/nodes', { node: o.node || null });
+            const commands = [];
+            for (const command of queued.commands) {
+                const finished = await waitForCommand(command.commandId, Number(o.timeout));
+                commands.push({
+                    nodeId: command.nodeId,
+                    state: finished.state,
+                    result: JSON.parse(finished.result_json || '{}'),
+                });
+            }
+            output.nodeDiagnostics = { commands };
+            if (commands.some((command) => command.state !== 'succeeded'))
+                output.ok = false;
+        }
+        print(output);
+    });
+}

package/dist/commands/restore.d.ts

@@ -0,0 +1,2 @@
+import type { Command } from 'commander';
+export declare function registerRestore(program: Command): void;

package/dist/commands/restore.js

@@ -0,0 +1,85 @@
+import { collect, loadCfg, request } from "./context.js";
+export function registerRestore(program) {
+    const restore = program
+        .command('restore')
+        .description('restore services and volumes from backups');
+    restore
+        .command('run <target>')
+        .description('restore a service or volume from a backup. Without --confirm it only prints the plan. An in-place restore locks the project, stops the services that write to the target, takes a safety backup, restores, then starts them again')
+        .option('--from <id>', 'backup run id, restic snapshot id, or latest', 'latest')
+        .option('--scratch', 'restore a volume into a new volume next to the live one, touching nothing live')
+        .option('--no-safety-backup', 'skip the backup of the current data taken before an in-place restore')
+        .option('--writers <service>', 'service to stop during the restore, instead of discovering them (repeatable)', collect, [])
+        .option('--no-writers', 'restore without stopping any service')
+        .option('--confirm', 'perform the restore')
+        .option('--timeout <seconds>', 'give up waiting after this many seconds', '3600')
+        .action(async (target, o) => {
+        const project = (await loadCfg()).project || 'site';
+        const explicit = Array.isArray(o.writers) && o.writers.length > 0 ? o.writers : undefined;
+        const result = await request('POST', '/api/v1/restores', {
+            project,
+            target,
+            from: o.from,
+            scratch: o.scratch === true,
+            safetyBackup: o.safetyBackup,
+            writers: explicit,
+            noWriters: o.writers === false,
+            confirm: o.confirm === true,
+        });
+        const plan = result.plan;
+        console.log(`target:   ${plan.target} on ${plan.node}`);
+        console.log(`snapshot: ${plan.snapshotId}${plan.snapshotTime ? ` (${plan.snapshotTime})` : ''}`);
+        console.log(`mode:     ${plan.mode}${plan.scratchVolume ? ` into new volume ${plan.scratchVolume}` : ''}`);
+        if (plan.mode === 'in-place') {
+            console.log(`stops:    ${plan.writers.length ? plan.writers.join(', ') : 'nothing'}`);
+            console.log(`safety:   ${plan.safetyBackup ? 'a backup of the current data is taken first' : 'none (--no-safety-backup)'}`);
+        }
+        if (!result.confirmed) {
+            console.log('\nNothing was changed. Run again with --confirm to restore.');
+            return;
+        }
+        const deadline = Date.now() + Number(o.timeout) * 1000;
+        for (;;) {
+            await new Promise((resolve) => setTimeout(resolve, 2000));
+            const { restores } = await request('GET', `/api/v1/restores?project=${encodeURIComponent(project)}`);
+            const record = restores.find((item) => item.id === result.restoreId);
+            if (record?.state === 'succeeded') {
+                if (record.error)
+                    console.error(`warning: ${record.error}`);
+                console.log(`restore ${record.id} completed${record.scratch_volume ? `; files are in volume ${record.scratch_volume}` : ''}`);
+                return;
+            }
+            if (record && ['failed', 'interrupted', 'released'].includes(record.state)) {
+                console.error(`restore ${record.id} ${record.state}: ${record.error ?? ''}`);
+                if (record.state === 'interrupted')
+                    console.error(`the services stay stopped; run tug restore resume ${record.id} to try again or tug restore release ${record.id} to start them (data may be partly restored)`);
+                process.exitCode = 1;
+                return;
+            }
+            if (Date.now() > deadline) {
+                console.error(`timed out waiting; restore ${result.restoreId} continues on control (tug restore list)`);
+                process.exitCode = 1;
+                return;
+            }
+        }
+    });
+    restore
+        .command('list')
+        .description('show recent restores and their state')
+        .action(async () => {
+        const project = (await loadCfg()).project || 'site';
+        const { restores } = await request('GET', `/api/v1/restores?project=${encodeURIComponent(project)}`);
+        if (!restores.length)
+            console.log('no restores');
+        for (const r of restores)
+            console.log(`${r.started_at}\t${r.id}\t${r.target}\t${r.mode}\t${r.state}\t${r.snapshot_id.slice(0, 8)}${r.error ? `\t${r.error}` : ''}`);
+    });
+    restore
+        .command('resume <id>')
+        .description('run an interrupted restore again; the stopped services stay stopped meanwhile')
+        .action(async (id) => console.log(JSON.stringify(await request('POST', `/api/v1/restores/${encodeURIComponent(id)}/resume`))));
+    restore
+        .command('release <id>')
+        .description('start the services an interrupted restore stopped and unlock the project; the data may be partly restored')
+        .action(async (id) => console.log(JSON.stringify(await request('POST', `/api/v1/restores/${encodeURIComponent(id)}/release`))));
+}

package/dist/commands/secret.d.ts

@@ -0,0 +1,2 @@
+import type { Command } from 'commander';
+export declare function registerSecret(program: Command): void;

package/dist/commands/secret.js

@@ -0,0 +1,26 @@
+import { print, projectOf, request } from "./context.js";
+export function registerSecret(program) {
+    const secret = program
+        .command('secret')
+        .description('manage the project secrets that Compose files reference');
+    secret
+        .command('set <name>')
+        .description('set a secret, reading the value from stdin unless --value is given')
+        .option('--value <value>')
+        .action(async (name, o) => {
+        const value = o.value ??
+            (await new Promise((resolve) => {
+                process.stdout.write(`${name}: `);
+                process.stdin.once('data', (d) => resolve(String(d).trim()));
+            }));
+        print(await request('POST', '/api/v1/secrets', { project: await projectOf(o), name, value }));
+    });
+    secret
+        .command('list')
+        .description('list secret names and versions, never values')
+        .action(async (o) => print(await request('GET', `/api/v1/secrets?project=${encodeURIComponent(await projectOf(o))}`)));
+    secret
+        .command('remove <name>')
+        .description('delete a secret')
+        .action(async (name, o) => print(await request('DELETE', `/api/v1/secrets/${encodeURIComponent(name)}?project=${encodeURIComponent(await projectOf(o))}`)));
+}

package/dist/commands/service.d.ts

@@ -0,0 +1,2 @@
+import type { Command } from 'commander';
+export declare function registerService(program: Command): void;

package/dist/commands/service.js

@@ -0,0 +1,62 @@
+import { projectOf, request, runLive, waitForCommand } from "./context.js";
+export function registerService(program) {
+    program
+        .command('exec <service> [command...]')
+        .description('execute a command on a service container (default: sh)')
+        .option('-i, --interactive', 'forward stdin to the remote command')
+        .option('-t, --tty', 'allocate a terminal (requires a local terminal and agent TTY support)')
+        .option('--node <name>', 'select the node when a service has multiple placements')
+        .action(async (service, command, o) => {
+        await runLive('exec', { service, command, ...o });
+    });
+    program
+        .command('run <service> [command...]')
+        .description("run a one-off command in a new container from the service: same image, environment, secrets, network and mounts, removed afterwards (for example: tug run web -- wp core update-db). Exits with the command's exit code")
+        .option('--node <name>', 'select the node when a service has multiple placements')
+        .action(async (service, command, o) => {
+        if (!command.length)
+            throw new Error('run needs a command, such as: tug run web -- wp core update-db');
+        await runLive('exec', { service, command, node: o.node, oneOff: true });
+    });
+    program
+        .command('ps')
+        .description('print per-container state as JSON')
+        .option('--node <name>', 'select one node')
+        .action(async (o) => {
+        await runLive('ps', o);
+    });
+    program
+        .command('inspect <service>')
+        .description('print container details as JSON')
+        .option('--node <name>', 'select one node')
+        .action(async (service, o) => {
+        await runLive('inspect', { service, ...o });
+    });
+    program
+        .command('logs <service>')
+        .description('fetch service container logs from the node that runs the service')
+        .option('--tail <n>', 'number of lines', '200')
+        .option('-f, --follow', 'follow live logs')
+        .option('--node <name>', 'select a node for followed logs')
+        .action(async (svc, o) => {
+        if (o.follow) {
+            await runLive('logs', { service: svc, node: o.node, tail: o.tail });
+            return;
+        }
+        if (o.node)
+            throw new Error('--node requires --follow for logs');
+        const created = await request('POST', '/api/v1/logs', {
+            project: await projectOf(o),
+            service: svc,
+            tail: Number(o.tail),
+        });
+        const cmd = await waitForCommand(created.commandId, 60);
+        const result = JSON.parse(cmd.result_json || '{}');
+        if (result.stdout)
+            process.stdout.write(result.stdout);
+        if (result.stderr)
+            process.stderr.write(result.stderr);
+        if (result.error)
+            throw new Error(result.error);
+    });
+}

package/dist/commands/volume.d.ts

@@ -0,0 +1,2 @@
+import type { Command } from 'commander';
+export declare function registerVolume(program: Command): void;

package/dist/commands/volume.js

@@ -0,0 +1,13 @@
+import { print, projectOf, request } from "./context.js";
+export function registerVolume(program) {
+    const volume = program.command('volume').description("list and remove a project's volumes");
+    volume
+        .command('list')
+        .description('list the volumes of the project and where they live')
+        .action(async (o) => print(await request('GET', `/api/v1/volumes?project=${encodeURIComponent(await projectOf(o))}`)));
+    volume
+        .command('remove <name>')
+        .description('remove a volume from the project')
+        .option('--delete-data', 'delete underlying Docker volume data')
+        .action(async (name, o) => print(await request('DELETE', `/api/v1/volumes/${encodeURIComponent(name)}?project=${encodeURIComponent(await projectOf(o))}&deleteData=${Boolean(o.deleteData)}`)));
+}

package/dist/control.js

@@ -29,7 +29,7 @@ import { parseStatusReport } from "./status-report.js";
 import { ACTIVE_DEPLOYMENT_STATES, LIVE_MAX_FRAME_BYTES, LIVE_MAX_BUFFERED_BYTES, LIVE_MAX_SESSIONS, LIVE_START_TIMEOUT_MS, BACKUP_COMMAND_TYPES, } from "./protocol.js";
 import { BACKUP_SECRET_PROJECT, isReservedProject, parseBackupDestination, projectRepository, repositoryPassword, redactSecrets, } from "./backup-destination.js";
 import { createBackupService } from "./backup-scheduler.js";
-import { createRestoreService } from "./backup-restore.js";
+import { createRestoreService, discoverWriters } from "./backup-restore.js";
 import { AUTO_ROLLBACK_ACTOR, ROUTE_ROLES, PROJECT_NAME_PATTERN, authenticate, hasRole, assertProject, bodyProject, createGuard, createCredential, listCredentials, parseCredentialRequest, queryProject, recordEvent, revokeCredential, } from "./credentials.js";
 class WakeHub {
     cursors = new Map();
@@ -758,6 +758,7 @@ export async function createApp(options = {}) {
         WHERE ds.project_id=? AND ds.deployment_id != ? AND d.state IN (${states}) ORDER BY ds.created_at DESC LIMIT 1`, project, current?.current_deployment_id || '');
             if (!previousDs)
                 throw httpError(404, 'no previous deployment found');
+            assertArtifactsAvailable(JSON.parse(previousDs.state_json), `roll back to ${previousDs.version}`);
             const ts = now();
             const depId = id('dep');
             const previousState = JSON.parse(previousDs.state_json);
@@ -934,10 +935,17 @@ export async function createApp(options = {}) {
         const global = ext.global === true;
         const labels = Array.isArray(placement.labels) ? placement.labels : [];
         const matched = nodeInfos.filter((node) => labels.every((label) => node.labels.includes(label)));
-        const pool = labels.length ? matched : nodeInfos;
+        // Explicit `placement.nodes` (without labels or global, which deploy resolves into nodes) bound the pool; scaling never re-homes a service off them.
+        const pinned = !labels.length && !global && Array.isArray(placement.nodes)
+            ? placement.nodes
+            : [];
+        const candidates = pinned.length
+            ? matched.filter((node) => pinned.includes(node.name))
+            : matched;
+        const pool = candidates;
         if (!pool.length)
-            throw httpError(400, labels.length
-                ? 'no healthy nodes match the service placement labels'
+            throw httpError(400, labels.length || pinned.length
+                ? 'no healthy nodes match the service placement'
                 : 'no nodes available for scaling');
         const selected = global ? pool : pool.slice(0, Math.min(replicas, pool.length));
         state.services[service].placement = {
@@ -1008,7 +1016,7 @@ export async function createApp(options = {}) {
             return res.status(400).json({ error: "mode must be 'create' or 'replace'" });
         try {
             // The certificate names the immutable node ID: the existing node's, or a fresh one for a new name. A name freed by
-            // node-remove therefore never inherits the old ID, and certificates for the removed node cannot match the new one.
+            // `tug node remove` therefore never inherits the old ID, and certificates for the removed node cannot match the new one.
             let nodeId = db.get('SELECT * FROM nodes WHERE name=?', name)?.id ?? id('node');
             let signed = await pki.signClientCsr(body.csr, identityUri('node', nodeId), NODE_LIFETIME_MS);
             // Signing is asynchronous: re-read, and re-sign once if a concurrent request created the node meanwhile.
@@ -1024,7 +1032,7 @@ export async function createApp(options = {}) {
                 if (existing) {
                     if (existing.key_hash !== signed.keyHash)
                         return res.status(409).json({
-                            error: `node ${name} already exists; use node-replace-identity to install a new key`,
+                            error: `node ${name} already exists; use tug node replace to install a new key`,
                         });
                 }
                 else {
@@ -1137,7 +1145,7 @@ export async function createApp(options = {}) {
             const signed = await pki.signClientCsr(req.body?.csr, identityUri('node', node.id), NODE_LIFETIME_MS);
             if (signed.keyHash !== node.key_hash)
                 return res.status(403).json({
-                    error: 'renewal requires a CSR for the node key; use node-replace-identity to change keys',
+                    error: 'renewal requires a CSR for the node key; use tug node replace to change keys',
                 });
             const last = node.last_renewed_at ? Date.parse(node.last_renewed_at) : 0;
             if (Date.now() - last < NODE_RENEW_MIN_INTERVAL_MS())
@@ -1150,7 +1158,7 @@ export async function createApp(options = {}) {
             sendError(res, e);
         }
     });
-    // Blocks a node's key without deleting its metadata or changing placement. It returns only through node-replace-identity.
+    // Blocks a node's key without deleting its metadata or changing placement. It returns only through `tug node replace`.
     api('post', '/api/v1/nodes/:name/disable', null, async (req, res) => {
         const node = findNode(String(req.params.name));
         if (!node)
@@ -1536,7 +1544,70 @@ export async function createApp(options = {}) {
     }
     // Previous-process builds cannot finish after a restart; clear their unpublished files.
     db.run("UPDATE deployments SET state='build_failed',error_message='control restarted during build' WHERE state IN ('uploading','building')");
+    /** Archives deleted by `image prune`, or missing from a backup taken without images, can no longer be deployed. */
+    async function markMissingArtifacts() {
+        for (const row of db.all("SELECT * FROM builds WHERE state='complete' AND artifact_path IS NOT NULL"))
+            if (!(await fs.stat(path.join(DATA_DIR, row.artifact_path)).catch(() => null)))
+                db.run("UPDATE builds SET state='pruned',artifact_path=NULL,updated_at=? WHERE id=?", now(), row.id);
+    }
+    /** Throws a 409 when a desired state needs a build archive that was pruned. */
+    function assertArtifactsAvailable(state, action) {
+        for (const [name, service] of Object.entries(state.services)) {
+            if (!service.artifact)
+                continue;
+            const build = db.get('SELECT * FROM builds WHERE id=?', service.artifact.buildId);
+            if (build?.state === 'pruned')
+                throw httpError(409, `cannot ${action}: the build archive of ${name} was pruned (tug image prune); deploy it again instead`);
+        }
+    }
+    /**
+     * Deletes build archives no longer needed: everything except the newest `keep` desired states of each project,
+     * the current deployment and any version a node still reports. The desired states stay, so history lists them,
+     * but rolling back to a pruned one is refused.
+     */
+    async function pruneArtifacts(keep, dryRun) {
+        const kept = new Set();
+        const reported = db
+            .all('SELECT project_id,status_json FROM node_statuses')
+            .map((row) => ({
+            project: row.project_id,
+            ...JSON.parse(row.status_json),
+        }));
+        const current = new Set(db
+            .all('SELECT current_deployment_id FROM projects')
+            .map((row) => row.current_deployment_id));
+        const rows = db.all('SELECT * FROM desired_states ORDER BY created_at DESC');
+        const seen = new Map();
+        for (const row of rows) {
+            const index = seen.get(row.project_id) ?? 0;
+            seen.set(row.project_id, index + 1);
+            const state = JSON.parse(row.state_json);
+            const lagging = reported.some((r) => r.project === row.project_id &&
+                r.observedVersion === state.version &&
+                (!r.observedRevision || r.observedRevision === state.revision));
+            if (index >= keep && !lagging && !current.has(row.deployment_id))
+                continue;
+            for (const service of Object.values(state.services))
+                if (service.artifact)
+                    kept.add(service.artifact.buildId);
+        }
+        const pruned = [];
+        for (const build of db.all("SELECT * FROM builds WHERE state='complete' AND artifact_path IS NOT NULL")) {
+            if (kept.has(build.id))
+                continue;
+            pruned.push({ id: build.id, service: build.service, sizeBytes: build.size_bytes });
+            if (dryRun)
+                continue;
+            await fs.rm(path.join(DATA_DIR, build.artifact_path), { force: true });
+            db.run("UPDATE builds SET state='pruned',artifact_path=NULL,updated_at=? WHERE id=?", now(), build.id);
+            const shared = db.get("SELECT 1 FROM builds WHERE image_ref=? AND state='complete'", build.image_ref);
+            if (!shared)
+                await images.run('docker', ['image', 'rm', build.image_ref]).catch(() => { });
+        }
+        return { pruned, freedBytes: pruned.reduce((sum, build) => sum + build.sizeBytes, 0) };
+    }
     await collectArtifacts();
+    await markMissingArtifacts();
     const receiveDeployment = upload.any();
     api('post', '/api/v1/deployments', null, (req, res) => {
         receiveDeployment(req, res, (error) => {
@@ -1547,6 +1618,25 @@ export async function createApp(options = {}) {
             });
         });
     });
+    /**
+     * Services that keep data on a node and are placed on different nodes by `compose` than by the current
+     * deployment. Deploying such a change starts the service empty on the new node: the data does not move.
+     */
+    function movedServices(project, compose) {
+        const current = currentDesired(project);
+        if (!current)
+            return [];
+        const before = JSON.parse(current.state_json);
+        return Object.entries(compose.services || {})
+            .filter(([name, svc]) => {
+            const old = before.services?.[name]?.placement?.nodes ?? [];
+            const next = svc['x-tug']?.placement?.nodes ?? [];
+            const keepsData = Boolean(svc['x-tug']?.backup) ||
+                (svc.volumes ?? []).some((mount) => typeof mount === 'object' && mount.type === 'volume');
+            return keepsData && old.length > 0 && next.length > 0 && old.some((n) => !next.includes(n));
+        })
+            .map(([name]) => name);
+    }
     async function deployRequest(req, res, uploadError) {
         const files = (Array.isArray(req.files) ? req.files : []);
         let responseBody;
@@ -1620,6 +1710,10 @@ export async function createApp(options = {}) {
                 throw httpError(400, errMessage(e));
             }
             assertIngressHostsFree(project, compose);
+            const acknowledgedMoves = Array.isArray(body.moves) ? body.moves.map(String) : [];
+            const unacknowledged = movedServices(project, compose).filter((name) => !acknowledgedMoves.includes(name));
+            if (unacknowledged.length)
+                throw httpError(409, `${unacknowledged.join(', ')} would start empty on a different node: its data does not move. Back it up, then deploy with --move ${unacknowledged.join(' --move ')} to hold the services that write to it, and run tug restore run <target> --confirm`);
             // The CLI interpolates control secrets as sentinels; a reference to a secret that is gone would deploy an empty value.
             const referenced = sentinelNames(compose);
             if (referenced.includes(REGISTRY_AUTH_SECRET))
@@ -1631,7 +1725,7 @@ export async function createApp(options = {}) {
                     .map((row) => row.name));
                 const missing = referenced.filter((name) => !known.has(name));
                 if (missing.length)
-                    throw httpError(400, `secrets not found in project ${project}: ${missing.join(', ')}; create them with tug secret-set`);
+                    throw httpError(400, `secrets not found in project ${project}: ${missing.join(', ')}; create them with tug secret set`);
             }
             const builtServices = Object.entries(compose.services)
                 .filter(([, service]) => service.build)
@@ -1672,7 +1766,7 @@ export async function createApp(options = {}) {
                     continue;
                 for (const secret of [ing.cert, ing.key])
                     if (!db.get('SELECT 1 FROM secrets WHERE project_id=? AND name=?', project, secret ?? null))
-                        throw httpError(400, `${svcName}: custom TLS secret ${secret} not found in project ${project}; set it with secret-set first`);
+                        throw httpError(400, `${svcName}: custom TLS secret ${secret} not found in project ${project}; set it with tug secret set first`);
             }
             lock = acquireProjectLock(project, 'deploy');
             const active = db.get(`SELECT * FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) LIMIT 1`, project);
@@ -1770,6 +1864,11 @@ export async function createApp(options = {}) {
                         !db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now()))
                         throw httpError(409, 'deployment was cancelled or its build lease was lost');
                     db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, publishingId, version, JSON.stringify(desired), ts);
+                    // The moved service starts empty: keep what writes to it stopped until its data is restored.
+                    for (const moved of acknowledgedMoves.filter((name) => compose.services?.[name]))
+                        for (const writer of discoverWriters(compose, `service:${moved}`))
+                            if (!acknowledgedMoves.includes(writer))
+                                db.run('INSERT OR REPLACE INTO service_holds(project_id,service,reason,restore_id,created_at) VALUES(?,?,?,?,?)', project, writer, `moved ${moved}`, null, ts);
                     for (const [volumeName, volumeSpec] of Object.entries(compose.volumes || {})) {
                         const lp = volumeSpec?.['x-tug'] || {};
                         const nodeName = lp.placement?.nodes?.[0] || null;
@@ -1933,6 +2032,31 @@ export async function createApp(options = {}) {
             releaseLock(lock);
         }
     });
+    api('post', '/api/v1/holds/release', bodyProject, async (req, res) => withWriter(async () => {
+        const project = String(req.body?.project || 'site');
+        const services = Array.isArray(req.body?.services)
+            ? req.body.services.map(String)
+            : [];
+        // Only holds a deploy placed for a move; a restore's holds are dropped by `restore release`.
+        const rows = db.all('SELECT service FROM service_holds WHERE project_id=? AND restore_id IS NULL', project);
+        const released = rows
+            .map((row) => row.service)
+            .filter((service) => !services.length || services.includes(service));
+        for (const service of released)
+            db.run('DELETE FROM service_holds WHERE project_id=? AND service=? AND restore_id IS NULL', project, service);
+        wakeAll(project);
+        res.json({ ok: true, project, released });
+    }));
+    api('post', '/api/v1/images/prune', null, async (req, res) => withWriter(async () => {
+        const keep = req.body?.keep === undefined ? 5 : Number(req.body.keep);
+        if (!Number.isInteger(keep) || keep < 1)
+            return res.status(400).json({ error: 'keep must be a positive integer' });
+        res.json({
+            dryRun: Boolean(req.body?.dryRun),
+            keep,
+            ...(await pruneArtifacts(keep, Boolean(req.body?.dryRun))),
+        });
+    }));
     api('post', '/api/v1/control/backups/run', null, async (req, res) => withWriter(async () => {
         const backupDir = process.env.TUG_CONTROL_BACKUP_DIR || path.join(DATA_DIR, 'control-backups');
         // The archive holds the master key and the CA key, so nothing here is ever readable beyond the control user.
@@ -1954,14 +2078,16 @@ export async function createApp(options = {}) {
                 recursive: true,
                 filter: (src) => path.basename(src) !== 'server.json',
             });
-            await fs.cp(artifactDir, path.join(staging, 'images'), { recursive: true });
-            await fs.writeFile(path.join(staging, 'metadata.json'), JSON.stringify({ backupId, createdAt: ts, type: 'control' }, null, 2), { mode: 0o600 });
+            const withImages = req.body?.images !== false;
+            if (withImages)
+                await fs.cp(artifactDir, path.join(staging, 'images'), { recursive: true });
+            await fs.writeFile(path.join(staging, 'metadata.json'), JSON.stringify({ backupId, createdAt: ts, type: 'control', images: withImages }, null, 2), { mode: 0o600 });
             await tar.c({ gzip: true, file: artifactRef, cwd: staging, mode: 0o600 }, [
                 'tug.db',
                 'master.key',
                 'metadata.json',
                 'pki',
-                'images',
+                ...(withImages ? ['images'] : []),
             ]);
         }
         catch (error) {
@@ -2007,11 +2133,16 @@ export async function createApp(options = {}) {
             for (const secret of snapshot.all('SELECT * FROM secrets'))
                 decrypt(secret.encrypted_value, key);
             const states = snapshot.all('SELECT * FROM desired_states');
-            for (const stateRow of states) {
+            const meta = JSON.parse(await fs.readFile(path.join(restoreDir, 'metadata.json'), 'utf8').catch(() => '{}'));
+            // A backup taken without images restores the archive-less builds as pruned on the next start.
+            for (const stateRow of meta.images === false ? [] : states) {
                 const state = JSON.parse(stateRow.state_json);
                 for (const service of Object.values(state.services)) {
                     if (!service.artifact)
                         continue;
+                    // `image prune` already removed this one before the backup.
+                    if (snapshot.get("SELECT 1 FROM builds WHERE id=? AND state='pruned'", service.artifact.buildId))
+                        continue;
                     const build = snapshot.get('SELECT * FROM builds WHERE id=? AND state=?', service.artifact.buildId, 'complete');
                     if (!build?.artifact_path ||
                         !/^images\/build_[a-zA-Z0-9_-]+\.tar\.gz$/.test(build.artifact_path))

package/dist/credentials.js

@@ -53,6 +53,8 @@ export const ROUTE_ROLES = {
     'DELETE /api/v1/backup/destination': 'admin',
     'GET /api/v1/backup/repository-password': 'admin',
     'POST /api/v1/control/backups/run': 'admin',
+    'POST /api/v1/images/prune': 'admin',
+    'POST /api/v1/holds/release': 'admin',
     'POST /api/v1/control/restore': 'admin',
     'GET /api/v1/diagnostics': 'admin',
     'POST /api/v1/diagnostics/nodes': 'admin',

package/dist/diagnostics.js

@@ -50,7 +50,7 @@ export function diagnoseNode(node, nowMs, staleSeconds) {
     // A disabled node's certificate is irrelevant until it is replaced, so it never fails the diagnostics.
     const report = connectivity === 'disabled' ? warnings : problems;
     if (certificate?.state === 'expired')
-        report.push(`certificate expired ${certificate.notAfter}; run node-replace-identity ${node.name}`);
+        report.push(`certificate expired ${certificate.notAfter}; run tug node replace ${node.name}`);
     else if (certificate?.state === 'expiring')
         warnings.push(`certificate expires in ${certificate.daysLeft} days and has not renewed; check the agent logs on node ${node.name}`);
     if (clockSkew.state === 'observed') {

package/dist/docker-api.d.ts

@@ -40,6 +40,7 @@ export interface ContainerInspect {
         NetworkMode?: string;
         Binds?: string[];
         Mounts?: unknown[];
+        ExtraHosts?: string[];
     };
     State?: {
         Status?: string;

package/dist/live.js

@@ -312,6 +312,7 @@ export async function startOneOff(docker, template, command) {
             NetworkMode: host.NetworkMode,
             ...(host.Binds?.length ? { Binds: host.Binds } : {}),
             ...(host.Mounts?.length ? { Mounts: host.Mounts } : {}),
+            ...(host.ExtraHosts?.length ? { ExtraHosts: host.ExtraHosts } : {}),
         },
     });
     const remove = () => docker.removeContainer(id).catch(() => { });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wpelevator/tug",
-  "version": "0.10.0",
+  "version": "0.12.0",
   "description": "Portable VPS deployments with Docker Compose",
   "repository": {
     "type": "git",