@wpelevator/tug 0.4.0 → 0.6.0

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { Command, Option } from 'commander';
 import { openAsBlob } from 'node:fs';
-import { resolveCliConfig, integerOption, normalizeControlUrl } from "./cli-config.js";
+import { applyConfigDefaults, integerOption, normalizeControlUrl, nodeSettings, resolveCliConfig, writeInitConfig, } from "./cli-config.js";
 import { dockerRunSnippet, enrollmentCloudInit, shellQuote as sh, HOST_NETWORK_FLAG, MESH_RUN_FLAGS, controlPortCheck, localImageBuildArgs, meshPortCheck, } from "./cli-install.js";
 import { createSourceArchive, prepareLocalImages } from "./cli-build.js";
 import { saveImageArchive } from "./artifacts.js";
@@ -22,29 +22,32 @@ import { createCliLiveTransport } from "./cli-live-transport.js";
 // built image (`node dist/cli.js`; tools/scripts/copy-templates.js puts them there).
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const templatesDir = path.join(__dirname, 'templates');
+const CONFIG_HELP = 'CLI config file, YAML or JSON (default: TUG_CONFIG, else tug.yml, tug.yaml, tug.json or .tug.json)';
 const program = new Command();
 program
     .name('tug')
     .description('Portable VPS deployments with Docker Compose')
     .version(packageVersion())
     .enablePositionalOptions()
-    .option('--config <file>', 'CLI JSON config file (default: TUG_CONFIG or .tug.json)')
+    .option('--config <file>', CONFIG_HELP)
     .option('--control-url <url>', 'control API URL (overrides TUG_CONTROL_URL and config)')
     .option('--project <name>', 'project (overrides TUG_PROJECT and config; default: site)');
 let resolvedConfig;
-const cfgPath = () => resolvedConfig.file;
 async function loadCfg() {
     return resolvedConfig;
 }
 const collect = (v, a) => [...a, v];
-async function saveCfg(c) {
-    await fs.writeFile(cfgPath(), JSON.stringify(c, null, 2), { mode: 0o600 });
+async function readStdin() {
+    const chunks = [];
+    for await (const chunk of process.stdin)
+        chunks.push(Buffer.from(chunk));
+    return Buffer.concat(chunks).toString('utf8');
 }
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 async function request(method, route, body, multipart = false) {
     const c = await loadCfg();
     const base = c.controlUrl;
-    const tok = c.operatorToken;
+    const tok = c.token;
     const headers = tok ? { Authorization: `Bearer ${tok}` } : {};
     if (!multipart)
         headers['content-type'] = 'application/json';
@@ -180,11 +183,15 @@ async function waitForDeployment(deploymentId, timeoutSeconds = 300) {
 program
     .command('init')
     .option('--token <token>', 'operator token (prefer TUG_TOKEN to avoid process-list exposure)')
-    .action(async (o) => {
+    .description('create the config file if missing and store the operator token in its git-ignored .local overlay')
+    .action(async (o, command) => {
     const cfg = await loadCfg();
-    const operatorToken = o.token || cfg.operatorToken || token('tug_op');
-    await saveCfg({ controlUrl: cfg.controlUrl, operatorToken, project: cfg.project });
-    console.log(`wrote ${cfgPath()}`);
+    const { controlUrl, project } = command.optsWithGlobals();
+    for (const file of await writeInitConfig(cfg, o.token || cfg.token || token('tug_op'), {
+        controlUrl,
+        project,
+    }))
+        console.log(`wrote ${file}`);
 });
 program
     .command('node-register <name>')
@@ -201,12 +208,6 @@ program
     });
     console.log(JSON.stringify(r, null, 2));
 });
-program
-    .command('node-rotate-registry-token <name>')
-    .description('issue a new registry pull token for a node, invalidating the old one')
-    .action(async (name) => {
-    console.log(JSON.stringify(await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/rotate-registry-token`, {}), null, 2));
-});
 program
     .command('node-rotate-wg-key <name>')
     .description('generate a new WireGuard keypair on the node; peers pick up the new public key on their next poll')
@@ -234,12 +235,6 @@ program
         enabled: !o.disable,
     }), null, 2));
 });
-program
-    .command('node-revoke-registry-token <name>')
-    .description("revoke a node's registry pull token, blocking future pulls until rotated")
-    .action(async (name) => {
-    console.log(JSON.stringify(await request('POST', `/api/v1/nodes/${encodeURIComponent(name)}/revoke-registry-token`, {}), null, 2));
-});
 program
     .command('scale <service> <replicas>')
     .description('scale a service to N replicas across matching nodes')
@@ -263,9 +258,6 @@ program
     .option('--token <token>', 'operator token')
     .option('--port <port>', 'published API port', '8080')
     .option('--mesh', 'enable the WireGuard mesh on control (TUG_MESH=true); install nodes with --mesh too')
-    .option('--registry <host>', 'registry host used in desired images; bring your own registry, Tug does not provision one')
-    .option('--registry-auth-path <path>', 'container path to an htpasswd file tug-control should manage credentials in, if your registry uses one')
-    .option('--registry-container <name>', "docker container name to restart when a node's registry credentials change, if your registry uses an htpasswd file")
     .action(async (sshTarget, o) => {
     const operatorToken = o.token || process.env.TUG_TOKEN || token('tug_op');
     if (o.installDocker)
@@ -274,16 +266,12 @@ program
     const image = resolveImage(o.image, o.buildLocalImage);
     if (o.buildLocalImage)
         await transferTugImage(sshTarget, image);
-    const registryHost = o.registry || '';
     const yamlValue = (value) => JSON.stringify(value);
     const compose = await loadTemplate(templatesDir, 'control-compose.yml', {
         IMAGE: yamlValue(image),
         OPERATOR_TOKEN: yamlValue(operatorToken),
-        REGISTRY_HOST: yamlValue(registryHost),
         PORT: o.port,
         MESH: yamlValue(o.mesh ? 'true' : 'false'),
-        REGISTRY_AUTH_PATH: yamlValue(o.registryAuthPath || ''),
-        REGISTRY_CONTAINER: yamlValue(o.registryContainer || ''),
     });
     const remote = await loadTemplate(templatesDir, 'control-install.sh', {
         COMPOSE: compose,
@@ -295,26 +283,27 @@ program
         sshTarget,
         operatorToken,
         url: `http://${remoteHost(sshTarget)}:${o.port}`,
-        registry: registryHost,
     }, null, 2));
 });
 program
-    .command('node-install <sshTarget>')
-    .description('register and install an agent container on a Docker-capable host over SSH')
+    .command('node-install [sshTarget]')
+    .description('register and install an agent container on a Docker-capable host over SSH (the SSH target defaults to nodes.<name>.ssh in the config file)')
     .requiredOption('--name <name>')
     .option('--address <addr>')
     .option('--public-address <addr>')
     .option('--label <label>', 'node label (repeatable)', collect, [])
     .option('--agent-control-url <url>', 'control URL reachable by the agent (default: resolved operator control URL)')
-    .option('--registry <host>', 'registry host for docker login')
     .option('--http-only-dev', 'allow plaintext HTTP control URL for local/private development')
     .option('--mesh', 'join the WireGuard mesh (agent runs with host networking and NET_ADMIN; control needs TUG_MESH=true)')
     .option('--wg-port <port>', 'WireGuard UDP port checked by preflight', '51820')
     .option('--image <tag>', "Tug image to run (default: ghcr.io/wpelevator/tug:<this CLI's version>)")
     .option('--install-docker', 'install missing Docker Engine and Compose on supported Debian, Ubuntu, Fedora and RHEL-compatible hosts (root required)')
     .option('--build-local-image', 'build the image from the current checkout and copy it to the host over SSH (tags it tug:local unless --image is set); for development')
-    .action(async (sshTarget, o) => {
+    .action(async (sshArgument, o) => {
     const cfg = await loadCfg();
+    const sshTarget = sshArgument ?? nodeSettings(cfg, o.name).ssh;
+    if (!sshTarget)
+        throw new Error(`node-install needs an SSH target: pass <ssh-target> or set nodes.${o.name}.ssh in ${cfg.file}`);
     const controlUrl = normalizeControlUrl(o.agentControlUrl ?? cfg.controlUrl);
     assertHttps(controlUrl, o.httpOnlyDev);
     if (o.installDocker)
@@ -338,17 +327,13 @@ program
         NODE_NAME: o.name,
         AGENT_TOKEN: registered.agentToken,
     })) + (o.mesh ? 'TUG_MESH=true\n' : '');
-    const dockerLogin = o.registry
-        ? `printf %s ${sh(registered.registryToken)} | docker login ${sh(o.registry)} --username ${sh(o.name)} --password-stdin`
-        : 'true';
     const remote = await loadTemplate(templatesDir, 'node-install.sh', {
         ENV: env,
-        DOCKER_LOGIN: dockerLogin,
         IMAGE_SH: sh(image),
         RUN_FLAGS: `${o.mesh ? MESH_RUN_FLAGS : HOST_NETWORK_FLAG} `,
     });
     runLocal('ssh', ['--', sshTarget, remote]);
-    console.log(JSON.stringify({ ok: true, node: registered.node, registryToken: registered.registryToken }, null, 2));
+    console.log(JSON.stringify({ ok: true, node: registered.node }, null, 2));
 });
 program
     .command('node-enroll')
@@ -419,6 +404,172 @@ program
 program
     .command('secret-remove <name>')
     .action(async (name, o) => console.log(await request('DELETE', `/api/v1/secrets/${encodeURIComponent(name)}?project=${encodeURIComponent(o.project || (await loadCfg()).project || 'site')}`)));
+const formatBytes = (bytes) => bytes === null
+    ? '-'
+    : bytes < 1024 ** 2
+        ? `${Math.ceil(bytes / 1024)} KiB`
+        : `${(bytes / 1024 ** 2).toFixed(1)} MiB`;
+const BACKUP_TERMINAL = ['succeeded', 'failed', 'missed', 'cancelled'];
+program
+    .command('backup-run [targets...]')
+    .description('back up the named services or volumes now, or every backup target of the project; waits and exits nonzero if any backup fails')
+    .option('--no-wait', 'return once the backups are queued')
+    .option('--timeout <seconds>', 'give up waiting after this many seconds', '3600')
+    .action(async (targets, o) => {
+    const project = (await loadCfg()).project || 'site';
+    const queued = await request('POST', '/api/v1/backups/run', { project, targets: targets.length ? targets : undefined });
+    for (const message of queued.errors)
+        console.error(message);
+    for (const run of queued.runs)
+        console.log(`queued ${run.target} as ${run.runId}`);
+    let failed = queued.errors.length > 0;
+    if (o.wait) {
+        const deadline = Date.now() + Number(o.timeout) * 1000;
+        const pending = new Set(queued.runs.map((run) => run.runId));
+        while (pending.size && Date.now() < deadline) {
+            await new Promise((resolve) => setTimeout(resolve, 2000));
+            const listing = await request('GET', `/api/v1/backups?project=${encodeURIComponent(project)}&limit=200`);
+            for (const run of listing.runs)
+                if (pending.has(run.id) && BACKUP_TERMINAL.includes(run.state)) {
+                    pending.delete(run.id);
+                    const ok = run.state === 'succeeded';
+                    failed ||= !ok;
+                    console.log(ok
+                        ? `${run.target}: snapshot ${run.snapshot_id?.slice(0, 8)}, ${formatBytes(run.added_bytes)} added`
+                        : `${run.target}: ${run.state}${run.error ? ` (${run.error})` : ''}`);
+                }
+        }
+        if (pending.size) {
+            console.error(`timed out waiting for ${pending.size} backup(s); they keep running on the node`);
+            failed = true;
+        }
+    }
+    if (failed)
+        process.exitCode = 1;
+});
+program
+    .command('backup-list')
+    .description('show each backup target with its last success and next run, then recent runs')
+    .option('--limit <n>', 'number of recent runs to show', '20')
+    .action(async (o) => {
+    const project = (await loadCfg()).project || 'site';
+    const listing = await request('GET', `/api/v1/backups?project=${encodeURIComponent(project)}&limit=${encodeURIComponent(o.limit)}`);
+    if (!listing.destinationConfigured)
+        console.error('no backup destination is configured; run tug backup-destination-set');
+    if (!listing.targets.length)
+        console.log('no backup targets; add x-tug.backup to a database service or volume');
+    for (const t of listing.targets)
+        console.log(`${t.target}\t${t.node ?? '-'}\t${t.schedule}\tlast success ${t.lastSuccessAt ?? 'never'}\tnext ${t.enabled ? t.nextRunAt : 'disabled'}`);
+    for (const r of listing.runs)
+        console.log(`${r.finished_at ?? r.queued_at}\t${r.target}\t${r.trigger}\t${r.state}\t${r.snapshot_id?.slice(0, 8) ?? '-'}\t${formatBytes(r.size_bytes)}${r.error ? `\t${r.error}` : ''}`);
+});
+program
+    .command('restore <target>')
+    .description('restore a service or volume from a backup. Without --confirm it only prints the plan. An in-place restore locks the project, stops the services that write to the target, takes a safety backup, restores, then starts them again')
+    .option('--from <id>', 'backup run id, restic snapshot id, or latest', 'latest')
+    .option('--scratch', 'restore a volume into a new volume next to the live one, touching nothing live')
+    .option('--no-safety-backup', 'skip the backup of the current data taken before an in-place restore')
+    .option('--writers <service>', 'service to stop during the restore, instead of discovering them (repeatable)', collect, [])
+    .option('--no-writers', 'restore without stopping any service')
+    .option('--confirm', 'perform the restore')
+    .option('--timeout <seconds>', 'give up waiting after this many seconds', '3600')
+    .action(async (target, o) => {
+    const project = (await loadCfg()).project || 'site';
+    const explicit = Array.isArray(o.writers) && o.writers.length > 0 ? o.writers : undefined;
+    const result = await request('POST', '/api/v1/restores', {
+        project,
+        target,
+        from: o.from,
+        scratch: o.scratch === true,
+        safetyBackup: o.safetyBackup,
+        writers: explicit,
+        noWriters: o.writers === false,
+        confirm: o.confirm === true,
+    });
+    const plan = result.plan;
+    console.log(`target:   ${plan.target} on ${plan.node}`);
+    console.log(`snapshot: ${plan.snapshotId}${plan.snapshotTime ? ` (${plan.snapshotTime})` : ''}`);
+    console.log(`mode:     ${plan.mode}${plan.scratchVolume ? ` into new volume ${plan.scratchVolume}` : ''}`);
+    if (plan.mode === 'in-place') {
+        console.log(`stops:    ${plan.writers.length ? plan.writers.join(', ') : 'nothing'}`);
+        console.log(`safety:   ${plan.safetyBackup ? 'a backup of the current data is taken first' : 'none (--no-safety-backup)'}`);
+    }
+    if (!result.confirmed) {
+        console.log('\nNothing was changed. Run again with --confirm to restore.');
+        return;
+    }
+    const deadline = Date.now() + Number(o.timeout) * 1000;
+    for (;;) {
+        await new Promise((resolve) => setTimeout(resolve, 2000));
+        const { restores } = await request('GET', `/api/v1/restores?project=${encodeURIComponent(project)}`);
+        const restore = restores.find((item) => item.id === result.restoreId);
+        if (restore?.state === 'succeeded') {
+            console.log(`restore ${restore.id} completed${restore.scratch_volume ? `; files are in volume ${restore.scratch_volume}` : ''}`);
+            return;
+        }
+        if (restore && ['failed', 'interrupted', 'released'].includes(restore.state)) {
+            console.error(`restore ${restore.id} ${restore.state}: ${restore.error ?? ''}`);
+            if (restore.state === 'interrupted')
+                console.error(`the services stay stopped; run tug restore-resume ${restore.id} to try again or tug restore-release ${restore.id} to start them (data may be partly restored)`);
+            process.exitCode = 1;
+            return;
+        }
+        if (Date.now() > deadline) {
+            console.error(`timed out waiting; restore ${result.restoreId} continues on control (tug restore-list)`);
+            process.exitCode = 1;
+            return;
+        }
+    }
+});
+program
+    .command('restore-list')
+    .description('show recent restores and their state')
+    .action(async () => {
+    const project = (await loadCfg()).project || 'site';
+    const { restores } = await request('GET', `/api/v1/restores?project=${encodeURIComponent(project)}`);
+    if (!restores.length)
+        console.log('no restores');
+    for (const r of restores)
+        console.log(`${r.started_at}\t${r.id}\t${r.target}\t${r.mode}\t${r.state}\t${r.snapshot_id.slice(0, 8)}${r.error ? `\t${r.error}` : ''}`);
+});
+program
+    .command('restore-resume <id>')
+    .description('run an interrupted restore again; the stopped services stay stopped meanwhile')
+    .action(async (id) => console.log(JSON.stringify(await request('POST', `/api/v1/restores/${encodeURIComponent(id)}/resume`))));
+program
+    .command('restore-release <id>')
+    .description('start the services an interrupted restore stopped and unlock the project; the data may be partly restored')
+    .action(async (id) => console.log(JSON.stringify(await request('POST', `/api/v1/restores/${encodeURIComponent(id)}/release`))));
+program
+    .command('backup-destination-set <url>')
+    .description('set where backups go, such as s3:https://s3.example.com/bucket/tug-backups; credentials are stored encrypted on control')
+    .option('--env <name>', 'credential variable for the backend, such as AWS_ACCESS_KEY_ID (repeatable); each value is read from the variable of that name in your environment, else from a line on stdin', collect, [])
+    .action(async (url, o) => {
+    const missing = o.env.filter((name) => process.env[name] === undefined);
+    const lines = missing.length
+        ? (await readStdin()).split(/\r?\n/).filter((line) => line.length > 0)
+        : [];
+    if (lines.length < missing.length)
+        throw new Error(`expected a value on stdin for ${missing.slice(lines.length).join(', ')}`);
+    const env = Object.fromEntries(o.env.map((name) => [name, process.env[name] ?? lines[missing.indexOf(name)]]));
+    console.log(JSON.stringify(await request('POST', '/api/v1/backup/destination', { url, env })));
+});
+program
+    .command('backup-destination-show')
+    .description('show the backup destination and the names of its credential variables, never values')
+    .action(async () => console.log(JSON.stringify(await request('GET', '/api/v1/backup/destination'), null, 2)));
+program
+    .command('backup-destination-remove')
+    .description('remove the backup destination and its stored credentials; existing backups stay in the repository')
+    .action(async () => console.log(JSON.stringify(await request('DELETE', '/api/v1/backup/destination'))));
+program
+    .command('backup-key')
+    .description("print the project's repository and its password for offline escrow; anyone holding the password and destination credentials can read the project's backups")
+    .action(async () => {
+    const c = await loadCfg();
+    const project = c.project || 'site';
+    console.log(JSON.stringify(await request('GET', `/api/v1/backup/key?project=${encodeURIComponent(project)}`), null, 2));
+});
 program
     .command('token-create <name>')
     .description('create a named API token for CI or a teammate; the secret is printed once and cannot be shown again')
@@ -450,7 +601,10 @@ program
 async function makeContext() {
     const tmp = path.join(os.tmpdir(), `tug-${Date.now()}.tgz`);
     try {
-        await createSourceArchive(process.cwd(), tmp, [cfgPath()]);
+        await createSourceArchive(process.cwd(), tmp, [
+            resolvedConfig.file,
+            resolvedConfig.localFile,
+        ]);
     }
     catch (error) {
         await fs.rm(tmp, { force: true });
@@ -463,7 +617,6 @@ program
     .option('-f, --file <file>', 'compose file', 'docker-compose.yml')
     .option('--version <version>', 'display version (default: git-<Compose hash>)')
     .option('--build-local', 'build application images locally and upload them without source')
-    .option('--no-build-upload', 'deploy published images without uploading source; conflicts with --build-local')
     .option('--no-wait', 'return after desired state is created')
     .option('--timeout <seconds>', 'wait timeout in seconds', '300')
     .action(async (o) => {
@@ -488,8 +641,7 @@ program
             await fs.rm(directory, { recursive: true, force: true });
         }
     }
-    else if (o.buildUpload &&
-        Object.values(compose.services).some((service) => service.build)) {
+    else if (Object.values(compose.services).some((service) => service.build)) {
         const tarball = await makeContext();
         try {
             const form = new FormData();
@@ -636,7 +788,7 @@ program
 });
 async function runLive(operation, input) {
     const cfg = await loadCfg();
-    process.exitCode = await runCliLive(parseLiveRequest(operation, { ...input, project: cfg.project || 'site' }), createCliLiveTransport(cfg.controlUrl, cfg.operatorToken));
+    process.exitCode = await runCliLive(parseLiveRequest(operation, { ...input, project: cfg.project || 'site' }), createCliLiveTransport(cfg.controlUrl, cfg.token));
 }
 program
     .command('exec <service> [command...]')
@@ -647,6 +799,15 @@ program
     .action(async (service, command, o) => {
     await runLive('exec', { service, command, ...o });
 });
+program
+    .command('run <service> [command...]')
+    .description("run a one-off command in a new container from the service: same image, environment, secrets, network and mounts, removed afterwards (for example: tug run web -- wp core update-db). Exits with the command's exit code")
+    .option('--node <name>', 'select the node when a service has multiple placements')
+    .action(async (service, command, o) => {
+    if (!command.length)
+        throw new Error('run needs a command, such as: tug run web -- wp core update-db');
+    await runLive('exec', { service, command, node: o.node, oneOff: true });
+});
 program
     .command('ps')
     .description('print per-container state as JSON')
@@ -705,10 +866,6 @@ program.hook('preAction', async (_root, command) => {
         if (options[key] !== undefined)
             integerOption(String(options[key]), '--' + key.replace(/[A-Z]/g, (c) => '-' + c.toLowerCase()), min, max);
     }
-    if (options.buildLocal && options.buildUpload === false)
-        throw new Error('--build-local and --no-build-upload cannot be combined');
-    if ((options.registryAuthPath || options.registryContainer) && !options.registry)
-        throw new Error('--registry-auth-path and --registry-container require --registry');
     if (command.name() === 'control-restore' && !options.confirm)
         throw new Error('control-restore requires --confirm');
     const positional = command.name() === 'exec' ? command.args.slice(0, 1) : command.args;
@@ -732,10 +889,21 @@ program.hook('preAction', async (_root, command) => {
         connection.project = undefined;
     }
     resolvedConfig = await resolveCliConfig(connection, process.env, process.cwd(), command.name() === 'init');
+    // Config values are defaults for options not given on the command line; nodes are only read by the
+    // commands that create or register one, never by commands that talk to an existing node.
+    const nodeName = command.name() === 'node-register'
+        ? command.args[0]
+        : ['node-install', 'node-enroll'].includes(command.name())
+            ? options.name
+            : undefined;
+    applyConfigDefaults(command, options, {
+        ...resolvedConfig.commandDefaults,
+        ...(command.name().startsWith('node-') ? nodeSettings(resolvedConfig, nodeName) : {}),
+    });
 });
 for (const command of program.commands) {
     command
-        .option('--config <file>', 'CLI JSON config file (default: TUG_CONFIG or .tug.json)')
+        .option('--config <file>', CONFIG_HELP)
         .option('--control-url <url>', 'control API URL (overrides TUG_CONTROL_URL and config)');
     if (!command.options.some((option) => option.long === '--project'))
         command.option('--project <name>', 'project (overrides TUG_PROJECT and config; default: site)');

package/dist/compose.d.ts

@@ -39,7 +39,6 @@ export interface BuildDesiredStateOptions {
     project: string;
     version: string;
     compose: ComposeFile;
-    registry: string;
     revision?: string;
     defaultNode?: string | null;
     nodes?: Array<{
@@ -47,7 +46,7 @@ export interface BuildDesiredStateOptions {
         labels?: string[];
     }>;
 }
-export declare function buildDesiredState({ project, version, compose, registry, revision, defaultNode, nodes, }: BuildDesiredStateOptions): DesiredState;
+export declare function buildDesiredState({ project, version, compose, revision, defaultNode, nodes, }: BuildDesiredStateOptions): DesiredState;
 /** `extra_hosts` entries (`name:ip`) for services that run on other nodes, from health-aware endpoints. */
 export declare function crossNodeHosts(project: string, endpoints: Record<string, ServiceEndpoint[]>, local: Set<string>): string[];
 /**

package/dist/compose.js

@@ -1,4 +1,5 @@
 import crypto from 'node:crypto';
+import { resolveBackups, validateBackupDefaults } from "./backup-config.js";
 import { isDuration } from "./util.js";
 import { sentinelNames } from "./normalize.js";
 export function composeHash(compose) {
@@ -166,10 +167,11 @@ function validateProjectExtension(compose, errors) {
         return;
     }
     for (const key of Object.keys(ext))
-        if (key !== 'rollback')
-            errors.push(`x-tug.${key} is not supported (supported: rollback)`);
+        if (key !== 'rollback' && key !== 'backup')
+            errors.push(`x-tug.${key} is not supported (supported: rollback, backup)`);
     if (ext.rollback !== undefined && !['auto', 'manual'].includes(ext.rollback))
         errors.push('x-tug.rollback must be auto or manual');
+    validateBackupDefaults(ext.backup, errors);
 }
 /** Service keys the agent can create a container from; anything else is rejected with its name. */
 export const SUPPORTED_SERVICE_KEYS = [
@@ -302,10 +304,11 @@ export function validateCompose(compose, nodes = []) {
     }
     validateTopLevel(compose, errors);
     validateProjectExtension(compose, errors);
+    errors.push(...resolveBackups(compose).errors);
     if (errors.length)
         throw new Error(errors.join('; '));
 }
-export function buildDesiredState({ project, version, compose, registry, revision, defaultNode = null, nodes, }) {
+export function buildDesiredState({ project, version, compose, revision, defaultNode = null, nodes, }) {
     const services = {};
     for (const [name, src] of Object.entries(compose.services || {})) {
         const lp = src['x-tug'] || {};
@@ -323,9 +326,7 @@ export function buildDesiredState({ project, version, compose, registry, revisio
             }
             placement = { ...placement, nodes: selected.map((node) => node.name) };
         }
-        const image = src.build
-            ? `${registry ? `${registry}/` : 'tug-build/'}${project}/${name}:${revision || version}`
-            : src.image;
+        const image = src.build ? `tug-build/${project}/${name}:${revision || version}` : src.image;
         services[name] = {
             image,
             build: src.build || null,

package/dist/container-spec.d.ts

@@ -2,6 +2,8 @@ import type { CreateContainerBody } from './docker-api.ts';
 import type { DesiredState, ServiceEndpoint } from './protocol.ts';
 import type { PortBind } from './compose.ts';
 export declare const LABEL_PROJECT = "tug.project";
+/** Labels helper containers (backup jobs, one-off runs) so a restarted agent can clean up after them. */
+export declare const LABEL_JOB = "tug.job";
 export declare const LABEL_SERVICE = "tug.service";
 export declare const LABEL_REVISION = "tug.revision";
 export declare const LABEL_HASH = "tug.config-hash";
@@ -41,7 +43,6 @@ export interface ContainerSpecOptions {
     /** Secret versions by name; they are part of the hash instead of the values. */
     secretVersions?: Record<string, number>;
     endpoints?: Record<string, ServiceEndpoint[]>;
-    nodeAddresses?: Record<string, string>;
     /** Host IP for published ports that name none; without it `ports` stay as written. */
     bind?: PortBind;
 }
@@ -49,7 +50,7 @@ export interface ContainerSpecOptions {
 export declare function composeDurationMs(value: unknown): number | undefined;
 /** Compose sizes: bytes as a number, or `64m` / `1g` style strings. */
 export declare function sizeBytes(value: unknown): number | undefined;
-export declare function containerSpecs(desired: DesiredState, node: string, { secrets, secretVersions, endpoints, nodeAddresses, bind, }?: ContainerSpecOptions): NodePlan;
+export declare function containerSpecs(desired: DesiredState, node: string, { secrets, secretVersions, endpoints, bind }?: ContainerSpecOptions): NodePlan;
 /** Start order: dependencies first, ties by service name; a cycle is an error. */
 export declare function startOrder(specs: Record<string, ContainerSpec>): string[];
 /** `body` with the host side of its (single) published port replaced; used for rolling revisions. */

package/dist/container-spec.js

@@ -4,6 +4,8 @@ import crypto from 'node:crypto';
 import { bindPorts, crossNodeHosts } from "./compose.js";
 import { sentinelNames, substituteSentinels } from "./normalize.js";
 export const LABEL_PROJECT = 'tug.project';
+/** Labels helper containers (backup jobs, one-off runs) so a restarted agent can clean up after them. */
+export const LABEL_JOB = 'tug.job';
 export const LABEL_SERVICE = 'tug.service';
 export const LABEL_REVISION = 'tug.revision';
 export const LABEL_HASH = 'tug.config-hash';
@@ -105,8 +107,8 @@ function portMaps(svc, ports) {
         exposed[`${port.split('/')[0]}/${port.split('/')[1] || 'tcp'}`] = {};
     return { exposed, bindings };
 }
-/** Volume definitions this node needs, with NFS volumes mounted from the exporter's directory or address. */
-function volumeSpecs(desired, node, used, nodeAddresses) {
+/** Volume definitions this node needs. */
+function volumeSpecs(desired, node, used) {
     const out = [];
     for (const key of [...used].sort()) {
         const def = desired.compose.volumes?.[key] || {};
@@ -128,42 +130,15 @@ function volumeSpecs(desired, node, used, nodeAddresses) {
         }
         const name = volumeName(desired.project, key);
         const nodes = xe.placement?.nodes || [];
-        const isNfs = xe.type === 'nfs' || xe.driver === 'nfs' || (xe.type === 'shared' && xe.driver === 'nfs');
-        const plain = () => ({
+        if (nodes.length && !nodes.includes(node))
+            continue;
+        out.push({
             name,
             driver: String(def.driver || 'local'),
             opts: def.driver_opts || {},
             labels,
             external: false,
         });
-        if (isNfs) {
-            const exporter = nodes[0];
-            const exportPath = xe.export || `/srv/tug/${key}`;
-            if (!exporter)
-                out.push(plain());
-            else if (nodes.includes(node))
-                // The exporter mounts the exported directory directly; an NFS server on this node must export it.
-                out.push({
-                    name,
-                    driver: 'local',
-                    opts: { type: 'none', o: 'bind', device: exportPath },
-                    labels,
-                    external: false,
-                });
-            else {
-                const addr = xe.address || nodeAddresses[exporter] || exporter;
-                out.push({
-                    name,
-                    driver: 'local',
-                    opts: { type: 'nfs', o: `addr=${addr},rw,nolock,soft`, device: `:${exportPath}` },
-                    labels,
-                    external: false,
-                });
-            }
-            continue;
-        }
-        if (!nodes.length || nodes.includes(node))
-            out.push(plain());
     }
     return out;
 }
@@ -176,7 +151,7 @@ function walkStrings(value, fn) {
         return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, walkStrings(v, fn)]));
     return value;
 }
-export function containerSpecs(desired, node, { secrets = {}, secretVersions = {}, endpoints = {}, nodeAddresses = {}, bind, } = {}) {
+export function containerSpecs(desired, node, { secrets = {}, secretVersions = {}, endpoints = {}, bind } = {}) {
     const { project } = desired;
     const network = networkName(project);
     const placed = new Set();
@@ -322,7 +297,7 @@ export function containerSpecs(desired, node, { secrets = {}, secretVersions = {
             dependsOn,
         };
     }
-    return { network, volumes: volumeSpecs(desired, node, usedVolumes, nodeAddresses), specs };
+    return { network, volumes: volumeSpecs(desired, node, usedVolumes), specs };
 }
 /** Start order: dependencies first, ties by service name; a cycle is an error. */
 export function startOrder(specs) {

package/dist/control.d.ts

@@ -1,18 +1,20 @@
 #!/usr/bin/env node
 import express from 'express';
+import { resolveImage } from './registry.ts';
 import { saveImageArchive, inspectImage, validateImageArchive } from './artifacts.ts';
 import https from 'node:https';
 import http from 'node:http';
 import { openDb } from './db.ts';
 import type { NodeRow } from './db.ts';
-import { buildAndPush, run as runCommand } from './build.ts';
+import { buildImage, run as runCommand } from './build.ts';
 declare module 'express-serve-static-core' {
     interface Request {
         node?: NodeRow;
     }
 }
 export interface ControlImageOperations {
-    build: typeof buildAndPush;
+    resolve: typeof resolveImage;
+    build: typeof buildImage;
     save: typeof saveImageArchive;
     inspect: typeof inspectImage;
     validate: typeof validateImageArchive;