@wpelevator/tug 0.5.0 → 0.6.0

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { Command, Option } from 'commander';
 import { openAsBlob } from 'node:fs';
-import { resolveCliConfig, integerOption, normalizeControlUrl } from "./cli-config.js";
+import { applyConfigDefaults, integerOption, normalizeControlUrl, nodeSettings, resolveCliConfig, writeInitConfig, } from "./cli-config.js";
 import { dockerRunSnippet, enrollmentCloudInit, shellQuote as sh, HOST_NETWORK_FLAG, MESH_RUN_FLAGS, controlPortCheck, localImageBuildArgs, meshPortCheck, } from "./cli-install.js";
 import { createSourceArchive, prepareLocalImages } from "./cli-build.js";
 import { saveImageArchive } from "./artifacts.js";
@@ -22,29 +22,32 @@ import { createCliLiveTransport } from "./cli-live-transport.js";
 // built image (`node dist/cli.js`; tools/scripts/copy-templates.js puts them there).
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const templatesDir = path.join(__dirname, 'templates');
+const CONFIG_HELP = 'CLI config file, YAML or JSON (default: TUG_CONFIG, else tug.yml, tug.yaml, tug.json or .tug.json)';
 const program = new Command();
 program
     .name('tug')
     .description('Portable VPS deployments with Docker Compose')
     .version(packageVersion())
     .enablePositionalOptions()
-    .option('--config <file>', 'CLI JSON config file (default: TUG_CONFIG or .tug.json)')
+    .option('--config <file>', CONFIG_HELP)
     .option('--control-url <url>', 'control API URL (overrides TUG_CONTROL_URL and config)')
     .option('--project <name>', 'project (overrides TUG_PROJECT and config; default: site)');
 let resolvedConfig;
-const cfgPath = () => resolvedConfig.file;
 async function loadCfg() {
     return resolvedConfig;
 }
 const collect = (v, a) => [...a, v];
-async function saveCfg(c) {
-    await fs.writeFile(cfgPath(), JSON.stringify(c, null, 2), { mode: 0o600 });
+async function readStdin() {
+    const chunks = [];
+    for await (const chunk of process.stdin)
+        chunks.push(Buffer.from(chunk));
+    return Buffer.concat(chunks).toString('utf8');
 }
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 async function request(method, route, body, multipart = false) {
     const c = await loadCfg();
     const base = c.controlUrl;
-    const tok = c.operatorToken;
+    const tok = c.token;
     const headers = tok ? { Authorization: `Bearer ${tok}` } : {};
     if (!multipart)
         headers['content-type'] = 'application/json';
@@ -180,11 +183,15 @@ async function waitForDeployment(deploymentId, timeoutSeconds = 300) {
 program
     .command('init')
     .option('--token <token>', 'operator token (prefer TUG_TOKEN to avoid process-list exposure)')
-    .action(async (o) => {
+    .description('create the config file if missing and store the operator token in its git-ignored .local overlay')
+    .action(async (o, command) => {
     const cfg = await loadCfg();
-    const operatorToken = o.token || cfg.operatorToken || token('tug_op');
-    await saveCfg({ controlUrl: cfg.controlUrl, operatorToken, project: cfg.project });
-    console.log(`wrote ${cfgPath()}`);
+    const { controlUrl, project } = command.optsWithGlobals();
+    for (const file of await writeInitConfig(cfg, o.token || cfg.token || token('tug_op'), {
+        controlUrl,
+        project,
+    }))
+        console.log(`wrote ${file}`);
 });
 program
     .command('node-register <name>')
@@ -279,8 +286,8 @@ program
     }, null, 2));
 });
 program
-    .command('node-install <sshTarget>')
-    .description('register and install an agent container on a Docker-capable host over SSH')
+    .command('node-install [sshTarget]')
+    .description('register and install an agent container on a Docker-capable host over SSH (the SSH target defaults to nodes.<name>.ssh in the config file)')
     .requiredOption('--name <name>')
     .option('--address <addr>')
     .option('--public-address <addr>')
@@ -292,8 +299,11 @@ program
     .option('--image <tag>', "Tug image to run (default: ghcr.io/wpelevator/tug:<this CLI's version>)")
     .option('--install-docker', 'install missing Docker Engine and Compose on supported Debian, Ubuntu, Fedora and RHEL-compatible hosts (root required)')
     .option('--build-local-image', 'build the image from the current checkout and copy it to the host over SSH (tags it tug:local unless --image is set); for development')
-    .action(async (sshTarget, o) => {
+    .action(async (sshArgument, o) => {
     const cfg = await loadCfg();
+    const sshTarget = sshArgument ?? nodeSettings(cfg, o.name).ssh;
+    if (!sshTarget)
+        throw new Error(`node-install needs an SSH target: pass <ssh-target> or set nodes.${o.name}.ssh in ${cfg.file}`);
     const controlUrl = normalizeControlUrl(o.agentControlUrl ?? cfg.controlUrl);
     assertHttps(controlUrl, o.httpOnlyDev);
     if (o.installDocker)
@@ -394,6 +404,172 @@ program
 program
     .command('secret-remove <name>')
     .action(async (name, o) => console.log(await request('DELETE', `/api/v1/secrets/${encodeURIComponent(name)}?project=${encodeURIComponent(o.project || (await loadCfg()).project || 'site')}`)));
+const formatBytes = (bytes) => bytes === null
+    ? '-'
+    : bytes < 1024 ** 2
+        ? `${Math.ceil(bytes / 1024)} KiB`
+        : `${(bytes / 1024 ** 2).toFixed(1)} MiB`;
+const BACKUP_TERMINAL = ['succeeded', 'failed', 'missed', 'cancelled'];
+program
+    .command('backup-run [targets...]')
+    .description('back up the named services or volumes now, or every backup target of the project; waits and exits nonzero if any backup fails')
+    .option('--no-wait', 'return once the backups are queued')
+    .option('--timeout <seconds>', 'give up waiting after this many seconds', '3600')
+    .action(async (targets, o) => {
+    const project = (await loadCfg()).project || 'site';
+    const queued = await request('POST', '/api/v1/backups/run', { project, targets: targets.length ? targets : undefined });
+    for (const message of queued.errors)
+        console.error(message);
+    for (const run of queued.runs)
+        console.log(`queued ${run.target} as ${run.runId}`);
+    let failed = queued.errors.length > 0;
+    if (o.wait) {
+        const deadline = Date.now() + Number(o.timeout) * 1000;
+        const pending = new Set(queued.runs.map((run) => run.runId));
+        while (pending.size && Date.now() < deadline) {
+            await new Promise((resolve) => setTimeout(resolve, 2000));
+            const listing = await request('GET', `/api/v1/backups?project=${encodeURIComponent(project)}&limit=200`);
+            for (const run of listing.runs)
+                if (pending.has(run.id) && BACKUP_TERMINAL.includes(run.state)) {
+                    pending.delete(run.id);
+                    const ok = run.state === 'succeeded';
+                    failed ||= !ok;
+                    console.log(ok
+                        ? `${run.target}: snapshot ${run.snapshot_id?.slice(0, 8)}, ${formatBytes(run.added_bytes)} added`
+                        : `${run.target}: ${run.state}${run.error ? ` (${run.error})` : ''}`);
+                }
+        }
+        if (pending.size) {
+            console.error(`timed out waiting for ${pending.size} backup(s); they keep running on the node`);
+            failed = true;
+        }
+    }
+    if (failed)
+        process.exitCode = 1;
+});
+program
+    .command('backup-list')
+    .description('show each backup target with its last success and next run, then recent runs')
+    .option('--limit <n>', 'number of recent runs to show', '20')
+    .action(async (o) => {
+    const project = (await loadCfg()).project || 'site';
+    const listing = await request('GET', `/api/v1/backups?project=${encodeURIComponent(project)}&limit=${encodeURIComponent(o.limit)}`);
+    if (!listing.destinationConfigured)
+        console.error('no backup destination is configured; run tug backup-destination-set');
+    if (!listing.targets.length)
+        console.log('no backup targets; add x-tug.backup to a database service or volume');
+    for (const t of listing.targets)
+        console.log(`${t.target}\t${t.node ?? '-'}\t${t.schedule}\tlast success ${t.lastSuccessAt ?? 'never'}\tnext ${t.enabled ? t.nextRunAt : 'disabled'}`);
+    for (const r of listing.runs)
+        console.log(`${r.finished_at ?? r.queued_at}\t${r.target}\t${r.trigger}\t${r.state}\t${r.snapshot_id?.slice(0, 8) ?? '-'}\t${formatBytes(r.size_bytes)}${r.error ? `\t${r.error}` : ''}`);
+});
+program
+    .command('restore <target>')
+    .description('restore a service or volume from a backup. Without --confirm it only prints the plan. An in-place restore locks the project, stops the services that write to the target, takes a safety backup, restores, then starts them again')
+    .option('--from <id>', 'backup run id, restic snapshot id, or latest', 'latest')
+    .option('--scratch', 'restore a volume into a new volume next to the live one, touching nothing live')
+    .option('--no-safety-backup', 'skip the backup of the current data taken before an in-place restore')
+    .option('--writers <service>', 'service to stop during the restore, instead of discovering them (repeatable)', collect, [])
+    .option('--no-writers', 'restore without stopping any service')
+    .option('--confirm', 'perform the restore')
+    .option('--timeout <seconds>', 'give up waiting after this many seconds', '3600')
+    .action(async (target, o) => {
+    const project = (await loadCfg()).project || 'site';
+    const explicit = Array.isArray(o.writers) && o.writers.length > 0 ? o.writers : undefined;
+    const result = await request('POST', '/api/v1/restores', {
+        project,
+        target,
+        from: o.from,
+        scratch: o.scratch === true,
+        safetyBackup: o.safetyBackup,
+        writers: explicit,
+        noWriters: o.writers === false,
+        confirm: o.confirm === true,
+    });
+    const plan = result.plan;
+    console.log(`target:   ${plan.target} on ${plan.node}`);
+    console.log(`snapshot: ${plan.snapshotId}${plan.snapshotTime ? ` (${plan.snapshotTime})` : ''}`);
+    console.log(`mode:     ${plan.mode}${plan.scratchVolume ? ` into new volume ${plan.scratchVolume}` : ''}`);
+    if (plan.mode === 'in-place') {
+        console.log(`stops:    ${plan.writers.length ? plan.writers.join(', ') : 'nothing'}`);
+        console.log(`safety:   ${plan.safetyBackup ? 'a backup of the current data is taken first' : 'none (--no-safety-backup)'}`);
+    }
+    if (!result.confirmed) {
+        console.log('\nNothing was changed. Run again with --confirm to restore.');
+        return;
+    }
+    const deadline = Date.now() + Number(o.timeout) * 1000;
+    for (;;) {
+        await new Promise((resolve) => setTimeout(resolve, 2000));
+        const { restores } = await request('GET', `/api/v1/restores?project=${encodeURIComponent(project)}`);
+        const restore = restores.find((item) => item.id === result.restoreId);
+        if (restore?.state === 'succeeded') {
+            console.log(`restore ${restore.id} completed${restore.scratch_volume ? `; files are in volume ${restore.scratch_volume}` : ''}`);
+            return;
+        }
+        if (restore && ['failed', 'interrupted', 'released'].includes(restore.state)) {
+            console.error(`restore ${restore.id} ${restore.state}: ${restore.error ?? ''}`);
+            if (restore.state === 'interrupted')
+                console.error(`the services stay stopped; run tug restore-resume ${restore.id} to try again or tug restore-release ${restore.id} to start them (data may be partly restored)`);
+            process.exitCode = 1;
+            return;
+        }
+        if (Date.now() > deadline) {
+            console.error(`timed out waiting; restore ${result.restoreId} continues on control (tug restore-list)`);
+            process.exitCode = 1;
+            return;
+        }
+    }
+});
+program
+    .command('restore-list')
+    .description('show recent restores and their state')
+    .action(async () => {
+    const project = (await loadCfg()).project || 'site';
+    const { restores } = await request('GET', `/api/v1/restores?project=${encodeURIComponent(project)}`);
+    if (!restores.length)
+        console.log('no restores');
+    for (const r of restores)
+        console.log(`${r.started_at}\t${r.id}\t${r.target}\t${r.mode}\t${r.state}\t${r.snapshot_id.slice(0, 8)}${r.error ? `\t${r.error}` : ''}`);
+});
+program
+    .command('restore-resume <id>')
+    .description('run an interrupted restore again; the stopped services stay stopped meanwhile')
+    .action(async (id) => console.log(JSON.stringify(await request('POST', `/api/v1/restores/${encodeURIComponent(id)}/resume`))));
+program
+    .command('restore-release <id>')
+    .description('start the services an interrupted restore stopped and unlock the project; the data may be partly restored')
+    .action(async (id) => console.log(JSON.stringify(await request('POST', `/api/v1/restores/${encodeURIComponent(id)}/release`))));
+program
+    .command('backup-destination-set <url>')
+    .description('set where backups go, such as s3:https://s3.example.com/bucket/tug-backups; credentials are stored encrypted on control')
+    .option('--env <name>', 'credential variable for the backend, such as AWS_ACCESS_KEY_ID (repeatable); each value is read from the variable of that name in your environment, else from a line on stdin', collect, [])
+    .action(async (url, o) => {
+    const missing = o.env.filter((name) => process.env[name] === undefined);
+    const lines = missing.length
+        ? (await readStdin()).split(/\r?\n/).filter((line) => line.length > 0)
+        : [];
+    if (lines.length < missing.length)
+        throw new Error(`expected a value on stdin for ${missing.slice(lines.length).join(', ')}`);
+    const env = Object.fromEntries(o.env.map((name) => [name, process.env[name] ?? lines[missing.indexOf(name)]]));
+    console.log(JSON.stringify(await request('POST', '/api/v1/backup/destination', { url, env })));
+});
+program
+    .command('backup-destination-show')
+    .description('show the backup destination and the names of its credential variables, never values')
+    .action(async () => console.log(JSON.stringify(await request('GET', '/api/v1/backup/destination'), null, 2)));
+program
+    .command('backup-destination-remove')
+    .description('remove the backup destination and its stored credentials; existing backups stay in the repository')
+    .action(async () => console.log(JSON.stringify(await request('DELETE', '/api/v1/backup/destination'))));
+program
+    .command('backup-key')
+    .description("print the project's repository and its password for offline escrow; anyone holding the password and destination credentials can read the project's backups")
+    .action(async () => {
+    const c = await loadCfg();
+    const project = c.project || 'site';
+    console.log(JSON.stringify(await request('GET', `/api/v1/backup/key?project=${encodeURIComponent(project)}`), null, 2));
+});
 program
     .command('token-create <name>')
     .description('create a named API token for CI or a teammate; the secret is printed once and cannot be shown again')
@@ -425,7 +601,10 @@ program
 async function makeContext() {
     const tmp = path.join(os.tmpdir(), `tug-${Date.now()}.tgz`);
     try {
-        await createSourceArchive(process.cwd(), tmp, [cfgPath()]);
+        await createSourceArchive(process.cwd(), tmp, [
+            resolvedConfig.file,
+            resolvedConfig.localFile,
+        ]);
     }
     catch (error) {
         await fs.rm(tmp, { force: true });
@@ -609,7 +788,7 @@ program
 });
 async function runLive(operation, input) {
     const cfg = await loadCfg();
-    process.exitCode = await runCliLive(parseLiveRequest(operation, { ...input, project: cfg.project || 'site' }), createCliLiveTransport(cfg.controlUrl, cfg.operatorToken));
+    process.exitCode = await runCliLive(parseLiveRequest(operation, { ...input, project: cfg.project || 'site' }), createCliLiveTransport(cfg.controlUrl, cfg.token));
 }
 program
     .command('exec <service> [command...]')
@@ -620,6 +799,15 @@ program
     .action(async (service, command, o) => {
     await runLive('exec', { service, command, ...o });
 });
+program
+    .command('run <service> [command...]')
+    .description("run a one-off command in a new container from the service: same image, environment, secrets, network and mounts, removed afterwards (for example: tug run web -- wp core update-db). Exits with the command's exit code")
+    .option('--node <name>', 'select the node when a service has multiple placements')
+    .action(async (service, command, o) => {
+    if (!command.length)
+        throw new Error('run needs a command, such as: tug run web -- wp core update-db');
+    await runLive('exec', { service, command, node: o.node, oneOff: true });
+});
 program
     .command('ps')
     .description('print per-container state as JSON')
@@ -701,10 +889,21 @@ program.hook('preAction', async (_root, command) => {
         connection.project = undefined;
     }
     resolvedConfig = await resolveCliConfig(connection, process.env, process.cwd(), command.name() === 'init');
+    // Config values are defaults for options not given on the command line; nodes are only read by the
+    // commands that create or register one, never by commands that talk to an existing node.
+    const nodeName = command.name() === 'node-register'
+        ? command.args[0]
+        : ['node-install', 'node-enroll'].includes(command.name())
+            ? options.name
+            : undefined;
+    applyConfigDefaults(command, options, {
+        ...resolvedConfig.commandDefaults,
+        ...(command.name().startsWith('node-') ? nodeSettings(resolvedConfig, nodeName) : {}),
+    });
 });
 for (const command of program.commands) {
     command
-        .option('--config <file>', 'CLI JSON config file (default: TUG_CONFIG or .tug.json)')
+        .option('--config <file>', CONFIG_HELP)
         .option('--control-url <url>', 'control API URL (overrides TUG_CONTROL_URL and config)');
     if (!command.options.some((option) => option.long === '--project'))
         command.option('--project <name>', 'project (overrides TUG_PROJECT and config; default: site)');

package/dist/compose.js

@@ -1,4 +1,5 @@
 import crypto from 'node:crypto';
+import { resolveBackups, validateBackupDefaults } from "./backup-config.js";
 import { isDuration } from "./util.js";
 import { sentinelNames } from "./normalize.js";
 export function composeHash(compose) {
@@ -166,10 +167,11 @@ function validateProjectExtension(compose, errors) {
         return;
     }
     for (const key of Object.keys(ext))
-        if (key !== 'rollback')
-            errors.push(`x-tug.${key} is not supported (supported: rollback)`);
+        if (key !== 'rollback' && key !== 'backup')
+            errors.push(`x-tug.${key} is not supported (supported: rollback, backup)`);
     if (ext.rollback !== undefined && !['auto', 'manual'].includes(ext.rollback))
         errors.push('x-tug.rollback must be auto or manual');
+    validateBackupDefaults(ext.backup, errors);
 }
 /** Service keys the agent can create a container from; anything else is rejected with its name. */
 export const SUPPORTED_SERVICE_KEYS = [
@@ -302,6 +304,7 @@ export function validateCompose(compose, nodes = []) {
     }
     validateTopLevel(compose, errors);
     validateProjectExtension(compose, errors);
+    errors.push(...resolveBackups(compose).errors);
     if (errors.length)
         throw new Error(errors.join('; '));
 }

package/dist/container-spec.d.ts

@@ -2,6 +2,8 @@ import type { CreateContainerBody } from './docker-api.ts';
 import type { DesiredState, ServiceEndpoint } from './protocol.ts';
 import type { PortBind } from './compose.ts';
 export declare const LABEL_PROJECT = "tug.project";
+/** Labels helper containers (backup jobs, one-off runs) so a restarted agent can clean up after them. */
+export declare const LABEL_JOB = "tug.job";
 export declare const LABEL_SERVICE = "tug.service";
 export declare const LABEL_REVISION = "tug.revision";
 export declare const LABEL_HASH = "tug.config-hash";
@@ -41,7 +43,6 @@ export interface ContainerSpecOptions {
     /** Secret versions by name; they are part of the hash instead of the values. */
     secretVersions?: Record<string, number>;
     endpoints?: Record<string, ServiceEndpoint[]>;
-    nodeAddresses?: Record<string, string>;
     /** Host IP for published ports that name none; without it `ports` stay as written. */
     bind?: PortBind;
 }
@@ -49,7 +50,7 @@ export interface ContainerSpecOptions {
 export declare function composeDurationMs(value: unknown): number | undefined;
 /** Compose sizes: bytes as a number, or `64m` / `1g` style strings. */
 export declare function sizeBytes(value: unknown): number | undefined;
-export declare function containerSpecs(desired: DesiredState, node: string, { secrets, secretVersions, endpoints, nodeAddresses, bind, }?: ContainerSpecOptions): NodePlan;
+export declare function containerSpecs(desired: DesiredState, node: string, { secrets, secretVersions, endpoints, bind }?: ContainerSpecOptions): NodePlan;
 /** Start order: dependencies first, ties by service name; a cycle is an error. */
 export declare function startOrder(specs: Record<string, ContainerSpec>): string[];
 /** `body` with the host side of its (single) published port replaced; used for rolling revisions. */

package/dist/container-spec.js

@@ -4,6 +4,8 @@ import crypto from 'node:crypto';
 import { bindPorts, crossNodeHosts } from "./compose.js";
 import { sentinelNames, substituteSentinels } from "./normalize.js";
 export const LABEL_PROJECT = 'tug.project';
+/** Labels helper containers (backup jobs, one-off runs) so a restarted agent can clean up after them. */
+export const LABEL_JOB = 'tug.job';
 export const LABEL_SERVICE = 'tug.service';
 export const LABEL_REVISION = 'tug.revision';
 export const LABEL_HASH = 'tug.config-hash';
@@ -105,8 +107,8 @@ function portMaps(svc, ports) {
         exposed[`${port.split('/')[0]}/${port.split('/')[1] || 'tcp'}`] = {};
     return { exposed, bindings };
 }
-/** Volume definitions this node needs, with NFS volumes mounted from the exporter's directory or address. */
-function volumeSpecs(desired, node, used, nodeAddresses) {
+/** Volume definitions this node needs. */
+function volumeSpecs(desired, node, used) {
     const out = [];
     for (const key of [...used].sort()) {
         const def = desired.compose.volumes?.[key] || {};
@@ -128,42 +130,15 @@ function volumeSpecs(desired, node, used, nodeAddresses) {
         }
         const name = volumeName(desired.project, key);
         const nodes = xe.placement?.nodes || [];
-        const isNfs = xe.type === 'nfs' || xe.driver === 'nfs' || (xe.type === 'shared' && xe.driver === 'nfs');
-        const plain = () => ({
+        if (nodes.length && !nodes.includes(node))
+            continue;
+        out.push({
             name,
             driver: String(def.driver || 'local'),
             opts: def.driver_opts || {},
             labels,
             external: false,
         });
-        if (isNfs) {
-            const exporter = nodes[0];
-            const exportPath = xe.export || `/srv/tug/${key}`;
-            if (!exporter)
-                out.push(plain());
-            else if (nodes.includes(node))
-                // The exporter mounts the exported directory directly; an NFS server on this node must export it.
-                out.push({
-                    name,
-                    driver: 'local',
-                    opts: { type: 'none', o: 'bind', device: exportPath },
-                    labels,
-                    external: false,
-                });
-            else {
-                const addr = xe.address || nodeAddresses[exporter] || exporter;
-                out.push({
-                    name,
-                    driver: 'local',
-                    opts: { type: 'nfs', o: `addr=${addr},rw,nolock,soft`, device: `:${exportPath}` },
-                    labels,
-                    external: false,
-                });
-            }
-            continue;
-        }
-        if (!nodes.length || nodes.includes(node))
-            out.push(plain());
     }
     return out;
 }
@@ -176,7 +151,7 @@ function walkStrings(value, fn) {
         return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, walkStrings(v, fn)]));
     return value;
 }
-export function containerSpecs(desired, node, { secrets = {}, secretVersions = {}, endpoints = {}, nodeAddresses = {}, bind, } = {}) {
+export function containerSpecs(desired, node, { secrets = {}, secretVersions = {}, endpoints = {}, bind } = {}) {
     const { project } = desired;
     const network = networkName(project);
     const placed = new Set();
@@ -322,7 +297,7 @@ export function containerSpecs(desired, node, { secrets = {}, secretVersions = {
             dependsOn,
         };
     }
-    return { network, volumes: volumeSpecs(desired, node, usedVolumes, nodeAddresses), specs };
+    return { network, volumes: volumeSpecs(desired, node, usedVolumes), specs };
 }
 /** Start order: dependencies first, ties by service name; a cycle is an error. */
 export function startOrder(specs) {