@wpelevator/tug 0.5.0 → 0.6.0

package/README.md

@@ -61,18 +61,18 @@ npm install -g @wpelevator/tug
 The CLI installs the matching `ghcr.io/wpelevator/tug:<version>` image on your hosts.
 
 1. Install the control plane: `tug control-install root@control.example.com --install-docker`. The flag installs missing Docker Engine and Compose on supported Debian, Ubuntu, Fedora and RHEL-compatible hosts; omit it when Docker is ready. Put HTTPS in front of the control API as described in [Control plane](docs/control-plane.md).
-2. Point the CLI at it: `tug init --control-url https://control.example.com --token "$TUG_TOKEN"`.
+2. Point the CLI at it: `tug init --control-url https://control.example.com --token "$TUG_TOKEN"`. It writes `tug.yml` (commit it, and list your servers in it) and keeps the token in the git-ignored `tug.local.yml`; see [CLI reference](docs/cli.md#config-file).
 3. Set the secrets your services need: `tug secret-set DB_PASSWORD`.
 4. Add nodes with `tug node-install root@app1.example.com --name app1 --install-docker`, or `tug node-enroll --name app1` and run the printed Docker command. For a bare cloud host, `tug node-enroll --name app1 --install-docker` prints cloud-init user-data.
 5. Deploy: `tug deploy -f docker-compose.yml`. It waits until every node reports the new version healthy.
-6. Operate with `tug status`, `events`, `exec`, `logs`, `ps`, `inspect`, `restart`, `rollback` and `doctor`.
+6. Operate with `tug status`, `events`, `exec`, `run`, `logs`, `ps`, `inspect`, `restart`, `rollback`, `backup-run`, `restore` and `doctor`.
 
 ## Documentation
 
 - [Architecture](docs/architecture.md): components, reconciliation, workflow diagrams, secrets and runtime paths.
 - [Compose extensions](docs/compose.md): `x-tug` placement, health checks, volumes and limitations.
 - [Control plane](docs/control-plane.md): install, TLS, image delivery, backup and restore.
-- [Backups](docs/backups.md): interim WordPress backup sidecar with restic, per-node and NFS variants, and a restore drill until `x-tug.backup` (M11).
+- [Backups](docs/backups.md): interim WordPress backup sidecar with restic, per-node variants, and a restore drill until `x-tug.backup` (M11).
 - [Access control](docs/access-control.md): scoped API tokens, roles, audit trail and a GitHub Actions deploy example.
 - [Networking](docs/networking.md): WireGuard mesh, service DNS, ingress and TLS, firewall notes.
 - [Nodes](docs/nodes.md): SSH install, self-service enrollment, registration and status.

package/dist/agent-jobs.d.ts

@@ -0,0 +1,41 @@
+import type { DockerApi } from './docker-api.ts';
+import type { BackupCredentialsResponse, Command, CommandResult, JobStatus } from './protocol.ts';
+export interface JobHost {
+    docker: DockerApi;
+    stateDir: string;
+    fetchCredentials(commandId: string): Promise<BackupCredentialsResponse>;
+    /** Parallel jobs on this node; extra jobs wait queued. */
+    concurrency?: number;
+}
+/**
+ * The agent's background jobs. Reconciliation must not wait for an hour-long backup, so commands of
+ * these types are queued here, report a heartbeat through `statuses()`, and deliver their result
+ * through `drain()` on a later poll, even if the agent restarted in between.
+ */
+export declare class JobRunner {
+    private readonly host;
+    private readonly active;
+    private readonly queue;
+    private results;
+    private ready;
+    private settled;
+    constructor(host: JobHost);
+    /** Called whenever a job finishes, so the agent can report its result without waiting a full interval. */
+    onSettled(callback: () => void): void;
+    private file;
+    private save;
+    /** First use: remove helpers of a previous run and fail the jobs it never finished. */
+    private init;
+    /** Starts cleanup of a previous process's leftovers; safe to call more than once. */
+    warm(): Promise<void>;
+    /** Queues a job; the command is acknowledged immediately and its result arrives later. */
+    start(command: Command): Promise<void>;
+    private pump;
+    private execute;
+    /** Jobs that are running or waiting, for the status report's heartbeat. */
+    statuses(): JobStatus[];
+    /** Finished results to put in the next status report. They are removed only after `acknowledge`. */
+    pending(): CommandResult[];
+    /** Forgets results control has stored. */
+    acknowledge(delivered: CommandResult[]): Promise<void>;
+}

package/dist/agent-jobs.js

@@ -0,0 +1,161 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { runBackupJob, removeStaleJobContainers } from "./backup-job.js";
+import { redactSecrets } from "./backup-destination.js";
+import { ensureDir, errMessage, now } from "./util.js";
+const RESULTS_FILE = 'job-results.json';
+/**
+ * The agent's background jobs. Reconciliation must not wait for an hour-long backup, so commands of
+ * these types are queued here, report a heartbeat through `statuses()`, and deliver their result
+ * through `drain()` on a later poll, even if the agent restarted in between.
+ */
+export class JobRunner {
+    host;
+    active = new Map();
+    queue = [];
+    results = [];
+    ready;
+    settled;
+    constructor(host) {
+        this.host = host;
+    }
+    /** Called whenever a job finishes, so the agent can report its result without waiting a full interval. */
+    onSettled(callback) {
+        this.settled = callback;
+    }
+    file() {
+        return path.join(this.host.stateDir, RESULTS_FILE);
+    }
+    async save() {
+        const data = {
+            results: this.results,
+            running: [...this.active.keys(), ...this.queue.map((job) => job.command.id)],
+        };
+        await ensureDir(this.host.stateDir);
+        await fs.writeFile(this.file(), JSON.stringify(data), { mode: 0o600 });
+    }
+    /** First use: remove helpers of a previous run and fail the jobs it never finished. */
+    init() {
+        this.ready ??= (async () => {
+            await removeStaleJobContainers(this.host.docker).catch(() => 0);
+            try {
+                const saved = JSON.parse(await fs.readFile(this.file(), 'utf8'));
+                this.results = saved.results ?? [];
+                for (const commandId of saved.running ?? [])
+                    this.results.push({
+                        commandId,
+                        state: 'failed',
+                        result: { error: 'the agent restarted while the job was running' },
+                    });
+                if (saved.running?.length)
+                    await this.save();
+            }
+            catch {
+                /* no saved state */
+            }
+        })();
+        return this.ready;
+    }
+    /** Starts cleanup of a previous process's leftovers; safe to call more than once. */
+    warm() {
+        return this.init();
+    }
+    /** Queues a job; the command is acknowledged immediately and its result arrives later. */
+    async start(command) {
+        await this.init();
+        let payload;
+        try {
+            payload = JSON.parse(command.payload_json ?? '{}');
+            if (!payload.runId || !payload.repository || !payload.resticImage)
+                throw new Error('the job payload is incomplete');
+        }
+        catch (error) {
+            this.results.push({
+                commandId: command.id,
+                state: 'failed',
+                result: { error: errMessage(error) },
+            });
+            await this.save();
+            return;
+        }
+        this.queue.push({ command, payload });
+        await this.save();
+        this.pump();
+    }
+    pump() {
+        const limit = this.host.concurrency ?? 1;
+        while (this.active.size < limit && this.queue.length) {
+            const job = this.queue.shift();
+            void this.execute(job.command, job.payload);
+        }
+    }
+    async execute(command, payload) {
+        const controller = new AbortController();
+        const status = {
+            commandId: command.id,
+            runId: payload.runId,
+            phase: 'pulling',
+            startedAt: now(),
+            controller,
+        };
+        this.active.set(command.id, status);
+        const timeout = setTimeout(() => controller.abort(new Error(`the job exceeded its ${payload.limits.timeoutSec}s limit`)), payload.limits.timeoutSec * 1000);
+        timeout.unref();
+        let credentials;
+        try {
+            credentials = await this.host.fetchCredentials(command.id);
+            const result = await runBackupJob(payload, command.id, {
+                docker: this.host.docker,
+                credentials,
+                signal: controller.signal,
+                onPhase: (phase, bytes) => {
+                    status.phase = phase;
+                    if (bytes !== undefined)
+                        status.bytes = bytes;
+                },
+            }, command.type === 'backup.check'
+                ? 'backup.check'
+                : command.type === 'backup.restore'
+                    ? 'backup.restore'
+                    : 'backup.run');
+            this.results.push({ commandId: command.id, state: 'succeeded', result });
+        }
+        catch (error) {
+            const values = credentials ? Object.values(credentials.env) : [];
+            this.results.push({
+                commandId: command.id,
+                state: 'failed',
+                result: { error: redactSecrets(errMessage(controller.signal.reason ?? error), values) },
+            });
+        }
+        finally {
+            clearTimeout(timeout);
+            this.active.delete(command.id);
+            await this.save().catch(() => { });
+            this.settled?.();
+            this.pump();
+        }
+    }
+    /** Jobs that are running or waiting, for the status report's heartbeat. */
+    statuses() {
+        return [
+            ...[...this.active.values()].map(({ controller: _controller, ...status }) => status),
+            ...this.queue.map(({ command, payload }) => ({
+                commandId: command.id,
+                runId: payload.runId,
+                phase: 'queued',
+                startedAt: now(),
+            })),
+        ];
+    }
+    /** Finished results to put in the next status report. They are removed only after `acknowledge`. */
+    pending() {
+        return [...this.results];
+    }
+    /** Forgets results control has stored. */
+    async acknowledge(delivered) {
+        const ids = new Set(delivered.map((result) => result.commandId));
+        this.results = this.results.filter((result) => !ids.has(result.commandId));
+        await this.save().catch(() => { });
+    }
+}

package/dist/agent.d.ts

@@ -85,7 +85,7 @@ export declare function pruneManagedImages(ctx: AgentContext, retained: string[]
 export declare function resolveBindAddress(address: string | null, lookup?: (host: string) => Promise<{
     address: string;
 }>): Promise<string>;
-export declare function reconcile(desiredState: DesiredState, ctx?: AgentContext, endpoints?: Record<string, ServiceEndpoint[]>, ports?: PortBinding): Promise<ProjectStatus>;
+export declare function reconcile(desiredState: DesiredState, ctx?: AgentContext, endpoints?: Record<string, ServiceEndpoint[]>, ports?: PortBinding, held?: string[]): Promise<ProjectStatus>;
 /** Reports this node's WireGuard public key when control has not seen the current one yet. */
 export declare function reportWgKey(ctx: AgentContext): Promise<void>;
 export declare function pollOnce(ctx?: AgentContext): Promise<ProjectStatus[]>;

package/dist/agent.js

@@ -12,8 +12,9 @@ import dns from 'node:dns/promises';
 import { WebSocket } from 'ws';
 import { createHash } from 'node:crypto';
 import { addressBoundServices, isRolling, serviceSecretVersions } from "./compose.js";
-import { containerSpecs, networkName, volumeName } from "./container-spec.js";
+import { LABEL_PROJECT, LABEL_SERVICE, containerSpecs, networkName, volumeName, } from "./container-spec.js";
 import { API_VERSION, DockerEngine, isNotFound } from "./docker-api.js";
+import { JobRunner } from "./agent-jobs.js";
 import { LIVE_MAX_FRAME_BYTES, LIVE_START_TIMEOUT_MS } from "./protocol.js";
 import { LiveOperationEngine } from "./live.js";
 import { activeRollingStates, clearRolloutState, ensureProjectResources, loadRolloutState, projectContainers, reconcileRecreate, reconcileRolling, sweepOrphans, } from "./reconciler.js";
@@ -46,6 +47,23 @@ export function defaultContext() {
         rolloutPorts: process.env.TUG_ROLLOUT_PORTS,
     };
 }
+const jobRunners = new Map();
+/** One job runner per state directory, so a background backup outlives the polls that started it. */
+function jobRunnerFor(ctx) {
+    let runner = jobRunners.get(ctx.stateDir);
+    if (!runner) {
+        runner = new JobRunner({
+            docker: ctx.docker,
+            stateDir: ctx.stateDir,
+            concurrency: Number(process.env.TUG_BACKUP_CONCURRENCY) || 1,
+            fetchCredentials: (commandId) => api(ctx, `/agent/v1/backup/credentials?command=${encodeURIComponent(commandId)}`),
+        });
+        jobRunners.set(ctx.stateDir, runner);
+        // Clear helper containers a previous agent process left behind, before any live session can start one.
+        void runner.warm();
+    }
+    return runner;
+}
 const dbsByStateDir = new Map();
 async function agentDb(stateDir) {
     let db = dbsByStateDir.get(stateDir);
@@ -426,6 +444,7 @@ export class AgentLiveOperations {
                     container,
                     command: request.command,
                     interactive: request.interactive,
+                    oneOff: request.oneOff,
                     tty: request.tty,
                     columns: request.columns,
                     rows: request.rows,
@@ -802,7 +821,15 @@ export async function resolveBindAddress(address, lookup = (host) => dns.lookup(
 }
 // Last public-port warning per project, so the event is sent when it changes rather than on every poll.
 const publicPortWarnings = new Map();
-export async function reconcile(desiredState, ctx = defaultContext(), endpoints = {}, ports = { address: null, loopback: {} }) {
+/** Stops every running container of a service, including the revisions of a rolling service. */
+async function stopService(docker, project, service) {
+    const running = await docker.listContainers({
+        label: [`${LABEL_PROJECT}=${project}`, `${LABEL_SERVICE}=${service}`],
+    });
+    for (const container of running)
+        await docker.stopContainer(container.Id);
+}
+export async function reconcile(desiredState, ctx = defaultContext(), endpoints = {}, ports = { address: null, loopback: {} }, held = []) {
     const dir = path.join(ctx.stateDir, 'projects', desiredState.project);
     await ensureDir(dir);
     const serviceNames = Object.keys(containerSpecs(desiredState, ctx.nodeName).specs);
@@ -839,9 +866,13 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
     const currentSecretVersions = serviceSecretVersions(desiredState.compose, serviceNames, sec.versions || {});
     const db = await agentDb(ctx.stateDir);
     const previousRow = db.get('SELECT version, revision FROM applied_state WHERE project = ?', desiredState.project);
-    const rolling = serviceNames.filter((service) => isRolling(desiredState.compose.services[service]));
+    // A held service is stopped and left alone (a restore is rewriting its data) until control drops the hold.
+    const heldSet = new Set(held.filter((service) => serviceNames.includes(service)));
+    for (const service of heldSet)
+        serviceStates[service].rolloutState = 'held';
+    const rolling = serviceNames.filter((service) => !heldSet.has(service) && isRolling(desiredState.compose.services[service]));
     const rollingSet = new Set(rolling);
-    const mainServices = serviceNames.filter((service) => !rollingSet.has(service));
+    const mainServices = serviceNames.filter((service) => !rollingSet.has(service) && !heldSet.has(service));
     // Published images are pulled only for a new desired revision. Pulling on every poll would let a tag that moved
     // upstream (`mariadb:11`) recreate the container without a deploy.
     const sameDesiredVersion = previousRow
@@ -903,6 +934,8 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
         try {
             await ensureProjectResources(ctx.docker, desiredState.project, plan);
             await sweepOrphans(ctx.docker, desiredState.project, new Set(serviceNames));
+            for (const service of heldSet)
+                await stopService(ctx.docker, desiredState.project, service);
             if (mainServices.length)
                 await reconcileRecreate(ctx.docker, {
                     desired: desiredState,
@@ -1051,7 +1084,7 @@ export async function pollOnce(ctx = defaultContext()) {
     const projects = [];
     for (const desired of work.desiredStates || (work.desiredState ? [work.desiredState] : [])) {
         try {
-            projects.push(await reconcile(desired, ctx, work.serviceEndpoints?.[desired.project], work.ports));
+            projects.push(await reconcile(desired, ctx, work.serviceEndpoints?.[desired.project], work.ports, work.holds?.[desired.project]));
         }
         catch (e) {
             projects.push({
@@ -1072,7 +1105,13 @@ export async function pollOnce(ctx = defaultContext()) {
             await removeIngress(ingressCtx).catch(() => false);
     }
     const commandResults = [];
+    const jobs = jobRunnerFor(ctx);
     for (const command of work.commands || []) {
+        // Backups can run for an hour, so they run in the background and report through later polls.
+        if (command.type.startsWith('backup.')) {
+            await jobs.start(command);
+            continue;
+        }
         try {
             commandResults.push(await executeCommand(command, ctx));
         }
@@ -1084,7 +1123,12 @@ export async function pollOnce(ctx = defaultContext()) {
             });
         }
     }
+    const jobResults = jobs.pending();
+    commandResults.push(...jobResults);
     const report = { node: ctx.nodeName, projects, commandResults };
+    const running = jobs.statuses();
+    if (running.length)
+        report.jobs = running;
     if (ingress)
         report.ingress = ingress;
     if (work.retainedImages && projects.every((project) => project.health !== 'failed'))
@@ -1094,6 +1138,7 @@ export async function pollOnce(ctx = defaultContext()) {
         headers: { 'content-type': 'application/json' },
         body: JSON.stringify(report),
     });
+    await jobs.acknowledge(jobResults);
     return projects;
 }
 const CREDS_FILE = () => path.join(STATE_DIR, 'agent.json');

package/dist/backup-config.d.ts

@@ -0,0 +1,47 @@
+import type { BackupRetention, BackupType, ComposeFile } from './protocol.ts';
+/** One backup target after defaults and placement are resolved. */
+export interface ResolvedBackup {
+    /** `service:db` or `volume:uploads`, unique within a project. */
+    target: string;
+    kind: 'service' | 'volume';
+    name: string;
+    /** `command` for a service, `volume` for a volume. */
+    type: BackupType;
+    schedule: string;
+    retention: BackupRetention;
+    jitterSeconds: number;
+    timeoutSec: number;
+    /** Service targets: the command run in the service's container, and the snapshot file it is stored as. */
+    command?: string[];
+    /** Run in the service container to restore; the stored file arrives on its standard input. */
+    restore?: string[];
+    filename?: string;
+    verify?: string;
+    /** Volume targets. */
+    exclude?: string[];
+    /** Services held during a restore; unset means "discover from the Compose file". */
+    writers?: string[];
+    /** The one node the backup runs on, when it is known from literal placement. Label placement resolves at deploy time. */
+    node?: string;
+}
+export declare const DEFAULT_RETENTION: BackupRetention;
+export declare const DEFAULT_BACKUP_TIMEOUT_SEC: number;
+/** `30m`, `2h`, `90s` or a number of seconds. */
+export declare function parseBackupTimeout(value: unknown): number | null;
+/** Deterministic delay in `[0, 900)` seconds so many sites on one node do not start together. */
+export declare function defaultJitterSeconds(project: string, target: string): number;
+/** Validates the project-level defaults in `x-tug.backup`. */
+export declare function validateBackupDefaults(value: unknown, errors: string[]): void;
+/** The argv of a `command`: a list is used as is, a string runs through `sh -c` like a shell one-liner. */
+export declare function commandArgv(command: unknown): string[] | null;
+interface Resolved {
+    targets: ResolvedBackup[];
+    errors: string[];
+}
+/**
+ * Resolves and validates every `x-tug.backup` in a Compose file. Tug knows nothing about what a service
+ * is: a service backup is a command whose output is stored, a volume backup is its files. `project`
+ * seeds the default jitter. Errors name the key, so they can be shown to the operator as they are.
+ */
+export declare function resolveBackups(compose: ComposeFile, project?: string): Resolved;
+export {};

package/dist/backup-config.js

@@ -0,0 +1,256 @@
+import { createHash } from 'node:crypto';
+import { parseCron } from "./cron.js";
+export const DEFAULT_RETENTION = { daily: 7, weekly: 4, monthly: 6 };
+export const DEFAULT_BACKUP_TIMEOUT_SEC = 2 * 3600;
+const MIN_TIMEOUT_SEC = 60;
+const MAX_TIMEOUT_SEC = 24 * 3600;
+const MAX_JITTER_SEC = 3600;
+const RETENTION_KEYS = ['last', 'hourly', 'daily', 'weekly', 'monthly', 'yearly'];
+const COMMON_KEYS = ['schedule', 'retention', 'jitter', 'timeout', 'writers'];
+const SERVICE_KEYS = [...COMMON_KEYS, 'command', 'restore', 'filename', 'verify'];
+const VOLUME_KEYS = [...COMMON_KEYS, 'exclude'];
+const FILENAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
+const isObject = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
+/** `30m`, `2h`, `90s` or a number of seconds. */
+export function parseBackupTimeout(value) {
+    if (typeof value === 'number')
+        return Number.isInteger(value) ? value : null;
+    const match = typeof value === 'string' ? value.match(/^(\d+)(s|m|h)$/) : null;
+    if (!match)
+        return null;
+    return Number(match[1]) * { s: 1, m: 60, h: 3600 }[match[2]];
+}
+/** Deterministic delay in `[0, 900)` seconds so many sites on one node do not start together. */
+export function defaultJitterSeconds(project, target) {
+    const digest = createHash('sha256').update(`${project}\0${target}`).digest();
+    return digest.readUInt32BE(0) % 900;
+}
+function validateRetention(label, value, errors) {
+    if (!isObject(value)) {
+        errors.push(`${label}.retention must be an object such as { daily: 7, weekly: 4 }`);
+        return undefined;
+    }
+    const retention = {};
+    for (const [key, count] of Object.entries(value)) {
+        if (!RETENTION_KEYS.includes(key))
+            errors.push(`${label}.retention.${key} is not supported (supported: ${RETENTION_KEYS.join(', ')})`);
+        else if (!Number.isInteger(count) || count < 1)
+            errors.push(`${label}.retention.${key} must be a whole number of at least 1`);
+        else
+            retention[key] = count;
+    }
+    if (!Object.keys(value).length)
+        errors.push(`${label}.retention is empty, which would keep no snapshots; set at least one of ${RETENTION_KEYS.join(', ')}`);
+    return retention;
+}
+function validateSchedule(label, value, errors) {
+    if (typeof value !== 'string') {
+        errors.push(`${label}.schedule must be a cron string such as "0 3 * * *" or @daily`);
+        return undefined;
+    }
+    try {
+        parseCron(value);
+    }
+    catch (error) {
+        errors.push(`${label}.schedule: ${error instanceof Error ? error.message : String(error)}`);
+        return undefined;
+    }
+    return value.trim();
+}
+/** Validates the project-level defaults in `x-tug.backup`. */
+export function validateBackupDefaults(value, errors) {
+    if (value === undefined)
+        return;
+    if (!isObject(value)) {
+        errors.push('x-tug.backup must be an object');
+        return;
+    }
+    for (const key of Object.keys(value))
+        if (key !== 'schedule' && key !== 'retention')
+            errors.push(`x-tug.backup.${key} is not supported (supported: schedule, retention)`);
+    if (value.schedule !== undefined)
+        validateSchedule('x-tug.backup', value.schedule, errors);
+    if (value.retention !== undefined)
+        validateRetention('x-tug.backup', value.retention, errors);
+}
+function normalizeConfig(config) {
+    if (config === false)
+        return null;
+    if (config === true)
+        return {};
+    if (typeof config === 'string')
+        return { schedule: config };
+    return config;
+}
+/** The argv of a `command`: a list is used as is, a string runs through `sh -c` like a shell one-liner. */
+export function commandArgv(command) {
+    if (typeof command === 'string')
+        return command.trim() ? ['sh', '-c', command] : null;
+    return Array.isArray(command) &&
+        command.length &&
+        command.every((part) => typeof part === 'string' && part)
+        ? command
+        : null;
+}
+function serviceNodes(svc) {
+    const lp = svc['x-tug'] || {};
+    const placement = lp.placement || {};
+    return {
+        nodes: placement.nodes || [],
+        dynamic: Array.isArray(placement.labels) && placement.labels.length > 0,
+        global: lp.global === true || placement.global === true,
+    };
+}
+function mountedVolumes(svc) {
+    return (svc.volumes || []).flatMap((mount) => typeof mount === 'object' && mount && mount.type === 'volume' && mount.source
+        ? [String(mount.source)]
+        : []);
+}
+/**
+ * Resolves and validates every `x-tug.backup` in a Compose file. Tug knows nothing about what a service
+ * is: a service backup is a command whose output is stored, a volume backup is its files. `project`
+ * seeds the default jitter. Errors name the key, so they can be shown to the operator as they are.
+ */
+export function resolveBackups(compose, project = '') {
+    const errors = [];
+    const targets = [];
+    const defaults = compose['x-tug']?.backup;
+    const defaultSchedule = isObject(defaults) ? defaults.schedule : undefined;
+    const defaultRetention = (isObject(defaults) && isObject(defaults.retention) ? defaults.retention : DEFAULT_RETENTION);
+    const common = (label, kind, name, spec) => {
+        const allowed = kind === 'service' ? SERVICE_KEYS : VOLUME_KEYS;
+        for (const key of Object.keys(spec))
+            if (!allowed.includes(key))
+                errors.push(`${label}.${key} is not supported${key === 'command' || key === 'restore' || key === 'filename' || key === 'verify'
+                    ? ' on a volume; a volume backup archives its files'
+                    : key === 'exclude'
+                        ? ' on a service; it applies to volumes'
+                        : ''} (supported: ${allowed.join(', ')})`);
+        const before = errors.length;
+        const schedule = spec.schedule !== undefined
+            ? validateSchedule(label, spec.schedule, errors)
+            : typeof defaultSchedule === 'string'
+                ? defaultSchedule.trim()
+                : undefined;
+        if (spec.schedule === undefined && schedule === undefined)
+            errors.push(`${label} needs a schedule: set ${label}.schedule or x-tug.backup.schedule`);
+        const retention = spec.retention !== undefined
+            ? validateRetention(label, spec.retention, errors)
+            : defaultRetention;
+        let jitterSeconds = defaultJitterSeconds(project, `${kind}:${name}`);
+        if (spec.jitter !== undefined) {
+            if (!Number.isInteger(spec.jitter) || spec.jitter < 0 || spec.jitter > MAX_JITTER_SEC)
+                errors.push(`${label}.jitter must be a whole number of seconds from 0 to ${MAX_JITTER_SEC}`);
+            else
+                jitterSeconds = spec.jitter;
+        }
+        let timeoutSec = DEFAULT_BACKUP_TIMEOUT_SEC;
+        if (spec.timeout !== undefined) {
+            const parsed = parseBackupTimeout(spec.timeout);
+            if (parsed === null || parsed < MIN_TIMEOUT_SEC || parsed > MAX_TIMEOUT_SEC)
+                errors.push(`${label}.timeout must be a duration from 1m to 24h such as 30m or 2h`);
+            else
+                timeoutSec = parsed;
+        }
+        if (spec.writers !== undefined) {
+            if (!Array.isArray(spec.writers) || spec.writers.some((w) => typeof w !== 'string'))
+                errors.push(`${label}.writers must be a list of service names`);
+            else
+                for (const writer of spec.writers)
+                    if (!compose.services[writer])
+                        errors.push(`${label}.writers: ${writer} is not a service`);
+        }
+        if (errors.length > before || !schedule || !retention)
+            return null;
+        return { schedule, retention, jitterSeconds, timeoutSec };
+    };
+    for (const [name, svc] of Object.entries(compose.services)) {
+        const config = svc['x-tug']?.backup;
+        if (config === undefined)
+            continue;
+        const label = `${name}: x-tug.backup`;
+        const spec = normalizeConfig(config);
+        if (spec === null)
+            continue;
+        if (typeof spec !== 'object' || Array.isArray(spec)) {
+            errors.push(`${label} must be true, false, a cron string or an object`);
+            continue;
+        }
+        const before = errors.length;
+        const base = common(label, 'service', name, spec);
+        const argv = spec.command === undefined ? null : commandArgv(spec.command);
+        if (spec.command === undefined)
+            errors.push(`${label}.command is required: the command to run in the ${name} container, whose output is stored (see the recipes in docs/backups.md)`);
+        else if (!argv)
+            errors.push(`${label}.command must be a non-empty string or list of strings`);
+        const restoreArgv = spec.restore === undefined ? undefined : commandArgv(spec.restore);
+        if (spec.restore !== undefined && !restoreArgv)
+            errors.push(`${label}.restore must be a non-empty string or list of strings`);
+        if (spec.filename !== undefined &&
+            (typeof spec.filename !== 'string' || !FILENAME_RE.test(spec.filename)))
+            errors.push(`${label}.filename must be a plain file name such as ${name}.sql`);
+        if (spec.verify !== undefined && (typeof spec.verify !== 'string' || !spec.verify))
+            errors.push(`${label}.verify must be text that appears at the end of the output`);
+        const { nodes, dynamic, global } = serviceNodes(svc);
+        const replicas = svc['x-tug']?.replicas ?? 1;
+        if (global || replicas > 1 || nodes.length > 1)
+            errors.push(`${label} needs a single instance; ${name} is global, replicated or placed on several nodes`);
+        if (errors.length > before || !base || !argv)
+            continue;
+        targets.push({
+            target: `service:${name}`,
+            kind: 'service',
+            name,
+            type: 'command',
+            ...base,
+            command: argv,
+            restore: restoreArgv ?? undefined,
+            filename: spec.filename ?? `${name}.out`,
+            verify: spec.verify,
+            writers: spec.writers,
+            node: !dynamic && nodes.length === 1 ? nodes[0] : undefined,
+        });
+    }
+    for (const [name, def] of Object.entries(compose.volumes || {})) {
+        const config = isObject(def) ? def['x-tug']?.backup : undefined;
+        if (config === undefined)
+            continue;
+        const label = `volumes.${name}: x-tug.backup`;
+        const spec = normalizeConfig(config);
+        if (spec === null)
+            continue;
+        if (typeof spec !== 'object' || Array.isArray(spec)) {
+            errors.push(`${label} must be true, false, a cron string or an object`);
+            continue;
+        }
+        const before = errors.length;
+        const base = common(label, 'volume', name, spec);
+        if (spec.exclude !== undefined &&
+            (!Array.isArray(spec.exclude) ||
+                spec.exclude.some((pattern) => typeof pattern !== 'string' || !pattern)))
+            errors.push(`${label}.exclude must be a list of restic exclude patterns`);
+        const volumeExt = isObject(def) ? def['x-tug'] : undefined;
+        const placed = volumeExt?.placement?.nodes || [];
+        const mounting = Object.entries(compose.services).filter(([, svc]) => mountedVolumes(svc).includes(name));
+        const mountingNodes = [...new Set(mounting.flatMap(([, svc]) => serviceNodes(svc).nodes))];
+        const dynamicPlacement = mounting.some(([, svc]) => serviceNodes(svc).dynamic || serviceNodes(svc).global);
+        const candidates = placed.length ? placed : mountingNodes;
+        if (candidates.length > 1)
+            errors.push(`${label} needs one node: volume ${name} is ${placed.length ? 'placed' : 'used'} on ${candidates.join(', ')}; pin it with x-tug.placement.nodes or back up per node`);
+        else if (!placed.length && dynamicPlacement)
+            errors.push(`${label} needs x-tug.placement.nodes on volume ${name}, since its services are placed by label or global`);
+        if (errors.length > before || !base)
+            continue;
+        targets.push({
+            target: `volume:${name}`,
+            kind: 'volume',
+            name,
+            type: 'volume',
+            ...base,
+            exclude: spec.exclude,
+            writers: spec.writers,
+            node: candidates.length === 1 ? candidates[0] : undefined,
+        });
+    }
+    return { targets, errors };
+}