@wpelevator/tug 0.5.0 → 0.7.0

package/README.md

@@ -61,18 +61,18 @@ npm install -g @wpelevator/tug
 The CLI installs the matching `ghcr.io/wpelevator/tug:<version>` image on your hosts.
 
 1. Install the control plane: `tug control-install root@control.example.com --install-docker`. The flag installs missing Docker Engine and Compose on supported Debian, Ubuntu, Fedora and RHEL-compatible hosts; omit it when Docker is ready. Put HTTPS in front of the control API as described in [Control plane](docs/control-plane.md).
-2. Point the CLI at it: `tug init --control-url https://control.example.com --token "$TUG_TOKEN"`.
+2. Point the CLI at it: `tug init --control-url https://control.example.com --token "$TUG_TOKEN"`. It writes `tug.yml` (commit it, and list your servers in it) and keeps the token in the git-ignored `tug.local.yml`; see [CLI reference](docs/cli.md#config-file).
 3. Set the secrets your services need: `tug secret-set DB_PASSWORD`.
 4. Add nodes with `tug node-install root@app1.example.com --name app1 --install-docker`, or `tug node-enroll --name app1` and run the printed Docker command. For a bare cloud host, `tug node-enroll --name app1 --install-docker` prints cloud-init user-data.
 5. Deploy: `tug deploy -f docker-compose.yml`. It waits until every node reports the new version healthy.
-6. Operate with `tug status`, `events`, `exec`, `logs`, `ps`, `inspect`, `restart`, `rollback` and `doctor`.
+6. Operate with `tug status`, `events`, `exec`, `run`, `logs`, `ps`, `inspect`, `restart`, `rollback`, `backup-run`, `restore` and `doctor`.
 
 ## Documentation
 
 - [Architecture](docs/architecture.md): components, reconciliation, workflow diagrams, secrets and runtime paths.
 - [Compose extensions](docs/compose.md): `x-tug` placement, health checks, volumes and limitations.
 - [Control plane](docs/control-plane.md): install, TLS, image delivery, backup and restore.
-- [Backups](docs/backups.md): interim WordPress backup sidecar with restic, per-node and NFS variants, and a restore drill until `x-tug.backup` (M11).
+- [Backups](docs/backups.md): scheduled restic backups with `x-tug.backup`, restore, recipes for common databases, and a restore drill. See also the [WordPress example](examples/wordpress/README.md).
 - [Access control](docs/access-control.md): scoped API tokens, roles, audit trail and a GitHub Actions deploy example.
 - [Networking](docs/networking.md): WireGuard mesh, service DNS, ingress and TLS, firewall notes.
 - [Nodes](docs/nodes.md): SSH install, self-service enrollment, registration and status.

package/dist/agent-jobs.d.ts

@@ -0,0 +1,41 @@
+import type { DockerApi } from './docker-api.ts';
+import type { BackupCredentialsResponse, Command, CommandResult, JobStatus } from './protocol.ts';
+export interface JobHost {
+    docker: DockerApi;
+    stateDir: string;
+    fetchCredentials(commandId: string): Promise<BackupCredentialsResponse>;
+    /** Parallel jobs on this node; extra jobs wait queued. */
+    concurrency?: number;
+}
+/**
+ * The agent's background jobs. Reconciliation must not wait for an hour-long backup, so commands of
+ * these types are queued here, report a heartbeat through `statuses()`, and deliver their result
+ * through `drain()` on a later poll, even if the agent restarted in between.
+ */
+export declare class JobRunner {
+    private readonly host;
+    private readonly active;
+    private readonly queue;
+    private results;
+    private ready;
+    private settled;
+    constructor(host: JobHost);
+    /** Called whenever a job finishes, so the agent can report its result without waiting a full interval. */
+    onSettled(callback: () => void): void;
+    private file;
+    private save;
+    /** First use: remove helpers of a previous run and fail the jobs it never finished. */
+    private init;
+    /** Starts cleanup of a previous process's leftovers; safe to call more than once. */
+    warm(): Promise<void>;
+    /** Queues a job; the command is acknowledged immediately and its result arrives later. */
+    start(command: Command): Promise<void>;
+    private pump;
+    private execute;
+    /** Jobs that are running or waiting, for the status report's heartbeat. */
+    statuses(): JobStatus[];
+    /** Finished results to put in the next status report. They are removed only after `acknowledge`. */
+    pending(): CommandResult[];
+    /** Forgets results control has stored. */
+    acknowledge(delivered: CommandResult[]): Promise<void>;
+}

package/dist/agent-jobs.js

@@ -0,0 +1,161 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { runBackupJob, removeStaleJobContainers } from "./backup-job.js";
+import { redactSecrets } from "./backup-destination.js";
+import { ensureDir, errMessage, now } from "./util.js";
+const RESULTS_FILE = 'job-results.json';
+/**
+ * The agent's background jobs. Reconciliation must not wait for an hour-long backup, so commands of
+ * these types are queued here, report a heartbeat through `statuses()`, and deliver their result
+ * through `drain()` on a later poll, even if the agent restarted in between.
+ */
+export class JobRunner {
+    host;
+    active = new Map();
+    queue = [];
+    results = [];
+    ready;
+    settled;
+    constructor(host) {
+        this.host = host;
+    }
+    /** Called whenever a job finishes, so the agent can report its result without waiting a full interval. */
+    onSettled(callback) {
+        this.settled = callback;
+    }
+    file() {
+        return path.join(this.host.stateDir, RESULTS_FILE);
+    }
+    async save() {
+        const data = {
+            results: this.results,
+            running: [...this.active.keys(), ...this.queue.map((job) => job.command.id)],
+        };
+        await ensureDir(this.host.stateDir);
+        await fs.writeFile(this.file(), JSON.stringify(data), { mode: 0o600 });
+    }
+    /** First use: remove helpers of a previous run and fail the jobs it never finished. */
+    init() {
+        this.ready ??= (async () => {
+            await removeStaleJobContainers(this.host.docker).catch(() => 0);
+            try {
+                const saved = JSON.parse(await fs.readFile(this.file(), 'utf8'));
+                this.results = saved.results ?? [];
+                for (const commandId of saved.running ?? [])
+                    this.results.push({
+                        commandId,
+                        state: 'failed',
+                        result: { error: 'the agent restarted while the job was running' },
+                    });
+                if (saved.running?.length)
+                    await this.save();
+            }
+            catch {
+                /* no saved state */
+            }
+        })();
+        return this.ready;
+    }
+    /** Starts cleanup of a previous process's leftovers; safe to call more than once. */
+    warm() {
+        return this.init();
+    }
+    /** Queues a job; the command is acknowledged immediately and its result arrives later. */
+    async start(command) {
+        await this.init();
+        let payload;
+        try {
+            payload = JSON.parse(command.payload_json ?? '{}');
+            if (!payload.runId || !payload.repository || !payload.resticImage)
+                throw new Error('the job payload is incomplete');
+        }
+        catch (error) {
+            this.results.push({
+                commandId: command.id,
+                state: 'failed',
+                result: { error: errMessage(error) },
+            });
+            await this.save();
+            return;
+        }
+        this.queue.push({ command, payload });
+        await this.save();
+        this.pump();
+    }
+    pump() {
+        const limit = this.host.concurrency ?? 1;
+        while (this.active.size < limit && this.queue.length) {
+            const job = this.queue.shift();
+            void this.execute(job.command, job.payload);
+        }
+    }
+    async execute(command, payload) {
+        const controller = new AbortController();
+        const status = {
+            commandId: command.id,
+            runId: payload.runId,
+            phase: 'pulling',
+            startedAt: now(),
+            controller,
+        };
+        this.active.set(command.id, status);
+        const timeout = setTimeout(() => controller.abort(new Error(`the job exceeded its ${payload.limits.timeoutSec}s limit`)), payload.limits.timeoutSec * 1000);
+        timeout.unref();
+        let credentials;
+        try {
+            credentials = await this.host.fetchCredentials(command.id);
+            const result = await runBackupJob(payload, command.id, {
+                docker: this.host.docker,
+                credentials,
+                signal: controller.signal,
+                onPhase: (phase, bytes) => {
+                    status.phase = phase;
+                    if (bytes !== undefined)
+                        status.bytes = bytes;
+                },
+            }, command.type === 'backup.check'
+                ? 'backup.check'
+                : command.type === 'backup.restore'
+                    ? 'backup.restore'
+                    : 'backup.run');
+            this.results.push({ commandId: command.id, state: 'succeeded', result });
+        }
+        catch (error) {
+            const values = credentials ? Object.values(credentials.env) : [];
+            this.results.push({
+                commandId: command.id,
+                state: 'failed',
+                result: { error: redactSecrets(errMessage(controller.signal.reason ?? error), values) },
+            });
+        }
+        finally {
+            clearTimeout(timeout);
+            this.active.delete(command.id);
+            await this.save().catch(() => { });
+            this.settled?.();
+            this.pump();
+        }
+    }
+    /** Jobs that are running or waiting, for the status report's heartbeat. */
+    statuses() {
+        return [
+            ...[...this.active.values()].map(({ controller: _controller, ...status }) => status),
+            ...this.queue.map(({ command, payload }) => ({
+                commandId: command.id,
+                runId: payload.runId,
+                phase: 'queued',
+                startedAt: now(),
+            })),
+        ];
+    }
+    /** Finished results to put in the next status report. They are removed only after `acknowledge`. */
+    pending() {
+        return [...this.results];
+    }
+    /** Forgets results control has stored. */
+    async acknowledge(delivered) {
+        const ids = new Set(delivered.map((result) => result.commandId));
+        this.results = this.results.filter((result) => !ids.has(result.commandId));
+        await this.save().catch(() => { });
+    }
+}

package/dist/agent.d.ts

@@ -66,6 +66,13 @@ export declare class AgentLiveOperations {
 }
 /** The container's address on the project network, which health checks dial. */
 export declare function containerIp(ctx: AgentContext, project: string, container: string): Promise<string>;
+/** True when this process runs inside a container: Docker, containerd, Podman or Kubernetes. */
+export declare function runsInContainer(): boolean;
+/**
+ * Health checks dial the container's address on the project network. An agent that runs as a plain host process
+ * cannot reach it on a machine where Docker lives in a VM (macOS, Windows), and the bare timeout does not say why.
+ */
+export declare function outsideContainerHint(health: HealthCheck, inContainer?: boolean): string;
 export declare function waitForHealth(ctx: AgentContext, project: string, container: string, service: string, health: HealthCheck | null | undefined): Promise<void>;
 export { parseDuration };
 export interface PublicListener {
@@ -85,7 +92,7 @@ export declare function pruneManagedImages(ctx: AgentContext, retained: string[]
 export declare function resolveBindAddress(address: string | null, lookup?: (host: string) => Promise<{
     address: string;
 }>): Promise<string>;
-export declare function reconcile(desiredState: DesiredState, ctx?: AgentContext, endpoints?: Record<string, ServiceEndpoint[]>, ports?: PortBinding): Promise<ProjectStatus>;
+export declare function reconcile(desiredState: DesiredState, ctx?: AgentContext, endpoints?: Record<string, ServiceEndpoint[]>, ports?: PortBinding, held?: string[]): Promise<ProjectStatus>;
 /** Reports this node's WireGuard public key when control has not seen the current one yet. */
 export declare function reportWgKey(ctx: AgentContext): Promise<void>;
 export declare function pollOnce(ctx?: AgentContext): Promise<ProjectStatus[]>;

package/dist/agent.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import fs from 'node:fs/promises';
 import { imageLocation } from "./registry.js";
-import { createReadStream, createWriteStream } from 'node:fs';
+import { createReadStream, createWriteStream, existsSync, readFileSync } from 'node:fs';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { validateImageArchive, verifyDownload } from "./artifacts.js";
@@ -12,8 +12,9 @@ import dns from 'node:dns/promises';
 import { WebSocket } from 'ws';
 import { createHash } from 'node:crypto';
 import { addressBoundServices, isRolling, serviceSecretVersions } from "./compose.js";
-import { containerSpecs, networkName, volumeName } from "./container-spec.js";
+import { LABEL_PROJECT, LABEL_SERVICE, containerSpecs, networkName, volumeName, } from "./container-spec.js";
 import { API_VERSION, DockerEngine, isNotFound } from "./docker-api.js";
+import { JobRunner } from "./agent-jobs.js";
 import { LIVE_MAX_FRAME_BYTES, LIVE_START_TIMEOUT_MS } from "./protocol.js";
 import { LiveOperationEngine } from "./live.js";
 import { activeRollingStates, clearRolloutState, ensureProjectResources, loadRolloutState, projectContainers, reconcileRecreate, reconcileRolling, sweepOrphans, } from "./reconciler.js";
@@ -46,6 +47,23 @@ export function defaultContext() {
         rolloutPorts: process.env.TUG_ROLLOUT_PORTS,
     };
 }
+const jobRunners = new Map();
+/** One job runner per state directory, so a background backup outlives the polls that started it. */
+function jobRunnerFor(ctx) {
+    let runner = jobRunners.get(ctx.stateDir);
+    if (!runner) {
+        runner = new JobRunner({
+            docker: ctx.docker,
+            stateDir: ctx.stateDir,
+            concurrency: Number(process.env.TUG_BACKUP_CONCURRENCY) || 1,
+            fetchCredentials: (commandId) => api(ctx, `/agent/v1/backup/credentials?command=${encodeURIComponent(commandId)}`),
+        });
+        jobRunners.set(ctx.stateDir, runner);
+        // Clear helper containers a previous agent process left behind, before any live session can start one.
+        void runner.warm();
+    }
+    return runner;
+}
 const dbsByStateDir = new Map();
 async function agentDb(stateDir) {
     let db = dbsByStateDir.get(stateDir);
@@ -426,6 +444,7 @@ export class AgentLiveOperations {
                     container,
                     command: request.command,
                     interactive: request.interactive,
+                    oneOff: request.oneOff,
                     tty: request.tty,
                     columns: request.columns,
                     rows: request.rows,
@@ -466,6 +485,30 @@ function tcpCheck(host, port, timeoutMs = 2000) {
         s.on('error', reject);
     });
 }
+let containerized;
+/** True when this process runs inside a container: Docker, containerd, Podman or Kubernetes. */
+export function runsInContainer() {
+    containerized ??= (() => {
+        if (process.env.container || existsSync('/.dockerenv'))
+            return true;
+        try {
+            return /docker|containerd|kubepods|libpod/.test(readFileSync('/proc/1/cgroup', 'utf8'));
+        }
+        catch {
+            return false;
+        }
+    })();
+    return containerized;
+}
+/**
+ * Health checks dial the container's address on the project network. An agent that runs as a plain host process
+ * cannot reach it on a machine where Docker lives in a VM (macOS, Windows), and the bare timeout does not say why.
+ */
+export function outsideContainerHint(health, inContainer = runsInContainer()) {
+    if (inContainer || health.type === 'docker')
+        return '';
+    return '; this agent is not running in a container, so it may be unable to reach container addresses (a host process on macOS cannot); run it with `docker run` as `tug node-install` does';
+}
 export async function waitForHealth(ctx, project, container, service, health) {
     if (!ctx.apply || !health)
         return;
@@ -499,7 +542,7 @@ export async function waitForHealth(ctx, project, container, service, health) {
             await new Promise((r) => setTimeout(r, intervalMs));
         }
     }
-    throw new Error(`${service}: ${describeHealthCheck(health)} failed after ${retries} attempt${retries === 1 ? '' : 's'}: ${describeError(last)}`);
+    throw new Error(`${service}: ${describeHealthCheck(health)} failed after ${retries} attempt${retries === 1 ? '' : 's'}: ${describeError(last)}${outsideContainerHint(health)}`);
 }
 /** Node's fetch reports every network failure as "fetch failed"; the system error that explains it is on `cause`. */
 function describeError(error) {
@@ -802,7 +845,15 @@ export async function resolveBindAddress(address, lookup = (host) => dns.lookup(
 }
 // Last public-port warning per project, so the event is sent when it changes rather than on every poll.
 const publicPortWarnings = new Map();
-export async function reconcile(desiredState, ctx = defaultContext(), endpoints = {}, ports = { address: null, loopback: {} }) {
+/** Stops every running container of a service, including the revisions of a rolling service. */
+async function stopService(docker, project, service) {
+    const running = await docker.listContainers({
+        label: [`${LABEL_PROJECT}=${project}`, `${LABEL_SERVICE}=${service}`],
+    });
+    for (const container of running)
+        await docker.stopContainer(container.Id);
+}
+export async function reconcile(desiredState, ctx = defaultContext(), endpoints = {}, ports = { address: null, loopback: {} }, held = []) {
     const dir = path.join(ctx.stateDir, 'projects', desiredState.project);
     await ensureDir(dir);
     const serviceNames = Object.keys(containerSpecs(desiredState, ctx.nodeName).specs);
@@ -839,9 +890,13 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
     const currentSecretVersions = serviceSecretVersions(desiredState.compose, serviceNames, sec.versions || {});
     const db = await agentDb(ctx.stateDir);
     const previousRow = db.get('SELECT version, revision FROM applied_state WHERE project = ?', desiredState.project);
-    const rolling = serviceNames.filter((service) => isRolling(desiredState.compose.services[service]));
+    // A held service is stopped and left alone (a restore is rewriting its data) until control drops the hold.
+    const heldSet = new Set(held.filter((service) => serviceNames.includes(service)));
+    for (const service of heldSet)
+        serviceStates[service].rolloutState = 'held';
+    const rolling = serviceNames.filter((service) => !heldSet.has(service) && isRolling(desiredState.compose.services[service]));
     const rollingSet = new Set(rolling);
-    const mainServices = serviceNames.filter((service) => !rollingSet.has(service));
+    const mainServices = serviceNames.filter((service) => !rollingSet.has(service) && !heldSet.has(service));
     // Published images are pulled only for a new desired revision. Pulling on every poll would let a tag that moved
     // upstream (`mariadb:11`) recreate the container without a deploy.
     const sameDesiredVersion = previousRow
@@ -903,6 +958,8 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
         try {
             await ensureProjectResources(ctx.docker, desiredState.project, plan);
             await sweepOrphans(ctx.docker, desiredState.project, new Set(serviceNames));
+            for (const service of heldSet)
+                await stopService(ctx.docker, desiredState.project, service);
             if (mainServices.length)
                 await reconcileRecreate(ctx.docker, {
                     desired: desiredState,
@@ -1051,7 +1108,7 @@ export async function pollOnce(ctx = defaultContext()) {
     const projects = [];
     for (const desired of work.desiredStates || (work.desiredState ? [work.desiredState] : [])) {
         try {
-            projects.push(await reconcile(desired, ctx, work.serviceEndpoints?.[desired.project], work.ports));
+            projects.push(await reconcile(desired, ctx, work.serviceEndpoints?.[desired.project], work.ports, work.holds?.[desired.project]));
         }
         catch (e) {
             projects.push({
@@ -1072,7 +1129,13 @@ export async function pollOnce(ctx = defaultContext()) {
             await removeIngress(ingressCtx).catch(() => false);
     }
     const commandResults = [];
+    const jobs = jobRunnerFor(ctx);
     for (const command of work.commands || []) {
+        // Backups can run for an hour, so they run in the background and report through later polls.
+        if (command.type.startsWith('backup.')) {
+            await jobs.start(command);
+            continue;
+        }
         try {
             commandResults.push(await executeCommand(command, ctx));
         }
@@ -1084,7 +1147,12 @@ export async function pollOnce(ctx = defaultContext()) {
             });
         }
     }
+    const jobResults = jobs.pending();
+    commandResults.push(...jobResults);
     const report = { node: ctx.nodeName, projects, commandResults };
+    const running = jobs.statuses();
+    if (running.length)
+        report.jobs = running;
     if (ingress)
         report.ingress = ingress;
     if (work.retainedImages && projects.every((project) => project.health !== 'failed'))
@@ -1094,6 +1162,7 @@ export async function pollOnce(ctx = defaultContext()) {
         headers: { 'content-type': 'application/json' },
         body: JSON.stringify(report),
     });
+    await jobs.acknowledge(jobResults);
     return projects;
 }
 const CREDS_FILE = () => path.join(STATE_DIR, 'agent.json');

package/dist/backup-check.d.ts

@@ -0,0 +1,11 @@
+/** The part of `GET /api/v1/backups` that `tug backup-list --check` judges. */
+export interface BackupCheckInput {
+    destinationConfigured: boolean;
+    targets: {
+        target: string;
+        enabled: boolean;
+        problem?: string | null;
+    }[];
+}
+/** What a monitor should be told: one line per problem, empty when every backup target is healthy. */
+export declare function backupCheckProblems(listing: BackupCheckInput): string[];

package/dist/backup-check.js

@@ -0,0 +1,10 @@
+/** What a monitor should be told: one line per problem, empty when every backup target is healthy. */
+export function backupCheckProblems(listing) {
+    const active = listing.targets.filter((target) => target.enabled);
+    const problems = active
+        .filter((target) => target.problem)
+        .map((target) => `${target.target}: ${target.problem}`);
+    if (active.length && !listing.destinationConfigured)
+        problems.unshift('no backup destination is configured');
+    return problems;
+}

package/dist/backup-config.d.ts

@@ -0,0 +1,52 @@
+import type { BackupRetention, BackupType, ComposeFile } from './protocol.ts';
+/** One backup target after defaults and placement are resolved. */
+export interface ResolvedBackup {
+    /** `service:db` or `volume:uploads`, unique within a project. */
+    target: string;
+    kind: 'service' | 'volume';
+    name: string;
+    /** `command` for a service, `volume` for a volume. */
+    type: BackupType;
+    schedule: string;
+    retention: BackupRetention;
+    jitterSeconds: number;
+    timeoutSec: number;
+    /** Memory limit of the restic helper container. */
+    memory: string;
+    /** Service targets: the command run in the service's container, and the snapshot file it is stored as. */
+    command?: string[];
+    /** Run in the service container to restore; the stored file arrives on its standard input. */
+    restore?: string[];
+    filename?: string;
+    verify?: string;
+    /** Volume targets. */
+    exclude?: string[];
+    /** Services held during a restore; unset means "discover from the Compose file". */
+    writers?: string[];
+    /** The one node the backup runs on, when it is known from literal placement. Label placement resolves at deploy time. */
+    node?: string;
+}
+export declare const DEFAULT_RETENTION: BackupRetention;
+export declare const DEFAULT_BACKUP_TIMEOUT_SEC: number;
+export declare const DEFAULT_BACKUP_MEMORY = "512m";
+/** `30m`, `2h`, `90s` or a number of seconds. */
+export declare function parseBackupTimeout(value: unknown): number | null;
+/** Bytes in a Docker-style memory size (`256m`, `1g`, `1048576`), or null when it is not one. */
+export declare function parseMemorySize(value: unknown): number | null;
+/** Deterministic delay in `[0, 900)` seconds so many sites on one node do not start together. */
+export declare function defaultJitterSeconds(project: string, target: string): number;
+/** Validates the project-level defaults in `x-tug.backup`. */
+export declare function validateBackupDefaults(value: unknown, errors: string[]): void;
+/** The argv of a `command`: a list is used as is, a string runs through `sh -c` like a shell one-liner. */
+export declare function commandArgv(command: unknown): string[] | null;
+interface Resolved {
+    targets: ResolvedBackup[];
+    errors: string[];
+}
+/**
+ * Resolves and validates every `x-tug.backup` in a Compose file. Tug knows nothing about what a service
+ * is: a service backup is a command whose output is stored, a volume backup is its files. `project`
+ * seeds the default jitter. Errors name the key, so they can be shown to the operator as they are.
+ */
+export declare function resolveBackups(compose: ComposeFile, project?: string): Resolved;
+export {};