@wpelevator/tug 0.5.0 → 0.7.0

package/dist/live.js

@@ -1,5 +1,6 @@
 import { EventEmitter } from 'node:events';
 import { LIVE_MAX_FRAME_BYTES } from "./protocol.js";
+import { LABEL_JOB } from "./container-spec.js";
 const identifier = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,255}$/;
 function object(value) {
     return typeof value === 'object' && value !== null && !Array.isArray(value);
@@ -22,6 +23,8 @@ function validate(value) {
             !value.command.every((arg) => typeof arg === 'string' && !arg.includes('\0') && Buffer.byteLength(arg) <= 32768) ||
             value.command[0].length === 0)
             throw new Error('Invalid exec command');
+        if (value.oneOff !== undefined && typeof value.oneOff !== 'boolean')
+            throw new Error('Invalid one-off flag');
         if (value.interactive !== undefined && typeof value.interactive !== 'boolean')
             throw new Error('Invalid interactive flag');
         if (value.tty !== undefined && typeof value.tty !== 'boolean')
@@ -187,6 +190,8 @@ export class LiveOperationEngine {
                 tail: request.tail ?? 100,
                 timestamps: request.timestamps,
             });
+        if (request.operation === 'exec' && request.oneOff)
+            return startOneOff(docker, request.container, request.command);
         const proc = await docker.execStart(request.container, {
             cmd: request.command,
             stdin: request.interactive !== false,
@@ -274,3 +279,82 @@ export class LiveOperationEngine {
         });
     }
 }
+/**
+ * Runs `command` in a new container cloned from `template`: the same image, environment (secrets included),
+ * user, working directory, network and mounts, with no published ports and none of the service's labels, so
+ * reconciliation never mistakes it for the service. The container is removed when the command ends.
+ */
+export async function startOneOff(docker, template, command) {
+    const source = await docker.inspectContainer(template);
+    const image = source.Config?.Image;
+    if (!image)
+        throw new Error('the service container has no image to run from');
+    const host = source.HostConfig ?? {};
+    const name = `tug-run-${Math.random().toString(36).slice(2, 10)}`;
+    const id = await docker.createContainer(name, {
+        Image: image,
+        Env: source.Config?.Env,
+        User: source.Config?.User || undefined,
+        WorkingDir: source.Config?.WorkingDir || undefined,
+        // Bypass the image's entrypoint script, like `docker exec`, so the command is exactly what was typed.
+        Entrypoint: command,
+        Cmd: [],
+        Labels: { [LABEL_JOB]: 'run' },
+        HostConfig: {
+            NetworkMode: host.NetworkMode,
+            ...(host.Binds?.length ? { Binds: host.Binds } : {}),
+            ...(host.Mounts?.length ? { Mounts: host.Mounts } : {}),
+        },
+    });
+    const remove = () => docker.removeContainer(id).catch(() => { });
+    try {
+        await docker.startContainer(id);
+        const logs = await docker.logs(id, { follow: true });
+        let killed = false;
+        const proc = Object.assign(new EventEmitter(), {
+            stdin: null,
+            stdout: logs.stdout,
+            stderr: logs.stderr,
+            kill: () => {
+                killed = true;
+                logs.kill();
+                void remove();
+                return true;
+            },
+        });
+        const drained = (stream) => new Promise((resolve) => {
+            if (stream.readableEnded || stream.destroyed)
+                return resolve();
+            stream.once('end', resolve);
+            stream.once('close', resolve);
+        });
+        // The exit code comes from the container, not from the log stream, which only ends once it is consumed.
+        void (async () => {
+            let code = -1;
+            while (!killed) {
+                const state = (await docker.inspectContainer(id).catch(() => undefined))?.State;
+                if (!state)
+                    break;
+                if (!state.Running && state.Status !== 'created') {
+                    code = state.ExitCode ?? -1;
+                    break;
+                }
+                await new Promise((resolve) => setTimeout(resolve, 100));
+            }
+            if (killed)
+                return;
+            // Give the consumer a moment to read the last output before the container goes away.
+            await Promise.race([
+                Promise.all([drained(logs.stdout), drained(logs.stderr)]),
+                new Promise((resolve) => setTimeout(resolve, 1500)),
+            ]);
+            await remove();
+            proc.emit('close', code, null);
+        })();
+        return proc;
+    }
+    catch (error) {
+        await remove();
+        throw error;
+    }
+}

package/dist/normalize.d.ts

@@ -10,6 +10,12 @@ export declare function sentinelNames(value: unknown): string[];
 export declare function substituteSentinels(value: string, secrets: Record<string, string>): string;
 /** Arguments of the `docker compose` call that renders `file` canonically (as YAML, which keeps `x-tug`). */
 export declare function composeConfigArgs(file: string, project?: string): string[];
+/**
+ * `docker compose config` writes every literal `$` as `$$` so its output can be fed back to Compose. Tug does not
+ * interpolate again, so a command like `sh -c 'echo $$HOME'` would reach the container with the doubled dollar,
+ * where a shell reads `$$` as its own process ID. Undoing the escaping is exact: output never contains a bare `$`.
+ */
+export declare function unescapeDollars<T>(value: T): T;
 /**
  * Renders `file` with Compose and returns the canonical definition. Every name in `secretNames` is set to
  * its sentinel in the environment, so it wins over a same-named local variable.

package/dist/normalize.js

@@ -41,6 +41,20 @@ export function composeConfigArgs(file, project) {
         '--no-path-resolution',
     ];
 }
+/**
+ * `docker compose config` writes every literal `$` as `$$` so its output can be fed back to Compose. Tug does not
+ * interpolate again, so a command like `sh -c 'echo $$HOME'` would reach the container with the doubled dollar,
+ * where a shell reads `$$` as its own process ID. Undoing the escaping is exact: output never contains a bare `$`.
+ */
+export function unescapeDollars(value) {
+    if (typeof value === 'string')
+        return value.replaceAll('$$', '$');
+    if (Array.isArray(value))
+        return value.map(unescapeDollars);
+    if (value && typeof value === 'object')
+        return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, unescapeDollars(item)]));
+    return value;
+}
 /**
  * Renders `file` with Compose and returns the canonical definition. Every name in `secretNames` is set to
  * its sentinel in the environment, so it wins over a same-named local variable.
@@ -71,5 +85,5 @@ export async function normalizeCompose(file, secretNames, options = {}) {
     }
     if (!parsed || typeof parsed !== 'object' || !parsed.services)
         throw new Error(`${file} defines no services`);
-    return parsed;
+    return unescapeDollars(parsed);
 }

package/dist/protocol.d.ts

@@ -24,6 +24,42 @@ export interface IngressSpec {
     cert?: string;
     key?: string;
 }
+/** What a backup captures: the output of a command run in a service, or the files of a volume. */
+export type BackupType = 'command' | 'volume';
+/** A restic keep policy; at least one key is required. */
+export interface BackupRetention {
+    last?: number;
+    hourly?: number;
+    daily?: number;
+    weekly?: number;
+    monthly?: number;
+    yearly?: number;
+}
+/** `x-tug.backup` on a service or top-level volume, as an object. */
+export interface BackupSpec {
+    /** Five-field cron in UTC, or `@hourly`, `@daily`, `@weekly`, `@monthly`, `@yearly`. */
+    schedule?: string;
+    retention?: BackupRetention;
+    /** Seconds of deterministic per-target delay after the scheduled minute. */
+    jitter?: number;
+    /** Hard limit for one run, such as `30m` or `2h`. */
+    timeout?: string | number;
+    /** Memory limit of the restic helper container, such as `256m` (default `512m`). */
+    memory?: string;
+    /** Services only: run in the service's container; its standard output is the backup. A string runs through `sh -c`. */
+    command?: string | string[];
+    /** Services only: run in the service container to restore; the stored file arrives on its standard input. */
+    restore?: string | string[];
+    /** Services only: name of the file inside the snapshot (default `<service>.out`). */
+    filename?: string;
+    /** Services only: text that must appear in the last 4 KiB of output, so a command cut short is not kept as a good backup. */
+    verify?: string;
+    /** Volumes only: restic exclude patterns. */
+    exclude?: string[];
+    /** Services to hold during a restore. */
+    writers?: string[];
+}
+export type BackupConfig = boolean | string | BackupSpec;
 export type RolloutStrategy = 'recreate' | 'rolling';
 /** `x-tug.rollout`: how a service is replaced on a new deployment. `recreate` (default) is `docker compose up -d`. */
 export interface RolloutSpec {
@@ -39,11 +75,14 @@ export interface ServiceExtension {
     health?: HealthCheck;
     ingress?: IngressSpec;
     rollout?: RolloutSpec;
+    backup?: BackupConfig;
 }
 /** The top-level `x-tug` extension of a Compose file. */
 export interface ProjectExtension {
     /** `auto` creates a rollback deployment when a deployment fails on health; `manual` (default) only reports it. */
     rollback?: 'auto' | 'manual';
+    /** Defaults for every `x-tug.backup` target; they enable nothing by themselves. */
+    backup?: Pick<BackupSpec, 'schedule' | 'retention'>;
 }
 /** The `x-tug` extension on a top-level Compose volume. */
 export interface VolumeExtension {
@@ -51,10 +90,7 @@ export interface VolumeExtension {
     driver?: string;
     persistent?: boolean;
     placement?: Placement;
-    /** NFS exporter address; control fills in the exporter's overlay IP (or node address) at deploy time. */
-    address?: string;
-    /** Directory exported by the NFS server on the exporter node (default `/srv/tug/<volume>`). */
-    export?: string;
+    backup?: BackupConfig;
 }
 export type BuildConfig = string | {
     context?: string;
@@ -130,7 +166,74 @@ export declare const ACTIVE_DEPLOYMENT_STATES: readonly ["uploading", "building"
 /** Deployment states a CLI wait loop stops on. */
 export declare const TERMINAL_DEPLOYMENT_STATES: readonly ["succeeded", "failed", "build_failed", "cancelled"];
 export type DeploymentState = (typeof ACTIVE_DEPLOYMENT_STATES)[number] | (typeof TERMINAL_DEPLOYMENT_STATES)[number];
-export type CommandType = 'diagnostics.run' | 'service.restart' | 'logs.fetch' | 'volume.delete' | 'wg.rotate-key';
+export type CommandType = 'diagnostics.run' | 'service.restart' | 'logs.fetch' | 'volume.delete' | 'wg.rotate-key' | 'backup.check' | 'backup.run' | 'backup.restore' | 'backup.list';
+/** Commands whose helper containers need the backup destination's credentials. */
+export declare const BACKUP_COMMAND_TYPES: readonly CommandType[];
+/** `GET /agent/v1/backup/credentials`: environment for the restic helper of one running backup command. */
+export interface BackupCredentialsResponse {
+    env: Record<string, string>;
+}
+/** Why a command may be refused or a run marked missed; recorded on the run. */
+export type BackupMissReason = 'node_offline' | 'control_down' | 'deadline';
+export type JobPhase = 'queued' | 'pulling' | 'checking' | 'dumping' | 'uploading' | 'forgetting' | 'restoring';
+/** Payload of `backup.check`, `backup.run` and `backup.restore`. Restic credentials are never in a payload. */
+export interface BackupJobPayload {
+    runId: string;
+    project: string;
+    /** `service:db` or `volume:uploads`; empty for `backup.check`. */
+    target: string;
+    type: BackupType;
+    /** The project's restic repository, `<destination>/<project>`. */
+    repository: string;
+    resticImage: string;
+    /** Used as restic's `--host`, so each node keeps its own snapshot chain. */
+    node: string;
+    tags: string[];
+    retention?: BackupRetention;
+    /** Command targets: what to run in the service's container, and how to name and check its output. */
+    command?: {
+        service: string;
+        argv: string[];
+        filename: string;
+        verify?: string;
+        /** Run in the service container with the snapshot's file on stdin when restoring. */
+        restoreArgv?: string[];
+    };
+    /** Volume targets: the Docker volume to mount read-only; it must already exist on the node. */
+    volume?: {
+        name: string;
+        dockerVolume: string;
+        exclude?: string[];
+    };
+    limits: {
+        memory: string;
+        timeoutSec: number;
+    };
+    restore?: {
+        snapshotId: string;
+        mode: 'in-place' | 'scratch';
+        scratchVolume?: string;
+    };
+}
+export interface BackupJobResult {
+    runId: string;
+    snapshotId?: string;
+    addedBytes?: number;
+    totalBytes?: number;
+    durationMs: number;
+    resticVersion?: string;
+    /** Repository state for `backup.check`. */
+    repository?: 'ok' | 'initialized';
+    warnings?: string[];
+}
+/** Live jobs in `StatusReport.jobs`, so control can tell "running" from "lost". */
+export interface JobStatus {
+    commandId: string;
+    runId: string;
+    phase: JobPhase;
+    startedAt: string;
+    bytes?: number;
+}
 /** A `commands` table row as delivered in `GET /agent/v1/work`. */
 export interface Command {
     id: string;
@@ -173,7 +276,7 @@ export type CommandResultPayload = {
     deleted: true;
 } | {
     publicKey: string;
-} | {
+} | BackupJobResult | {
     error: string;
 };
 export interface CommandResult {
@@ -206,6 +309,8 @@ export type LiveRequest = {
     tty: boolean;
     columns?: number;
     rows?: number;
+    /** Run the command in a new container cloned from the service's, instead of in the running one. */
+    oneOff?: boolean;
 } | {
     operation: 'logs';
     project: string;
@@ -327,6 +432,8 @@ export interface WorkResponse {
     ingress?: IngressWork;
     /** Host addresses for published ports that name no host IP. */
     ports?: PortBinding;
+    /** Per project, services that must stay stopped because a restore is rewriting their data. */
+    holds?: Record<string, string[]>;
 }
 /**
  * Where a node's published ports listen when a `ports` entry names no host IP. Entries with an
@@ -393,7 +500,9 @@ export interface IngressStatus {
     error?: string;
 }
 export type ProjectHealth = 'serving' | 'skipped' | 'failed';
-export type RolloutState = 'pending' | 'pulling' | 'starting' | 'health_checking' | 'serving';
+export type RolloutState = 'pending' | 'pulling' | 'starting' | 'health_checking' | 'serving'
+/** Stopped on purpose while a restore rewrites its data. */
+ | 'held';
 export interface ServiceState {
     rolloutState: RolloutState;
     image?: string;
@@ -419,6 +528,8 @@ export interface StatusReport {
     projects: ProjectStatus[];
     commandResults: CommandResult[];
     ingress?: IngressStatus;
+    /** Backup jobs still running on this node; absent when none. */
+    jobs?: JobStatus[];
 }
 export interface SecretsResponse {
     registryAuth?: Record<string, {

package/dist/protocol.js

@@ -16,6 +16,13 @@ export const TERMINAL_DEPLOYMENT_STATES = [
     'build_failed',
     'cancelled',
 ];
+/** Commands whose helper containers need the backup destination's credentials. */
+export const BACKUP_COMMAND_TYPES = [
+    'backup.check',
+    'backup.run',
+    'backup.restore',
+    'backup.list',
+];
 export const LIVE_MAX_FRAME_BYTES = 256 * 1024;
 export const LIVE_MAX_BUFFERED_BYTES = 1024 * 1024;
 export const LIVE_MAX_SESSIONS = 64;

package/dist/tokens.js

@@ -42,6 +42,16 @@ export const ROUTE_ROLES = {
     'POST /api/v1/nodes/:name/rotate-wg-key': 'admin',
     'POST /api/v1/nodes/:name/ingress': 'admin',
     'DELETE /api/v1/nodes/:name': 'admin',
+    'POST /api/v1/backups/run': 'deploy',
+    'GET /api/v1/backups': 'read',
+    'POST /api/v1/restores': 'admin',
+    'GET /api/v1/restores': 'read',
+    'POST /api/v1/restores/:id/release': 'admin',
+    'POST /api/v1/restores/:id/resume': 'admin',
+    'GET /api/v1/backup/destination': 'admin',
+    'POST /api/v1/backup/destination': 'admin',
+    'DELETE /api/v1/backup/destination': 'admin',
+    'GET /api/v1/backup/key': 'admin',
     'POST /api/v1/control/backups/run': 'admin',
     'POST /api/v1/control/restore': 'admin',
     'GET /api/v1/diagnostics': 'admin',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wpelevator/tug",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "Portable VPS deployments with Docker Compose",
   "repository": {
     "type": "git",
@@ -29,6 +29,7 @@
     "test:integration": "bash tools/scripts/integration-test.sh",
     "test:integration:network": "bash tools/scripts/integration-network-test.sh",
     "test:integration:agent": "node --test 'tests/integration/**/*.test.ts'",
+    "test:integration:backups": "node --test tests/integration/backup-job.test.ts tests/integration/run-job.test.ts",
     "test:smoke:dist": "npm run build && CLI_ENTRY=dist/cli.js node --test src/cli.templates.test.ts",
     "changeset": "changeset",
     "version-packages": "changeset version && npm install --package-lock-only --ignore-scripts",