@wpelevator/tug 0.5.0 → 0.7.0

package/dist/backup-config.js

@@ -0,0 +1,276 @@
+import { createHash } from 'node:crypto';
+import { parseCron } from "./cron.js";
+export const DEFAULT_RETENTION = { daily: 7, weekly: 4, monthly: 6 };
+export const DEFAULT_BACKUP_TIMEOUT_SEC = 2 * 3600;
+export const DEFAULT_BACKUP_MEMORY = '512m';
+/** Below this restic runs out of memory on any repository worth backing up. */
+const MIN_MEMORY_BYTES = 128 * 1024 ** 2;
+const MAX_MEMORY_BYTES = 16 * 1024 ** 3;
+const MIN_TIMEOUT_SEC = 60;
+const MAX_TIMEOUT_SEC = 24 * 3600;
+const MAX_JITTER_SEC = 3600;
+const RETENTION_KEYS = ['last', 'hourly', 'daily', 'weekly', 'monthly', 'yearly'];
+const COMMON_KEYS = ['schedule', 'retention', 'jitter', 'timeout', 'memory', 'writers'];
+const SERVICE_KEYS = [...COMMON_KEYS, 'command', 'restore', 'filename', 'verify'];
+const VOLUME_KEYS = [...COMMON_KEYS, 'exclude'];
+const FILENAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
+const isObject = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
+/** `30m`, `2h`, `90s` or a number of seconds. */
+export function parseBackupTimeout(value) {
+    if (typeof value === 'number')
+        return Number.isInteger(value) ? value : null;
+    const match = typeof value === 'string' ? value.match(/^(\d+)(s|m|h)$/) : null;
+    if (!match)
+        return null;
+    return Number(match[1]) * { s: 1, m: 60, h: 3600 }[match[2]];
+}
+/** Bytes in a Docker-style memory size (`256m`, `1g`, `1048576`), or null when it is not one. */
+export function parseMemorySize(value) {
+    const match = typeof value === 'string' ? /^(\d+)([kmg]?)$/i.exec(value.trim()) : null;
+    if (!match)
+        return null;
+    return (Number(match[1]) *
+        { '': 1, k: 1024, m: 1024 ** 2, g: 1024 ** 3 }[match[2].toLowerCase()]);
+}
+/** Deterministic delay in `[0, 900)` seconds so many sites on one node do not start together. */
+export function defaultJitterSeconds(project, target) {
+    const digest = createHash('sha256').update(`${project}\0${target}`).digest();
+    return digest.readUInt32BE(0) % 900;
+}
+function validateRetention(label, value, errors) {
+    if (!isObject(value)) {
+        errors.push(`${label}.retention must be an object such as { daily: 7, weekly: 4 }`);
+        return undefined;
+    }
+    const retention = {};
+    for (const [key, count] of Object.entries(value)) {
+        if (!RETENTION_KEYS.includes(key))
+            errors.push(`${label}.retention.${key} is not supported (supported: ${RETENTION_KEYS.join(', ')})`);
+        else if (!Number.isInteger(count) || count < 1)
+            errors.push(`${label}.retention.${key} must be a whole number of at least 1`);
+        else
+            retention[key] = count;
+    }
+    if (!Object.keys(value).length)
+        errors.push(`${label}.retention is empty, which would keep no snapshots; set at least one of ${RETENTION_KEYS.join(', ')}`);
+    return retention;
+}
+function validateSchedule(label, value, errors) {
+    if (typeof value !== 'string') {
+        errors.push(`${label}.schedule must be a cron string such as "0 3 * * *" or @daily`);
+        return undefined;
+    }
+    try {
+        parseCron(value);
+    }
+    catch (error) {
+        errors.push(`${label}.schedule: ${error instanceof Error ? error.message : String(error)}`);
+        return undefined;
+    }
+    return value.trim();
+}
+/** Validates the project-level defaults in `x-tug.backup`. */
+export function validateBackupDefaults(value, errors) {
+    if (value === undefined)
+        return;
+    if (!isObject(value)) {
+        errors.push('x-tug.backup must be an object');
+        return;
+    }
+    for (const key of Object.keys(value))
+        if (key !== 'schedule' && key !== 'retention')
+            errors.push(`x-tug.backup.${key} is not supported (supported: schedule, retention)`);
+    if (value.schedule !== undefined)
+        validateSchedule('x-tug.backup', value.schedule, errors);
+    if (value.retention !== undefined)
+        validateRetention('x-tug.backup', value.retention, errors);
+}
+function normalizeConfig(config) {
+    if (config === false)
+        return null;
+    if (config === true)
+        return {};
+    if (typeof config === 'string')
+        return { schedule: config };
+    return config;
+}
+/** The argv of a `command`: a list is used as is, a string runs through `sh -c` like a shell one-liner. */
+export function commandArgv(command) {
+    if (typeof command === 'string')
+        return command.trim() ? ['sh', '-c', command] : null;
+    return Array.isArray(command) &&
+        command.length &&
+        command.every((part) => typeof part === 'string' && part)
+        ? command
+        : null;
+}
+function serviceNodes(svc) {
+    const lp = svc['x-tug'] || {};
+    const placement = lp.placement || {};
+    return {
+        nodes: placement.nodes || [],
+        dynamic: Array.isArray(placement.labels) && placement.labels.length > 0,
+        global: lp.global === true || placement.global === true,
+    };
+}
+function mountedVolumes(svc) {
+    return (svc.volumes || []).flatMap((mount) => typeof mount === 'object' && mount && mount.type === 'volume' && mount.source
+        ? [String(mount.source)]
+        : []);
+}
+/**
+ * Resolves and validates every `x-tug.backup` in a Compose file. Tug knows nothing about what a service
+ * is: a service backup is a command whose output is stored, a volume backup is its files. `project`
+ * seeds the default jitter. Errors name the key, so they can be shown to the operator as they are.
+ */
+export function resolveBackups(compose, project = '') {
+    const errors = [];
+    const targets = [];
+    const defaults = compose['x-tug']?.backup;
+    const defaultSchedule = isObject(defaults) ? defaults.schedule : undefined;
+    const defaultRetention = (isObject(defaults) && isObject(defaults.retention) ? defaults.retention : DEFAULT_RETENTION);
+    const common = (label, kind, name, spec) => {
+        const allowed = kind === 'service' ? SERVICE_KEYS : VOLUME_KEYS;
+        for (const key of Object.keys(spec))
+            if (!allowed.includes(key))
+                errors.push(`${label}.${key} is not supported${key === 'command' || key === 'restore' || key === 'filename' || key === 'verify'
+                    ? ' on a volume; a volume backup archives its files'
+                    : key === 'exclude'
+                        ? ' on a service; it applies to volumes'
+                        : ''} (supported: ${allowed.join(', ')})`);
+        const before = errors.length;
+        const schedule = spec.schedule !== undefined
+            ? validateSchedule(label, spec.schedule, errors)
+            : typeof defaultSchedule === 'string'
+                ? defaultSchedule.trim()
+                : undefined;
+        if (spec.schedule === undefined && schedule === undefined)
+            errors.push(`${label} needs a schedule: set ${label}.schedule or x-tug.backup.schedule`);
+        const retention = spec.retention !== undefined
+            ? validateRetention(label, spec.retention, errors)
+            : defaultRetention;
+        let jitterSeconds = defaultJitterSeconds(project, `${kind}:${name}`);
+        if (spec.jitter !== undefined) {
+            if (!Number.isInteger(spec.jitter) || spec.jitter < 0 || spec.jitter > MAX_JITTER_SEC)
+                errors.push(`${label}.jitter must be a whole number of seconds from 0 to ${MAX_JITTER_SEC}`);
+            else
+                jitterSeconds = spec.jitter;
+        }
+        let timeoutSec = DEFAULT_BACKUP_TIMEOUT_SEC;
+        if (spec.timeout !== undefined) {
+            const parsed = parseBackupTimeout(spec.timeout);
+            if (parsed === null || parsed < MIN_TIMEOUT_SEC || parsed > MAX_TIMEOUT_SEC)
+                errors.push(`${label}.timeout must be a duration from 1m to 24h such as 30m or 2h`);
+            else
+                timeoutSec = parsed;
+        }
+        let memory = DEFAULT_BACKUP_MEMORY;
+        if (spec.memory !== undefined) {
+            const bytes = parseMemorySize(spec.memory);
+            if (bytes === null || bytes < MIN_MEMORY_BYTES || bytes > MAX_MEMORY_BYTES)
+                errors.push(`${label}.memory must be a size from 128m to 16g such as 256m or 1g`);
+            else
+                memory = String(spec.memory).trim().toLowerCase();
+        }
+        if (spec.writers !== undefined) {
+            if (!Array.isArray(spec.writers) || spec.writers.some((w) => typeof w !== 'string'))
+                errors.push(`${label}.writers must be a list of service names`);
+            else
+                for (const writer of spec.writers)
+                    if (!compose.services[writer])
+                        errors.push(`${label}.writers: ${writer} is not a service`);
+        }
+        if (errors.length > before || !schedule || !retention)
+            return null;
+        return { schedule, retention, jitterSeconds, timeoutSec, memory };
+    };
+    for (const [name, svc] of Object.entries(compose.services)) {
+        const config = svc['x-tug']?.backup;
+        if (config === undefined)
+            continue;
+        const label = `${name}: x-tug.backup`;
+        const spec = normalizeConfig(config);
+        if (spec === null)
+            continue;
+        if (typeof spec !== 'object' || Array.isArray(spec)) {
+            errors.push(`${label} must be true, false, a cron string or an object`);
+            continue;
+        }
+        const before = errors.length;
+        const base = common(label, 'service', name, spec);
+        const argv = spec.command === undefined ? null : commandArgv(spec.command);
+        if (spec.command === undefined)
+            errors.push(`${label}.command is required: the command to run in the ${name} container, whose output is stored (see the recipes in docs/backups.md)`);
+        else if (!argv)
+            errors.push(`${label}.command must be a non-empty string or list of strings`);
+        const restoreArgv = spec.restore === undefined ? undefined : commandArgv(spec.restore);
+        if (spec.restore !== undefined && !restoreArgv)
+            errors.push(`${label}.restore must be a non-empty string or list of strings`);
+        if (spec.filename !== undefined &&
+            (typeof spec.filename !== 'string' || !FILENAME_RE.test(spec.filename)))
+            errors.push(`${label}.filename must be a plain file name such as ${name}.sql`);
+        if (spec.verify !== undefined && (typeof spec.verify !== 'string' || !spec.verify))
+            errors.push(`${label}.verify must be text that appears at the end of the output`);
+        const { nodes, dynamic, global } = serviceNodes(svc);
+        const replicas = svc['x-tug']?.replicas ?? 1;
+        if (global || replicas > 1 || nodes.length > 1)
+            errors.push(`${label} needs a single instance; ${name} is global, replicated or placed on several nodes`);
+        if (errors.length > before || !base || !argv)
+            continue;
+        targets.push({
+            target: `service:${name}`,
+            kind: 'service',
+            name,
+            type: 'command',
+            ...base,
+            command: argv,
+            restore: restoreArgv ?? undefined,
+            filename: spec.filename ?? `${name}.out`,
+            verify: spec.verify,
+            writers: spec.writers,
+            node: !dynamic && nodes.length === 1 ? nodes[0] : undefined,
+        });
+    }
+    for (const [name, def] of Object.entries(compose.volumes || {})) {
+        const config = isObject(def) ? def['x-tug']?.backup : undefined;
+        if (config === undefined)
+            continue;
+        const label = `volumes.${name}: x-tug.backup`;
+        const spec = normalizeConfig(config);
+        if (spec === null)
+            continue;
+        if (typeof spec !== 'object' || Array.isArray(spec)) {
+            errors.push(`${label} must be true, false, a cron string or an object`);
+            continue;
+        }
+        const before = errors.length;
+        const base = common(label, 'volume', name, spec);
+        if (spec.exclude !== undefined &&
+            (!Array.isArray(spec.exclude) ||
+                spec.exclude.some((pattern) => typeof pattern !== 'string' || !pattern)))
+            errors.push(`${label}.exclude must be a list of restic exclude patterns`);
+        const volumeExt = isObject(def) ? def['x-tug'] : undefined;
+        const placed = volumeExt?.placement?.nodes || [];
+        const mounting = Object.entries(compose.services).filter(([, svc]) => mountedVolumes(svc).includes(name));
+        const mountingNodes = [...new Set(mounting.flatMap(([, svc]) => serviceNodes(svc).nodes))];
+        const dynamicPlacement = mounting.some(([, svc]) => serviceNodes(svc).dynamic || serviceNodes(svc).global);
+        const candidates = placed.length ? placed : mountingNodes;
+        if (candidates.length > 1)
+            errors.push(`${label} needs one node: volume ${name} is ${placed.length ? 'placed' : 'used'} on ${candidates.join(', ')}; pin it with x-tug.placement.nodes or back up per node`);
+        else if (!placed.length && dynamicPlacement)
+            errors.push(`${label} needs x-tug.placement.nodes on volume ${name}, since its services are placed by label or global`);
+        if (errors.length > before || !base)
+            continue;
+        targets.push({
+            target: `volume:${name}`,
+            kind: 'volume',
+            name,
+            type: 'volume',
+            ...base,
+            exclude: spec.exclude,
+            writers: spec.writers,
+            node: candidates.length === 1 ? candidates[0] : undefined,
+        });
+    }
+    return { targets, errors };
+}

package/dist/backup-destination.d.ts

@@ -0,0 +1,22 @@
+/** Reserved secrets-table project holding the backup destination's credentials. */
+export declare const BACKUP_SECRET_PROJECT = "_backup";
+export interface BackupDestinationInput {
+    url: string;
+    /** Variable name to value; stored encrypted and never returned. */
+    env: Record<string, string>;
+}
+export declare const isBackupProject: (project: string) => boolean;
+/** A name starting with `_` is reserved for control's own records and is never a deployable project. */
+export declare const isReservedProject: (project: string) => boolean;
+/** Throws an `Error` naming the first problem with a destination, or returns it normalized. */
+export declare function parseBackupDestination(input: unknown): BackupDestinationInput;
+/** The restic repository of one project: every node of the project shares it, other projects never do. */
+export declare function projectRepository(destinationUrl: string, project: string): string;
+/**
+ * The repository password of a project, derived from the control master key so nothing extra is stored and
+ * a node only ever holds the passwords of projects it runs backups for. Losing the master key loses every
+ * repository, which is why `tug backup-key` exists for offline escrow.
+ */
+export declare function repositoryPassword(masterKey: string, project: string): string;
+/** Removes every known secret value from text that is about to be stored, shown or put in an event. */
+export declare function redactSecrets(text: string, values: Iterable<string>): string;

package/dist/backup-destination.js

@@ -0,0 +1,72 @@
+import { createHmac } from 'node:crypto';
+/** Reserved secrets-table project holding the backup destination's credentials. */
+export const BACKUP_SECRET_PROJECT = '_backup';
+const MAX_ENV_VARIABLES = 16;
+const ENV_NAME_RE = /^[A-Z][A-Z0-9_]{0,63}$/;
+const RESERVED_ENV = ['RESTIC_PASSWORD', 'RESTIC_REPOSITORY', 'RESTIC_PASSWORD_FILE'];
+/** Shorter values would redact ordinary words out of error text. */
+const MIN_REDACTED_LENGTH = 4;
+const REQUIRED_ENV = {
+    s3: ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'],
+};
+export const isBackupProject = (project) => project === BACKUP_SECRET_PROJECT;
+/** A name starting with `_` is reserved for control's own records and is never a deployable project. */
+export const isReservedProject = (project) => project.startsWith('_');
+/** Throws an `Error` naming the first problem with a destination, or returns it normalized. */
+export function parseBackupDestination(input) {
+    const body = (input && typeof input === 'object' ? input : {});
+    const url = typeof body.url === 'string' ? body.url.trim().replace(/\/+$/, '') : '';
+    if (!url)
+        throw new Error('url is required, such as s3:https://s3.example.com/bucket/tug-backups');
+    if (/\s/.test(url))
+        throw new Error('url must not contain whitespace');
+    const scheme = url.startsWith('/') ? 'local' : url.split(':')[0];
+    if (!['s3', 'sftp', 'rest', 'local'].includes(scheme))
+        throw new Error('url must start with s3:, sftp:, rest: or be an absolute path (a path is for development only: it is read on each node, not shared)');
+    if (/:\/\/[^/@\s]*:[^/@\s]*@/.test(url))
+        throw new Error('url must not contain a username and password; pass credentials as env variables so they are stored encrypted');
+    if (scheme === 's3' && !/^s3:https?:\/\/[^/]+\/[^/]+/.test(url))
+        throw new Error('s3 urls look like s3:https://host/bucket[/prefix] and must name a bucket; AWS S3 uses s3:https://s3.amazonaws.com/bucket');
+    const rawEnv = body.env && typeof body.env === 'object' ? body.env : {};
+    const names = Object.keys(rawEnv);
+    if (names.length > MAX_ENV_VARIABLES)
+        throw new Error(`at most ${MAX_ENV_VARIABLES} env variables are supported`);
+    const env = {};
+    for (const name of names) {
+        if (!ENV_NAME_RE.test(name))
+            throw new Error(`env variable ${JSON.stringify(name)} must be upper case letters, digits and underscores`);
+        if (RESERVED_ENV.includes(name))
+            throw new Error(`${name} is managed by Tug and cannot be set`);
+        const value = rawEnv[name];
+        if (typeof value !== 'string' || !value)
+            throw new Error(`env variable ${name} needs a value`);
+        env[name] = value;
+    }
+    const missing = (REQUIRED_ENV[scheme] ?? []).filter((name) => !(name in env));
+    if (missing.length)
+        throw new Error(`${scheme} destinations need ${missing.join(' and ')}`);
+    return { url, env };
+}
+/** The restic repository of one project: every node of the project shares it, other projects never do. */
+export function projectRepository(destinationUrl, project) {
+    return `${destinationUrl.replace(/\/+$/, '')}/${project}`;
+}
+/**
+ * The repository password of a project, derived from the control master key so nothing extra is stored and
+ * a node only ever holds the passwords of projects it runs backups for. Losing the master key loses every
+ * repository, which is why `tug backup-key` exists for offline escrow.
+ */
+export function repositoryPassword(masterKey, project) {
+    return createHmac('sha256', masterKey).update(`tug-restic:${project}`).digest('hex');
+}
+/** Removes every known secret value from text that is about to be stored, shown or put in an event. */
+export function redactSecrets(text, values) {
+    const secrets = [...new Set(values)]
+        .filter((value) => value.length >= MIN_REDACTED_LENGTH)
+        .sort((a, b) => b.length - a.length);
+    let result = text;
+    for (const secret of secrets)
+        result = result.split(secret).join('[redacted]');
+    // A credential embedded in a URL, such as one echoed back in a backend error.
+    return result.replace(/([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s@]+@/gi, '$1[redacted]@');
+}

package/dist/backup-job.d.ts

@@ -0,0 +1,32 @@
+import type { DockerApi } from './docker-api.ts';
+import type { BackupCredentialsResponse, BackupJobPayload, BackupJobResult, BackupRetention, JobPhase } from './protocol.ts';
+/** Docker volume holding restic's cache across jobs, so repeated runs do not re-read the repository index. */
+export declare const RESTIC_CACHE_VOLUME = "tug_restic_cache";
+export interface JobContext {
+    docker: DockerApi;
+    credentials: BackupCredentialsResponse;
+    /** Called as the job moves between phases; feeds the agent's heartbeat. */
+    onPhase?: (phase: JobPhase, bytes?: number) => void;
+    /** Aborts the job (timeout or cancellation); helper containers are removed either way. */
+    signal?: AbortSignal;
+}
+/** `512m`, `2g` or a byte count. */
+export declare function memoryBytes(value: string): number;
+/** `restic forget` arguments applying a retention policy to one target of one node. */
+export declare function forgetArgs(payload: BackupJobPayload, retention: BackupRetention): string[];
+/** `restic backup` arguments; `stdinName` selects a dump stream, otherwise `/data` is archived. */
+export declare function backupArgs(payload: BackupJobPayload, stdinName?: string): string[];
+interface Summary {
+    snapshotId?: string;
+    addedBytes?: number;
+    totalBytes?: number;
+}
+/** The `summary` line restic prints at the end of `backup --json`. */
+export declare function parseSummary(output: string): Summary;
+export declare class JobError extends Error {
+}
+/** Runs one backup job to completion. Helper containers carry the `tug.job` label and are always removed. */
+export declare function runBackupJob(payload: BackupJobPayload, commandId: string, ctx: JobContext, kind?: 'backup.run' | 'backup.check' | 'backup.restore'): Promise<BackupJobResult>;
+/** Removes helper containers left behind by an agent that stopped mid-job. */
+export declare function removeStaleJobContainers(docker: DockerApi): Promise<number>;
+export {};