@matthesketh/fleet 1.8.0 → 1.11.0

package/dist/core/backup/repo.d.ts

@@ -0,0 +1,71 @@
+import { ChildProcessWithoutNullStreams } from 'node:child_process';
+import { FleetError } from '../errors.js';
+import { Retention, SnapshotInfo, RepoStats } from './types.js';
+export declare const SFTP_HOST_ALIAS: string;
+export declare class ResticError extends FleetError {
+}
+/** true when the backend rejects deletions/rewrites (rest-server --append-only
+ *  in particular). primary-side prune must skip on such backends — pruning
+ *  happens locally on the backup vps via its own cron. */
+export declare function isAppendOnly(): boolean;
+export declare function initRepo(app: string): void;
+export interface BackupOptions {
+    paths: string[];
+    excludes?: string[];
+    tags?: string[];
+    /** add a host=... override (useful for restoring tests). */
+    hostname?: string;
+    /** when set, restic gets --stdin --stdin-filename and paths are ignored
+     *  (restic does not accept both). */
+    stdinFilename?: string;
+    /** small in-memory payloads piped via process stdin (~1mb safe). */
+    stdinData?: string;
+    /** for large/streaming payloads: a shell command whose stdout is piped
+     *  into `restic backup --stdin` via `sh -c "<cmd> | restic ..."`. avoids
+     *  the spawnSync 1mb buffer ceiling so multi-gb dumps work. */
+    stdinCommand?: string;
+    /** dry-run: restic walks paths and reports what would be added, no upload. */
+    dryRun?: boolean;
+}
+export declare function snapshot(app: string, opts: BackupOptions, timeoutMs?: number): SnapshotInfo;
+export declare function listSnapshots(app: string): SnapshotInfo[];
+export interface RestoreOptions {
+    snapshotId: string;
+    target: string;
+    /** restore only these subpaths (restic --include). */
+    include?: string[];
+    /** dry-run: list files that would be restored without writing. */
+    dryRun?: boolean;
+    /** post-restore integrity check: re-walks the restored tree and confirms
+     *  file contents match the snapshot's chunk hashes (restic --verify). */
+    verify?: boolean;
+}
+export interface TreeEntry {
+    name: string;
+    type: 'dir' | 'file';
+    path: string;
+    size: number;
+    mtime: string;
+}
+/** parses `restic ls --json` output into the direct children of dirPath.
+ *  restic ls recurses, so deeper descendants are filtered out here. */
+export declare function parseLsOutput(stdout: string, dirPath: string): TreeEntry[];
+/** lists the immediate children of dirPath within a snapshot. */
+export declare function lsTree(app: string, snapshotId: string, dirPath: string): TreeEntry[];
+/** the argv (after `restic`) for dumping a file from a snapshot. */
+export declare function dumpFileArgs(app: string, snapshotId: string, filePath: string): string[];
+/** spawns `restic dump`; caller pipes child.stdout to an http response and
+ *  must handle 'error' / non-zero 'close'. streaming avoids buffering a
+ *  potentially multi-gb file in node memory. */
+export declare function dumpFileSpawn(app: string, snapshotId: string, filePath: string): ChildProcessWithoutNullStreams;
+export declare function restore(app: string, opts: RestoreOptions, timeoutMs?: number): void;
+/** orthogonal integrity check: walks the restic repo, recomputes chunk
+ *  hashes, asserts the index is consistent. catches bit-rot on the
+ *  backup-vps disk. callable independently or right after a restore. */
+export declare function checkIntegrity(app: string, readDataPercent?: number): {
+    ok: boolean;
+    output: string;
+};
+export declare function prune(app: string, retention: Retention): void;
+export declare function check(app: string): boolean;
+export declare function stats(app: string): RepoStats | null;

package/dist/core/backup/repo.js

@@ -0,0 +1,256 @@
+import { spawn } from 'node:child_process';
+import { FleetError } from '../errors.js';
+import { execSafe } from '../exec.js';
+import { passwordCommandFor } from './unlock.js';
+export const SFTP_HOST_ALIAS = process.env.FLEET_BACKUP_SFTP_ALIAS ?? 'backup-vps-sftp';
+export class ResticError extends FleetError {
+}
+/** when fleet_backup_base_url is set we use it as the backend base — that
+ *  enables rest:// + append-only protection. when unset we fall back to the
+ *  legacy sftp alias. the systemd unit loads the url + rest creds from
+ *  encrypted credstore so they never sit on disk in plaintext. */
+function repoUri(app) {
+    const base = process.env.FLEET_BACKUP_BASE_URL;
+    if (base) {
+        return base.endsWith('/') ? `${base}${app}` : `${base}/${app}`;
+    }
+    return `sftp:${SFTP_HOST_ALIAS}:${app}`;
+}
+/** true when the backend rejects deletions/rewrites (rest-server --append-only
+ *  in particular). primary-side prune must skip on such backends — pruning
+ *  happens locally on the backup vps via its own cron. */
+export function isAppendOnly() {
+    return (process.env.FLEET_BACKUP_BASE_URL ?? '').startsWith('rest:');
+}
+function resticEnv(app) {
+    // restic respects restic_rest_username/password env vars; the wrapper
+    // populates them from the encrypted credstore.
+    const env = { RESTIC_PASSWORD_COMMAND: passwordCommandFor(app) };
+    if (process.env.RESTIC_REST_USERNAME)
+        env.RESTIC_REST_USERNAME = process.env.RESTIC_REST_USERNAME;
+    if (process.env.RESTIC_REST_PASSWORD)
+        env.RESTIC_REST_PASSWORD = process.env.RESTIC_REST_PASSWORD;
+    return env;
+}
+function runRestic(app, args, timeoutMs = 60_000) {
+    const r = execSafe('restic', ['-r', repoUri(app), ...args], {
+        timeout: timeoutMs,
+        env: resticEnv(app),
+    });
+    return { stdout: r.stdout, stderr: r.stderr, ok: r.ok };
+}
+export function initRepo(app) {
+    // if it already exists, snapshots --no-lock succeeds; only init when it doesn't.
+    const probe = runRestic(app, ['snapshots', '--no-lock', '--quiet'], 15_000);
+    if (probe.ok)
+        return;
+    const init = runRestic(app, ['init'], 30_000);
+    if (!init.ok)
+        throw new ResticError(`restic init failed: ${init.stderr}`);
+}
+/** sh single-quote escape — wraps s in '...' with embedded quotes as '\''. */
+function shellQuote(s) {
+    return `'${s.replace(/'/g, "'\\''")}'`;
+}
+export function snapshot(app, opts, timeoutMs = 30 * 60_000) {
+    const args = ['backup'];
+    if (opts.dryRun)
+        args.push('--dry-run');
+    for (const ex of opts.excludes ?? []) {
+        args.push('--exclude', ex);
+    }
+    for (const tag of opts.tags ?? []) {
+        args.push('--tag', tag);
+    }
+    if (opts.hostname) {
+        args.push('--host', opts.hostname);
+    }
+    if (opts.stdinFilename) {
+        // restic --stdin reads from stdin only; positional paths must be omitted.
+        args.push('--stdin', '--stdin-filename', opts.stdinFilename);
+    }
+    else {
+        args.push(...opts.paths);
+    }
+    args.push('--json');
+    let r;
+    if (opts.stdinCommand) {
+        // stream via bash — kernel pipes between the dump cmd and restic, so
+        // the dump bytes never enter node's spawnSync buffer. bash is used
+        // explicitly (rather than /bin/sh which is dash on ubuntu) because
+        // we need `pipefail` to surface dump errors that would otherwise be
+        // masked by restic's exit code.
+        const resticInvocation = ['restic', '-r', repoUri(app), ...args]
+            .map(shellQuote)
+            .join(' ');
+        const fullCmd = `set -eo pipefail; ${opts.stdinCommand} | ${resticInvocation}`;
+        if (process.env.FLEET_DEBUG_DUMP === '1') {
+            process.stderr.write(`[fleet-debug] bash -c: ${fullCmd}\n`);
+        }
+        r = execSafe('bash', ['-c', fullCmd], {
+            timeout: timeoutMs,
+            env: resticEnv(app),
+        });
+    }
+    else {
+        r = execSafe('restic', ['-r', repoUri(app), ...args], {
+            timeout: timeoutMs,
+            env: resticEnv(app),
+            input: opts.stdinData,
+        });
+    }
+    if (!r.ok)
+        throw new ResticError(`restic backup failed: ${r.stderr || r.stdout}`);
+    // last json line is the summary with snapshot_id (or counters for dry-run)
+    const lines = r.stdout.split('\n').filter(Boolean);
+    for (let i = lines.length - 1; i >= 0; i--) {
+        try {
+            const obj = JSON.parse(lines[i]);
+            if (obj.message_type === 'summary') {
+                return {
+                    id: obj.snapshot_id ?? 'dry-run',
+                    shortId: (obj.snapshot_id ?? 'dry-run').slice(0, 8),
+                    time: new Date().toISOString(),
+                    hostname: opts.hostname ?? '',
+                    paths: opts.paths,
+                    tags: opts.tags ?? [],
+                    sizeBytes: obj.data_added,
+                };
+            }
+        }
+        catch { /* not json, skip */ }
+    }
+    throw new ResticError(`could not parse restic snapshot summary`);
+}
+export function listSnapshots(app) {
+    const r = runRestic(app, ['snapshots', '--json'], 15_000);
+    if (!r.ok) {
+        if (r.stderr.includes('does not exist'))
+            return [];
+        throw new ResticError(`restic snapshots failed: ${r.stderr}`);
+    }
+    if (!r.stdout)
+        return [];
+    const arr = JSON.parse(r.stdout);
+    return arr.map(s => ({
+        id: s.id,
+        shortId: s.short_id,
+        time: s.time,
+        hostname: s.hostname,
+        paths: s.paths,
+        tags: s.tags ?? [],
+    }));
+}
+/** parses `restic ls --json` output into the direct children of dirPath.
+ *  restic ls recurses, so deeper descendants are filtered out here. */
+export function parseLsOutput(stdout, dirPath) {
+    const base = dirPath === '/' ? '' : dirPath.replace(/\/+$/, '');
+    const entries = [];
+    for (const line of stdout.split('\n')) {
+        if (!line)
+            continue;
+        let node;
+        try {
+            node = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (node.struct_type !== 'node')
+            continue;
+        const path = typeof node.path === 'string' ? node.path : '';
+        if (!path || !path.startsWith(base + '/'))
+            continue;
+        const rest = path.slice(base.length + 1);
+        if (rest.length === 0 || rest.includes('/'))
+            continue;
+        entries.push({
+            name: typeof node.name === 'string' ? node.name : rest,
+            type: node.type === 'dir' ? 'dir' : 'file',
+            path,
+            size: typeof node.size === 'number' ? node.size : 0,
+            mtime: typeof node.mtime === 'string' ? node.mtime : '',
+        });
+    }
+    entries.sort((a, b) => a.type === b.type ? a.name.localeCompare(b.name) : a.type === 'dir' ? -1 : 1);
+    return entries;
+}
+/** lists the immediate children of dirPath within a snapshot. */
+export function lsTree(app, snapshotId, dirPath) {
+    const r = runRestic(app, ['ls', snapshotId, '--json', dirPath], 60_000);
+    if (!r.ok) {
+        throw new ResticError(`restic ls failed for ${dirPath}: ${r.stderr || r.stdout}`);
+    }
+    return parseLsOutput(r.stdout, dirPath);
+}
+/** the argv (after `restic`) for dumping a file from a snapshot. */
+export function dumpFileArgs(app, snapshotId, filePath) {
+    return ['-r', repoUri(app), 'dump', snapshotId, filePath];
+}
+/** spawns `restic dump`; caller pipes child.stdout to an http response and
+ *  must handle 'error' / non-zero 'close'. streaming avoids buffering a
+ *  potentially multi-gb file in node memory. */
+export function dumpFileSpawn(app, snapshotId, filePath) {
+    return spawn('restic', dumpFileArgs(app, snapshotId, filePath), {
+        env: { ...process.env, ...resticEnv(app) },
+    });
+}
+export function restore(app, opts, timeoutMs = 60 * 60_000) {
+    const args = ['restore', opts.snapshotId, '--target', opts.target];
+    if (opts.dryRun)
+        args.push('--dry-run');
+    if (opts.verify)
+        args.push('--verify');
+    for (const inc of opts.include ?? []) {
+        args.push('--include', inc);
+    }
+    const r = runRestic(app, args, timeoutMs);
+    if (!r.ok)
+        throw new ResticError(`restic restore failed: ${r.stderr}`);
+}
+/** orthogonal integrity check: walks the restic repo, recomputes chunk
+ *  hashes, asserts the index is consistent. catches bit-rot on the
+ *  backup-vps disk. callable independently or right after a restore. */
+export function checkIntegrity(app, readDataPercent = 5) {
+    const r = runRestic(app, ['check', `--read-data-subset=${readDataPercent}%`], 30 * 60_000);
+    return { ok: r.ok, output: r.ok ? r.stdout : `${r.stdout}\n${r.stderr}`.trim() };
+}
+export function prune(app, retention) {
+    const args = ['forget', '--prune'];
+    if (retention.hourly)
+        args.push('--keep-hourly', String(retention.hourly));
+    if (retention.daily)
+        args.push('--keep-daily', String(retention.daily));
+    if (retention.weekly)
+        args.push('--keep-weekly', String(retention.weekly));
+    if (retention.monthly)
+        args.push('--keep-monthly', String(retention.monthly));
+    if (retention.yearly)
+        args.push('--keep-yearly', String(retention.yearly));
+    if (args.length === 2) {
+        throw new ResticError('retention is empty — refusing to forget all snapshots');
+    }
+    const r = runRestic(app, args, 5 * 60_000);
+    if (!r.ok)
+        throw new ResticError(`restic forget failed: ${r.stderr}`);
+}
+export function check(app) {
+    const r = runRestic(app, ['check', '--read-data-subset=5%'], 10 * 60_000);
+    return r.ok;
+}
+export function stats(app) {
+    const r = runRestic(app, ['stats', 'latest', '--json'], 15_000);
+    if (!r.ok || !r.stdout)
+        return null;
+    try {
+        const obj = JSON.parse(r.stdout);
+        return {
+            totalSize: obj.total_size ?? 0,
+            totalFileCount: obj.total_file_count ?? 0,
+            snapshotCount: obj.snapshots_count ?? 0,
+        };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/core/backup/schedule.d.ts

@@ -0,0 +1,17 @@
+import { Schedule } from './types.js';
+export declare const SYSTEMD_UNIT_DIR: string;
+export declare function timerUnitName(app: string): string;
+export declare function serviceUnitName(app: string): string;
+export interface ScheduleInstallResult {
+    timerPath: string;
+    timerContent: string;
+    sharedServicePath: string;
+    sharedServiceContent: string;
+    sharedServiceWrote: boolean;
+}
+/** plans the timer + service units. returns the unit file contents so callers
+ *  can show a dry-run. set apply=true to actually write+enable. */
+export declare function installScheduleUnits(app: string, schedule: Schedule, opts?: {
+    apply?: boolean;
+}): ScheduleInstallResult;
+export declare function disableSchedule(app: string): void;

package/dist/core/backup/schedule.js

@@ -0,0 +1,90 @@
+import { writeFileSync, existsSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { FleetError } from '../errors.js';
+import { execSafe } from '../exec.js';
+export const SYSTEMD_UNIT_DIR = process.env.FLEET_SYSTEMD_UNIT_DIR ?? '/etc/systemd/system';
+function onCalendarFor(schedule) {
+    if (schedule === 'hourly')
+        return 'hourly';
+    if (schedule === 'daily')
+        return 'daily';
+    if (schedule === 'weekly')
+        return 'weekly';
+    return schedule;
+}
+export function timerUnitName(app) {
+    return `fleet-backup@${app}.timer`;
+}
+export function serviceUnitName(app) {
+    return `fleet-backup@${app}.service`;
+}
+/** plans the timer + service units. returns the unit file contents so callers
+ *  can show a dry-run. set apply=true to actually write+enable. */
+export function installScheduleUnits(app, schedule, opts = {}) {
+    const sharedServicePath = join(SYSTEMD_UNIT_DIR, 'fleet-backup@.service');
+    const sharedServiceContent = sharedServiceUnit();
+    const timerPath = join(SYSTEMD_UNIT_DIR, timerUnitName(app));
+    const timerContent = perAppTimerUnit(app, schedule);
+    const sharedServiceWrote = !existsSync(sharedServicePath);
+    if (!opts.apply) {
+        return { timerPath, timerContent, sharedServicePath, sharedServiceContent, sharedServiceWrote };
+    }
+    if (!existsSync(SYSTEMD_UNIT_DIR)) {
+        mkdirSync(SYSTEMD_UNIT_DIR, { recursive: true });
+    }
+    if (sharedServiceWrote) {
+        writeFileSync(sharedServicePath, sharedServiceContent, { mode: 0o644 });
+    }
+    writeFileSync(timerPath, timerContent, { mode: 0o644 });
+    const reload = execSafe('systemctl', ['daemon-reload'], { timeout: 5_000 });
+    if (!reload.ok)
+        throw new FleetError(`systemctl daemon-reload failed: ${reload.stderr}`);
+    const enable = execSafe('systemctl', ['enable', '--now', timerUnitName(app)], { timeout: 5_000 });
+    if (!enable.ok)
+        throw new FleetError(`enable timer failed: ${enable.stderr}`);
+    return { timerPath, timerContent, sharedServicePath, sharedServiceContent, sharedServiceWrote };
+}
+export function disableSchedule(app) {
+    execSafe('systemctl', ['disable', '--now', timerUnitName(app)], { timeout: 5_000 });
+}
+function sharedServiceUnit() {
+    // the wrapper loads the rest backend url (with embedded user:pass) from
+    // systemd-creds (host-key sealed at rest) and exports it as
+    // FLEET_BACKUP_BASE_URL. fleet writes via the append-only rest backend.
+    // legacy sftp backend still works if the wrapper / credstore entry are
+    // absent.
+    return `[Unit]
+Description=fleet backup for %i
+Documentation=fleet backup --help
+After=network-online.target docker.service wg-quick@wg0.service
+Wants=network-online.target wg-quick@wg0.service
+
+[Service]
+Type=oneshot
+LoadCredentialEncrypted=mx-url:/etc/credstore.encrypted/mx-url
+ExecStart=/usr/local/sbin/fleet-backup-wrapper %i
+# big snapshots (multi-gb dumps) need headroom; default 90s would kill them.
+TimeoutStartSec=4h
+TimeoutStopSec=5min
+Nice=10
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+
+[Install]
+WantedBy=multi-user.target
+`;
+}
+function perAppTimerUnit(app, schedule) {
+    return `[Unit]
+Description=fleet backup timer for ${app}
+
+[Timer]
+OnCalendar=${onCalendarFor(schedule)}
+Persistent=true
+RandomizedDelaySec=10m
+Unit=${serviceUnitName(app)}
+
+[Install]
+WantedBy=timers.target
+`;
+}

package/dist/core/backup/sensitive.d.ts

@@ -0,0 +1,5 @@
+/** classifies a snapshot-relative path as holding secret material or not.
+ *  used by the backup explorer to refuse view/download of sensitive files
+ *  while still allowing them to be restored to a staging dir. */
+export type Sensitivity = 'sensitive' | 'normal';
+export declare function classify(path: string): Sensitivity;

package/dist/core/backup/sensitive.js

@@ -0,0 +1,37 @@
+/** classifies a snapshot-relative path as holding secret material or not.
+ *  used by the backup explorer to refuse view/download of sensitive files
+ *  while still allowing them to be restored to a staging dir. */
+// matched case-insensitively against the full path.
+const SENSITIVE_PATTERNS = [
+    // key material
+    /\/\.ssh\//,
+    /\/\.gnupg\//,
+    /^\/etc\/ssh\//,
+    /^\/etc\/letsencrypt\//,
+    /\.pem$/,
+    /\.key$/,
+    // cloud + credential stores
+    /\/\.aws\//,
+    /\/\.gcloud\//,
+    /\/\.azure\//,
+    /\/\.kube\//,
+    /\/\.docker\//,
+    /\/\.secrets\//,
+    /\/credentials/,
+    /\/\.token/,
+    /\/\.npmrc$/,
+    /\/\.git-credentials$/,
+    // fleet + agent secrets
+    /\/\.claude/,
+    /^\/var\/lib\/fleet\//,
+    // database dumps
+    /\.pg\.sql$/,
+    /\.mysql\.sql$/,
+    /\.mongo\.archive$/,
+    /\.rdb$/,
+    /\.sql$/,
+];
+export function classify(path) {
+    const p = path.toLowerCase();
+    return SENSITIVE_PATTERNS.some(re => re.test(p)) ? 'sensitive' : 'normal';
+}

package/dist/core/backup/status.d.ts

@@ -0,0 +1,3 @@
+import { StatusReport } from './statuspage.js';
+/** gathers per-app snapshot counts, last-snapshot time and repo size. */
+export declare function buildStatusReport(): StatusReport;

package/dist/core/backup/status.js

@@ -0,0 +1,29 @@
+import { listConfiguredApps, loadConfig } from './config.js';
+import { listSnapshots, stats, isAppendOnly } from './repo.js';
+/** gathers per-app snapshot counts, last-snapshot time and repo size. */
+export function buildStatusReport() {
+    const apps = listConfiguredApps();
+    const entries = [];
+    for (const app of apps) {
+        const cfg = loadConfig(app);
+        if (!cfg)
+            continue;
+        const snaps = listSnapshots(app);
+        const last = snaps[snaps.length - 1];
+        const st = stats(app);
+        entries.push({
+            app,
+            schedule: cfg.schedule,
+            disabled: !!cfg.disabled,
+            snapshotCount: snaps.length,
+            lastSnapshotAt: last?.time ?? null,
+            totalSize: st?.totalSize ?? null,
+        });
+    }
+    return {
+        generatedAt: new Date().toISOString(),
+        backend: (process.env.FLEET_BACKUP_BASE_URL ?? '').startsWith('rest:') ? 'rest' : 'sftp',
+        appendOnly: isAppendOnly(),
+        apps: entries,
+    };
+}

package/dist/core/backup/statuspage.d.ts

@@ -0,0 +1,23 @@
+/** read-only backup status dashboard. rendered to static html by a systemd
+ *  timer and served at the /backups route (ip-restricted in nginx). */
+export interface StatusEntry {
+    app: string;
+    schedule: string;
+    disabled: boolean;
+    snapshotCount: number;
+    lastSnapshotAt: string | null;
+    totalSize: number | null;
+}
+export interface StatusReport {
+    generatedAt: string;
+    backend: 'rest' | 'sftp';
+    appendOnly: boolean;
+    apps: StatusEntry[];
+}
+export type Health = 'ok' | 'stale' | 'missing' | 'disabled';
+export declare function healthOf(entry: StatusEntry, now?: number): Health;
+export declare function humanBytes(n: number | null): string;
+export declare function relativeTime(iso: string | null, now?: number): string;
+/** renders the full standalone html page. no external assets — inline css so
+ *  it works as a flat file behind nginx. */
+export declare function renderStatusHtml(report: StatusReport, now?: number): string;