@wpelevator/tug 0.6.0 → 0.8.0

package/dist/agent.js

@@ -1,15 +1,16 @@
 #!/usr/bin/env node
 import fs from 'node:fs/promises';
 import { imageLocation } from "./registry.js";
-import { createReadStream, createWriteStream } from 'node:fs';
+import { createReadStream, createWriteStream, existsSync, readFileSync } from 'node:fs';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { validateImageArchive, verifyDownload } from "./artifacts.js";
 import path from 'node:path';
-import { spawnSync } from 'node:child_process';
 import net from 'node:net';
 import dns from 'node:dns/promises';
 import { WebSocket } from 'ws';
+import { agentIdentityCsr, createTransport, describeMissingIdentity, installIdentity, loadAgentDocument, renewCertificate, renewalTime, } from "./agent-credential.js";
+import { clientTlsOptions, parseCredential } from "./pki.js";
 import { createHash } from 'node:crypto';
 import { addressBoundServices, isRolling, serviceSecretVersions } from "./compose.js";
 import { LABEL_PROJECT, LABEL_SERVICE, containerSpecs, networkName, volumeName, } from "./container-spec.js";
@@ -23,24 +24,27 @@ import { openAgentDb } from "./db.js";
 import { createHostRunner } from "./runners.js";
 import { CADDY_CONTAINER, reconcileIngress, removeIngress } from "./ingress.js";
 import { WG_INTERFACE, ensureWgKeys, readReportedKey, reconcileMesh, rotateWgKeys, writeReportedKey, } from "./mesh.js";
-const CONTROL_URL_FROM_ENV = Boolean(process.env.TUG_CONTROL_URL);
-let CONTROL_URL = process.env.TUG_CONTROL_URL || 'http://localhost:8080';
-let TOKEN = process.env.TUG_AGENT_TOKEN;
+// Connect-only override (for example an SSH tunnel); the certificate is still verified against the credential's public URL.
+const CONTROL_URL_OVERRIDE = process.env.TUG_CONTROL_URL;
 let NODE_NAME = process.env.TUG_NODE_NAME || process.env.HOSTNAME || 'node1';
 const STATE_DIR = process.env.TUG_STATE_DIR || '/var/lib/tug';
 const APPLY = process.env.TUG_APPLY !== 'false';
 const INTERVAL = Number(process.env.TUG_POLL_INTERVAL_MS || 5000);
 // Opt-in WireGuard mesh: needs `wg`/`ip` in the agent image, host networking and NET_ADMIN.
 const MESH = process.env.TUG_MESH === 'true';
-export function defaultContext() {
+let activeDocument;
+export function defaultContext(doc = activeDocument) {
+    if (!doc)
+        throw new Error(describeMissingIdentity());
+    const transport = createTransport(doc, CONTROL_URL_OVERRIDE);
     return {
         docker: new DockerEngine(),
-        http: fetch,
-        nodeName: NODE_NAME,
+        http: transport.http,
+        nodeName: doc.nodeName,
         stateDir: STATE_DIR,
         apply: APPLY,
-        controlUrl: CONTROL_URL,
-        token: TOKEN,
+        controlUrl: transport.base,
+        credential: doc,
         host: createHostRunner(),
         mesh: MESH,
         caddyImage: process.env.TUG_CADDY_IMAGE,
@@ -74,18 +78,10 @@ async function agentDb(stateDir) {
     }
     return db;
 }
-function authHeaders(ctx) {
-    if (!ctx.token)
-        throw new Error('TUG_AGENT_TOKEN required');
-    return { Authorization: `Bearer ${ctx.token}` };
-}
 async function api(ctx, route, opts = {}) {
     const res = await ctx.http(`${ctx.controlUrl}${route}`, {
         ...opts,
-        headers: {
-            ...authHeaders(ctx),
-            ...(opts.headers || {}),
-        },
+        headers: opts.headers || {},
     });
     if (!res.ok)
         throw new Error(`${opts.method || 'GET'} ${route}: ${res.status} ${await res.text()}`);
@@ -100,10 +96,15 @@ export class AgentWakeChannel {
     pending = false;
     waiter;
     handlers;
-    constructor(ctx, createSocket = (url, token) => new WebSocket(url, {
-        headers: { Authorization: `Bearer ${token}` },
-        maxPayload: LIVE_MAX_FRAME_BYTES,
-    }), handlers = {}) {
+    constructor(ctx, createSocket = (url, credential) => {
+        if (!credential)
+            throw new Error('agent credential required');
+        return new WebSocket(url, {
+            ...clientTlsOptions(credential),
+            maxPayload: LIVE_MAX_FRAME_BYTES,
+            followRedirects: false,
+        });
+    }, handlers = {}) {
         this.ctx = ctx;
         this.createSocket = createSocket;
         this.handlers = handlers;
@@ -115,13 +116,11 @@ export class AgentWakeChannel {
     async connect() {
         if (this.socket?.readyState === WebSocket.OPEN)
             return true;
-        if (!this.ctx.token)
-            throw new Error('TUG_AGENT_TOKEN required');
         const url = new URL('/agent/v1/stream', this.ctx.controlUrl);
         url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
         return new Promise((resolve) => {
             let settled = false;
-            const socket = this.createSocket(url.toString(), this.ctx.token);
+            const socket = this.createSocket(url.toString(), this.ctx.credential);
             this.socket = socket;
             const finish = (connected) => {
                 if (settled)
@@ -485,6 +484,30 @@ function tcpCheck(host, port, timeoutMs = 2000) {
         s.on('error', reject);
     });
 }
+let containerized;
+/** True when this process runs inside a container: Docker, containerd, Podman or Kubernetes. */
+export function runsInContainer() {
+    containerized ??= (() => {
+        if (process.env.container || existsSync('/.dockerenv'))
+            return true;
+        try {
+            return /docker|containerd|kubepods|libpod/.test(readFileSync('/proc/1/cgroup', 'utf8'));
+        }
+        catch {
+            return false;
+        }
+    })();
+    return containerized;
+}
+/**
+ * Health checks dial the container's address on the project network. An agent that runs as a plain host process
+ * cannot reach it on a machine where Docker lives in a VM (macOS, Windows), and the bare timeout does not say why.
+ */
+export function outsideContainerHint(health, inContainer = runsInContainer()) {
+    if (inContainer || health.type === 'docker')
+        return '';
+    return '; this agent is not running in a container, so it may be unable to reach container addresses (a host process on macOS cannot); run it with `docker run` as `tug node-install` does';
+}
 export async function waitForHealth(ctx, project, container, service, health) {
     if (!ctx.apply || !health)
         return;
@@ -518,7 +541,7 @@ export async function waitForHealth(ctx, project, container, service, health) {
             await new Promise((r) => setTimeout(r, intervalMs));
         }
     }
-    throw new Error(`${service}: ${describeHealthCheck(health)} failed after ${retries} attempt${retries === 1 ? '' : 's'}: ${describeError(last)}`);
+    throw new Error(`${service}: ${describeHealthCheck(health)} failed after ${retries} attempt${retries === 1 ? '' : 's'}: ${describeError(last)}${outsideContainerHint(health)}`);
 }
 /** Node's fetch reports every network failure as "fetch failed"; the system error that explains it is on `cause`. */
 function describeError(error) {
@@ -762,7 +785,7 @@ export async function ensureArtifact(ctx, artifact) {
     const downloadDir = await fs.mkdtemp(path.join(ctx.stateDir, 'image-download-'));
     const file = path.join(downloadDir, 'image.tar.gz');
     try {
-        const response = await ctx.http(`${ctx.controlUrl}/agent/v1/images/${encodeURIComponent(artifact.buildId)}`, { headers: authHeaders(ctx), redirect: 'error', signal: AbortSignal.timeout(10 * 60 * 1000) });
+        const response = await ctx.http(`${ctx.controlUrl}/agent/v1/images/${encodeURIComponent(artifact.buildId)}`, { redirect: 'error', signal: AbortSignal.timeout(10 * 60 * 1000) });
         if (!response.ok || !response.body)
             throw new Error(`image download failed: HTTP ${response.status}`);
         await pipeline(Readable.fromWeb(response.body), verifyDownload(artifact), createWriteStream(file, { mode: 0o600 }));
@@ -1125,7 +1148,12 @@ export async function pollOnce(ctx = defaultContext()) {
     }
     const jobResults = jobs.pending();
     commandResults.push(...jobResults);
-    const report = { node: ctx.nodeName, projects, commandResults };
+    const report = {
+        node: ctx.nodeName,
+        sentAt: new Date().toISOString(),
+        projects,
+        commandResults,
+    };
     const running = jobs.statuses();
     if (running.length)
         report.jobs = running;
@@ -1141,19 +1169,6 @@ export async function pollOnce(ctx = defaultContext()) {
     await jobs.acknowledge(jobResults);
     return projects;
 }
-const CREDS_FILE = () => path.join(STATE_DIR, 'agent.json');
-async function loadCreds() {
-    try {
-        return JSON.parse(await fs.readFile(CREDS_FILE(), 'utf8'));
-    }
-    catch {
-        return null;
-    }
-}
-async function saveCreds(creds) {
-    await ensureDir(STATE_DIR);
-    await fs.writeFile(CREDS_FILE(), JSON.stringify(creds), { mode: 0o600 });
-}
 async function readJoinKey() {
     if (process.env.TUG_JOIN_KEY)
         return process.env.TUG_JOIN_KEY.trim();
@@ -1168,84 +1183,90 @@ async function readJoinKey() {
     }
     return null;
 }
+/** Enrolls with a single-use bootstrap certificate; a retry reuses the pending key, which control accepts by key. */
 async function enroll(joinKey) {
     let decoded;
     try {
         decoded = JSON.parse(Buffer.from(joinKey, 'base64url').toString('utf8'));
+        if (decoded.v !== 2)
+            throw new Error('unsupported join key version');
+        parseCredential(decoded.credential);
     }
     catch (e) {
-        throw new Error('invalid TUG_JOIN_KEY (expected base64url-encoded JSON)', { cause: e });
+        throw new Error('invalid join key (expected a base64url-encoded v2 join key)', { cause: e });
     }
-    if (!decoded.enrollToken)
-        throw new Error('join key is missing enrollToken');
-    const controlUrl = !CONTROL_URL_FROM_ENV && decoded.controlUrl ? decoded.controlUrl : CONTROL_URL;
-    // node's fetch reads NODE_EXTRA_CA_CERTS only at startup, so apply an embedded CA by
-    // persisting it and re-execing ourselves once with the env var set.
-    if (decoded.ca && !process.env.NODE_EXTRA_CA_CERTS) {
-        await ensureDir(STATE_DIR);
-        const caPath = path.join(STATE_DIR, 'control-ca.pem');
-        await fs.writeFile(caPath, decoded.ca);
-        const r = spawnSync(process.execPath, process.argv.slice(1), {
-            stdio: 'inherit',
-            env: { ...process.env, NODE_EXTRA_CA_CERTS: caPath },
-        });
-        process.exit(r.status ?? 1);
-    }
-    console.log(`enrolling with ${controlUrl} as ${NODE_NAME}`);
+    const transport = createTransport(decoded.credential, CONTROL_URL_OVERRIDE || decoded.connectUrl);
+    console.log(`enrolling with ${decoded.credential.url} as ${decoded.name}`);
+    const csr = await agentIdentityCsr(STATE_DIR);
     const wgPublicKey = MESH ? (await ensureWgKeys(STATE_DIR)).publicKey : undefined;
-    const res = await fetch(`${controlUrl}/agent/v1/enroll`, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-            enrollToken: decoded.enrollToken,
-            name: NODE_NAME,
-            address: process.env.TUG_NODE_ADDRESS,
-            publicAddress: process.env.TUG_NODE_PUBLIC_ADDRESS,
-            wgPublicKey,
-            labels: decoded.labels ||
-                (process.env.TUG_LABELS
-                    ? process.env.TUG_LABELS.split(',')
-                        .map((s) => s.trim())
-                        .filter(Boolean)
-                    : undefined),
-        }),
-    });
-    if (!res.ok)
-        throw new Error(`enroll failed: ${res.status} ${await res.text()}`);
-    const out = (await res.json());
-    const creds = {
-        agentToken: out.agentToken,
-        nodeName: out.node?.name || NODE_NAME,
-        controlUrl,
+    const body = {
+        csr,
+        address: process.env.TUG_NODE_ADDRESS,
+        publicAddress: process.env.TUG_NODE_PUBLIC_ADDRESS,
+        wgPublicKey,
     };
-    await saveCreds(creds);
-    if (wgPublicKey)
-        await writeReportedKey(STATE_DIR, wgPublicKey);
-    console.log(`enrolled as node ${creds.nodeName}`);
-    return creds;
+    try {
+        const res = await transport.http(`${transport.base}/agent/v1/enroll`, {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify(body),
+        });
+        if (!res.ok)
+            throw new Error(`enroll failed: ${res.status} ${await res.text()}`);
+        const doc = await installIdentity(STATE_DIR, (await res.json()));
+        if (wgPublicKey)
+            await writeReportedKey(STATE_DIR, wgPublicKey);
+        console.log(`enrolled as node ${doc.nodeName}`);
+        return doc;
+    }
+    finally {
+        void transport.close();
+    }
 }
-// Establish durable credentials before polling: explicit env token (SSH/push install) wins,
-// then credentials persisted on the state volume, then a one-time enroll via the join key.
+/** Loads the persisted identity, or enrolls with a join key. Without either the agent refuses to start. */
 export async function loadOrEnroll() {
-    let creds = TOKEN
-        ? { agentToken: TOKEN, nodeName: NODE_NAME, controlUrl: CONTROL_URL }
-        : await loadCreds();
-    if (!creds) {
+    let doc = await loadAgentDocument(STATE_DIR);
+    if (!doc) {
         const joinKey = await readJoinKey();
         if (!joinKey)
-            throw new Error('no TUG_AGENT_TOKEN, no persisted credentials, and no TUG_JOIN_KEY to enroll with');
-        creds = await enroll(joinKey);
+            throw new Error(describeMissingIdentity());
+        doc = await enroll(joinKey);
     }
-    TOKEN = creds.agentToken;
-    NODE_NAME = creds.nodeName || NODE_NAME;
-    if (!CONTROL_URL_FROM_ENV && creds.controlUrl)
-        CONTROL_URL = creds.controlUrl;
-    return creds;
+    NODE_NAME = doc.nodeName;
+    activeDocument = doc;
+    return doc;
 }
+const RENEW_RETRY_MS = 15 * 60_000;
 export async function main() {
-    await loadOrEnroll();
-    console.log(`tug-agent node=${NODE_NAME} control=${CONTROL_URL} apply=${APPLY}`);
-    const ctx = defaultContext();
+    const doc = await loadOrEnroll();
+    const ctx = defaultContext(doc);
+    console.log(`tug-agent node=${NODE_NAME} control=${ctx.controlUrl} apply=${APPLY}`);
+    let renewAt = renewalTime(doc.cert);
+    /** Renews with the same key, saves, then swaps the client and reconnects. Failure keeps the old identity until it expires. */
+    const renewIfDue = async () => {
+        if (Date.now() < renewAt || !ctx.credential)
+            return;
+        try {
+            const current = ctx.credential;
+            const next = await renewCertificate(STATE_DIR, current, (csr) => api(ctx, '/agent/v1/renew', {
+                method: 'POST',
+                headers: { 'content-type': 'application/json' },
+                body: JSON.stringify({ csr }),
+            }));
+            const transport = createTransport(next, CONTROL_URL_OVERRIDE);
+            const previous = ctx.http;
+            ctx.http = transport.http;
+            ctx.credential = next;
+            void previous;
+            renewAt = renewalTime(next.cert);
+            wake.close();
+            console.log('node certificate renewed');
+        }
+        catch (e) {
+            renewAt = Date.now() + RENEW_RETRY_MS;
+            console.error(`certificate renewal failed (keeping the current one): ${errMessage(e)}`);
+        }
+    };
     const wake = new AgentWakeChannel(ctx, undefined, {
         onLive: (frame) => live.handle(frame),
         onDisconnect: () => live.dispose(),
@@ -1254,6 +1275,7 @@ export async function main() {
         wake.send(frame);
     });
     for (;;) {
+        await renewIfDue();
         try {
             await pollOnce(ctx);
         }

package/dist/backup-check.d.ts

@@ -0,0 +1,11 @@
+/** The part of `GET /api/v1/backups` that `tug backup-list --check` judges. */
+export interface BackupCheckInput {
+    destinationConfigured: boolean;
+    targets: {
+        target: string;
+        enabled: boolean;
+        problem?: string | null;
+    }[];
+}
+/** What a monitor should be told: one line per problem, empty when every backup target is healthy. */
+export declare function backupCheckProblems(listing: BackupCheckInput): string[];

package/dist/backup-check.js

@@ -0,0 +1,10 @@
+/** What a monitor should be told: one line per problem, empty when every backup target is healthy. */
+export function backupCheckProblems(listing) {
+    const active = listing.targets.filter((target) => target.enabled);
+    const problems = active
+        .filter((target) => target.problem)
+        .map((target) => `${target.target}: ${target.problem}`);
+    if (active.length && !listing.destinationConfigured)
+        problems.unshift('no backup destination is configured');
+    return problems;
+}

package/dist/backup-config.d.ts

@@ -11,6 +11,8 @@ export interface ResolvedBackup {
     retention: BackupRetention;
     jitterSeconds: number;
     timeoutSec: number;
+    /** Memory limit of the restic helper container. */
+    memory: string;
     /** Service targets: the command run in the service's container, and the snapshot file it is stored as. */
     command?: string[];
     /** Run in the service container to restore; the stored file arrives on its standard input. */
@@ -26,8 +28,11 @@ export interface ResolvedBackup {
 }
 export declare const DEFAULT_RETENTION: BackupRetention;
 export declare const DEFAULT_BACKUP_TIMEOUT_SEC: number;
+export declare const DEFAULT_BACKUP_MEMORY = "512m";
 /** `30m`, `2h`, `90s` or a number of seconds. */
 export declare function parseBackupTimeout(value: unknown): number | null;
+/** Bytes in a Docker-style memory size (`256m`, `1g`, `1048576`), or null when it is not one. */
+export declare function parseMemorySize(value: unknown): number | null;
 /** Deterministic delay in `[0, 900)` seconds so many sites on one node do not start together. */
 export declare function defaultJitterSeconds(project: string, target: string): number;
 /** Validates the project-level defaults in `x-tug.backup`. */

package/dist/backup-config.js

@@ -2,11 +2,15 @@ import { createHash } from 'node:crypto';
 import { parseCron } from "./cron.js";
 export const DEFAULT_RETENTION = { daily: 7, weekly: 4, monthly: 6 };
 export const DEFAULT_BACKUP_TIMEOUT_SEC = 2 * 3600;
+export const DEFAULT_BACKUP_MEMORY = '512m';
+/** Below this restic runs out of memory on any repository worth backing up. */
+const MIN_MEMORY_BYTES = 128 * 1024 ** 2;
+const MAX_MEMORY_BYTES = 16 * 1024 ** 3;
 const MIN_TIMEOUT_SEC = 60;
 const MAX_TIMEOUT_SEC = 24 * 3600;
 const MAX_JITTER_SEC = 3600;
 const RETENTION_KEYS = ['last', 'hourly', 'daily', 'weekly', 'monthly', 'yearly'];
-const COMMON_KEYS = ['schedule', 'retention', 'jitter', 'timeout', 'writers'];
+const COMMON_KEYS = ['schedule', 'retention', 'jitter', 'timeout', 'memory', 'writers'];
 const SERVICE_KEYS = [...COMMON_KEYS, 'command', 'restore', 'filename', 'verify'];
 const VOLUME_KEYS = [...COMMON_KEYS, 'exclude'];
 const FILENAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
@@ -20,6 +24,14 @@ export function parseBackupTimeout(value) {
         return null;
     return Number(match[1]) * { s: 1, m: 60, h: 3600 }[match[2]];
 }
+/** Bytes in a Docker-style memory size (`256m`, `1g`, `1048576`), or null when it is not one. */
+export function parseMemorySize(value) {
+    const match = typeof value === 'string' ? /^(\d+)([kmg]?)$/i.exec(value.trim()) : null;
+    if (!match)
+        return null;
+    return (Number(match[1]) *
+        { '': 1, k: 1024, m: 1024 ** 2, g: 1024 ** 3 }[match[2].toLowerCase()]);
+}
 /** Deterministic delay in `[0, 900)` seconds so many sites on one node do not start together. */
 export function defaultJitterSeconds(project, target) {
     const digest = createHash('sha256').update(`${project}\0${target}`).digest();
@@ -98,7 +110,7 @@ function serviceNodes(svc) {
     return {
         nodes: placement.nodes || [],
         dynamic: Array.isArray(placement.labels) && placement.labels.length > 0,
-        global: lp.global === true || placement.global === true,
+        global: lp.global === true,
     };
 }
 function mountedVolumes(svc) {
@@ -152,6 +164,14 @@ export function resolveBackups(compose, project = '') {
             else
                 timeoutSec = parsed;
         }
+        let memory = DEFAULT_BACKUP_MEMORY;
+        if (spec.memory !== undefined) {
+            const bytes = parseMemorySize(spec.memory);
+            if (bytes === null || bytes < MIN_MEMORY_BYTES || bytes > MAX_MEMORY_BYTES)
+                errors.push(`${label}.memory must be a size from 128m to 16g such as 256m or 1g`);
+            else
+                memory = String(spec.memory).trim().toLowerCase();
+        }
         if (spec.writers !== undefined) {
             if (!Array.isArray(spec.writers) || spec.writers.some((w) => typeof w !== 'string'))
                 errors.push(`${label}.writers must be a list of service names`);
@@ -162,7 +182,7 @@ export function resolveBackups(compose, project = '') {
         }
         if (errors.length > before || !schedule || !retention)
             return null;
-        return { schedule, retention, jitterSeconds, timeoutSec };
+        return { schedule, retention, jitterSeconds, timeoutSec, memory };
     };
     for (const [name, svc] of Object.entries(compose.services)) {
         const config = svc['x-tug']?.backup;

package/dist/backup-destination.d.ts

@@ -15,7 +15,7 @@ export declare function projectRepository(destinationUrl: string, project: strin
 /**
  * The repository password of a project, derived from the control master key so nothing extra is stored and
  * a node only ever holds the passwords of projects it runs backups for. Losing the master key loses every
- * repository, which is why `tug backup-key` exists for offline escrow.
+ * repository, which is why `tug backup-repository-password` exists for offline escrow.
  */
 export declare function repositoryPassword(masterKey: string, project: string): string;
 /** Removes every known secret value from text that is about to be stored, shown or put in an event. */

package/dist/backup-destination.js

@@ -54,7 +54,7 @@ export function projectRepository(destinationUrl, project) {
 /**
  * The repository password of a project, derived from the control master key so nothing extra is stored and
  * a node only ever holds the passwords of projects it runs backups for. Losing the master key loses every
- * repository, which is why `tug backup-key` exists for offline escrow.
+ * repository, which is why `tug backup-repository-password` exists for offline escrow.
  */
 export function repositoryPassword(masterKey, project) {
     return createHmac('sha256', masterKey).update(`tug-restic:${project}`).digest('hex');

package/dist/backup-restore.js

@@ -3,6 +3,10 @@ import { volumeName } from "./container-spec.js";
 const MINUTE = 60_000;
 const LOCK_TTL_MS = 15 * MINUTE;
 const HOLD_TIMEOUT_MS = 5 * MINUTE;
+/** How long a finished restore waits for its writers to serve again before it reports that they did not. */
+const VERIFY_TIMEOUT_MS = 5 * MINUTE;
+/** Restore states in which the project must stay locked. */
+const LOCKED_STATES = ['locking', 'holding', 'safety_backup', 'restoring', 'interrupted'];
 const TERMINAL = ['succeeded', 'failed', 'released'];
 /**
  * The services that write to a target. For a service they are the services that depend on it, transitively;
@@ -59,8 +63,9 @@ export function createRestoreService(deps) {
         if (!latest && !from.startsWith('bkr_'))
             return { snapshotId: from, time: null };
         const run = latest
-            ? db.get(`SELECT * FROM backup_runs WHERE project_id=? AND target=? AND state='succeeded' AND snapshot_id IS NOT NULL
-           ORDER BY finished_at DESC LIMIT 1`, project, target)
+            ? // A safety snapshot records the state just before a restore, often a damaged one, so it is never "the latest backup".
+                db.get(`SELECT * FROM backup_runs WHERE project_id=? AND target=? AND state='succeeded' AND snapshot_id IS NOT NULL
+           AND trigger != 'pre-restore' ORDER BY finished_at DESC LIMIT 1`, project, target)
             : db.get("SELECT * FROM backup_runs WHERE id=? AND project_id=? AND target=? AND state='succeeded' AND snapshot_id IS NOT NULL", from, project, target);
         if (!run?.snapshot_id)
             return {
@@ -170,7 +175,8 @@ export function createRestoreService(deps) {
             ...(restorePlan.scratchVolume ? { scratchVolume: restorePlan.scratchVolume } : {}),
         };
     }
-    const heldOnNodes = (restore, writers) => {
+    /** True when every writer reports `rolloutState` on every node it is placed on. */
+    const writersReport = (restore, writers, rolloutState) => {
         const desired = deployed(restore.project_id);
         if (!desired)
             return false;
@@ -179,7 +185,7 @@ export function createRestoreService(deps) {
             return nodes.every((node) => {
                 const row = db.get('SELECT s.* FROM node_statuses s JOIN nodes n ON n.id=s.node_id WHERE n.name=? AND s.project_id=?', node, restore.project_id);
                 const status = row ? JSON.parse(row.status_json) : undefined;
-                return status?.serviceStates?.[service]?.rolloutState === 'held';
+                return status?.serviceStates?.[service]?.rolloutState === rolloutState;
             });
         });
     };
@@ -210,12 +216,12 @@ export function createRestoreService(deps) {
             const age = clock().getTime() -
                 new Date(restore.state_changed_at ?? restore.started_at ?? stamp()).getTime();
             // Keep the project locked while a restore is unfinished, across a control restart.
-            if (restore.mode === 'in-place')
+            if (restore.mode === 'in-place' && LOCKED_STATES.includes(restore.state))
                 lock(restore);
             const writers = JSON.parse(restore.writers_json);
             switch (restore.state) {
                 case 'holding': {
-                    if (!heldOnNodes(restore, writers)) {
+                    if (!writersReport(restore, writers, 'held')) {
                         if (age > HOLD_TIMEOUT_MS)
                             fail(restore, 'the writers did not stop in time', false);
                         break;
@@ -247,8 +253,11 @@ export function createRestoreService(deps) {
                     if (command.state === 'succeeded') {
                         if (restore.mode === 'in-place')
                             release(restore);
-                        move(restore, 'succeeded');
-                        event('info', 'restore.completed', restore, `restore of ${restore.target} completed${restore.scratch_volume ? ` into volume ${restore.scratch_volume}` : ''}`, { scratchVolume: restore.scratch_volume });
+                        // The data is back; the restore is done when the services that were stopped are serving again.
+                        if (restore.mode === 'in-place' && writers.length)
+                            move(restore, 'verifying');
+                        else
+                            complete(restore);
                     }
                     else {
                         const error = JSON.parse(command.result_json ?? '{}').error ??
@@ -257,10 +266,21 @@ export function createRestoreService(deps) {
                     }
                     break;
                 }
+                case 'verifying': {
+                    if (writersReport(restore, writers, 'serving'))
+                        complete(restore);
+                    else if (age > VERIFY_TIMEOUT_MS)
+                        complete(restore, `the data was restored, but ${writers.join(', ')} did not report serving within ${VERIFY_TIMEOUT_MS / MINUTE} minutes; check tug status`);
+                    break;
+                }
                 default:
             }
         }
     }
+    function complete(restore, warning) {
+        move(restore, 'succeeded', warning ? { error: warning } : {});
+        event(warning ? 'warn' : 'info', 'restore.completed', restore, `restore of ${restore.target} completed${restore.scratch_volume ? ` into volume ${restore.scratch_volume}` : ''}${warning ? `, but ${warning}` : ''}`, { scratchVolume: restore.scratch_volume, ...(warning ? { warning } : {}) });
+    }
     /** Drops the holds and the lock of an interrupted restore; the data may be partly restored. */
     function releaseInterrupted(restoreId) {
         const restore = db.get('SELECT * FROM restore_runs WHERE id=?', restoreId);

package/dist/backup-scheduler.d.ts

@@ -1,6 +1,6 @@
 import type { ResolvedBackup } from './backup-config.ts';
 import type { Db } from './db.ts';
-import type { BackupRunRow } from './db.ts';
+import type { BackupRunRow, BackupScheduleRow } from './db.ts';
 import type { BackupJobPayload, CommandResultPayload, CommandType, DesiredState, JobStatus } from './protocol.ts';
 /** Pinned by tag; override on control with `TUG_RESTIC_IMAGE` (a digest reference is recommended for production). */
 export declare const DEFAULT_RESTIC_IMAGE = "restic/restic:0.17.3";
@@ -25,7 +25,13 @@ export interface TargetStatus {
     lastRunAt: string | null;
     lastSuccessAt: string | null;
     lastStatus: string | null;
+    /** Why this target needs attention (stale, last run failed or missed), or null when it is healthy. */
+    problem: string | null;
 }
+/** A target that has not succeeded for two periods is stale, even when every run failed alike. */
+export declare function isStale(schedule: Pick<BackupScheduleRow, 'schedule' | 'last_success_at' | 'created_at'>, at: Date): boolean;
+/** What a monitor should know about a target: null when it is healthy or switched off. */
+export declare function backupProblem(row: BackupScheduleRow, at: Date): string | null;
 /** Where a backup target runs, from the resolved desired state (labels are already resolved there). */
 export declare function targetNode(target: ResolvedBackup, desired: DesiredState): string | undefined;
 export declare function createBackupService(deps: BackupServiceDeps): {

package/dist/backup-scheduler.js

@@ -14,6 +14,24 @@ const MAX_DEADLINE_MS = 6 * 3600_000;
 const MANUAL_DEADLINE_MS = 30 * MINUTE;
 const STALE_REPEAT_MS = 24 * 3600_000;
 const MAX_CATCH_UP_SLOTS = 20_000;
+/** A target that has not succeeded for two periods is stale, even when every run failed alike. */
+export function isStale(schedule, at) {
+    const period = cronPeriodMs(schedule.schedule, at);
+    if (!period)
+        return false;
+    const since = new Date(schedule.last_success_at ?? schedule.created_at ?? at.toISOString()).getTime();
+    return at.getTime() - since > 2 * period + 10 * MINUTE;
+}
+/** What a monitor should know about a target: null when it is healthy or switched off. */
+export function backupProblem(row, at) {
+    if (!row.enabled)
+        return null;
+    if (isStale(row, at))
+        return `no successful backup since ${row.last_success_at ?? 'it was configured'}`;
+    if (row.last_status === 'failed' || row.last_status === 'missed')
+        return `the last run ${row.last_status}`;
+    return null;
+}
 function parse(json, fallback) {
     try {
         return json ? JSON.parse(json) : fallback;
@@ -113,7 +131,7 @@ export function createBackupService(deps) {
                 `trigger:${trigger}`,
             ],
             retention: config.retention,
-            limits: { memory: DEFAULT_MEMORY, timeoutSec: config.timeoutSec },
+            limits: { memory: config.memory ?? DEFAULT_MEMORY, timeoutSec: config.timeoutSec },
         };
         if (config.kind === 'service') {
             payload.command = {
@@ -283,11 +301,7 @@ export function createBackupService(deps) {
         }
         // Independent of single runs: a target that has not succeeded for two periods is stale, even when each run failed alike.
         for (const schedule of db.all('SELECT * FROM backup_schedules WHERE enabled=1')) {
-            const period = cronPeriodMs(schedule.schedule, at);
-            if (!period)
-                continue;
-            const since = new Date(schedule.last_success_at ?? schedule.created_at ?? iso).getTime();
-            if (at.getTime() - since <= 2 * period + 10 * MINUTE)
+            if (!isStale(schedule, at))
                 continue;
             if (schedule.stale_notified_at &&
                 at.getTime() - new Date(schedule.stale_notified_at).getTime() < STALE_REPEAT_MS)
@@ -376,6 +390,7 @@ export function createBackupService(deps) {
             lastRunAt: row.last_run_at,
             lastSuccessAt: row.last_success_at,
             lastStatus: row.last_status,
+            problem: backupProblem(row, clock()),
         }));
         const runs = db.all('SELECT * FROM backup_runs WHERE project_id=? ORDER BY COALESCE(finished_at,queued_at) DESC LIMIT ?', project, limit);
         return { targets, runs };