@wpelevator/tug 0.5.0 → 0.6.0

package/dist/control.js

@@ -21,7 +21,10 @@ import { sentinelNames } from "./normalize.js";
 import { ensureDir, errMessage, hashToken, id, now, token } from "./util.js";
 import { DEFAULT_MESH_CIDR, DEFAULT_WG_PORT, WG_INTERFACE, WG_KEEPALIVE, allocateOverlayIp, isWgKey, parseCidr, wgEndpoint, } from "./mesh.js";
 import { buildIngressWork, resolveServiceEndpoints } from "./ingress.js";
-import { ACTIVE_DEPLOYMENT_STATES, LIVE_MAX_FRAME_BYTES, LIVE_MAX_BUFFERED_BYTES, LIVE_MAX_SESSIONS, LIVE_START_TIMEOUT_MS, } from "./protocol.js";
+import { ACTIVE_DEPLOYMENT_STATES, LIVE_MAX_FRAME_BYTES, LIVE_MAX_BUFFERED_BYTES, LIVE_MAX_SESSIONS, LIVE_START_TIMEOUT_MS, BACKUP_COMMAND_TYPES, } from "./protocol.js";
+import { BACKUP_SECRET_PROJECT, isReservedProject, parseBackupDestination, projectRepository, repositoryPassword, redactSecrets, } from "./backup-destination.js";
+import { createBackupService } from "./backup-scheduler.js";
+import { createRestoreService } from "./backup-restore.js";
 import { AUTO_ROLLBACK_ACTOR, ROUTE_ROLES, PROJECT_NAME_PATTERN, authenticate, hasRole, assertProject, bodyProject, createGuard, createToken, listTokens, parseTokenRequest, queryProject, recordEvent, revokeToken, } from "./tokens.js";
 class WakeHub {
     cursors = new Map();
@@ -119,7 +122,18 @@ function parseLiveRequest(value) {
         Number(value.tail) <= 10_000)
         return value;
     if (value.operation === 'exec' &&
-        frameKeys(value, [...common, 'service', 'command', 'interactive', 'tty', 'columns', 'rows']) &&
+        frameKeys(value, [
+            ...common,
+            'service',
+            'command',
+            'interactive',
+            'tty',
+            'columns',
+            'rows',
+            'oneOff',
+        ]) &&
+        (value.oneOff === undefined || value.oneOff === true) &&
+        (value.oneOff !== true || (value.interactive === false && value.tty === false)) &&
         Array.isArray(value.command) &&
         value.command.length > 0 &&
         value.command.length <= 256 &&
@@ -705,6 +719,17 @@ export async function createApp(options = {}) {
         const node = nodes.find((node) => node.name === (request.node || [...eligible][0]));
         if (!node)
             throw httpError(404, 'no eligible node for this operation');
+        if (request.operation === 'exec' && request.oneOff)
+            audit(actor, {
+                type: 'job.run',
+                project: request.project,
+                message: `${actor.name} ran a one-off ${request.service} container on ${node.name}: ${request.command.join(' ').slice(0, 200)}`,
+                metadata: {
+                    service: request.service,
+                    node: node.name,
+                    command: request.command.slice(0, 64),
+                },
+            });
         return node;
     }, options.liveStartTimeoutMs);
     streamRuntimes.set(app, {
@@ -958,10 +983,10 @@ export async function createApp(options = {}) {
     function findNode(nameOrId) {
         return db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', nameOrId, nameOrId);
     }
-    function queueCommand(nodeId, projectId, type, payload = {}) {
+    function queueCommand(nodeId, projectId, type, payload = {}, deadline = null) {
         const commandId = id('cmd');
         const ts = now();
-        db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', commandId, nodeId, projectId, type, 'pending', JSON.stringify(payload), 0, ts, null, ts, ts);
+        db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', commandId, nodeId, projectId, type, 'pending', JSON.stringify(payload), 0, ts, deadline, ts, ts);
         wakeHub.wake(nodeId, projectId ?? undefined);
         return commandId;
     }
@@ -1060,8 +1085,188 @@ export async function createApp(options = {}) {
         });
         res.json({ ok: true, name: req.params.name });
     });
+    const backupDestination = () => db.get("SELECT * FROM backup_destination WHERE id='default'");
+    const backupEnv = () => {
+        const row = backupDestination();
+        if (!row)
+            return {};
+        const env = {};
+        for (const name of JSON.parse(row.env_names_json)) {
+            const secret = db.get('SELECT encrypted_value FROM secrets WHERE project_id=? AND name=?', BACKUP_SECRET_PROJECT, name);
+            if (secret)
+                env[name] = decrypt(secret.encrypted_value, masterKey);
+        }
+        return env;
+    };
+    const backups = createBackupService({
+        db,
+        queueCommand,
+        resticImage: process.env.TUG_RESTIC_IMAGE,
+        // Defensive: the agent already redacts, but a stored error must never carry a destination credential.
+        redact: (text) => redactSecrets(text, Object.values(backupEnv())),
+    });
+    app.locals.backups = backups;
+    const restores = createRestoreService({ db, backups, wake: (project) => wakeAll(project) });
+    app.locals.restores = restores;
+    const backupTickMs = Number(process.env.TUG_BACKUP_TICK_MS) || 30_000;
+    const backupTimer = setInterval(() => {
+        try {
+            backups.tick();
+            restores.advance();
+        }
+        catch (e) {
+            console.error(`backup scheduler failed: ${errMessage(e)}`);
+        }
+    }, backupTickMs);
+    backupTimer.unref();
+    try {
+        backups.tick(); // catch up on slots missed while control was down
+        restores.advance(); // re-adopt unfinished restores and keep their projects locked
+    }
+    catch (e) {
+        console.error(`backup scheduler failed: ${errMessage(e)}`);
+    }
+    api('post', '/api/v1/backups/run', bodyProject, async (req, res) => {
+        const project = String(req.body.project || 'site');
+        const targets = req.body.targets;
+        if (targets !== undefined &&
+            (!Array.isArray(targets) || targets.some((t) => typeof t !== 'string')))
+            return res.status(400).json({ error: 'targets must be a list of service or volume names' });
+        if (!db.get('SELECT 1 FROM projects WHERE id=?', project))
+            return res.status(404).json({ error: `project ${project} not found` });
+        const result = backups.runNow(project, targets, req.actor.name);
+        if (!result.runs.length)
+            return res.status(result.errors.length ? 409 : 404).json({
+                error: result.errors.join('; ') || `project ${project} has no backup targets`,
+            });
+        res.json(result);
+    });
+    api('post', '/api/v1/restores', bodyProject, async (req, res) => {
+        const body = req.body ?? {};
+        const project = String(body.project || 'site');
+        if (typeof body.target !== 'string' || !body.target)
+            return res
+                .status(400)
+                .json({ error: 'target is required, such as service:db or volume:uploads' });
+        const target = body.target.includes(':')
+            ? body.target
+            : (db.get('SELECT target FROM backup_schedules WHERE project_id=? AND enabled=1 AND target LIKE ?', project, `%:${body.target}`)?.target ?? body.target);
+        const request = {
+            project,
+            target,
+            from: typeof body.from === 'string' ? body.from : undefined,
+            scratch: body.scratch === true,
+            safetyBackup: body.safetyBackup !== false,
+            writers: Array.isArray(body.writers) ? body.writers.map(String) : undefined,
+            noWriters: body.noWriters === true,
+            actor: req.actor.name,
+        };
+        const plan = restores.plan(request);
+        if ('error' in plan)
+            return res.status(plan.status).json({ error: plan.error });
+        if (body.confirm !== true)
+            return res.json({ confirmed: false, plan });
+        const started = restores.start(request, plan);
+        if ('error' in started)
+            return res.status(started.status).json({ error: started.error });
+        res.json({ confirmed: true, plan, ...started });
+    });
+    api('get', '/api/v1/restores', queryProject, async (req, res) => res.json({ restores: restores.list(String(req.query.project || 'site')) }));
+    api('post', '/api/v1/restores/:id/release', null, async (req, res) => {
+        const result = restores.releaseInterrupted(req.params.id);
+        if ('error' in result)
+            return res.status(result.status).json({ error: result.error });
+        res.json(result);
+    });
+    api('post', '/api/v1/restores/:id/resume', null, async (req, res) => {
+        const result = restores.resume(req.params.id);
+        if ('error' in result)
+            return res.status(result.status).json({ error: result.error });
+        res.json(result);
+    });
+    api('get', '/api/v1/backups', queryProject, async (req, res) => {
+        const project = String(req.query.project || 'site');
+        const limit = Math.min(Math.max(Number(req.query.limit) || 50, 1), 500);
+        res.json({
+            project,
+            destinationConfigured: Boolean(backupDestination()),
+            ...backups.list(project, limit),
+        });
+    });
+    api('get', '/api/v1/backup/destination', null, async (_req, res) => {
+        const row = backupDestination();
+        res.json(row
+            ? {
+                configured: true,
+                url: row.url,
+                envNames: JSON.parse(row.env_names_json),
+                checkedAt: row.checked_at,
+                checkResult: row.check_result_json ? JSON.parse(row.check_result_json) : null,
+                updatedAt: row.updated_at,
+            }
+            : { configured: false });
+    });
+    api('post', '/api/v1/backup/destination', null, async (req, res) => {
+        let parsed;
+        try {
+            parsed = parseBackupDestination(req.body);
+        }
+        catch (e) {
+            return res.status(400).json({ error: errMessage(e) });
+        }
+        const ts = now();
+        db.exec('BEGIN IMMEDIATE');
+        try {
+            db.run('DELETE FROM secrets WHERE project_id=?', BACKUP_SECRET_PROJECT);
+            for (const [name, value] of Object.entries(parsed.env))
+                db.run('INSERT INTO secrets(id,project_id,name,encrypted_value,version,created_at,updated_at) VALUES(?,?,?,?,?,?,?)', id('sec'), BACKUP_SECRET_PROJECT, name, encrypt(value, masterKey), 1, ts, ts);
+            db.run(`INSERT INTO backup_destination(id,url,env_names_json,checked_at,check_result_json,updated_at) VALUES('default',?,?,NULL,NULL,?)
+        ON CONFLICT(id) DO UPDATE SET url=excluded.url,env_names_json=excluded.env_names_json,checked_at=NULL,check_result_json=NULL,updated_at=excluded.updated_at`, parsed.url, JSON.stringify(Object.keys(parsed.env).sort()), ts);
+            db.exec('COMMIT');
+        }
+        catch (e) {
+            db.exec('ROLLBACK');
+            throw e;
+        }
+        audit(req.actor, {
+            type: 'backup.destination_set',
+            message: `backup destination set to ${parsed.url} by ${req.actor.name}`,
+            metadata: { url: parsed.url, envNames: Object.keys(parsed.env).sort() },
+        });
+        res.json({ ok: true, url: parsed.url, envNames: Object.keys(parsed.env).sort() });
+    });
+    api('delete', '/api/v1/backup/destination', null, async (req, res) => {
+        db.run('DELETE FROM secrets WHERE project_id=?', BACKUP_SECRET_PROJECT);
+        const removed = db.run("DELETE FROM backup_destination WHERE id='default'");
+        if (removed.changes)
+            audit(req.actor, {
+                type: 'backup.destination_removed',
+                message: `backup destination removed by ${req.actor.name}`,
+            });
+        res.json({ ok: true, removed: removed.changes > 0 });
+    });
+    api('get', '/api/v1/backup/key', null, async (req, res) => {
+        const project = String(req.query.project || '');
+        if (!PROJECT_NAME_PATTERN.test(project) || isReservedProject(project))
+            return res.status(400).json({ error: 'project is required' });
+        if (!db.get('SELECT 1 FROM projects WHERE id=?', project))
+            return res.status(404).json({ error: `project ${project} not found` });
+        const destination = backupDestination();
+        audit(req.actor, {
+            type: 'backup.key_read',
+            project,
+            message: `repository password for ${project} read by ${req.actor.name}`,
+        });
+        res.json({
+            project,
+            repository: destination ? projectRepository(destination.url, project) : null,
+            password: repositoryPassword(masterKey, project),
+        });
+    });
     api('post', '/api/v1/secrets', bodyProject, async (req, res) => {
         const { project = 'site', name, value } = req.body;
+        if (isReservedProject(String(project)))
+            return res.status(400).json({ error: `project names starting with _ are reserved` });
         if (!name || value == null)
             return res.status(400).json({ error: 'name and value required' });
         let lock;
@@ -1087,11 +1292,15 @@ export async function createApp(options = {}) {
             releaseLock(lock);
         }
     });
-    api('get', '/api/v1/secrets', queryProject, async (req, res) => res.json({
-        secrets: db.all('SELECT project_id,name,version,updated_at FROM secrets WHERE project_id=? ORDER BY name', String(req.query.project || 'site')),
-    }));
+    api('get', '/api/v1/secrets', queryProject, async (req, res) => isReservedProject(String(req.query.project || 'site'))
+        ? res.status(400).json({ error: 'project names starting with _ are reserved' })
+        : res.json({
+            secrets: db.all('SELECT project_id,name,version,updated_at FROM secrets WHERE project_id=? ORDER BY name', String(req.query.project || 'site')),
+        }));
     api('delete', '/api/v1/secrets/:name', queryProject, async (req, res) => {
         const project = String(req.query.project || 'site');
+        if (isReservedProject(project))
+            return res.status(400).json({ error: 'project names starting with _ are reserved' });
         const removed = db.run('DELETE FROM secrets WHERE project_id=? AND name=?', project, req.params.name);
         if (removed.changes) {
             wakeAll(project);
@@ -1208,18 +1417,6 @@ export async function createApp(options = {}) {
                 }
             }
             const nodes = nodeRows.map((row) => ({ name: row.name }));
-            for (const vol of Object.values(compose.volumes || {})) {
-                const xe = vol?.['x-tug'];
-                if (!xe ||
-                    (xe.type !== 'nfs' && xe.driver !== 'nfs') ||
-                    xe.address ||
-                    !xe.placement?.nodes?.[0])
-                    continue;
-                const exporter = db.get('SELECT address,overlay_ip FROM nodes WHERE name=?', xe.placement.nodes[0]);
-                const addr = (MESH && exporter?.overlay_ip) || exporter?.address;
-                if (addr)
-                    xe.address = addr;
-            }
             try {
                 validateCompose(compose, nodes.map((n) => n.name));
             }
@@ -1876,6 +2073,8 @@ export async function createApp(options = {}) {
         const node = db.get('SELECT * FROM nodes WHERE id=?', req.node.id);
         const allStates = currentDesiredStates();
         const desiredStates = allStates.filter((st) => Object.values(st.services).some((svc) => (svc.placement?.nodes || []).includes(node.name) || !svc.placement?.nodes?.length));
+        // A command past its deadline is never delivered late; the backup scheduler turns it into a missed run.
+        db.run("UPDATE commands SET state='expired',updated_at=? WHERE node_id=? AND state='pending' AND deadline IS NOT NULL AND deadline<?", now(), node.id, now());
         const commands = db.all('SELECT * FROM commands WHERE node_id=? AND state=?', node.id, 'pending');
         if (commands.length)
             db.run(`UPDATE commands SET state='running',attempts=attempts+1,updated_at=? WHERE node_id=? AND state='pending'`, now(), node.id);
@@ -1896,6 +2095,9 @@ export async function createApp(options = {}) {
         if (network)
             work.network = network;
         work.ports = portBindingFor(node, desiredStates);
+        const holds = restores.holds();
+        if (Object.keys(holds).length)
+            work.holds = holds;
         if (SERVICE_DNS) {
             const all = endpointsFor(allStates);
             work.serviceEndpoints = Object.fromEntries(desiredStates.map((st) => [st.project, all[st.project] || {}]));
@@ -1911,13 +2113,23 @@ export async function createApp(options = {}) {
             db.run(`INSERT INTO node_statuses(id,node_id,project_id,observed_version,status_json,health,updated_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(node_id,project_id) DO UPDATE SET observed_version=excluded.observed_version,status_json=excluded.status_json,health=excluded.health,updated_at=excluded.updated_at`, id('ns'), req.node.id, st.project, st.observedVersion, JSON.stringify(st), st.health || 'unknown', ts);
             updateDeploymentConvergence(st.project);
         }
+        if (report.jobs?.length)
+            backups.onHeartbeat(report.jobs);
         for (const cr of (report.commandResults || [])) {
             const result = cr.result || {};
+            if (cr.commandId && cr.state)
+                backups.onResult(cr.commandId, cr.state, result);
             db.run('UPDATE commands SET state=?,result_json=?,updated_at=? WHERE id=? AND node_id=?', cr.state || 'succeeded', JSON.stringify(result), ts, cr.commandId ?? null, req.node.id);
         }
         if (report.ingress)
             db.run('UPDATE nodes SET ingress_status_json=? WHERE id=?', JSON.stringify({ ...report.ingress, reportedAt: ts }), req.node.id);
         db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', ts, 'online', req.node.id);
+        try {
+            restores.advance();
+        }
+        catch (e) {
+            console.error(`restore failed to advance: ${errMessage(e)}`);
+        }
         const nextEndpointSnapshot = JSON.stringify(endpointsFor(currentDesiredStates()));
         if (nextEndpointSnapshot !== endpointSnapshot) {
             endpointSnapshot = nextEndpointSnapshot;
@@ -1938,6 +2150,28 @@ export async function createApp(options = {}) {
             }
         res.json({ ok: true });
     });
+    // Credentials for a backup helper are released only while its command is running on the calling node, and
+    // never ride in the command payload, events or results.
+    get('/agent/v1/backup/credentials', authAgent, async (req, res) => {
+        const command = db.get('SELECT node_id,project_id,type,state FROM commands WHERE id=?', String(req.query.command || ''));
+        if (!command)
+            return res.status(404).json({ error: 'command not found' });
+        if (command.node_id !== req.node.id)
+            return res.status(403).json({ error: 'command belongs to another node' });
+        if (!BACKUP_COMMAND_TYPES.includes(command.type))
+            return res.status(403).json({ error: 'command does not use backup credentials' });
+        if (command.state !== 'running')
+            return res.status(409).json({ error: `command is ${command.state}, not running` });
+        if (!command.project_id || !backupDestination())
+            return res.status(409).json({ error: 'no backup destination is configured' });
+        const response = {
+            env: {
+                ...backupEnv(),
+                RESTIC_PASSWORD: repositoryPassword(masterKey, command.project_id),
+            },
+        };
+        res.json(response);
+    });
     get('/agent/v1/secrets', authAgent, async (req, res) => {
         const project = String(req.query.project || 'site');
         let rows = db.all('SELECT name,encrypted_value,version FROM secrets WHERE project_id=?', project);

package/dist/cron.d.ts

@@ -0,0 +1,17 @@
+/** A small UTC cron parser for `x-tug.backup.schedule`: five fields, lists, ranges, steps and `@` aliases. */
+export interface CronSchedule {
+    minutes: ReadonlySet<number>;
+    hours: ReadonlySet<number>;
+    days: ReadonlySet<number>;
+    months: ReadonlySet<number>;
+    weekdays: ReadonlySet<number>;
+    /** True when the field was `*`; cron matches day-of-month OR day-of-week when neither is. */
+    anyDay: boolean;
+    anyWeekday: boolean;
+}
+/** Parse a five-field cron expression or an `@hourly`-style alias. Throws a readable `Error` on bad input. */
+export declare function parseCron(expression: string): CronSchedule;
+/** The first matching minute strictly after `after` (UTC), or `null` when the schedule never fires (such as Feb 31). */
+export declare function nextCronRun(schedule: CronSchedule | string, after: Date): Date | null;
+/** Approximate gap between runs in milliseconds, from the next two matches after `from`; used to size deadlines and staleness. */
+export declare function cronPeriodMs(schedule: CronSchedule | string, from: Date): number | null;

package/dist/cron.js

@@ -0,0 +1,124 @@
+/** A small UTC cron parser for `x-tug.backup.schedule`: five fields, lists, ranges, steps and `@` aliases. */
+const ALIASES = {
+    '@hourly': '0 * * * *',
+    '@daily': '0 0 * * *',
+    '@midnight': '0 0 * * *',
+    '@weekly': '0 0 * * 0',
+    '@monthly': '0 0 1 * *',
+    '@yearly': '0 0 1 1 *',
+    '@annually': '0 0 1 1 *',
+};
+const FIELDS = [
+    { name: 'minute', min: 0, max: 59 },
+    { name: 'hour', min: 0, max: 23 },
+    { name: 'day of month', min: 1, max: 31 },
+    { name: 'month', min: 1, max: 12 },
+    { name: 'day of week', min: 0, max: 7 },
+];
+function parseNumber(text, field) {
+    if (!/^\d+$/.test(text))
+        throw new Error(`${field.name} "${text}" is not a number`);
+    const value = Number(text);
+    if (value < field.min || value > field.max)
+        throw new Error(`${field.name} ${value} is outside ${field.min}-${field.max}`);
+    return value;
+}
+function parseField(text, field) {
+    const values = new Set();
+    for (const part of text.split(',')) {
+        if (!part)
+            throw new Error(`${field.name} has an empty list item`);
+        const [range, stepText, extra] = part.split('/');
+        if (extra !== undefined)
+            throw new Error(`${field.name} "${part}" has more than one step`);
+        let step = 1;
+        if (stepText !== undefined) {
+            step = parseNumber(stepText, { name: `${field.name} step`, min: 1, max: field.max });
+        }
+        let low;
+        let high;
+        if (range === '*') {
+            low = field.min;
+            high = field.name === 'day of week' ? 6 : field.max;
+        }
+        else if (range.includes('-')) {
+            const [from, to, more] = range.split('-');
+            if (more !== undefined)
+                throw new Error(`${field.name} range "${range}" is malformed`);
+            low = parseNumber(from, field);
+            high = parseNumber(to, field);
+            if (low > high)
+                throw new Error(`${field.name} range "${range}" runs backwards`);
+        }
+        else {
+            low = parseNumber(range, field);
+            high = stepText === undefined ? low : field.max;
+        }
+        for (let value = low; value <= high; value += step)
+            values.add(value);
+    }
+    return values;
+}
+/** Parse a five-field cron expression or an `@hourly`-style alias. Throws a readable `Error` on bad input. */
+export function parseCron(expression) {
+    if (typeof expression !== 'string')
+        throw new Error('schedule must be a string');
+    const text = ALIASES[expression.trim().toLowerCase()] ?? expression.trim();
+    const parts = text.split(/\s+/);
+    if (parts.length !== 5)
+        throw new Error(`schedule "${expression}" must have five fields (minute hour day month weekday) or be one of ${Object.keys(ALIASES).join(', ')}`);
+    const [minutes, hours, days, months, weekdays] = parts.map((part, index) => parseField(part, FIELDS[index]));
+    // Both 0 and 7 mean Sunday.
+    if (weekdays.delete(7))
+        weekdays.add(0);
+    return {
+        minutes,
+        hours,
+        days,
+        months,
+        weekdays,
+        anyDay: parts[2].startsWith('*'),
+        anyWeekday: parts[4].startsWith('*'),
+    };
+}
+function dayMatches(schedule, date) {
+    const dayOk = schedule.days.has(date.getUTCDate());
+    const weekdayOk = schedule.weekdays.has(date.getUTCDay());
+    if (!schedule.anyDay && !schedule.anyWeekday)
+        return dayOk || weekdayOk;
+    return dayOk && weekdayOk;
+}
+/** Longest gap between two matches is a Feb 29 across a skipped century leap year: eight years. */
+const SEARCH_YEARS = 9;
+/** The first matching minute strictly after `after` (UTC), or `null` when the schedule never fires (such as Feb 31). */
+export function nextCronRun(schedule, after) {
+    const parsed = typeof schedule === 'string' ? parseCron(schedule) : schedule;
+    const next = new Date(Math.floor(after.getTime() / 60_000) * 60_000 + 60_000);
+    const limit = next.getUTCFullYear() + SEARCH_YEARS;
+    while (next.getUTCFullYear() <= limit) {
+        if (!parsed.months.has(next.getUTCMonth() + 1)) {
+            next.setUTCMonth(next.getUTCMonth() + 1, 1);
+            next.setUTCHours(0, 0, 0, 0);
+        }
+        else if (!dayMatches(parsed, next)) {
+            next.setUTCDate(next.getUTCDate() + 1);
+            next.setUTCHours(0, 0, 0, 0);
+        }
+        else if (!parsed.hours.has(next.getUTCHours())) {
+            next.setUTCHours(next.getUTCHours() + 1, 0, 0, 0);
+        }
+        else if (!parsed.minutes.has(next.getUTCMinutes())) {
+            next.setUTCMinutes(next.getUTCMinutes() + 1, 0, 0);
+        }
+        else {
+            return next;
+        }
+    }
+    return null;
+}
+/** Approximate gap between runs in milliseconds, from the next two matches after `from`; used to size deadlines and staleness. */
+export function cronPeriodMs(schedule, from) {
+    const first = nextCronRun(schedule, from);
+    const second = first && nextCronRun(schedule, first);
+    return first && second ? second.getTime() - first.getTime() : null;
+}

package/dist/db.d.ts

@@ -65,6 +65,76 @@ export interface NodeStatusRow {
     health: string;
     updated_at: string;
 }
+export interface BackupScheduleRow {
+    id: string;
+    project_id: string;
+    target: string;
+    type: string;
+    node_id: string | null;
+    config_json: string;
+    schedule: string;
+    jitter_seconds: number;
+    next_run_at: string;
+    last_run_at: string | null;
+    last_success_at: string | null;
+    last_status: string | null;
+    stale_notified_at: string | null;
+    deployment_id: string | null;
+    enabled: number;
+    created_at: string | null;
+    updated_at: string | null;
+}
+export interface BackupRunRow {
+    id: string;
+    schedule_id: string | null;
+    project_id: string;
+    target: string;
+    type: string;
+    node_id: string | null;
+    command_id: string | null;
+    trigger: string;
+    state: string;
+    scheduled_for: string | null;
+    queued_at: string | null;
+    started_at: string | null;
+    finished_at: string | null;
+    snapshot_id: string | null;
+    size_bytes: number | null;
+    added_bytes: number | null;
+    duration_ms: number | null;
+    restic_version: string | null;
+    error: string | null;
+    actor: string | null;
+    deployment_version: string | null;
+    metadata_json: string | null;
+}
+export interface RestoreRunRow {
+    id: string;
+    project_id: string;
+    target: string;
+    node_id: string | null;
+    snapshot_id: string;
+    mode: 'in-place' | 'scratch';
+    scratch_volume: string | null;
+    safety_backup: number;
+    safety_run_id: string | null;
+    writers_json: string;
+    state: string;
+    command_id: string | null;
+    actor: string | null;
+    started_at: string | null;
+    state_changed_at: string | null;
+    finished_at: string | null;
+    error: string | null;
+}
+export interface BackupDestinationRow {
+    id: 'default';
+    url: string;
+    env_names_json: string;
+    checked_at: string | null;
+    check_result_json: string | null;
+    updated_at: string;
+}
 export interface SecretRow {
     id: string;
     project_id: string;

package/dist/db.js

@@ -70,6 +70,13 @@ CREATE INDEX IF NOT EXISTS events_project_created ON events(project_id, created_
 CREATE TABLE IF NOT EXISTS api_tokens(id TEXT PRIMARY KEY, name TEXT UNIQUE NOT NULL, token_hash TEXT UNIQUE NOT NULL, role TEXT NOT NULL, projects_json TEXT, created_at TEXT, created_by TEXT, last_used_at TEXT, expires_at TEXT, revoked_at TEXT);
 CREATE INDEX IF NOT EXISTS events_by_time ON events(created_at);
 CREATE TABLE IF NOT EXISTS locks(name TEXT PRIMARY KEY, owner TEXT, expires_at TEXT, created_at TEXT);
+CREATE TABLE IF NOT EXISTS backup_destination(id TEXT PRIMARY KEY CHECK(id='default'), url TEXT NOT NULL, env_names_json TEXT NOT NULL, checked_at TEXT, check_result_json TEXT, updated_at TEXT);
+CREATE TABLE IF NOT EXISTS backup_schedules(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, target TEXT NOT NULL, type TEXT NOT NULL, node_id TEXT, config_json TEXT NOT NULL, schedule TEXT NOT NULL, jitter_seconds INTEGER NOT NULL, next_run_at TEXT NOT NULL, last_run_at TEXT, last_success_at TEXT, last_status TEXT, stale_notified_at TEXT, deployment_id TEXT, enabled INTEGER NOT NULL DEFAULT 1, created_at TEXT, updated_at TEXT, UNIQUE(project_id, target));
+CREATE TABLE IF NOT EXISTS backup_runs(id TEXT PRIMARY KEY, schedule_id TEXT, project_id TEXT NOT NULL, target TEXT NOT NULL, type TEXT NOT NULL, node_id TEXT, command_id TEXT, trigger TEXT NOT NULL, state TEXT NOT NULL, scheduled_for TEXT, queued_at TEXT, started_at TEXT, finished_at TEXT, snapshot_id TEXT, size_bytes INTEGER, added_bytes INTEGER, duration_ms INTEGER, restic_version TEXT, error TEXT, actor TEXT, deployment_version TEXT, metadata_json TEXT);
+CREATE TABLE IF NOT EXISTS restore_runs(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, target TEXT NOT NULL, node_id TEXT, snapshot_id TEXT NOT NULL, mode TEXT NOT NULL, scratch_volume TEXT, safety_backup INTEGER NOT NULL DEFAULT 1, safety_run_id TEXT, writers_json TEXT NOT NULL DEFAULT '[]', state TEXT NOT NULL, command_id TEXT, actor TEXT, started_at TEXT, state_changed_at TEXT, finished_at TEXT, error TEXT);
+CREATE TABLE IF NOT EXISTS service_holds(project_id TEXT NOT NULL, service TEXT NOT NULL, reason TEXT NOT NULL, restore_id TEXT, created_at TEXT, PRIMARY KEY(project_id, service));
+CREATE UNIQUE INDEX IF NOT EXISTS backup_runs_slot ON backup_runs(schedule_id, scheduled_for) WHERE trigger='schedule';
+CREATE INDEX IF NOT EXISTS backup_runs_target ON backup_runs(project_id, target, finished_at);
 `);
     const row = db.get('SELECT value FROM meta WHERE key = ?', 'schema_created_at');
     if (!row)

package/dist/docker-api.d.ts

@@ -33,6 +33,13 @@ export interface ContainerInspect {
         Image?: string;
         Labels?: Record<string, string>;
         Env?: string[];
+        User?: string;
+        WorkingDir?: string;
+    };
+    HostConfig?: {
+        NetworkMode?: string;
+        Binds?: string[];
+        Mounts?: unknown[];
     };
     State?: {
         Status?: string;

package/dist/live.d.ts

@@ -18,6 +18,8 @@ export type LiveRequest = {
     tty?: boolean;
     columns?: number;
     rows?: number;
+    /** `container` is only the template: run the command in a new container with its image, environment and mounts. */
+    oneOff?: boolean;
 } | {
     operation: 'logs';
     sessionId: string;
@@ -81,3 +83,9 @@ export declare class LiveOperationEngine {
     private output;
     private finish;
 }
+/**
+ * Runs `command` in a new container cloned from `template`: the same image, environment (secrets included),
+ * user, working directory, network and mounts, with no published ports and none of the service's labels, so
+ * reconciliation never mistakes it for the service. The container is removed when the command ends.
+ */
+export declare function startOneOff(docker: DockerApi, template: string, command: string[]): Promise<LiveProcess>;