@hivemagnus/magnus-server 1.1.2 → 1.1.3

package/agent-version.json

@@ -1,3 +1,3 @@
 {
-  "version": "1.1.2"
+  "version": "1.1.3"
 }

package/dist/cli/agents.js

@@ -1,24 +1,43 @@
 /** `magnus-server agents upgrade`: the console's "Upgrade agents" action from a shell. */
 import { UsageError } from './args.js';
 export const AGENTS_USAGE = `Usage: magnus-server agents upgrade [options] [--env-file FILE]
+       magnus-server agents stop RUN_ID [--env-file FILE]
 
-Upgrades agents to the version this server ships with (or --to), one canary
+Upgrades agents to the version bundled with this server (or --to), one canary
 first, then MAGNUS_AGENT_UPGRADE_CONCURRENCY at a time. Each agent installs the
 exact version beside the running one and switches back by itself if the new
-version does not report a heartbeat in time. Stops at the first failure.
+version does not report a heartbeat in time. A revert, a release problem or an
+agent that goes silent stops the run; an agent that refuses before changing
+anything (busy, enrollment, unreachable) is reported as refused and the run
+goes on.
 
 Options:
   --to X.Y.Z            Agent version (default: the one bundled with this server)
   --integrity sha512-…  npm integrity of that version (default: read from the registry)
   --nodes a,b           Only these nodes (names, host names or ids)
+  --canary NODE         Upgrade this node first (default: the first eligible one)
+  --allow-prerelease    Required for a prerelease version such as 1.2.0-rc.1
+  --retry-reverted      Also offer nodes that already reverted this version
+  --retry RUN_ID        Retry the failed, refused, skipped and interrupted nodes of a run
   --dry-run             Show which agents would be upgraded, and why others would not
 
+Ctrl-C asks the run to stop after the nodes in progress; a second Ctrl-C leaves
+at once, and the run is then shown as interrupted.
+
 Agents are called over TLS. NODE_EXTRA_CA_CERTS from --env-file is honoured, so
 the command trusts the same CA as the service.`;
 export function parseAgentsArgs(args) {
     const [sub, ...rest] = args;
     if (!sub || sub === '--help' || sub === '-h' || sub === 'help')
         return 'help';
+    if (sub === 'stop') {
+        const [id, ...extra] = rest;
+        if (id === '--help' || id === '-h')
+            return 'help';
+        if (!id || id.startsWith('--') || extra.length)
+            throw new UsageError('agents stop takes one RUN_ID');
+        return { stop: id, dryRun: false };
+    }
     if (sub !== 'upgrade')
         throw new UsageError(`unknown agents subcommand "${sub}"`);
     const options = { dryRun: false };
@@ -37,6 +56,14 @@ export function parseAgentsArgs(args) {
             options.integrity = value();
         else if (name === '--nodes')
             options.nodes = value().split(',').map(node => node.trim()).filter(Boolean);
+        else if (name === '--canary')
+            options.canary = value();
+        else if (name === '--retry')
+            options.retry = value();
+        else if (name === '--allow-prerelease')
+            options.allowPrerelease = true;
+        else if (name === '--retry-reverted')
+            options.retryReverted = true;
         else if (name === '--dry-run')
             options.dryRun = true;
         else if (name === '--help' || name === '-h')
@@ -44,8 +71,13 @@ export function parseAgentsArgs(args) {
         else
             throw new UsageError(`unknown option ${name}`);
     }
+    if (options.nodes?.length === 0)
+        throw new UsageError('--nodes needs at least one node');
+    if (options.retry && (options.to || options.integrity || options.nodes || options.dryRun))
+        throw new UsageError('--retry reuses the version, integrity and nodes of that run; drop --to, --integrity, --nodes and --dry-run');
     return options;
 }
+const nodeLine = (node) => `  ${node.name}: ${node.state}${node.detail ? ` (${node.code ? `${node.code}: ` : ''}${node.detail})` : node.code ? ` (${node.code})` : ''}`;
 export async function commandAgents(args) {
     const options = parseAgentsArgs(args);
     if (options === 'help') {
@@ -56,33 +88,32 @@ export async function commandAgents(args) {
     const { disconnectRedis } = await import('../queue/redis.js');
     const service = await import('../services/agent-upgrade-service.js');
     try {
-        let nodeIds;
-        if (options.nodes) {
+        if (options.stop) {
+            const run = await service.stopUpgradeRun(options.stop, 'system:magnus-server-cli');
+            console.log(`Asked run ${run.id} to stop; it stops before its next batch.`);
+            return 0;
+        }
+        const resolveNodes = async (names) => {
             const rows = await getDb().node.findMany({
-                where: { OR: [{ id: { in: options.nodes } }, { name: { in: options.nodes } }, { hostname: { in: options.nodes } }] },
+                where: { OR: [{ id: { in: names } }, { name: { in: names } }, { hostname: { in: names } }] },
                 select: { id: true, name: true, hostname: true },
             });
-            const unknown = options.nodes.filter(node => !rows.some(row => row.id === node || row.name === node || row.hostname === node));
-            if (unknown.length) {
-                console.error(`Unknown node(s): ${unknown.join(', ')}`);
+            const unknown = names.filter(node => !rows.some(row => row.id === node || row.name === node || row.hostname === node));
+            return { ids: rows.map(row => row.id), unknown, byName: (name) => rows.find(row => row.id === name || row.name === name || row.hostname === name)?.id };
+        };
+        let nodeIds;
+        let canaryNodeId;
+        const wanted = [...options.nodes ?? [], ...options.canary ? [options.canary] : []];
+        if (wanted.length) {
+            const resolved = await resolveNodes(wanted);
+            if (resolved.unknown.length) {
+                console.error(`Unknown node(s): ${resolved.unknown.join(', ')}`);
                 return 1;
             }
-            nodeIds = rows.map(row => row.id);
-        }
-        const release = await service.resolveAgentRelease({ version: options.to, integrity: options.integrity });
-        const candidates = await service.listUpgradeCandidates(release.version, nodeIds);
-        console.log(`magnus-agent ${release.version} (${release.integrity.slice(0, 19)}…)`);
-        for (const candidate of candidates) {
-            console.log(`  ${candidate.eligible ? '→' : '·'} ${candidate.name.padEnd(28)} ${candidate.version.padEnd(12)} ${candidate.eligible ? 'will upgrade' : candidate.reason}`);
-        }
-        const eligible = candidates.filter(candidate => candidate.eligible);
-        if (options.dryRun) {
-            console.log(`Dry run: ${eligible.length} of ${candidates.length} agent(s) would be upgraded.`);
-            return 0;
-        }
-        if (eligible.length === 0) {
-            console.log('Nothing to upgrade.');
-            return candidates.some(candidate => candidate.reason !== 'already on this version') ? 1 : 0;
+            if (options.nodes)
+                nodeIds = options.nodes.map(name => resolved.byName(name));
+            if (options.canary)
+                canaryNodeId = resolved.byName(options.canary);
         }
         const deps = service.defaultDeps();
         const seen = new Map();
@@ -93,15 +124,57 @@ export async function commandAgents(args) {
                     continue;
                 seen.set(node.nodeId, node.state);
                 if (node.state !== 'pending')
-                    console.log(`  ${node.name}: ${node.state}${node.detail ? ` (${node.detail})` : ''}`);
+                    console.log(nodeLine(node));
             }
             await save(run);
         };
-        const { done } = await service.startUpgradeRun({ version: release.version, integrity: release.integrity, nodeIds: eligible.map(node => node.nodeId),
-            actorId: 'system:magnus-server-cli', deps });
-        const run = await done;
-        console.log(`${run.status === 'succeeded' ? 'Done' : 'Stopped'}: ${run.message ?? run.status}`);
-        return run.status === 'succeeded' ? 0 : 1;
+        let started;
+        if (options.retry) {
+            started = await service.retryUpgradeRun(options.retry, { actorId: 'system:magnus-server-cli', deps, canaryNodeId,
+                allowRetryReverted: options.retryReverted, allowPrerelease: options.allowPrerelease });
+            console.log(`Retrying run ${options.retry} as ${started.run.id}: ${started.run.nodes.length} agent(s) to ${started.run.version}`);
+        }
+        else {
+            const release = await service.resolveAgentRelease({ version: options.to, integrity: options.integrity });
+            const candidates = await service.listUpgradeCandidates(release.version, nodeIds, { allowRetryReverted: options.retryReverted });
+            console.log(`magnus-agent ${release.version} (${release.integrity.slice(0, 19)}…)`);
+            for (const candidate of candidates) {
+                console.log(`  ${candidate.eligible ? '→' : '·'} ${candidate.name.padEnd(28)} ${candidate.version.padEnd(12)} ${candidate.eligible ? 'will upgrade' : candidate.reason}`);
+            }
+            const eligible = candidates.filter(candidate => candidate.eligible);
+            if (options.dryRun) {
+                console.log(`Dry run: ${eligible.length} of ${candidates.length} agent(s) would be upgraded.`);
+                return 0;
+            }
+            if (eligible.length === 0) {
+                console.log('Nothing to upgrade.');
+                return candidates.some(candidate => candidate.reason !== 'already on this version') ? 1 : 0;
+            }
+            started = await service.startUpgradeRun({ version: release.version, integrity: release.integrity, nodeIds: eligible.map(node => node.nodeId),
+                canaryNodeId, allowPrerelease: options.allowPrerelease, allowRetryReverted: options.retryReverted, actorId: 'system:magnus-server-cli', deps });
+        }
+        let interrupts = 0;
+        const onSigint = () => {
+            if (++interrupts > 1) {
+                console.error('Leaving now; the run shows as interrupted once its lease runs out.');
+                process.exit(130);
+            }
+            console.error('Stopping after the nodes in progress (Ctrl-C again to leave now)…');
+            void service.stopUpgradeRun(started.run.id, 'system:magnus-server-cli').catch(() => undefined);
+        };
+        process.on('SIGINT', onSigint);
+        try {
+            const run = await started.done;
+            for (const node of run.excluded ?? [])
+                if (options.nodes || options.retry)
+                    console.log(`  ${node.name}: not upgraded (${node.reason})`);
+            console.log(`${run.status === 'succeeded' ? 'Done' : run.status === 'stopped' ? 'Stopped' : 'Failed'}: ${run.message ?? run.status}`);
+            console.log(`Run ${run.id}${run.nodes.every(node => node.state === 'confirmed') ? '' : `; retry the rest with: magnus-server agents upgrade --retry ${run.id}`}`);
+            return run.status === 'succeeded' ? 0 : 1;
+        }
+        finally {
+            process.off('SIGINT', onSigint);
+        }
     }
     finally {
         await disconnectDb().catch(() => undefined);

package/dist/cli/args.js

@@ -8,6 +8,7 @@ Commands:
   migrate       Apply pending migrations (prisma migrate deploy) and base data
   migrations    Show applied and pending migrations with their impact notes
   backup        Dump PostgreSQL (pg_dump -Fc) into MAGNUS_BACKUP_DIR and print the restore command
+  restore FILE  Replace the database with a dump (stop the server first; asks unless --yes)
   upgrade       Upgrade a \`magnus init\` project (runs \`magnus upgrade\`; see --help there)
   rollback      Switch a \`magnus init\` project back to the previous version
   agents        Agent fleet commands: \`agents upgrade [--to X.Y.Z] [--nodes a,b] [--dry-run]\`
@@ -20,14 +21,15 @@ Options:
   --migrate           start: apply pending migrations before listening (MAGNUS_AUTO_MIGRATE adds a backup and lock)
   --format <fmt>      env-schema (default json), check and migrations (default table): json or table
   --label <text>      backup: label in the dump file name
+  --yes               restore: do not ask for confirmation
   -h, --help          Show this help
   -v, --version       Print the version
 `;
-const COMMANDS = new Set(['start', 'check', 'migrate', 'migrations', 'backup', 'upgrade', 'rollback', 'agents', 'env-schema', 'help', 'version']);
+const COMMANDS = new Set(['start', 'check', 'migrate', 'migrations', 'backup', 'restore', 'upgrade', 'rollback', 'agents', 'env-schema', 'help', 'version']);
 /** Commands whose remaining arguments belong to another tool. */
 const PASSTHROUGH = new Set(['upgrade', 'rollback', 'agents']);
 export function parseArgs(argv) {
-    const options = { command: 'start', envFiles: [], migrate: false, format: 'json', formatGiven: false, passthrough: [] };
+    const options = { command: 'start', envFiles: [], migrate: false, format: 'json', formatGiven: false, passthrough: [], yes: false };
     let commandSeen = false;
     for (let index = 0; index < argv.length; index++) {
         const arg = argv[index];
@@ -62,8 +64,12 @@ export function parseArgs(argv) {
         }
         else if (arg === '--label' || arg.startsWith('--label='))
             options.label = value('--label');
+        else if (arg === '--yes')
+            options.yes = true;
         else if (arg.startsWith('-'))
             throw new UsageError(`unknown option ${arg}`);
+        else if (commandSeen && options.command === 'restore' && options.file === undefined)
+            options.file = arg;
         else if (!commandSeen && COMMANDS.has(arg)) {
             if (options.command !== 'help' && options.command !== 'version')
                 options.command = arg;
@@ -72,6 +78,8 @@ export function parseArgs(argv) {
         else
             throw new UsageError(`unknown command "${arg}"`);
     }
+    if (options.command === 'restore' && !options.file)
+        throw new UsageError('restore needs the dump file');
     return options;
 }
 //# sourceMappingURL=args.js.map

package/dist/cli/auto-migrate.js

@@ -3,16 +3,21 @@
  *
  * Runs before the server listens. A session advisory lock serializes every
  * instance sharing the database, so only one dumps and migrates while the
- * others wait and then find nothing pending. A failed backup stops startup:
- * migrations never run without a dump unless the operator turned the dump off.
+ * others wait (up to MAGNUS_AUTO_MIGRATE_LOCK_TIMEOUT) and then find nothing
+ * pending. A failed backup stops startup: migrations never run without a dump
+ * unless the operator turned the dump off. A database with a failed migration
+ * is refused before any dump, and a restart loop reuses its recent dump, so a
+ * crash-looping service cannot fill the disk.
  */
 import { join } from 'node:path';
 import pg from 'pg';
-import { backupDatabase, describeMigration, migrationsNeedingAttention, readMigrationState, } from './migrations.js';
+import { backupDatabase, backupKeep, describeMigration, failedMigrationAdvice, findReusableBackup, restoreCommand, migrationsNeedingAttention, readMigrationState, } from './migrations.js';
 /** Key for `pg_advisory_lock(hashtextextended(key, 0))`; distinct from Prisma's own migrate lock. */
 export const MIGRATE_LOCK_KEY = 'magnus:migrate';
 /** Pure decision, separated for tests. */
 export function decideAutoMigrate(state, settings) {
+    if (state.failed.length > 0)
+        return { action: 'refuse', reason: failedMigrationAdvice(state.failed) };
     if (state.pending.length === 0)
         return { action: 'skip', reason: `${state.packaged.length} migration(s) already applied` };
     const attention = migrationsNeedingAttention(state.pending);
@@ -26,19 +31,61 @@ export function decideAutoMigrate(state, settings) {
     // A fresh database has nothing to back up.
     return { action: 'migrate', backup: settings.autoMigrateBackup && !state.fresh };
 }
+/** A dump this recent of the same database state is reused by the next attempt. */
+export const BACKUP_REUSE_MS = 60 * 60_000;
+export function autoMigrateLimits(env = process.env) {
+    const seconds = Number(env['MAGNUS_AUTO_MIGRATE_LOCK_TIMEOUT'] ?? '');
+    return { lockTimeoutMs: (Number.isFinite(seconds) && seconds > 0 ? seconds : 600) * 1000, keepBackups: backupKeep(env) };
+}
+/**
+ * The connection migrations and dumps use: MAGNUS_MIGRATE_DATABASE_URL when
+ * set (a direct connection past a pooler), else DATABASE_URL.
+ */
+export function migrateDatabaseUrl(env, fallback) {
+    return env['MAGNUS_MIGRATE_DATABASE_URL']?.trim() || fallback;
+}
+/** Advisory locks do not survive PgBouncer transaction pooling. */
+export function poolerWarning(url) {
+    try {
+        return new URL(url).searchParams.get('pgbouncer') === 'true'
+            ? 'DATABASE_URL goes through PgBouncer (pgbouncer=true): the migrate lock and prisma migrate need a direct connection; set MAGNUS_MIGRATE_DATABASE_URL.'
+            : null;
+    }
+    catch {
+        return null;
+    }
+}
 export async function autoMigrate(options) {
     const log = options.log ?? ((line) => console.log(line));
+    const now = options.now ?? (() => new Date());
+    const sleep = options.sleep ?? ((ms) => new Promise(resolve => setTimeout(resolve, ms)));
+    const limits = { ...autoMigrateLimits(), ...options.limits };
+    const pooler = poolerWarning(options.databaseUrl);
+    if (pooler)
+        log(`[auto-migrate] warning: ${pooler}`);
     const client = new pg.Client({ connectionString: options.databaseUrl, connectionTimeoutMillis: 10_000 });
     await client.connect();
     let locked = false;
     try {
-        const tried = await client.query('SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS locked', [MIGRATE_LOCK_KEY]);
-        if (!tried.rows[0]?.locked) {
-            log('[auto-migrate] another instance is migrating; waiting for it to finish');
-            await client.query('SELECT pg_advisory_lock(hashtextextended($1, 0))', [MIGRATE_LOCK_KEY]);
+        const deadline = now().getTime() + limits.lockTimeoutMs;
+        for (let attempt = 0;; attempt++) {
+            const tried = await client.query('SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS locked', [MIGRATE_LOCK_KEY]);
+            if (tried.rows[0]?.locked)
+                break;
+            if (attempt === 0)
+                log('[auto-migrate] another instance is migrating; waiting for it to finish');
+            if (now().getTime() >= deadline) {
+                throw new Error(`Auto-migrate gave up after ${Math.round(limits.lockTimeoutMs / 1000)} s waiting for another instance's migrate lock (MAGNUS_AUTO_MIGRATE_LOCK_TIMEOUT). `
+                    + 'If no other server is migrating, a stale session holds it: check pg_locks for advisory locks, or connect without a pooler (MAGNUS_MIGRATE_DATABASE_URL).');
+            }
+            await sleep(2_000);
         }
         locked = true;
         const state = await readMigrationState(client, options.migrationsDir);
+        if (state.unknown.length > 0) {
+            log(`[auto-migrate] warning: the database has ${state.unknown.length} migration(s) this version does not know (a newer release applied them): ${state.unknown.map(m => m.name).join(', ')}. `
+                + 'If this version misbehaves, upgrade again or restore the dump taken before that upgrade.');
+        }
         const decision = decideAutoMigrate(state, options.settings);
         if (decision.action === 'skip') {
             log(`[auto-migrate] nothing to do: ${decision.reason}`);
@@ -49,16 +96,27 @@ export async function autoMigrate(options) {
         log(`[auto-migrate] ${state.pending.length} pending: ${state.pending.map(describeMigration).join(', ')}`);
         let backup;
         if (decision.backup) {
-            backup = await backupDatabase({
-                databaseUrl: options.databaseUrl,
-                backupDir: options.backupDir,
-                label: `pre-migrate-${options.version}`,
-                client,
-                pgDump: options.pgDump,
-            }).catch((error) => {
-                throw new Error(`Auto-migrate stopped before migrating: the backup failed. ${error instanceof Error ? error.message : String(error)}`);
-            });
-            log(`[auto-migrate] backup written to ${backup.file}`);
+            const label = `pre-migrate-${options.version}`;
+            const recent = findReusableBackup(options.backupDir, label, state.lastApplied, BACKUP_REUSE_MS, now());
+            if (recent) {
+                backup = { file: recent, metadataFile: `${recent}.json`, restoreCommand: restoreCommand(recent), reused: true };
+                log(`[auto-migrate] reusing the dump of the same database state from the last attempt: ${recent}`);
+            }
+            else {
+                backup = await backupDatabase({
+                    databaseUrl: options.databaseUrl,
+                    backupDir: options.backupDir,
+                    label,
+                    client,
+                    pgDump: options.pgDump,
+                    lastApplied: state.lastApplied,
+                    keep: limits.keepBackups,
+                    now: now(),
+                }).catch((error) => {
+                    throw new Error(`Auto-migrate stopped before migrating: the backup failed. ${error instanceof Error ? error.message : String(error)}`);
+                });
+                log(`[auto-migrate] backup written to ${backup.file}`);
+            }
         }
         else if (!state.fresh) {
             log('[auto-migrate] MAGNUS_AUTO_MIGRATE_BACKUP=false: migrating without a dump');

package/dist/cli/env-catalog.js

@@ -102,8 +102,11 @@ export const ENV_CATALOG = [
     { name: 'MAGNUS_AUTO_MIGRATE_BACKUP', group: 'upgrades', default: 'true', description: 'Dump the database (pg_dump -Fc into MAGNUS_BACKUP_DIR) before an automatic migration. A failed dump stops startup. Set false only if you back up another way.' },
     { name: 'MAGNUS_AUTO_MIGRATE_ALLOW_SLOW', group: 'upgrades', default: 'false', description: 'Let auto-migrate apply migrations marked slow or one-way; otherwise startup stops and asks for `magnus upgrade`.' },
     { name: 'MAGNUS_BACKUP_DIR', group: 'upgrades', default: '$MAGNUS_HOME/backups', description: 'Directory for pre-migration database dumps (0700; dumps are 0600).' },
+    { name: 'MAGNUS_BACKUP_KEEP', group: 'upgrades', default: '5', description: 'Dumps kept in MAGNUS_BACKUP_DIR after each new one; older ones are deleted, except the newest dump of each label (each upgrade step keeps the dump its rollback needs).' },
+    { name: 'MAGNUS_AUTO_MIGRATE_LOCK_TIMEOUT', group: 'upgrades', default: '600', description: 'Seconds startup auto-migrate waits for another instance\'s migrate lock before it gives up and exits.' },
+    { name: 'MAGNUS_MIGRATE_DATABASE_URL', group: 'upgrades', secret: true, description: 'Direct PostgreSQL connection for migrations, dumps and restores when DATABASE_URL goes through a pooler such as PgBouncer (advisory locks need a session). Defaults to DATABASE_URL.' },
     { name: 'MAGNUS_PG_DUMP', group: 'upgrades', default: 'pg_dump', description: 'pg_dump binary used for pre-migration backups. Its major version must be at least the server\'s.' },
-    { name: 'MAGNUS_RUNTIME_FILE', group: 'upgrades', default: '$MAGNUS_HOME/run/server-runtime.json', description: 'File the server writes after it starts listening ({version, pid, port}); upgrade tooling reads it to confirm the new version is serving.' },
+    { name: 'MAGNUS_RUNTIME_FILE', group: 'upgrades', default: '$MAGNUS_HOME/run/server-runtime.json', description: 'File the server writes after it starts listening ({version, pid, port, host}); upgrade tooling reads it to confirm the new version is serving.' },
     { name: 'MAGNUS_UPDATE_CHECK', group: 'upgrades', default: 'true', description: 'Daily check of the npm registry for newer Magnus releases, shown in the console and `magnus-server check`. Sends no data beyond a registry GET. Set false to opt out.' },
     { name: 'MAGNUS_UPDATE_REGISTRY', group: 'upgrades', default: 'https://registry.npmjs.org', description: 'npm registry used by the update check.' },
     { name: 'MAGNUS_AGENT_UPGRADES', group: 'upgrades', default: 'true', description: 'Allow server-driven agent upgrades (console action, API and `magnus-server agents upgrade`). Agents must also allow it (MAGNUS_AGENT_SELF_UPGRADE).' },

package/dist/cli/migrations.js

@@ -3,9 +3,10 @@
  * `magnus-server migrate`, `check` and startup auto-migrate.
  */
 import { spawn } from 'node:child_process';
-import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { createRequire } from 'node:module';
-import { dirname, join } from 'node:path';
+import { basename, dirname, join } from 'node:path';
+import { restoreAsideSchema, restoreFinishSql, restoreRevertSql, restoreSwapSql } from '@hivemagnus/magnus-contracts';
 /** Packaged migration directory names, in apply order. */
 export function packagedMigrations(migrationsDir) {
     if (!existsSync(migrationsDir))
@@ -30,13 +31,27 @@ export function readMigrationImpact(migrationsDir, name) {
 export async function readMigrationState(client, migrationsDir) {
     const packaged = packagedMigrations(migrationsDir);
     const table = await client.query(`SELECT to_regclass('public._prisma_migrations') IS NOT NULL AS exists`);
-    const applied = table.rows[0]?.exists
-        ? new Set((await client.query('SELECT migration_name FROM _prisma_migrations WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL')).rows
-            .map(row => row.migration_name))
-        : new Set();
+    const rows = table.rows[0]?.exists
+        ? (await client.query('SELECT migration_name, finished_at, rolled_back_at FROM _prisma_migrations ORDER BY migration_name')).rows
+        : [];
+    const done = rows.filter(row => row.finished_at && !row.rolled_back_at);
+    const applied = new Set(done.map(row => row.migration_name));
+    const failed = [...new Set(rows.filter(row => !row.finished_at && !row.rolled_back_at && !applied.has(row.migration_name)).map(row => row.migration_name))];
+    const known = new Set(packaged);
     const pending = packaged.filter(name => !applied.has(name))
         .map(name => ({ name, ...readMigrationImpact(migrationsDir, name) }));
-    return { packaged, applied, pending, fresh: applied.size === 0 };
+    const unknown = done.filter(row => !known.has(row.migration_name))
+        .map(row => ({ name: row.migration_name, appliedAt: row.finished_at ? new Date(row.finished_at).toISOString() : null }));
+    const lastApplied = [...applied].sort().at(-1) ?? null;
+    return { packaged, applied, pending, fresh: applied.size === 0, failed, unknown, lastApplied };
+}
+/** Why Prisma will not migrate a database with failed rows, and the ways out. */
+export function failedMigrationAdvice(failed) {
+    return [
+        `migration(s) ${failed.join(', ')} failed earlier and are still marked as failed in _prisma_migrations; the database may be half-migrated.`,
+        'Restore the dump taken before that attempt (`magnus restore <dump>`, or `magnus-server restore <dump> --env-file <file>` with the server stopped),',
+        'or repair the schema by hand and then mark the migration with `npx prisma migrate resolve --rolled-back <name>` (or `--applied <name>`).',
+    ].join(' ');
 }
 /** Pending migrations an operator must confirm: slow ones, or ones only a restore can undo. */
 export function migrationsNeedingAttention(pending) {
@@ -67,14 +82,96 @@ export function majorOf(versionText) {
     const match = /(\d+)(?:\.\d+)?/.exec(versionText);
     return match ? Number(match[1]) : null;
 }
-/** A restore command that never contains the connection password. */
-export function restoreCommand(file) {
-    return `pg_restore --clean --if-exists --no-owner -d "$DATABASE_URL" '${file.replace(/'/g, `'\\''`)}'`;
+function shellQuote(value) {
+    return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+/** The command that restores a dump; it never contains the connection string. */
+export function restoreCommand(file, envFile = process.env['MAGNUS_ENV_FILE']) {
+    return `magnus-server restore ${shellQuote(file)}${envFile ? ` --env-file ${shellQuote(envFile)}` : ''}`;
+}
+/** Connection URL without its password, and the password for PGPASSWORD (never in argv). */
+export function splitPassword(databaseUrl) {
+    try {
+        const url = new URL(databaseUrl);
+        const password = decodeURIComponent(url.password);
+        url.password = '';
+        return { url: url.toString(), env: password ? { PGPASSWORD: password } : {} };
+    }
+    catch {
+        return { url: databaseUrl, env: {} };
+    }
+}
+/** pg_restore next to the configured pg_dump (same client version), else from PATH. */
+export function pgRestoreFor(pgDump) {
+    return pgDump && pgDump.includes('/') ? join(dirname(pgDump), 'pg_restore') : 'pg_restore';
+}
+function readMetadata(file) {
+    try {
+        return JSON.parse(readFileSync(`${file}.json`, 'utf8'));
+    }
+    catch {
+        return null;
+    }
+}
+function dumpsIn(dir) {
+    if (!existsSync(dir))
+        return [];
+    return readdirSync(dir).filter(name => name.endsWith('.dump')).map(name => join(dir, name));
+}
+/**
+ * A dump taken within `maxAgeMs` with the same label and the same newest
+ * applied migration: the database is in the state it captured (a restart loop
+ * after a failed migrate), so dumping again would only fill the disk.
+ */
+export function findReusableBackup(dir, label, lastApplied, maxAgeMs, now = new Date()) {
+    let best = null;
+    for (const file of dumpsIn(dir)) {
+        const meta = readMetadata(file);
+        if (!meta?.createdAt || meta.label !== label || (meta.lastApplied ?? null) !== lastApplied || !('lastApplied' in meta))
+            continue;
+        const at = Date.parse(meta.createdAt);
+        if (!Number.isFinite(at) || now.getTime() - at > maxAgeMs || at > now.getTime())
+            continue;
+        if (statSync(file).size === 0)
+            continue;
+        if (!best || at > best.at)
+            best = { file, at };
+    }
+    return best?.file ?? null;
+}
+/**
+ * Keep the newest `keep` dumps; older ones go, except the newest dump of each
+ * label (the one a rollback of that step would restore). Returns removed files.
+ */
+export function pruneBackups(dir, keep) {
+    if (!Number.isInteger(keep) || keep < 1)
+        return [];
+    const dumps = dumpsIn(dir).map(file => {
+        const meta = readMetadata(file);
+        return { file, label: meta?.label ?? basename(file), at: Date.parse(meta?.createdAt ?? '') || statSync(file).mtimeMs };
+    }).sort((a, b) => b.at - a.at);
+    const newestPerLabel = new Set();
+    const removed = [];
+    dumps.forEach((dump, index) => {
+        const newest = !newestPerLabel.has(dump.label);
+        newestPerLabel.add(dump.label);
+        if (index < keep || newest)
+            return;
+        rmSync(dump.file, { force: true });
+        rmSync(`${dump.file}.json`, { force: true });
+        removed.push(dump.file);
+    });
+    return removed;
+}
+export function backupKeep(env = process.env) {
+    const value = Number.parseInt(env['MAGNUS_BACKUP_KEEP'] ?? '', 10);
+    return Number.isInteger(value) && value > 0 ? value : 5;
 }
 /**
  * `pg_dump -Fc` into the backup directory (0700 directory, 0600 dump).
  * Refuses when pg_dump is older than the server, which would produce an
- * unusable or partial dump.
+ * unusable or partial dump. The password goes through PGPASSWORD, never argv,
+ * and the dump is read back with `pg_restore --list` before it counts.
  */
 export async function backupDatabase(options) {
     const run = options.run ?? runCommand;
@@ -95,23 +192,64 @@ export async function backupDatabase(options) {
         throw new Error(`${pgDump} is PostgreSQL ${clientMajor} but the server is PostgreSQL ${serverMajor}; install the PostgreSQL ${serverMajor} client and set MAGNUS_PG_DUMP to it (on Debian and Ubuntu: /usr/lib/postgresql/${serverMajor}/bin/pg_dump).`);
     }
     mkdirSync(options.backupDir, { recursive: true, mode: 0o700 });
-    const stamp = (options.now ?? new Date()).toISOString().replace(/[:.]/g, '-');
+    const now = options.now ?? new Date();
+    const stamp = now.toISOString().replace(/[:.]/g, '-');
     const file = join(options.backupDir, `${stamp}-${options.label.replace(/[^A-Za-z0-9._-]+/g, '_')}.dump`);
-    const result = await run(pgDump, ['--format=custom', '--no-owner', '--file', file, '--dbname', options.databaseUrl], { capture: true });
-    if (result.code !== 0)
+    const { url, env } = splitPassword(options.databaseUrl);
+    const result = await run(pgDump, ['--format=custom', '--no-owner', '--file', file, '--dbname', url], { capture: true, env: { ...process.env, ...env } });
+    if (result.code !== 0) {
+        rmSync(file, { force: true });
         throw new Error(`pg_dump failed (exit ${result.code}): ${result.stderr.trim().split('\n').slice(-3).join(' ')}`);
+    }
     if (!existsSync(file) || statSync(file).size === 0)
         throw new Error(`pg_dump wrote no data to ${file}`);
     chmodSync(file, 0o600);
+    const listed = await run(pgRestoreFor(options.pgDump), ['--list', file], { capture: true }).catch(() => null);
+    if (listed && listed.code !== 0) {
+        rmSync(file, { force: true });
+        throw new Error(`the dump ${file} is not readable by pg_restore: ${(listed.stderr || listed.stdout).trim().split('\n').slice(-2).join(' ')}`);
+    }
     const metadataFile = `${file}.json`;
     const restore = restoreCommand(file);
     writeFileSync(metadataFile, `${JSON.stringify({
-        createdAt: (options.now ?? new Date()).toISOString(),
+        createdAt: now.toISOString(),
         label: options.label,
         postgresMajor: serverMajor,
         pgDumpMajor: clientMajor,
+        lastApplied: options.lastApplied ?? null,
+        verified: listed !== null,
         restoreCommand: restore,
     }, null, 2)}\n`, { mode: 0o600 });
+    if (options.keep)
+        pruneBackups(options.backupDir, options.keep);
     return { file, metadataFile, restoreCommand: restore };
 }
+/**
+ * Restore a dump into an empty `public` schema (see restoreSwapSql). The
+ * services must be stopped. On failure the original schema is put back.
+ */
+export async function restoreDatabase(options) {
+    const run = options.run ?? runCommand;
+    const log = options.log ?? (() => undefined);
+    const pgRestore = options.pgRestore ?? 'pg_restore';
+    if (!existsSync(options.file))
+        throw new Error(`${options.file} does not exist`);
+    const listed = await run(pgRestore, ['--list', options.file], { capture: true }).catch((error) => {
+        throw new Error(`${pgRestore} could not be started (install the PostgreSQL client tools or set MAGNUS_PG_DUMP): ${error instanceof Error ? error.message : String(error)}`);
+    });
+    if (listed.code !== 0)
+        throw new Error(`${options.file} is not a readable dump: ${(listed.stderr || listed.stdout).trim().split('\n').slice(-2).join(' ')}`);
+    const aside = restoreAsideSchema(options.now ?? new Date());
+    await options.client.query(restoreSwapSql(aside));
+    log(`  moved the current schema aside as ${aside}`);
+    const { url, env } = splitPassword(options.databaseUrl);
+    const result = await run(pgRestore, ['--single-transaction', '--exit-on-error', '--no-owner', '--dbname', url, options.file], { capture: true, env: { ...process.env, ...env } });
+    if (result.code !== 0) {
+        await options.client.query(restoreRevertSql(aside)).catch((error) => {
+            throw new Error(`pg_restore failed and putting the original schema back failed too; it is kept as ${aside}: ${error instanceof Error ? error.message : String(error)}`);
+        });
+        throw new Error(`pg_restore failed (exit ${result.code}); the database is unchanged: ${result.stderr.trim().split('\n').slice(-3).join(' ')}`);
+    }
+    await options.client.query(restoreFinishSql(aside));
+}
 //# sourceMappingURL=migrations.js.map