@omnitron-dev/omnitron 0.3.0 → 0.3.2

package/dist/services/backup.service.js

@@ -1,3 +1,17 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var BackupService_1;
+import { OMNITRON_HOME } from '../config/defaults.js';
 /**
  * BackupService — Database backup and restore automation
  *
@@ -12,31 +26,20 @@
  * - Scheduled automated backups (cron expressions)
  * - Automatic cleanup of old backups
  */
-var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-var __metadata = (this && this.__metadata) || function (k, v) {
-    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
-};
-var __param = (this && this.__param) || function (paramIndex, decorator) {
-    return function (target, key) { decorator(target, key, paramIndex); }
-};
-var BackupService_1;
 import { randomUUID } from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 import { Injectable, Inject, Optional } from '@omnitron-dev/titan/decorators';
 import { LOGGER_SERVICE_TOKEN } from '@omnitron-dev/titan/module/logger';
 import { DAEMON_STATE_STORE_TOKEN, PROJECT_SERVICE_TOKEN } from '../shared/tokens.js';
-import { expandPath } from '../shared/paths.js';
 import { getEnv } from '../shared/env-config.js';
 import { resolveOmnitronPgConfig } from '../database/connection.js';
 import { ensurePrivateDir, sealFile, sealDirContents } from '../shared/private-files.js';
 import { dumpToFile, restoreFromFile, formatBackupSize, formatUtc, resolveBackupId } from './backup-pipeline.js';
 import { parseSchedule, nextCronDelay, describeSchedule, } from './backup-schedule.js';
+import { inspectStackContainers } from '../infrastructure/container-runtime.js';
+import { recoveryState } from '../daemon/recovery-state.js';
+import { backupStorageObjects, restoreStorageObjects, validateStorageBucket } from './backup-storage.js';
 /**
  * A schedule's first run after a start is never sooner than this — the
  * stacks whose databases it dumps are still coming up — and every further
@@ -255,6 +258,34 @@ let BackupService = class BackupService {
     store;
     projects;
     static SCHEDULES_KV_KEY = 'backup:schedules';
+    activeOperations = 0;
+    quiescing = false;
+    activeOperationCount() { return this.activeOperations; }
+    prepareHandoff() {
+        if (this.activeOperations)
+            throw new Error('Backup or restore is active; finish it before replacing the backend');
+        this.quiescing = true;
+        this.dispose();
+    }
+    async resumeAfterHandoff() {
+        if (!this.quiescing)
+            return;
+        this.quiescing = false;
+        await this.restoreSchedules().catch(error => {
+            this.logger.error({ error: reasonOf(error) }, 'Backup schedules could not resume after backend handoff; inspect backup schedules');
+        });
+    }
+    async trackOperation(operation) {
+        if (this.quiescing)
+            throw new Error('Backup admission is paused for backend handoff');
+        this.activeOperations++;
+        try {
+            return await recoveryState.run('Backup or restore in progress', operation);
+        }
+        finally {
+            this.activeOperations--;
+        }
+    }
     /** Target → the last pass it ran. Beside the schedules, in the same store. */
     static PASSES_KV_KEY = 'backup:last-pass';
     backupDir;
@@ -291,7 +322,7 @@ let BackupService = class BackupService {
         this.store = store;
         this.projects = projects;
         this.logger = loggerModule.logger;
-        this.backupDir = expandPath('~/.omnitron/backups');
+        this.backupDir = path.join(OMNITRON_HOME, 'backups');
         // 0700 on the directory, 0600 on every file in it — see `private-files`.
         // The sweep reaches backwards on purpose: the files written before this
         // existed are the ones that have been world-readable the longest, and a
@@ -403,6 +434,9 @@ let BackupService = class BackupService {
     // Create backup
     // ===========================================================================
     async createBackup(database, options) {
+        return this.trackOperation(() => this.createDatabaseBackup(database, options));
+    }
+    async createDatabaseBackup(database, options) {
         const compress = options?.compress !== false;
         // Resolved before the file is named: the name is the database's, never
         // the `<project>/<stack>/<db>` key, whose slashes would be directories.
@@ -463,33 +497,39 @@ let BackupService = class BackupService {
     // ===========================================================================
     // Non-DB backup targets: storage objects, tor keys, daemon-state (secrets)
     // ===========================================================================
-    /** First running stack's provisioned InfrastructureService (for minio/tor). */
-    getRunningInfra() {
-        const projects = this.projects;
-        if (!projects)
-            return null;
-        try {
-            const mgr = projects.getInfraManager();
-            for (const p of projects.listProjects()) {
-                let stacks = [];
-                try {
-                    stacks = projects.getRunningStacks(p.name);
-                }
-                catch {
-                    continue;
-                }
-                for (const stack of stacks) {
-                    const infra = mgr.getInstance(p.name, stack);
-                    if (infra)
-                        return { infra, project: p.name, stack };
-                }
-            }
+    /** Retained infrastructure remains addressable while application writers are stopped. */
+    resourceInstances() {
+        return this.projects?.getInfraManager().listInstances() ?? [];
+    }
+    async resolveResource(service, scope) {
+        const targets = this.resourceInstances().filter(target => target.infra.getResolvedContainerName(service));
+        const selected = scope
+            ? targets.filter(target => target.project === scope.project && target.stack === scope.stack)
+            : targets;
+        if (selected.length !== 1) {
+            const wanted = scope ? `${scope.project}/${scope.stack}/${service}` : service;
+            throw new Error(`Cannot resolve backup target '${wanted}' on this host. Name one provisioned stack: ${targets.map(target => `${target.project}/${target.stack}`).join(', ') || '(none)'}`);
+        }
+        const target = selected[0];
+        const name = target.infra.getResolvedContainerName(service);
+        const container = (await inspectStackContainers(target, [name])).find(item => item.name === name);
+        if (!container)
+            throw new Error(`Backup container '${name}' is absent; refusing another allocation`);
+        return { ...target, container: container.id };
+    }
+    async createArtifactBackup(request) {
+        if (!request || typeof request.project !== 'string' || !request.project || typeof request.stack !== 'string' || !request.stack)
+            throw new Error('An artifact backup requires an explicit project and stack');
+        if (request.kind === 'storage-objects') {
+            validateStorageBucket(request.bucket);
+            return this.createStorageBackup(request, request.bucket);
         }
-        catch { /* no running stack */ }
-        return null;
+        if (request.kind === 'tor-keys')
+            return this.createTorKeysBackup(request);
+        throw new Error('Unsupported backup artifact kind');
     }
     /** Index a already-written backup file into the SQLite backups table. */
-    indexBackupFile(app, filepath, type, origin) {
+    indexBackupFile(app, filepath, type, origin, bucket) {
         // Every non-DB target — storage objects, tor keys, daemon-state — finishes
         // here, which makes this the one place their mode can be set once.
         sealFile(filepath);
@@ -500,43 +540,9 @@ let BackupService = class BackupService {
         const from = originFields(origin);
         this.store.insertBackupSync({
             id, app, path: filepath, size_bytes: stats.size, created_at: createdAt,
-            metadata: { filename, compressed: filename.endsWith('.gz'), type, ...from },
-        });
-        return { id, database: app, filename, size: stats.size, createdAt, compressed: filename.endsWith('.gz'), ...from };
-    }
-    /**
-     * Run a script inside a container with the MinIO credentials in its
-     * ENVIRONMENT rather than in the command text.
-     *
-     * `mc alias set _bk http://localhost:9000 ${ak} ${sk}` put a secret into a
-     * string that `/bin/sh -c` then parsed — twice, since the outer command
-     * wrapped it in `sh -c '<inner>'`. A secret containing a single quote ends
-     * the inner quoting and the remainder runs as shell, on the host, as the
-     * daemon user. Nothing validates what a MinIO secret may contain, and a
-     * generated one is exactly the kind of string that eventually holds a
-     * quote.
-     *
-     * `execFile` with an argv array means the arguments never meet a shell, and
-     * the script reads `"$MC_AK"` / `"$MC_SK"` — expanded by the inner shell,
-     * not re-parsed by it.
-     */
-    async execInMinio(container, ak, sk, script, timeoutMs = 600_000) {
-        const { execFile } = await import('node:child_process');
-        const args = [
-            'exec',
-            '-e', `MC_AK=${ak}`,
-            '-e', `MC_SK=${sk}`,
-            container,
-            'sh', '-c', script,
-        ];
-        await new Promise((resolve, reject) => {
-            execFile('docker', args, { timeout: timeoutMs, maxBuffer: 200 * 1024 * 1024 }, (err) => {
-                if (err)
-                    reject(err);
-                else
-                    resolve();
-            });
+            metadata: { filename, compressed: filename.endsWith('.gz'), type, ...from, ...(bucket ? { bucket } : {}) },
         });
+        return { id, database: app, filename, size: stats.size, createdAt, compressed: filename.endsWith('.gz'), ...from, ...(bucket ? { bucket } : {}) };
     }
     /** Run a shell command, rejecting on a non-zero exit. Returns its stdout. */
     async execShell(cmd, timeoutMs = 600_000) {
@@ -558,7 +564,7 @@ let BackupService = class BackupService {
      * Measured in `~/.omnitron/backups` 2026-09-14, four files that passed it:
      *
      *     main_2026-09-14T07-51…sql.gz          98 304 B  gzip header, body cut off
-     *     priceverse_2026-09-10T06-27…sql.gz      1.5 MB  same
+     *     quotes_2026-09-10T06-27…sql.gz      1.5 MB  same
      *     storage-objects_2026-07-02T10-22…gz       122 B  «OCI runtime exec failed:
      *                                                       "tar": not found in $PATH»
      *
@@ -644,65 +650,39 @@ let BackupService = class BackupService {
             }
         }
     }
-    async restoreStorageBackup(filepath) {
-        const running = this.getRunningInfra();
-        const conn = running?.infra.getConnectionInfo('minio');
-        const container = running?.infra.getResolvedContainerName('minio');
-        if (!running || !conn || !container)
-            throw new Error('minio not found in any running stack');
-        const ak = conn.accessKey ?? 'minioadmin';
-        const sk = conn.secretKey ?? 'minioadmin';
-        const stage = path.join(this.backupDir, `.storage-restore-${randomUUID().slice(0, 8)}`);
-        // Untar on the host, docker cp into the container, then mirror back into the
-        // bucket (minio has no tar, so staging happens host-side).
-        const inner = `mc alias set _bk http://localhost:9000 "$MC_AK" "$MC_SK" >/dev/null 2>&1; ` +
-            `mc mb --ignore-existing _bk/storage >/dev/null 2>&1; ` +
-            `mc mirror --overwrite --quiet /tmp/_bk_storage _bk/storage >/dev/null 2>&1; true`;
-        try {
-            await this.execShell(`rm -rf "${stage}" && mkdir -p "${stage}" && tar xzf "${filepath}" -C "${stage}" && ` +
-                `docker exec ${container} rm -rf /tmp/_bk_storage && docker cp "${stage}/_bk_storage" ${container}:/tmp/_bk_storage`);
-            await this.execInMinio(container, ak, sk, inner);
-        }
-        finally {
-            await this.execShell(`rm -rf "${stage}"`).catch(() => { });
-        }
+    storageTarget(target, bucket) {
+        validateStorageBucket(bucket);
+        const connection = target.infra.getConnectionInfo('minio');
+        if (typeof connection?.['accessKey'] !== 'string' || typeof connection['secretKey'] !== 'string')
+            throw new Error('MinIO credentials are unavailable for the selected stack');
+        return { container: target.container, accessKey: connection['accessKey'], secretKey: connection['secretKey'], bucket };
     }
-    async restoreTorKeysBackup(filepath) {
-        const container = this.getRunningInfra()?.infra.getResolvedContainerName('tor');
-        if (!container)
-            throw new Error('tor not found in any running stack');
-        await this.execShell(`docker exec -i ${container} sh -c 'tar xzf - -C /var/lib/tor' < "${filepath}"`);
-        this.logger.warn({}, 'Tor keys restored — restart the tor container to serve the restored onion');
-    }
-    /** Object-level backup of the minio `storage` bucket (mc mirror → tar.gz). */
-    async createStorageBackup() {
-        const running = this.getRunningInfra();
-        const conn = running?.infra.getConnectionInfo('minio');
-        const container = running?.infra.getResolvedContainerName('minio');
-        if (!running || !conn || !container)
-            throw new Error('minio not found in any running stack');
-        const ts = new Date().toISOString().replace(/[:.]/g, '-');
-        const filepath = path.join(this.backupDir, `storage-objects_${ts}_${randomUUID().slice(0, 8)}.tar.gz`);
-        const stage = path.join(this.backupDir, `.storage-stage-${randomUUID().slice(0, 8)}`);
-        const ak = conn.accessKey ?? 'minioadmin';
-        const sk = conn.secretKey ?? 'minioadmin';
-        // mc is bundled in the minio image (tar is NOT): mirror the bucket into a
-        // container temp dir, `docker cp` it to the host, then tar on the host.
-        // Object-level (not raw volume) so it survives minio storage-format changes.
-        const inner = `mc alias set _bk http://localhost:9000 "$MC_AK" "$MC_SK" >/dev/null 2>&1; ` +
-            `mc mb --ignore-existing _bk/storage >/dev/null 2>&1; ` +
-            `rm -rf /tmp/_bk_storage && mkdir -p /tmp/_bk_storage && ` +
-            `mc mirror --overwrite --quiet _bk/storage /tmp/_bk_storage >/dev/null 2>&1; true`;
-        this.logger.info({ container }, 'Backing up minio storage bucket');
-        try {
-            await this.execInMinio(container, ak, sk, inner);
-            await this.execToFile(`rm -rf "${stage}" && mkdir -p "${stage}" && docker cp ${container}:/tmp/_bk_storage "${stage}/" && ` +
-                `tar czf "${filepath}" -C "${stage}" _bk_storage`, filepath);
-        }
-        finally {
-            await this.execShell(`rm -rf "${stage}"`).catch(() => { });
-        }
-        return this.indexBackupFile('storage-objects', filepath, 'storage-objects', { project: running.project, stack: running.stack });
+    artifactOrigin(meta) {
+        if (meta.scope === 'control-plane' || !!meta.project !== !!meta.stack)
+            throw new Error('Invalid artifact backup origin; refusing an inferred restore target');
+        return meta.project && meta.stack ? { project: meta.project, stack: meta.stack } : undefined;
+    }
+    async restoreStorageBackup(filepath, meta) {
+        const target = await this.resolveResource('minio', this.artifactOrigin(meta));
+        await restoreStorageObjects(this.storageTarget(target, meta.bucket ?? 'storage'), filepath);
+    }
+    async restoreTorKeysBackup(filepath, meta) {
+        const target = await this.resolveResource('tor', this.artifactOrigin(meta));
+        await restoreFromFile('docker', ['exec', '-i', target.container, 'tar', 'xf', '-', '-C', '/var/lib/tor'], filepath, true);
+        this.logger.warn({ project: target.project, stack: target.stack }, 'Tor keys restored — restart that stack’s tor container to serve the restored onion');
+    }
+    /** Explicit bucket and stack; an omitted scope is accepted only for one unambiguous legacy target. */
+    async createStorageBackup(scope, bucket = 'storage') {
+        return this.trackOperation(() => this.captureStorageBackup(scope, bucket));
+    }
+    async captureStorageBackup(scope, bucket) {
+        validateStorageBucket(bucket);
+        const target = await this.resolveResource('minio', scope);
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        const filepath = path.join(this.backupDir, `storage-objects_${timestamp}_${randomUUID().slice(0, 8)}.tar.gz`);
+        this.logger.info({ project: target.project, stack: target.stack, bucket, container: target.container }, 'Backing up storage bucket');
+        await backupStorageObjects(this.storageTarget(target, bucket), filepath);
+        return this.indexBackupFile('storage-objects', filepath, 'storage-objects', { project: target.project, stack: target.stack }, bucket);
     }
     /**
      * Snapshot the Tor hidden-service keys (the .onion identity).
@@ -721,21 +701,25 @@ let BackupService = class BackupService {
      * is kept, so a hidden-service directory added later is still captured
      * without anyone remembering to update a list.
      */
-    async createTorKeysBackup() {
-        const running = this.getRunningInfra();
-        const container = running?.infra.getResolvedContainerName('tor');
-        if (!running || !container)
-            throw new Error('tor not found in any running stack');
+    async createTorKeysBackup(scope) {
+        return this.trackOperation(() => this.captureTorKeys(scope));
+    }
+    async captureTorKeys(scope) {
+        const running = await this.resolveResource('tor', scope);
+        const container = running.container;
         const ts = new Date().toISOString().replace(/[:.]/g, '-');
         const filepath = path.join(this.backupDir, `tor-keys_${ts}_${randomUUID().slice(0, 8)}.tar.gz`);
         this.logger.info({ container }, 'Backing up tor hidden-service keys');
-        const excludes = TOR_REGENERABLE_STATE.map((g) => `--exclude='${g}'`).join(' ');
-        await this.execToFile(`docker exec ${container} tar czf - ${excludes} -C /var/lib/tor . > "${filepath}"`, filepath);
+        const excludes = TOR_REGENERABLE_STATE.map(g => `--exclude=${g}`);
+        await dumpToFile('docker', ['exec', container, 'tar', 'cf', '-', ...excludes, '-C', '/var/lib/tor', '.'], filepath, true);
         return this.indexBackupFile('tor-keys', filepath, 'tor-keys', { project: running.project, stack: running.stack });
     }
     /** Online snapshot of the daemon-state DB (encrypted secrets + backup index). */
     async createSecretsBackup() {
-        const src = expandPath('~/.omnitron/data/daemon-state.db');
+        return this.trackOperation(() => this.captureSecrets());
+    }
+    async captureSecrets() {
+        const src = path.join(OMNITRON_HOME, 'data/daemon-state.db');
         if (!fs.existsSync(src))
             throw new Error('daemon-state.db not found');
         const ts = new Date().toISOString().replace(/[:.]/g, '-');
@@ -752,6 +736,9 @@ let BackupService = class BackupService {
      * captured, not fatal.
      */
     async createFullBackup(trigger = 'manual') {
+        return this.trackOperation(() => this.createFullBackupOnce(trigger));
+    }
+    async createFullBackupOnce(trigger = 'manual') {
         const startedAt = new Date();
         const results = [];
         for (const r of await this.backUpStackDatabases()) {
@@ -763,18 +750,34 @@ let BackupService = class BackupService {
             // took it: its one copy on the master, 2026-09-05T07:39Z, was made by
             // hand. `omnitron-pg` names it whatever the stacks' databases are called.
             [CONTROL_PLANE_DB, () => this.createBackup('omnitron-pg', { compress: true })],
-            ['storage-objects', () => this.createStorageBackup()],
-            ['tor-keys', () => this.createTorKeysBackup()],
             ['daemon-state', () => this.createSecretsBackup()],
         ];
-        for (const [target, fn] of extras) {
+        for (const resource of this.resourceInstances()) {
+            if (resource.infra.getResolvedContainerName('minio')) {
+                try {
+                    const buckets = resource.infra.getMinioBuckets();
+                    if (!buckets.length)
+                        throw new Error('No bucket inventory is declared; use an explicit artifact backup target');
+                    for (const bucket of buckets)
+                        extras.push([`storage-objects/${bucket}`, () => this.createStorageBackup(resource, bucket), resource]);
+                }
+                catch (error) {
+                    results.push({ target: 'storage-objects', project: resource.project, stack: resource.stack,
+                        ok: false, error: reasonOf(error) });
+                }
+            }
+            if (resource.infra.getResolvedContainerName('tor'))
+                extras.push(['tor-keys', () => this.createTorKeysBackup(resource), resource]);
+        }
+        for (const [target, fn, scope] of extras) {
             try {
                 const info = await fn();
-                results.push({ target, ok: true, id: info.id, size: info.size });
+                results.push({ target, ok: true, id: info.id, size: info.size, ...(info.project && info.stack ? { project: info.project, stack: info.stack } : {}) });
             }
             catch (err) {
-                results.push({ target, ok: false, error: err.message });
-                this.logger.error({ target, error: reasonOf(err) }, 'Backup failed');
+                const origin = scope ? { project: scope.project, stack: scope.stack } : {};
+                results.push({ target, ok: false, error: err.message, ...origin });
+                this.logger.error({ target, ...origin, error: reasonOf(err) }, 'Backup failed');
             }
         }
         this.finishPass('full', trigger, startedAt, results.map(qualifiedEntry));
@@ -903,6 +906,7 @@ let BackupService = class BackupService {
                 // Where it came from, when the row recorded it — see `BackupOrigin`.
                 ...(meta.project && meta.stack ? { project: meta.project, stack: meta.stack } : {}),
                 ...(meta.scope ? { scope: meta.scope } : {}),
+                ...(meta.bucket ? { bucket: meta.bucket } : {}),
             });
         }
         return backups;
@@ -911,6 +915,9 @@ let BackupService = class BackupService {
     // Restore backup
     // ===========================================================================
     async restoreBackup(backupId) {
+        return this.trackOperation(() => this.restoreBackupOnce(backupId));
+    }
+    async restoreBackupOnce(backupId) {
         this.migrateLegacyMetaIfPresent();
         // The full id, or a prefix of it that names one backup — what the CLI
         // prints. An ambiguous prefix is refused with every candidate named.
@@ -922,10 +929,10 @@ let BackupService = class BackupService {
         this.logger.info({ app: row.app, type, path: row.path }, 'Restoring backup');
         switch (type) {
             case 'storage-objects':
-                await this.restoreStorageBackup(row.path);
+                await this.restoreStorageBackup(row.path, meta);
                 break;
             case 'tor-keys':
-                await this.restoreTorKeysBackup(row.path);
+                await this.restoreTorKeysBackup(row.path, meta);
                 break;
             case 'daemon-state':
                 throw new Error('Refusing to restore daemon-state.db into a running daemon. Stop the daemon (`omnitron down`), ' +
@@ -1070,6 +1077,8 @@ let BackupService = class BackupService {
      * so a long pass does not push the cadence back, and passes never overlap.
      */
     armSchedule(database, plan, slot = 0) {
+        if (this.quiescing)
+            throw new Error('Backup admission is paused for backend handoff');
         const existing = this.schedules.get(database);
         if (existing) {
             existing.cancelled = true;
@@ -1084,7 +1093,7 @@ let BackupService = class BackupService {
             entry.nextRunAt = new Date(Date.now() + delay).toISOString();
             entry.timer = setTimeout(() => {
                 const startedAt = Date.now();
-                void this.runExclusive(() => this.runScheduledBackup(database)).then((outcome) => {
+                void this.runExclusive(() => entry.cancelled ? Promise.resolve('empty') : this.runScheduledBackup(database)).then((outcome) => {
                     if (plan.kind === 'interval') {
                         const backedUpSomething = outcome === 'ok' || outcome === 'partial';
                         const wait = backedUpSomething ? plan.intervalMs : Math.min(plan.intervalMs, RETRY_AFTER_NOTHING_MS);
@@ -1164,6 +1173,11 @@ let BackupService = class BackupService {
      * decides how soon the next attempt comes.
      */
     async runScheduledBackup(database) {
+        if (this.quiescing)
+            return 'empty';
+        return this.trackOperation(() => this.runScheduledPass(database));
+    }
+    async runScheduledPass(database) {
         let outcome = 'failed';
         try {
             if (database === 'full')
@@ -1185,7 +1199,9 @@ let BackupService = class BackupService {
                 }
                 outcome = this.finishPass(database, 'schedule', startedAt, [entry]).outcome;
             }
-            await this.pruneOldBackups(database).catch(() => { });
+            await this.pruneOldBackups(database).catch(error => {
+                this.logger.error({ database, error: reasonOf(error) }, 'Backup retention failed');
+            });
         }
         catch (err) {
             this.logger.error({ database, error: err.message }, 'Scheduled backup failed');
@@ -1230,7 +1246,8 @@ let BackupService = class BackupService {
             const byOrigin = new Map();
             for (const r of ofName) {
                 const origin = r.project && r.stack ? `${r.project}/${r.stack}` : r.scope === 'control-plane' ? 'control-plane' : '';
-                byOrigin.set(origin, [...(byOrigin.get(origin) ?? []), r]);
+                const resource = origin && r.database === 'storage-objects' ? `${origin}/${r.bucket ?? 'storage'}` : origin;
+                byOrigin.set(resource, [...(byOrigin.get(resource) ?? []), r]);
             }
             const doomed = [];
             for (const [origin, group] of byOrigin) {
@@ -1365,8 +1382,8 @@ let BackupService = class BackupService {
      *
      * A stack whose infrastructure is not on this host used to be skipped with
      * `if (!infra) continue`, and nothing else ever mentioned it. On the master
-     * that is daos/test, whose database container runs on 37.27.130.185: every
-     * `all` pass was `{"total":6}` — daos/dev's six — while `backup list`
+     * that is sample/test, whose database container runs on 37.27.130.185: every
+     * `all` pass was `{"total":6}` — sample/dev's six — while `backup list`
      * printed «Found 380 backup(s)» and `backup schedules` «all hourly».
      * Remote stacks are still not dumped from here; they are now named.
      */
@@ -1378,7 +1395,7 @@ let BackupService = class BackupService {
             return { databases, stacks };
         try {
             const infraManager = projects.getInfraManager();
-            for (const p of projects.listProjects()) {
+            for (const p of projects.listProjectRegistrations()) {
                 let running = [];
                 try {
                     running = projects.getRunningStacks(p.name);
@@ -1386,6 +1403,7 @@ let BackupService = class BackupService {
                 catch {
                     continue;
                 }
+                running = [...new Set([...running, ...this.resourceInstances().filter(item => item.project === p.name).map(item => item.stack)])];
                 for (const stack of running) {
                     const infra = infraManager.getInstance(p.name, stack);
                     if (!infra) {
@@ -1440,6 +1458,9 @@ let BackupService = class BackupService {
      * and — marked `skipped` — every running stack it could not reach.
      */
     async createAllBackups(trigger = 'manual') {
+        return this.trackOperation(() => this.createAllBackupsOnce(trigger));
+    }
+    async createAllBackupsOnce(trigger = 'manual') {
         const startedAt = new Date();
         const results = await this.backUpStackDatabases();
         this.finishPass('all', trigger, startedAt, results.map(({ database, ...rest }) => qualifiedEntry({ target: database, ...rest })));