@omnitron-dev/omnitron 0.2.0 → 0.3.1

package/dist/services/backup.rpc-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup.rpc-service.js","sourceRoot":"","sources":["../../src/services/backup.rpc-service.ts"],"names":[],"mappings":"AAAA;;;;GAIG;;;;;;;;;;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAIxD,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IACE;IAA7B,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;IAAG,CAAC;IAGjD,AAAN,KAAK,CAAC,YAAY,CAAC,IAA8C;QAC/D,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACnH,CAAC;IAGK,AAAN,KAAK,CAAC,WAAW,CAAC,IAA4B;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClD,CAAC;IAGK,AAAN,KAAK,CAAC,aAAa,CAAC,IAA0B;QAC5C,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAGK,AAAN,KAAK,CAAC,YAAY,CAAC,IAA0B;QAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAGK,AAAN,KAAK,CAAC,WAAW,CAAC,IAAwC;QACxD,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACzD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAGK,AAAN,KAAK,CAAC,WAAW,CAAC,IAA0B;QAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC;CACF,CAAA;AA/BO;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC;;;;oDAGxC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC;;;;mDAGzC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC;;;;qDAIxC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC;;;;oDAIxC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC;;;;mDAIxC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC;;;;mDAGzC;AAlCU,gBAAgB;IAD5B,OAAO,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC;;GACxB,gBAAgB,CAmC5B"}
+{"version":3,"file":"backup.rpc-service.js","sourceRoot":"","sources":["../../src/services/backup.rpc-service.ts"],"names":[],"mappings":"AAAA;;;;GAIG;;;;;;;;;;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAMxD,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IACE,OAAO;IAApC,YAA6B,OAAsB;uBAAtB,OAAO;IAAkB,CAAC;IAGjD,AAAN,KAAK,CAAC,YAAY,CAAC,IAA8C;QAC/D,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACnH,CAAC;IAED,qDAAqD;IAE/C,AAAN,KAAK,CAAC,gBAAgB;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;IACzC,CAAC;IAED,4EAA4E;IAEtE,AAAN,KAAK,CAAC,gBAAgB;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;IACzC,CAAC;IAGK,AAAN,KAAK,CAAC,WAAW,CAAC,IAA4B;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClD,CAAC;IAGK,AAAN,KAAK,CAAC,aAAa,CAAC,IAA0B;QAC5C,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAGK,AAAN,KAAK,CAAC,YAAY,CAAC,IAA0B;QAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAGK,AAAN,KAAK,CAAC,WAAW,CAAC,IAAwC;QACxD,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACzD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAGK,AAAN,KAAK,CAAC,WAAW,CAAC,IAA0B;QAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC;IAGK,AAAN,KAAK,CAAC,aAAa;QACjB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;IACtC,CAAC;IAGK,AAAN,KAAK,CAAC,cAAc,CAAC,IAA0B;QAC7C,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IAEG,AAAN,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;IAClC,CAAC;CACF,CAAA;AAhEO;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC;;;;oDAGxC;AAIK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC;;;;wDAGxC;AAIK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC;;;;wDAGxC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC;;;;mDAGzC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC;;;;qDAIxC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC;;;;oDAIxC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC;;;;mDAIxC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC;;;;mDAGzC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC;;;;qDAGzC;AAGK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC;;;;sDAIxC;AAQK;IADL,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC;;;;uDAGzC;AAnEU,gBAAgB;IAD5B,OAAO,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC;;GACxB,gBAAgB,CAoE5B"}

package/dist/services/backup.service.d.ts

@@ -12,34 +12,478 @@
  * - Scheduled automated backups (cron expressions)
  * - Automatic cleanup of old backups
  */
-import type { ILogger } from '@omnitron-dev/titan/module/logger';
-export interface BackupInfo {
-    id: string;
-    database: string;
-    filename: string;
-    size: number;
-    createdAt: string;
-    compressed: boolean;
+import { type ILoggerModule } from '@omnitron-dev/titan/module/logger';
+import type { DaemonStateStore } from '../daemon/daemon-state-store.service.js';
+import type { ProjectService } from './project.service.js';
+import type { BackupInfo } from '../shared/dto/backups.js';
+export type { BackupInfo, } from '../shared/dto/backups.js';
+/**
+ * How one pass ended. `empty` is its own word: a pass that found nothing to
+ * back up did not succeed, and must not read as if it had.
+ */
+export type BackupPassOutcome = 'ok' | 'partial' | 'failed' | 'empty';
+/** One entry of a pass: a database, or a `full`-only artefact. */
+export interface BackupPassEntry {
+    target: string;
+    ok: boolean;
+    id?: string;
+    size?: number;
+    error?: string;
+    /**
+     * A running stack this pass could not reach at all — its databases are not
+     * on this host. Reported with the pass, and left out of its outcome: it is
+     * a gap in what the master can do, not a dump that failed.
+     */
+    skipped?: boolean;
+}
+/**
+ * Where a backup came from. The index row used to hold id, database,
+ * filename, size, createdAt, compressed — no stack — so a listing could not
+ * say whose `main` a row was, and a restore went to whichever stack happened
+ * to be first.
+ */
+export type BackupOrigin = {
+    project: string;
+    stack: string;
+} | 'control-plane';
+/** A backup as listed: the wire DTO, plus where it came from. */
+export interface BackupRecord extends BackupInfo {
+    /** The project and stack a stack database belongs to. Absent on rows written before it was recorded. */
+    project?: string;
+    stack?: string;
+    /** `control-plane` for the daemon's own state (`omnitron`, `daemon-state`). */
+    scope?: 'stack' | 'control-plane';
+}
+/** A running stack, and whether `all` can reach its databases from this host. */
+export interface BackupStackCoverage {
+    project: string;
+    stack: string;
+    /** The databases `all` dumps for it; empty when it has none this host can reach. */
+    databases: string[];
+    /** Why its databases are not backed up from here; absent when they are. */
+    notBackedUp?: string;
+}
+/** What the last pass of one target did — persisted, so it outlives a restart. */
+export interface BackupPassRecord {
+    target: string;
+    trigger: 'schedule' | 'manual';
+    startedAt: string;
+    finishedAt: string;
+    outcome: BackupPassOutcome;
+    ok: number;
+    total: number;
+    failures: Array<{
+        target: string;
+        error: string;
+    }>;
+    /** Running stacks the pass could not reach, and why. */
+    notBackedUp?: Array<{
+        stack: string;
+        reason: string;
+    }>;
+}
+/** A configured schedule, and what it last did. */
+export interface BackupScheduleStatus {
+    target: string;
+    /** The specification as stored — cron, preset, or ms. */
+    spec: string;
+    /** `describeSchedule` of it, or null when the stored spec cannot be read. */
+    schedule: string | null;
+    /** Why the stored spec cannot be read: configured, and not running. */
+    error?: string;
+    armed: boolean;
+    /** The last pass this target ran, from any daemon run; null if none was recorded. */
+    lastPass: BackupPassRecord | null;
+    /** The target's newest backup in the index — what its next run is measured from. */
+    lastBackupAt: string | null;
+    /** When the armed timer fires, or null when the schedule is not armed. */
+    nextRunAt: string | null;
+}
+/** Everything `backup schedules` and `backup list` need beyond the index rows. */
+export interface BackupStatus {
+    schedules: BackupScheduleStatus[];
+    /** Every running stack, and whether this host backs its databases up. */
+    stacks: BackupStackCoverage[];
+}
+/**
+ * The outcome of a pass, from its entries. No entries is `empty`, never `ok`:
+ * «0 of 0 backed up» is a pass that protected nothing.
+ */
+export declare function passOutcome(entries: Array<{
+    ok: boolean;
+    skipped?: boolean;
+}>): BackupPassOutcome;
+export interface LeftoverSweep {
+    removed: Array<{
+        name: string;
+        bytes: number;
+        reason: string;
+    }>;
+    /** Sidecars left alone because the database they belong to is still there. */
+    besideBackups: {
+        files: number;
+        bytes: number;
+    };
+    /** Orphaned sidecars left alone because their WAL still holds frames. */
+    keptOrphans: string[];
 }
+/**
+ * Remove what an interrupted pass leaves in the backup directory.
+ *
+ * Measured on the master, 2026-09-23: 28 files, 79 300 253 B (75.6 MiB), that
+ * no index row names — so `backup list` never showed them and retention never
+ * pruned them:
+ *
+ *     23 zero-byte files               22 `tor-keys`, 1 `storage` (July–September)
+ *      2 `main_…sql.gz.partial`         7 150 144 B and 70 477 299 B — the daemon
+ *                                      restarted mid-dump at 13:15:43Z and
+ *                                      21:15:47Z on 09-21, and no catch of ours
+ *                                      runs in a process that is gone
+ *      3 truncated archives            the ones named on `execToFile`
+ *
+ * A `.partial` is by construction not a backup (`dumpToFile`, `execToFile`
+ * write under it and rename only a finished file), and a zero-byte file
+ * restores nothing, so both go. Nothing is in flight when this runs: it is
+ * called from the constructor, once, before any schedule is armed.
+ *
+ * The SQLite sidecars are more careful, because something may have a
+ * database open. Measured: 102 of them (51 `-shm` of 32 768 B, 51 `-wal` of
+ * 0 B). Apple's `sqlite3` keeps both files after it closes a WAL database, and
+ * a `daemon-state_*.db` backup IS one — `.backup` copies the WAL flag in the
+ * header from the live store. So every open of a backup left a pair: 3 pairs
+ * `….db.partial-shm/-wal` from our own integrity check, which ran on the
+ * staging name and then renamed the file away from them; 4 pairs whose `.db`
+ * retention had deleted, because `deleteBackup` removed only the named file;
+ * and 44 pairs beside backups that still exist, written by bulk opens at
+ * 2026-09-11 23:50 and 09-14 15:27 local — not by this code.
+ *
+ * Only an ORPHAN pair is removed — its database is gone, so nothing can open
+ * it — and only while its WAL holds no frames. A pair beside a live database
+ * is counted and kept: that one is not provably idle.
+ */
+export declare function sweepBackupLeftovers(dir: string): LeftoverSweep;
+/**
+ * Which databases a sweep should bound, given what is scheduled and what
+ * actually has backups.
+ *
+ * Split out because the choice is the whole point: pruning only the
+ * scheduled set means retention runs where the producer still runs and
+ * nowhere else, so anything dropped from the configuration keeps its history
+ * for ever.
+ */
+export declare function databasesToPrune(scheduled: string[], onDisk: string[]): string[];
 export declare class BackupService {
-    private readonly logger;
+    /**
+     * T-7 — backup metadata persistence moved off side-car .meta.json
+     * files onto the SQLite-backed `backups` table in
+     * DaemonStateStore. The .sql.gz bytes themselves stay on disk
+     * (large + streaming pg_restore needs random access), but the
+     * INDEX of `which backup exists, for which app, when, how big`
+     * lives in SQLite — atomic upserts, no torn-write window on
+     * kill -9 mid-write. One-shot migration of legacy .meta.json
+     * files happens on the first list/restore call.
+     */
+    private readonly store;
+    /**
+     * Optional so the daemon still boots (and omnitron-pg self-backup still
+     * works) even if the project subsystem is unavailable. When present it is
+     * the source of truth for the running stacks' real DB topology, replacing
+     * the old hardcoded omnitron-pg-only mappings.
+     */
+    private readonly projects?;
+    private static readonly SCHEDULES_KV_KEY;
+    /** Target → the last pass it ran. Beside the schedules, in the same store. */
+    private static readonly PASSES_KV_KEY;
     private readonly backupDir;
+    private readonly logger;
     private schedules;
-    constructor(logger: ILogger, backupDir?: string);
+    /**
+     * True once the legacy .meta.json scan + import has run for this
+     * process. Subsequent list/restore calls skip the directory walk.
+     */
+    private legacyMigrated;
+    /** The tail of the scheduled passes, which run one at a time — see `runExclusive`. */
+    private passQueue;
+    constructor(loggerModule: ILoggerModule, 
+    /**
+     * T-7 — backup metadata persistence moved off side-car .meta.json
+     * files onto the SQLite-backed `backups` table in
+     * DaemonStateStore. The .sql.gz bytes themselves stay on disk
+     * (large + streaming pg_restore needs random access), but the
+     * INDEX of `which backup exists, for which app, when, how big`
+     * lives in SQLite — atomic upserts, no torn-write window on
+     * kill -9 mid-write. One-shot migration of legacy .meta.json
+     * files happens on the first list/restore call.
+     */
+    store: DaemonStateStore, 
+    /**
+     * Optional so the daemon still boots (and omnitron-pg self-backup still
+     * works) even if the project subsystem is unavailable. When present it is
+     * the source of truth for the running stacks' real DB topology, replacing
+     * the old hardcoded omnitron-pg-only mappings.
+     */
+    projects?: ProjectService | undefined);
+    /**
+     * Once per start: remove what interrupted passes left behind, and name what
+     * is still on disk without an index row. See `sweepBackupLeftovers`.
+     */
+    private clearLeftovers;
+    /**
+     * Lazy one-shot import of legacy .meta.json side-cars into the
+     * SQLite `backups` table. Idempotent — flag-guarded so multiple
+     * concurrent listBackups() calls don't double-import. After this
+     * runs the .meta.json files are unlinked (the SQLite row is
+     * authoritative).
+     */
+    private migrateLegacyMetaIfPresent;
     createBackup(database: string, options?: {
         compress?: boolean;
-    }): Promise<BackupInfo>;
-    listBackups(database?: string): Promise<BackupInfo[]>;
+    }): Promise<BackupRecord>;
+    /** First running stack's provisioned InfrastructureService (for minio/tor). */
+    private getRunningInfra;
+    /** Index a already-written backup file into the SQLite backups table. */
+    private indexBackupFile;
+    /**
+     * Run a script inside a container with the MinIO credentials in its
+     * ENVIRONMENT rather than in the command text.
+     *
+     * `mc alias set _bk http://localhost:9000 ${ak} ${sk}` put a secret into a
+     * string that `/bin/sh -c` then parsed — twice, since the outer command
+     * wrapped it in `sh -c '<inner>'`. A secret containing a single quote ends
+     * the inner quoting and the remainder runs as shell, on the host, as the
+     * daemon user. Nothing validates what a MinIO secret may contain, and a
+     * generated one is exactly the kind of string that eventually holds a
+     * quote.
+     *
+     * `execFile` with an argv array means the arguments never meet a shell, and
+     * the script reads `"$MC_AK"` / `"$MC_SK"` — expanded by the inner shell,
+     * not re-parsed by it.
+     */
+    private execInMinio;
+    /** Run a shell command, rejecting on a non-zero exit. Returns its stdout. */
+    private execShell;
+    /**
+     * Run a command that produces a backup file, and publish the name only if
+     * the file is READABLE.
+     *
+     * The check was `size === 0`, and "not empty" is not "not corrupt".
+     * Measured in `~/.omnitron/backups` 2026-09-14, four files that passed it:
+     *
+     *     main_2026-09-14T07-51…sql.gz          98 304 B  gzip header, body cut off
+     *     quotes_2026-09-10T06-27…sql.gz      1.5 MB  same
+     *     storage-objects_2026-07-02T10-22…gz       122 B  «OCI runtime exec failed:
+     *                                                       "tar": not found in $PATH»
+     *
+     * The last one is the shape in miniature: a shell error message, sitting
+     * under a `.tar.gz` name, 122 bytes long and therefore "not empty". The
+     * property that matters is whether the archive opens; the property measured
+     * was how many bytes exist.
+     *
+     * The write also lands on the final name from its first byte, so a command
+     * interrupted half-way — a full disk, a killed container — leaves a
+     * truncated archive under the name a restore would pick up. Today's
+     * ENOSPC-mid-write is exactly that case.
+     *
+     * So: write to `<name>.partial`, verify, then rename. Rename within a
+     * directory is atomic, which makes the visible state binary — either the
+     * name is absent, or it names an archive that opened.
+     */
+    private execToFile;
+    /**
+     * Open the archive and fail if it does not open.
+     *
+     * Keyed on the FINAL name's extension, not the staging one — `.partial` is
+     * an implementation detail and would otherwise match nothing and verify
+     * nothing, which is the failure mode this whole method exists to remove.
+     *
+     * An extension nobody has taught this to check passes: the alternative is
+     * refusing a backup that was taken correctly, and a verification gap is
+     * better than losing a good archive. The three formats produced here are
+     * all covered.
+     */
+    private assertReadable;
+    private restoreStorageBackup;
+    private restoreTorKeysBackup;
+    /** Object-level backup of the minio `storage` bucket (mc mirror → tar.gz). */
+    createStorageBackup(): Promise<BackupInfo>;
+    /**
+     * Snapshot the Tor hidden-service keys (the .onion identity).
+     *
+     * `-C /var/lib/tor .` took the whole data directory, and what dominates that
+     * directory is Tor's DIRECTORY CACHE, not key material. Measured on this
+     * host: a 29 MiB archive of which `cached-microdescs` was 42.5 MiB
+     * uncompressed, `cached-microdescs.new` 16 MiB and
+     * `cached-microdesc-consensus` 3.5 MiB — against `hs_ed25519_secret_key`
+     * files of 96 bytes each. Forty such archives held 400 MiB, essentially all
+     * of it a public consensus any Tor client re-downloads in minutes, on a
+     * machine whose disk has filled before and taken the database and the onion
+     * with it.
+     *
+     * The exclusions are Tor's own regenerable state, by name. Everything else
+     * is kept, so a hidden-service directory added later is still captured
+     * without anyone remembering to update a list.
+     */
+    createTorKeysBackup(): Promise<BackupInfo>;
+    /** Online snapshot of the daemon-state DB (encrypted secrets + backup index). */
+    createSecretsBackup(): Promise<BackupInfo>;
+    /**
+     * Full backup: every stack DB + the control-plane DB + minio storage
+     * objects + tor keys + daemon-state (secrets). Per-target failures are
+     * captured, not fatal.
+     */
+    createFullBackup(trigger?: BackupPassRecord['trigger']): Promise<Array<{
+        target: string;
+        ok: boolean;
+        id?: string;
+        size?: number;
+        error?: string;
+    }>>;
+    /**
+     * Close a pass: one summary line at a level that matches what happened, and
+     * a persisted record `backup schedules` shows.
+     *
+     * The summary was always `info` — `createAllBackups complete
+     * {"total":6,"ok":5}` for the 06:04Z pass on the master that lost `main`,
+     * the same level and message as a pass that lost nothing — and
+     * `runScheduledBackup` discarded the results, its `error` reachable only by
+     * a throw that a per-entry catch never let happen.
+     */
+    private finishPass;
+    /** The last recorded pass of every target. */
+    private lastPasses;
+    /**
+     * What the schedules are and what each last did — what `backup schedules`
+     * prints, and what `backup list` warns from.
+     */
+    getStatus(): Promise<BackupStatus>;
+    listBackups(database?: string): Promise<BackupRecord[]>;
     restoreBackup(backupId: string): Promise<void>;
     deleteBackup(backupId: string): Promise<void>;
+    /**
+     * Arm and PERSIST a recurring backup. `database` may be a specific DB name,
+     * a "<project>/<stack>/<db>" key, or the special target "all" — which backs
+     * up every database of every running stack each tick. Persisted to state_kv
+     * so it survives daemon restarts (re-armed via restoreSchedules on boot).
+     */
     setSchedule(database: string, cron: string): Promise<void>;
+    /** Cancel + un-persist a schedule. */
+    removeSchedule(database: string): Promise<void>;
+    /** Re-arm every persisted schedule. Called once at daemon start. */
+    restoreSchedules(): Promise<void>;
+    listSchedules(): Promise<Record<string, string>>;
     getSchedule(database: string): Promise<string | null>;
+    /**
+     * In-memory timer arm (no persistence) — shared by setSchedule/restore.
+     *
+     * An interval schedule used to be `setInterval(run, intervalMs)` from the
+     * moment it was armed — the moment the daemon started — whatever had run
+     * before. Measured on the master: 83 daemon starts between 09-20 07:23Z and
+     * 09-23, the longest uptime 10 h 51 min, so the `full` daily pass never
+     * fired, and its newest `tor-keys`, `storage-objects` and `daemon-state`
+     * copies were 3 d 15 h old; the hourly `all` pass left 13 gaps over two
+     * hours in 74 h (the longest 5 h 43 min, 18 restarts inside it).
+     *
+     * Now the first run is due one interval after the target's newest backup
+     * in the index — immediately if that is already past, which after days of
+     * restarts it is — but never sooner than `FIRST_RUN_GRACE_MS`, and each
+     * further schedule armed by the same start one `FIRST_RUN_STAGGER_MS` later
+     * (`slot`). Every later run is measured from when the previous one STARTED,
+     * so a long pass does not push the cadence back, and passes never overlap.
+     */
+    private armSchedule;
+    /**
+     * Scheduled passes run one at a time. Two schedules due together — which is
+     * what every start after a long gap produces — would otherwise dump the same
+     * databases twice, concurrently, on a machine already under load.
+     */
+    private runExclusive;
+    /**
+     * When `target` last produced a backup, by the index — the newest row among
+     * the artefacts that target writes: `full` → the ones only it produces;
+     * `all` → any stack database; a database → that database. Null when none.
+     */
+    private lastBackupOf;
+    /**
+     * One scheduled tick. Extracted so both timer shapes share it, and so a
+     * failure is reported the same way from either. Returns how it ended, which
+     * decides how soon the next attempt comes.
+     */
+    private runScheduledBackup;
+    /**
+     * Retention: keep the most recent `keep` backups per database, delete older.
+     * Bounds unbounded growth from hourly schedules (~168/week/db otherwise).
+     *
+     * Per database OF EACH STACK, now that a row says which stack it came from:
+     * two stacks' `main` are two histories, and one count across both would
+     * keep half of each. A row from before the stack was recorded is pruned as
+     * rows always were — once `keep` newer rows of its database name exist —
+     * so the history already on disk is bounded exactly as before.
+     */
+    private pruneOldBackups;
     dispose(): void;
     private pgDumpDocker;
     private pgRestoreDocker;
     private pgDumpLocal;
     private pgRestoreLocal;
-    private resolveDbConfig;
-    private parseCronInterval;
+    /**
+     * The database a backup target names, resolved LIVE from the running
+     * stacks' real infrastructure (container, credentials, DB list):
+     *
+     *   `<project>/<stack>/<db>`  that database of that stack;
+     *   `omnitron-pg`             the control plane, always;
+     *   a bare name               the one running stack that has it — and a
+     *                             refusal naming every candidate when more than
+     *                             one does. It was "first stack wins", which
+     *                             dumps, and restores into, whichever stack the
+     *                             iteration happened to reach first;
+     *   `omnitron`                the control plane, when no stack has one.
+     */
+    private resolveDbTarget;
+    /**
+     * The control-plane database, from the resolver the daemon itself connects
+     * with rather than a second copy of its credentials — `localhost:5480`,
+     * `omnitron`/`omnitron` were written out here by hand. Dumped through the
+     * `omnitron-pg` container unless OMNITRON_DATABASE_URL points elsewhere.
+     */
+    private controlPlaneTarget;
+    /** Where a backup being restored goes: where it was taken from. */
+    private restoreTargetOf;
+    /**
+     * Every running stack, and every database of it this host can reach — from
+     * the provisioned InfrastructureService, so the backup set is always in
+     * lock-step with what omnitron provisioned.
+     *
+     * A stack whose infrastructure is not on this host used to be skipped with
+     * `if (!infra) continue`, and nothing else ever mentioned it. On the master
+     * that is sample/test, whose database container runs on 37.27.130.185: every
+     * `all` pass was `{"total":6}` — sample/dev's six — while `backup list`
+     * printed «Found 380 backup(s)» and `backup schedules` «all hourly».
+     * Remote stacks are still not dumped from here; they are now named.
+     */
+    private surveyStacks;
+    /** Why a running stack's databases cannot be reached from this host. */
+    private whyNotHere;
+    /**
+     * Back up every database of every running stack. A single DB failure is
+     * captured per-entry and does not abort the rest. Returns per-DB results,
+     * and — marked `skipped` — every running stack it could not reach.
+     */
+    createAllBackups(trigger?: BackupPassRecord['trigger']): Promise<Array<{
+        database: string;
+        ok: boolean;
+        id?: string;
+        size?: number;
+        error?: string;
+    }>>;
+    /**
+     * The database half of `all` and `full`, stack by stack. Each failure is
+     * logged where it happens, with the database and its reason: the 06:04Z
+     * pass on the master lost `main` to the 600 s dump timeout, and the only
+     * trace was a database missing from the next listing.
+     *
+     * Every database of every stack, by its `<project>/<stack>/<db>` key: by
+     * bare name, a second stack's `main` was never dumped at all.
+     */
+    private backUpStackDatabases;
 }
 //# sourceMappingURL=backup.service.d.ts.map

package/dist/services/backup.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup.service.d.ts","sourceRoot":"","sources":["../../src/services/backup.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAMjE,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,OAAO,CAAC;CACrB;AAYD,qBAAa,aAAa;IAKtB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJzB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,SAAS,CAAoC;gBAGlC,MAAM,EAAE,OAAO,EAChC,SAAS,CAAC,EAAE,MAAM;IAUd,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,CAAC;IAiDrF,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IA2BrD,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAyB9C,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB7C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB1D,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAI3D,OAAO,IAAI,IAAI;YAWD,YAAY;YA4BZ,eAAe;YAuBf,WAAW;YAsBX,cAAc;IAqB5B,OAAO,CAAC,eAAe;IA8BvB,OAAO,CAAC,iBAAiB;CAkB1B"}
+{"version":3,"file":"backup.service.d.ts","sourceRoot":"","sources":["../../src/services/backup.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,OAAO,EAAwB,KAAK,aAAa,EAAgB,MAAM,mCAAmC,CAAC;AAa3G,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EACV,UAAU,EACX,MAAM,0BAA0B,CAAC;AAMlC,YAAY,EACV,UAAU,GACX,MAAM,0BAA0B,CAAC;AAmClC;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG,IAAI,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;AAEtE,kEAAkE;AAClE,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,OAAO,CAAC;IACZ,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,MAAM,YAAY,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,eAAe,CAAC;AAEhF,iEAAiE;AACjE,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,wGAAwG;IACxG,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,KAAK,CAAC,EAAE,OAAO,GAAG,eAAe,CAAC;CACnC;AAED,iFAAiF;AACjF,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,oFAAoF;IACpF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,kFAAkF;AAClF,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,UAAU,GAAG,QAAQ,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnD,wDAAwD;IACxD,WAAW,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD;AAED,mDAAmD;AACnD,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,uEAAuE;IACvE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,qFAAqF;IACrF,QAAQ,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAClC,oFAAoF;IACpF,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,0EAA0E;IAC1E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,kFAAkF;AAClF,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,oBAAoB,EAAE,CAAC;IAClC,yEAAyE;IACzE,MAAM,EAAE,mBAAmB,EAAE,CAAC;CAC/B;AAYD;;;GAGG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,GAAG,iBAAiB,CAMjG;AAgFD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChE,8EAA8E;IAC9E,aAAa,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,yEAAyE;IACzE,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,CAmD/D;AAED;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAEhF;AAMD,qBACa,aAAa;IAmBtB;;;;;;;;;OASG;IAC+B,OAAO,CAAC,QAAQ,CAAC,KAAK;IACxD;;;;;OAKG;IACwC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAnCvE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAsB;IAC9D,8EAA8E;IAC9E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAsB;IAC3D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,SAAS,CAAoC;IACrD;;;OAGG;IACH,OAAO,CAAC,cAAc,CAAS;IAC/B,sFAAsF;IACtF,OAAO,CAAC,SAAS,CAAuC;IAIxD,YACgC,YAAY,EAAE,aAAa;IACzD;;;;;;;;;OASG;IACgD,KAAK,EAAE,gBAAgB;IAC1E;;;;;OAKG;IACyD,QAAQ,CAAC,EAAE,cAAc,YAAA,EAiBtF;IAED;;;OAGG;IACH,OAAO,CAAC,cAAc;IAoDtB;;;;;;OAMG;IACH,OAAO,CAAC,0BAA0B;IAyC5B,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,CAAC,CA0D5F;IAMD,+EAA+E;IAC/E,OAAO,CAAC,eAAe;IAiBvB,yEAAyE;IACzE,OAAO,CAAC,eAAe;IAgBvB;;;;;;;;;;;;;;;OAeG;YACW,WAAW;IAsBzB,6EAA6E;YAC/D,SAAS;IASvB;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;YACW,UAAU;IAsBxB;;;;;;;;;;;OAWG;YACW,cAAc;YAqCd,oBAAoB;YAyBpB,oBAAoB;IAOlC,8EAA8E;IACxE,mBAAmB,IAAI,OAAO,CAAC,UAAU,CAAC,CA+B/C;IAED;;;;;;;;;;;;;;;;OAgBG;IACG,mBAAmB,IAAI,OAAO,CAAC,UAAU,CAAC,CAa/C;IAED,iFAAiF;IAC3E,mBAAmB,IAAI,OAAO,CAAC,UAAU,CAAC,CAU/C;IAED;;;;OAIG;IACG,gBAAgB,CACpB,OAAO,GAAE,gBAAgB,CAAC,SAAS,CAAY,GAC9C,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,OAAO,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CA2B7F;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,UAAU;IAqDlB,8CAA8C;IAC9C,OAAO,CAAC,UAAU;IAQlB;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC,CAwBvC;IAMK,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CA2B5D;IAMK,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAwCnD;IAMK,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBlD;IAMD;;;;;OAKG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAW/D;IAED,sCAAsC;IAChC,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAapD;IAED,oEAAoE;IAC9D,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAqCtC;IAEK,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAErD;IAEK,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAE1D;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,WAAW;IA4CnB;;;;OAIG;IACH,OAAO,CAAC,YAAY;IAMpB;;;;OAIG;IACH,OAAO,CAAC,YAAY;IA8BpB;;;;OAIG;YACW,kBAAkB;IA0BhC;;;;;;;;;OASG;YACW,eAAe;IA6C7B,OAAO,IAAI,IAAI,CASd;YAMa,YAAY;YAcZ,eAAe;YAkBf,WAAW;YAYX,cAAc;IAc5B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,eAAe;IAgCvB;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAW1B,mEAAmE;IACnE,OAAO,CAAC,eAAe;IAiBvB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,YAAY;IA+CpB,wEAAwE;IACxE,OAAO,CAAC,UAAU;IAUlB;;;;OAIG;IACG,gBAAgB,CACpB,OAAO,GAAE,gBAAgB,CAAC,SAAS,CAAY,GAC9C,OAAO,CAAC,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,OAAO,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAK/F;IAED;;;;;;;;OAQG;YACW,oBAAoB;CAiCnC"}