@camstack/system 1.2.179 → 1.2.181
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +2 -2
- package/dist/addon-runner.mjs +2 -2
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.d.ts +56 -10
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +657 -62
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +658 -63
- package/dist/builtins/backup-orchestrator/run-queue.d.ts +103 -0
- package/dist/builtins/backup-orchestrator/staging-paths.d.ts +36 -0
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
- package/dist/builtins/device-manager/device-manager.addon.js +349 -65
- package/dist/builtins/device-manager/device-manager.addon.mjs +349 -65
- package/dist/builtins/device-manager/device-meta-actions.d.ts +8 -19
- package/dist/builtins/device-manager/device-provider-context.d.ts +4 -0
- package/dist/builtins/device-manager/migrate-device.d.ts +47 -3
- package/dist/builtins/device-manager/migrate-guard.d.ts +39 -0
- package/dist/builtins/device-manager/migrate-hardware-state.d.ts +24 -0
- package/dist/builtins/device-manager/migrate-reload.d.ts +8 -0
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +0 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +0 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
- package/dist/builtins/system-backup/system-backup.service.d.ts +55 -2
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-L_39vRmH.mjs → dist-DbnNk4Lw.mjs} +211 -2
- package/dist/{dist-C7qWwrK1.js → dist-whe0b4_3.js} +211 -2
- package/dist/index.js +530 -2
- package/dist/index.mjs +533 -5
- package/dist/kernel/addon-installer.d.ts +69 -0
- package/dist/kernel/deps/closure-dedupe.d.ts +105 -0
- package/dist/{manifest-system-deps-CMOGc5nC.mjs → manifest-system-deps-DOcMuUWK.mjs} +1 -1
- package/dist/{manifest-system-deps-PAIfmNun.js → manifest-system-deps-y5vZBO6q.js} +1 -1
- package/dist/{retired-settings-keys-3soU9pfP.js → retired-settings-keys-BdvmiaK7.js} +1 -1
- package/dist/{retired-settings-keys-CnbKAxmX.mjs → retired-settings-keys-CyQG8b-A.mjs} +1 -1
- package/package.json +1 -1
|
@@ -3,7 +3,7 @@ Object.defineProperties(exports, {
|
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
5
|
const require_chunk = require("../../chunk-Cek0wNdY.js");
|
|
6
|
-
const require_dist = require("../../dist-
|
|
6
|
+
const require_dist = require("../../dist-whe0b4_3.js");
|
|
7
7
|
let node_crypto = require("node:crypto");
|
|
8
8
|
let buffer = require("buffer");
|
|
9
9
|
let node_fs_promises = require("node:fs/promises");
|
|
@@ -31,6 +31,7 @@ let assert = require("assert");
|
|
|
31
31
|
assert = require_chunk.__toESM(assert, 1);
|
|
32
32
|
let events = require("events");
|
|
33
33
|
events = require_chunk.__toESM(events, 1);
|
|
34
|
+
let node_stream_promises = require("node:stream/promises");
|
|
34
35
|
let node_assert = require("node:assert");
|
|
35
36
|
node_assert = require_chunk.__toESM(node_assert, 1);
|
|
36
37
|
//#region node_modules/tar/dist/esm/index.min.js
|
|
@@ -3004,13 +3005,32 @@ var Ro = (s) => {
|
|
|
3004
3005
|
//#endregion
|
|
3005
3006
|
//#region src/builtins/system-backup/system-backup.service.ts
|
|
3006
3007
|
/**
|
|
3008
|
+
* Thrown when an archive build (or a backup run driving one) is
|
|
3009
|
+
* stopped via its `AbortSignal`. Callers use `instanceof` to tell a
|
|
3010
|
+
* deliberate operator cancel from a real failure — a cancelled run is
|
|
3011
|
+
* an outcome, not an error to alert on. (D342)
|
|
3012
|
+
*/
|
|
3013
|
+
var BackupCancelledError = class extends Error {
|
|
3014
|
+
constructor(message = "backup run cancelled") {
|
|
3015
|
+
super(message);
|
|
3016
|
+
this.name = "BackupCancelledError";
|
|
3017
|
+
}
|
|
3018
|
+
};
|
|
3019
|
+
/** Throw `BackupCancelledError` when the signal has fired. */
|
|
3020
|
+
function throwIfCancelled(signal) {
|
|
3021
|
+
if (signal?.aborted) throw new BackupCancelledError();
|
|
3022
|
+
}
|
|
3023
|
+
/**
|
|
3007
3024
|
* Default set of paths considered part of a "system backup". Includes
|
|
3008
3025
|
* everything that isn't pure backend code — config, sqlite, addons
|
|
3009
3026
|
* (runtime overlay), addons-data (per-addon private dir), TLS certs,
|
|
3010
3027
|
* auto-update marker.
|
|
3011
3028
|
*
|
|
3012
|
-
*
|
|
3013
|
-
* (too large or fully recreatable).
|
|
3029
|
+
* Anything not listed is excluded by omission — recordings/, media/,
|
|
3030
|
+
* models/, logs/, cache/ (too large or fully recreatable). Note this
|
|
3031
|
+
* covers only TOP-LEVEL trees: bulk nested INSIDE a listed location
|
|
3032
|
+
* is excluded via `DEFAULT_BACKUP_EXCLUDES` + the node_modules rule
|
|
3033
|
+
* below, enforced in the walk and the tar filter alike.
|
|
3014
3034
|
*/
|
|
3015
3035
|
var DEFAULT_BACKUP_LOCATIONS = [
|
|
3016
3036
|
"config.yaml",
|
|
@@ -3019,6 +3039,40 @@ var DEFAULT_BACKUP_LOCATIONS = [
|
|
|
3019
3039
|
"tls",
|
|
3020
3040
|
"auto-update.json"
|
|
3021
3041
|
];
|
|
3042
|
+
/**
|
|
3043
|
+
* Subtrees never captured by a system backup even though they live
|
|
3044
|
+
* under a `DEFAULT_BACKUP_LOCATIONS` entry. Paths are dataDir-relative
|
|
3045
|
+
* prefixes. Two lessons from the live hub (2026-09-04, D-record
|
|
3046
|
+
* "backup excludes"):
|
|
3047
|
+
*
|
|
3048
|
+
* - `addons-data/backup-orchestrator/staging` — the orchestrator's
|
|
3049
|
+
* LEGACY staging dir (now `<dataDir>/backup-staging`, outside every
|
|
3050
|
+
* location). Before Jul 28 every archive nested all previous
|
|
3051
|
+
* archives (175M → 503M → 4.5G); leaked files from that era must
|
|
3052
|
+
* never be re-swept into new backups.
|
|
3053
|
+
* - `addons-data/recorder/exports` — operator-exported clips (8.4 GB
|
|
3054
|
+
* on the hub). Media, not system state; a restore has no use for it.
|
|
3055
|
+
*/
|
|
3056
|
+
var DEFAULT_BACKUP_EXCLUDES = ["addons-data/backup-orchestrator/staging", "addons-data/recorder/exports"];
|
|
3057
|
+
/**
|
|
3058
|
+
* Directory name excluded wherever it appears in the tree.
|
|
3059
|
+
* `AddonInstaller` materialises runtime deps under
|
|
3060
|
+
* `addons/@camstack/<addon>/node_modules` (~196 MB on the hub); only
|
|
3061
|
+
* built `dist` + package.json + assets are restore-worthy — deps are
|
|
3062
|
+
* reinstalled from npm on demand. node_modules is NEVER archived.
|
|
3063
|
+
*/
|
|
3064
|
+
var EXCLUDED_DIR_NAME = "node_modules";
|
|
3065
|
+
/**
|
|
3066
|
+
* True when a dataDir-relative path must be omitted from archives,
|
|
3067
|
+
* manifests and location stats. Accepts tar-style paths (`./` prefix,
|
|
3068
|
+
* trailing `/`) and backslash separators.
|
|
3069
|
+
*/
|
|
3070
|
+
function isExcludedFromBackup(relPath) {
|
|
3071
|
+
const norm = relPath.replace(/\\/g, "/").replace(/^\.\//, "").replace(/\/+$/, "");
|
|
3072
|
+
if (norm.length === 0) return false;
|
|
3073
|
+
if (norm.split("/").includes(EXCLUDED_DIR_NAME)) return true;
|
|
3074
|
+
return DEFAULT_BACKUP_EXCLUDES.some((prefix) => norm === prefix || norm.startsWith(`${prefix}/`));
|
|
3075
|
+
}
|
|
3022
3076
|
/** Path of the manifest file embedded inside every archive. */
|
|
3023
3077
|
var ARCHIVE_MANIFEST_NAME = ".camstack-backup-manifest.json";
|
|
3024
3078
|
var DEFAULT_STAT_CACHE_TTL_MS = 6e4;
|
|
@@ -3071,7 +3125,7 @@ var SystemBackupService = class {
|
|
|
3071
3125
|
fileCount: 0,
|
|
3072
3126
|
present: false
|
|
3073
3127
|
};
|
|
3074
|
-
const walked = await walkAbs(abs, abs);
|
|
3128
|
+
const walked = await walkAbs(abs, abs, this.exclusionPredicate());
|
|
3075
3129
|
return {
|
|
3076
3130
|
name: loc,
|
|
3077
3131
|
sizeBytes: walked.reduce((acc, e) => acc + e.sizeBytes, 0),
|
|
@@ -3091,8 +3145,20 @@ var SystemBackupService = class {
|
|
|
3091
3145
|
*
|
|
3092
3146
|
* Locations that don't exist are silently skipped. Throws if zero
|
|
3093
3147
|
* locations would be included.
|
|
3148
|
+
*
|
|
3149
|
+
* Cancellation: `signal` aborts the build IN FLIGHT — the manifest
|
|
3150
|
+
* walk checks it per filesystem node and the tar phase runs through
|
|
3151
|
+
* `stream.pipeline(..., { signal })`, which destroys the pack and
|
|
3152
|
+
* the file sink immediately. On abort the half-written archive is
|
|
3153
|
+
* deleted here (never left for the caller to remember) and the call
|
|
3154
|
+
* rejects with {@link BackupCancelledError}. (D342)
|
|
3155
|
+
*
|
|
3156
|
+
* Progress: `onArchiveBytes` receives the cumulative compressed
|
|
3157
|
+
* bytes written to `archivePath` — the live "how large has staging
|
|
3158
|
+
* grown" counter `backup.listRuns` exposes.
|
|
3094
3159
|
*/
|
|
3095
3160
|
async createArchive(opts) {
|
|
3161
|
+
throwIfCancelled(opts.signal);
|
|
3096
3162
|
const requested = opts.locations ?? DEFAULT_BACKUP_LOCATIONS;
|
|
3097
3163
|
const present = [];
|
|
3098
3164
|
for (const loc of requested) {
|
|
@@ -3105,9 +3171,10 @@ var SystemBackupService = class {
|
|
|
3105
3171
|
await this.checkpointDatabasesUnder(present);
|
|
3106
3172
|
const entries = [];
|
|
3107
3173
|
for (const loc of present) {
|
|
3174
|
+
throwIfCancelled(opts.signal);
|
|
3108
3175
|
const abs = node_path.join(this.dataDir, loc);
|
|
3109
3176
|
const stat = await node_fs_promises.stat(abs);
|
|
3110
|
-
if (stat.isDirectory()) for (const e of await walkAbs(abs, this.dataDir)) entries.push(e);
|
|
3177
|
+
if (stat.isDirectory()) for (const e of await walkAbs(abs, this.dataDir, this.exclusionPredicate(), opts.signal)) entries.push(e);
|
|
3111
3178
|
else entries.push(toArchiveEntry(abs, this.dataDir, stat));
|
|
3112
3179
|
}
|
|
3113
3180
|
const manifest = {
|
|
@@ -3133,18 +3200,28 @@ var SystemBackupService = class {
|
|
|
3133
3200
|
const stagingRel = node_path.relative(this.dataDir, node_path.dirname(opts.archivePath));
|
|
3134
3201
|
const excludePrefix = stagingRel && !stagingRel.startsWith("..") && !node_path.isAbsolute(stagingRel) ? stagingRel : null;
|
|
3135
3202
|
try {
|
|
3136
|
-
|
|
3137
|
-
file: opts.archivePath,
|
|
3203
|
+
const pack = Kn({
|
|
3138
3204
|
gzip: true,
|
|
3139
3205
|
cwd: this.dataDir,
|
|
3140
3206
|
follow: true,
|
|
3141
3207
|
portable: true,
|
|
3142
3208
|
filter: (entryPath) => {
|
|
3143
|
-
if (!excludePrefix) return true;
|
|
3144
3209
|
const norm = entryPath.replace(/^\.\//, "").replace(/\/+$/, "");
|
|
3210
|
+
if (isExcludedFromBackup(norm)) return false;
|
|
3211
|
+
if (!excludePrefix) return true;
|
|
3145
3212
|
return norm !== excludePrefix && !norm.startsWith(`${excludePrefix}/`);
|
|
3146
3213
|
}
|
|
3147
3214
|
}, [ARCHIVE_MANIFEST_NAME, ...present]);
|
|
3215
|
+
const sink = node_fs.createWriteStream(opts.archivePath);
|
|
3216
|
+
try {
|
|
3217
|
+
await (0, node_stream_promises.pipeline)(pack, byteCountingPassthrough(opts.onArchiveBytes), sink, opts.signal ? { signal: opts.signal } : {});
|
|
3218
|
+
} catch (err) {
|
|
3219
|
+
try {
|
|
3220
|
+
node_fs.rmSync(opts.archivePath, { force: true });
|
|
3221
|
+
} catch {}
|
|
3222
|
+
if (opts.signal?.aborted) throw new BackupCancelledError();
|
|
3223
|
+
throw err;
|
|
3224
|
+
}
|
|
3148
3225
|
} finally {
|
|
3149
3226
|
try {
|
|
3150
3227
|
node_fs.rmSync(stagedInDataDir, { force: true });
|
|
@@ -3178,7 +3255,7 @@ var SystemBackupService = class {
|
|
|
3178
3255
|
*/
|
|
3179
3256
|
async checkpointDatabasesUnder(locations) {
|
|
3180
3257
|
const dbPaths = /* @__PURE__ */ new Set();
|
|
3181
|
-
for (const loc of locations) await collectWalDatabases(node_path.isAbsolute(loc) ? loc : node_path.join(this.dataDir, loc), dbPaths);
|
|
3258
|
+
for (const loc of locations) await collectWalDatabases(node_path.isAbsolute(loc) ? loc : node_path.join(this.dataDir, loc), dbPaths, this.exclusionPredicate());
|
|
3182
3259
|
if (dbPaths.size === 0) return;
|
|
3183
3260
|
let Database;
|
|
3184
3261
|
try {
|
|
@@ -3323,6 +3400,15 @@ var SystemBackupService = class {
|
|
|
3323
3400
|
getRestoreMarkerPath() {
|
|
3324
3401
|
return node_path.join(this.dataDir, ".pending-restore.json");
|
|
3325
3402
|
}
|
|
3403
|
+
/**
|
|
3404
|
+
* Absolute-path form of {@link isExcludedFromBackup}, anchored on
|
|
3405
|
+
* this service's dataDir. Shared by the manifest walk, the location
|
|
3406
|
+
* stats and the WAL-checkpoint scan so all three agree with the tar
|
|
3407
|
+
* filter about what a backup contains.
|
|
3408
|
+
*/
|
|
3409
|
+
exclusionPredicate() {
|
|
3410
|
+
return (abs) => isExcludedFromBackup(node_path.relative(this.dataDir, abs));
|
|
3411
|
+
}
|
|
3326
3412
|
};
|
|
3327
3413
|
/**
|
|
3328
3414
|
* Recursively collect every SQLite database under `abs` that is in WAL
|
|
@@ -3331,7 +3417,8 @@ var SystemBackupService = class {
|
|
|
3331
3417
|
* (unreadable dir, broken symlink) are skipped silently — this feeds a
|
|
3332
3418
|
* best-effort step.
|
|
3333
3419
|
*/
|
|
3334
|
-
async function collectWalDatabases(abs, out) {
|
|
3420
|
+
async function collectWalDatabases(abs, out, shouldSkip) {
|
|
3421
|
+
if (shouldSkip?.(abs)) return;
|
|
3335
3422
|
let stat;
|
|
3336
3423
|
try {
|
|
3337
3424
|
stat = await node_fs_promises.stat(abs);
|
|
@@ -3345,7 +3432,7 @@ async function collectWalDatabases(abs, out) {
|
|
|
3345
3432
|
} catch {
|
|
3346
3433
|
return;
|
|
3347
3434
|
}
|
|
3348
|
-
for (const child of children) await collectWalDatabases(node_path.join(abs, child), out);
|
|
3435
|
+
for (const child of children) await collectWalDatabases(node_path.join(abs, child), out, shouldSkip);
|
|
3349
3436
|
return;
|
|
3350
3437
|
}
|
|
3351
3438
|
if (abs.endsWith("-wal")) {
|
|
@@ -3359,12 +3446,14 @@ async function collectWalDatabases(abs, out) {
|
|
|
3359
3446
|
* path. Symlinks are dereferenced (matches the `follow: true` flag we
|
|
3360
3447
|
* pass to tar.create).
|
|
3361
3448
|
*/
|
|
3362
|
-
async function walkAbs(absRoot, dataDir) {
|
|
3449
|
+
async function walkAbs(absRoot, dataDir, shouldSkip, signal) {
|
|
3363
3450
|
const out = [];
|
|
3364
|
-
await walkRec(absRoot, dataDir, out);
|
|
3451
|
+
await walkRec(absRoot, dataDir, out, shouldSkip, signal);
|
|
3365
3452
|
return out;
|
|
3366
3453
|
}
|
|
3367
|
-
async function walkRec(abs, dataDir, out) {
|
|
3454
|
+
async function walkRec(abs, dataDir, out, shouldSkip, signal) {
|
|
3455
|
+
throwIfCancelled(signal);
|
|
3456
|
+
if (shouldSkip?.(abs)) return;
|
|
3368
3457
|
let stat;
|
|
3369
3458
|
try {
|
|
3370
3459
|
stat = await node_fs_promises.stat(abs);
|
|
@@ -3379,9 +3468,22 @@ async function walkRec(abs, dataDir, out) {
|
|
|
3379
3468
|
} catch {
|
|
3380
3469
|
return;
|
|
3381
3470
|
}
|
|
3382
|
-
for (const child of children) await walkRec(node_path.join(abs, child), dataDir, out);
|
|
3471
|
+
for (const child of children) await walkRec(node_path.join(abs, child), dataDir, out, shouldSkip, signal);
|
|
3383
3472
|
}
|
|
3384
3473
|
}
|
|
3474
|
+
/**
|
|
3475
|
+
* Pass-through transform that reports the CUMULATIVE bytes flowed so
|
|
3476
|
+
* far. Sits between the tar pack and the file sink so the caller's
|
|
3477
|
+
* progress counter tracks the compressed archive as it grows on disk.
|
|
3478
|
+
*/
|
|
3479
|
+
function byteCountingPassthrough(onBytes) {
|
|
3480
|
+
let total = 0;
|
|
3481
|
+
return new node_stream.Transform({ transform(chunk, _encoding, callback) {
|
|
3482
|
+
if (Buffer.isBuffer(chunk)) total += chunk.byteLength;
|
|
3483
|
+
onBytes?.(total);
|
|
3484
|
+
callback(null, chunk);
|
|
3485
|
+
} });
|
|
3486
|
+
}
|
|
3385
3487
|
function toArchiveEntry(abs, dataDir, stat) {
|
|
3386
3488
|
const rel = node_path.relative(dataDir, abs).replace(/\\/g, "/");
|
|
3387
3489
|
const kind = stat.isDirectory() ? "dir" : stat.isSymbolicLink() ? "symlink" : "file";
|
|
@@ -4601,6 +4703,257 @@ var LocationManifestLock = class {
|
|
|
4601
4703
|
}
|
|
4602
4704
|
};
|
|
4603
4705
|
//#endregion
|
|
4706
|
+
//#region src/builtins/backup-orchestrator/run-queue.ts
|
|
4707
|
+
/**
|
|
4708
|
+
* Backup run queue + per-run tracking (D342).
|
|
4709
|
+
*
|
|
4710
|
+
* Why a GLOBAL queue (concurrency 1), not per-destination: every run
|
|
4711
|
+
* walks the same `<nodeDataDir>` source tree and stages its tar.gz on
|
|
4712
|
+
* the same disk. On 2026-09-04 two schedule ticks built two multi-GB
|
|
4713
|
+
* archives concurrently — staging went 5.1 GB → 18 GB in ~12 minutes
|
|
4714
|
+
* with host load 62, and each run archived the other's growing tar
|
|
4715
|
+
* (pre-D338 staging placement). A per-destination lock would still
|
|
4716
|
+
* allow two concurrent walks of the same tree and two staged archives;
|
|
4717
|
+
* the destination write is the CHEAP half. One run at a time bounds
|
|
4718
|
+
* peak staging disk to a single archive and the walk cost to one
|
|
4719
|
+
* traversal.
|
|
4720
|
+
*
|
|
4721
|
+
* This subsumes the earlier single-flight `buildLock`
|
|
4722
|
+
* (`LocationManifestLock.run('archive-build', …)`): that lock
|
|
4723
|
+
* serialised builds but was invisible (nothing reported a waiting
|
|
4724
|
+
* run), unjoinable and uncancellable. The queue is the one
|
|
4725
|
+
* serialisation authority for archive builds — do not reintroduce a
|
|
4726
|
+
* second lock around the executor.
|
|
4727
|
+
*
|
|
4728
|
+
* Every run is OBSERVABLE while it exists: id, state, phase, byte
|
|
4729
|
+
* counters, per-destination outcome. The registry keeps queued +
|
|
4730
|
+
* active + a bounded tail of finished runs so `backup.listRuns` can
|
|
4731
|
+
* answer "what is running, how far along, how big" without `du`.
|
|
4732
|
+
*/
|
|
4733
|
+
/** Finished runs kept for `listRuns` history. Older ones are dropped. */
|
|
4734
|
+
var DEFAULT_MAX_FINISHED = 20;
|
|
4735
|
+
/**
|
|
4736
|
+
* FIFO queue with concurrency 1. `submit` never starts a second
|
|
4737
|
+
* executor while one is in flight; an identical request already
|
|
4738
|
+
* WAITING (same destinations/locations/label/retention) is joined
|
|
4739
|
+
* instead of queued twice — two schedule ticks due on the same minute
|
|
4740
|
+
* produce one run, not two.
|
|
4741
|
+
*/
|
|
4742
|
+
var BackupRunQueue = class {
|
|
4743
|
+
executor;
|
|
4744
|
+
runs = /* @__PURE__ */ new Map();
|
|
4745
|
+
pending = [];
|
|
4746
|
+
finished = [];
|
|
4747
|
+
activeId = null;
|
|
4748
|
+
maxFinished;
|
|
4749
|
+
now;
|
|
4750
|
+
constructor(executor, opts) {
|
|
4751
|
+
this.executor = executor;
|
|
4752
|
+
this.maxFinished = opts?.maxFinished ?? DEFAULT_MAX_FINISHED;
|
|
4753
|
+
this.now = opts?.now ?? Date.now;
|
|
4754
|
+
}
|
|
4755
|
+
submit(request) {
|
|
4756
|
+
const signature = requestSignature(request);
|
|
4757
|
+
const busy = this.activeId !== null || this.pending.length > 0;
|
|
4758
|
+
for (const id of this.pending) {
|
|
4759
|
+
const existing = this.runs.get(id);
|
|
4760
|
+
if (existing && existing.signature === signature) return {
|
|
4761
|
+
runId: id,
|
|
4762
|
+
queued: true,
|
|
4763
|
+
joined: true,
|
|
4764
|
+
result: existing.resultPromise
|
|
4765
|
+
};
|
|
4766
|
+
}
|
|
4767
|
+
const record = this.createRecord(request, signature);
|
|
4768
|
+
this.runs.set(record.snapshot.id, record);
|
|
4769
|
+
this.pending.push(record.snapshot.id);
|
|
4770
|
+
this.pump();
|
|
4771
|
+
return {
|
|
4772
|
+
runId: record.snapshot.id,
|
|
4773
|
+
queued: busy,
|
|
4774
|
+
joined: false,
|
|
4775
|
+
result: record.resultPromise
|
|
4776
|
+
};
|
|
4777
|
+
}
|
|
4778
|
+
/**
|
|
4779
|
+
* Cancel a run. Queued: removed before it ever starts (its result
|
|
4780
|
+
* rejects `BackupCancelledError`). Running: the abort signal fires —
|
|
4781
|
+
* the executor's tar/upload streams stop and its cleanup runs.
|
|
4782
|
+
* Unknown or already terminal: `false`.
|
|
4783
|
+
*/
|
|
4784
|
+
cancel(runId) {
|
|
4785
|
+
const record = this.runs.get(runId);
|
|
4786
|
+
if (!record) return false;
|
|
4787
|
+
if (record.snapshot.state === "queued") {
|
|
4788
|
+
const idx = this.pending.indexOf(runId);
|
|
4789
|
+
if (idx >= 0) this.pending.splice(idx, 1);
|
|
4790
|
+
this.finish(record, "cancelled");
|
|
4791
|
+
record.rejectResult(new BackupCancelledError(`backup run ${runId} cancelled while queued`));
|
|
4792
|
+
return true;
|
|
4793
|
+
}
|
|
4794
|
+
if (record.snapshot.state === "running") {
|
|
4795
|
+
record.controller.abort();
|
|
4796
|
+
return true;
|
|
4797
|
+
}
|
|
4798
|
+
return false;
|
|
4799
|
+
}
|
|
4800
|
+
/**
|
|
4801
|
+
* EXECUTION order, not timestamp order: the running run first, then
|
|
4802
|
+
* every queued run in the exact order the pump will start them (each
|
|
4803
|
+
* stamped with `queuePosition`, 1 = next), then the bounded finished
|
|
4804
|
+
* history newest-first. The queued section is read straight off
|
|
4805
|
+
* `this.pending` — a consumer sorting by `requestedAt` descending
|
|
4806
|
+
* would put the run that starts NEXT at the BOTTOM, which is
|
|
4807
|
+
* backwards exactly when the queue matters (more than one waiter).
|
|
4808
|
+
*/
|
|
4809
|
+
listRuns() {
|
|
4810
|
+
const running = [];
|
|
4811
|
+
const finished = [];
|
|
4812
|
+
for (const record of this.runs.values()) {
|
|
4813
|
+
const snap = record.snapshot;
|
|
4814
|
+
if (snap.state === "running") running.push(snap);
|
|
4815
|
+
else if (snap.state !== "queued") finished.push(snap);
|
|
4816
|
+
}
|
|
4817
|
+
const queued = [];
|
|
4818
|
+
for (const [index, id] of this.pending.entries()) {
|
|
4819
|
+
const record = this.runs.get(id);
|
|
4820
|
+
if (record) queued.push({
|
|
4821
|
+
...record.snapshot,
|
|
4822
|
+
queuePosition: index + 1
|
|
4823
|
+
});
|
|
4824
|
+
}
|
|
4825
|
+
finished.sort((a, b) => b.requestedAt - a.requestedAt);
|
|
4826
|
+
return [
|
|
4827
|
+
...running,
|
|
4828
|
+
...queued,
|
|
4829
|
+
...finished
|
|
4830
|
+
];
|
|
4831
|
+
}
|
|
4832
|
+
getRun(runId) {
|
|
4833
|
+
const record = this.runs.get(runId);
|
|
4834
|
+
if (!record) return void 0;
|
|
4835
|
+
const pendingIndex = this.pending.indexOf(runId);
|
|
4836
|
+
return pendingIndex >= 0 ? {
|
|
4837
|
+
...record.snapshot,
|
|
4838
|
+
queuePosition: pendingIndex + 1
|
|
4839
|
+
} : record.snapshot;
|
|
4840
|
+
}
|
|
4841
|
+
createRecord(request, signature) {
|
|
4842
|
+
let resolveResult;
|
|
4843
|
+
let rejectResult;
|
|
4844
|
+
const resultPromise = new Promise((res, rej) => {
|
|
4845
|
+
resolveResult = res;
|
|
4846
|
+
rejectResult = rej;
|
|
4847
|
+
});
|
|
4848
|
+
resultPromise.catch(() => void 0);
|
|
4849
|
+
return {
|
|
4850
|
+
snapshot: {
|
|
4851
|
+
id: (0, node_crypto.randomUUID)(),
|
|
4852
|
+
state: "queued",
|
|
4853
|
+
phase: "queued",
|
|
4854
|
+
destinationIds: request.destinations ?? [],
|
|
4855
|
+
...request.label !== void 0 ? { label: request.label } : {},
|
|
4856
|
+
requestedAt: this.now(),
|
|
4857
|
+
stagedBytes: 0,
|
|
4858
|
+
uploadedBytes: 0,
|
|
4859
|
+
completedDestinationIds: [],
|
|
4860
|
+
failedDestinationIds: []
|
|
4861
|
+
},
|
|
4862
|
+
request,
|
|
4863
|
+
signature,
|
|
4864
|
+
controller: new AbortController(),
|
|
4865
|
+
resultPromise,
|
|
4866
|
+
resolveResult,
|
|
4867
|
+
rejectResult
|
|
4868
|
+
};
|
|
4869
|
+
}
|
|
4870
|
+
pump() {
|
|
4871
|
+
if (this.activeId !== null) return;
|
|
4872
|
+
const nextId = this.pending.shift();
|
|
4873
|
+
if (nextId === void 0) return;
|
|
4874
|
+
const record = this.runs.get(nextId);
|
|
4875
|
+
if (!record) {
|
|
4876
|
+
this.pump();
|
|
4877
|
+
return;
|
|
4878
|
+
}
|
|
4879
|
+
this.activeId = nextId;
|
|
4880
|
+
this.patch(record, {
|
|
4881
|
+
state: "running",
|
|
4882
|
+
phase: "building",
|
|
4883
|
+
startedAt: this.now()
|
|
4884
|
+
});
|
|
4885
|
+
this.execute(record);
|
|
4886
|
+
}
|
|
4887
|
+
async execute(record) {
|
|
4888
|
+
const progress = this.progressFor(record);
|
|
4889
|
+
try {
|
|
4890
|
+
const value = await this.executor(record.request, record.controller.signal, progress);
|
|
4891
|
+
this.finish(record, record.controller.signal.aborted ? "cancelled" : "succeeded");
|
|
4892
|
+
record.resolveResult(value);
|
|
4893
|
+
} catch (err) {
|
|
4894
|
+
if (err instanceof BackupCancelledError || record.controller.signal.aborted) {
|
|
4895
|
+
this.finish(record, "cancelled");
|
|
4896
|
+
record.rejectResult(err instanceof BackupCancelledError ? err : new BackupCancelledError());
|
|
4897
|
+
} else {
|
|
4898
|
+
this.finish(record, "failed", err instanceof Error ? err.message : String(err));
|
|
4899
|
+
record.rejectResult(err);
|
|
4900
|
+
}
|
|
4901
|
+
} finally {
|
|
4902
|
+
this.activeId = null;
|
|
4903
|
+
this.pump();
|
|
4904
|
+
}
|
|
4905
|
+
}
|
|
4906
|
+
progressFor(record) {
|
|
4907
|
+
return {
|
|
4908
|
+
setPhase: (phase) => this.patch(record, { phase }),
|
|
4909
|
+
setDestinations: (ids) => this.patch(record, { destinationIds: [...ids] }),
|
|
4910
|
+
setStagedBytes: (bytes) => this.patch(record, { stagedBytes: bytes }),
|
|
4911
|
+
setArchiveSize: (bytes) => this.patch(record, { archiveSizeBytes: bytes }),
|
|
4912
|
+
setUploadedBytes: (bytes) => this.patch(record, { uploadedBytes: bytes }),
|
|
4913
|
+
destinationCompleted: (id) => this.patch(record, {
|
|
4914
|
+
completedDestinationIds: [...record.snapshot.completedDestinationIds, id],
|
|
4915
|
+
uploadedBytes: 0
|
|
4916
|
+
}),
|
|
4917
|
+
destinationFailed: (id) => this.patch(record, {
|
|
4918
|
+
failedDestinationIds: [...record.snapshot.failedDestinationIds, id],
|
|
4919
|
+
uploadedBytes: 0
|
|
4920
|
+
})
|
|
4921
|
+
};
|
|
4922
|
+
}
|
|
4923
|
+
patch(record, patch) {
|
|
4924
|
+
record.snapshot = {
|
|
4925
|
+
...record.snapshot,
|
|
4926
|
+
...patch
|
|
4927
|
+
};
|
|
4928
|
+
}
|
|
4929
|
+
finish(record, state, error) {
|
|
4930
|
+
this.patch(record, {
|
|
4931
|
+
state,
|
|
4932
|
+
phase: "done",
|
|
4933
|
+
finishedAt: this.now(),
|
|
4934
|
+
...error !== void 0 ? { error } : {}
|
|
4935
|
+
});
|
|
4936
|
+
this.finished.push(record.snapshot.id);
|
|
4937
|
+
while (this.finished.length > this.maxFinished) {
|
|
4938
|
+
const evicted = this.finished.shift();
|
|
4939
|
+
if (evicted !== void 0) this.runs.delete(evicted);
|
|
4940
|
+
}
|
|
4941
|
+
}
|
|
4942
|
+
};
|
|
4943
|
+
/**
|
|
4944
|
+
* Canonical signature for join-detection. Order-normalised so
|
|
4945
|
+
* `{ destinations: [a, b] }` and `{ destinations: [b, a] }` — the same
|
|
4946
|
+
* fan-out set — coalesce.
|
|
4947
|
+
*/
|
|
4948
|
+
function requestSignature(request) {
|
|
4949
|
+
return JSON.stringify({
|
|
4950
|
+
destinations: [...request.destinations ?? []].toSorted(),
|
|
4951
|
+
locations: [...request.locations ?? []].toSorted(),
|
|
4952
|
+
label: request.label ?? null,
|
|
4953
|
+
retentionCount: request.retentionCount ?? null
|
|
4954
|
+
});
|
|
4955
|
+
}
|
|
4956
|
+
//#endregion
|
|
4604
4957
|
//#region src/builtins/backup-orchestrator/schedule-store.ts
|
|
4605
4958
|
/**
|
|
4606
4959
|
* @durable class=config owner=backup-orchestrator
|
|
@@ -4777,6 +5130,107 @@ function mapRowToSchedule(row) {
|
|
|
4777
5130
|
};
|
|
4778
5131
|
}
|
|
4779
5132
|
//#endregion
|
|
5133
|
+
//#region src/builtins/backup-orchestrator/staging-paths.ts
|
|
5134
|
+
/** Basename of the staging dir under the node data root. */
|
|
5135
|
+
var STAGING_DIR_NAME = "backup-staging";
|
|
5136
|
+
/** Canonical staging dir for a node data root. */
|
|
5137
|
+
function resolveStagingDir(nodeDataDir) {
|
|
5138
|
+
return node_path.join(nodeDataDir, STAGING_DIR_NAME);
|
|
5139
|
+
}
|
|
5140
|
+
/**
|
|
5141
|
+
* Delete every top-level entry of `stagingDir` except those whose
|
|
5142
|
+
* resolved path appears in `keepPaths`. Per-entry failures are logged
|
|
5143
|
+
* and skipped — a locked file must not abort the sweep. A missing
|
|
5144
|
+
* staging dir is a clean no-op.
|
|
5145
|
+
*/
|
|
5146
|
+
async function sweepStagingDir(input) {
|
|
5147
|
+
const keep = new Set(input.keepPaths.map((p) => node_path.resolve(p)));
|
|
5148
|
+
let entries;
|
|
5149
|
+
try {
|
|
5150
|
+
entries = await node_fs_promises.readdir(input.stagingDir);
|
|
5151
|
+
} catch {
|
|
5152
|
+
return {
|
|
5153
|
+
removedCount: 0,
|
|
5154
|
+
freedBytes: 0,
|
|
5155
|
+
keptCount: 0
|
|
5156
|
+
};
|
|
5157
|
+
}
|
|
5158
|
+
let removedCount = 0;
|
|
5159
|
+
let freedBytes = 0;
|
|
5160
|
+
let keptCount = 0;
|
|
5161
|
+
for (const name of entries) {
|
|
5162
|
+
const abs = node_path.join(input.stagingDir, name);
|
|
5163
|
+
if (keep.has(node_path.resolve(abs))) {
|
|
5164
|
+
keptCount++;
|
|
5165
|
+
continue;
|
|
5166
|
+
}
|
|
5167
|
+
try {
|
|
5168
|
+
const footprint = await measureDirectoryFootprint(abs);
|
|
5169
|
+
await node_fs_promises.rm(abs, {
|
|
5170
|
+
recursive: true,
|
|
5171
|
+
force: true
|
|
5172
|
+
});
|
|
5173
|
+
removedCount++;
|
|
5174
|
+
freedBytes += footprint.totalBytes;
|
|
5175
|
+
} catch (err) {
|
|
5176
|
+
input.logger.warn("backup staging sweep: entry removal failed — skipping", { meta: {
|
|
5177
|
+
path: abs,
|
|
5178
|
+
error: err instanceof Error ? err.message : String(err)
|
|
5179
|
+
} });
|
|
5180
|
+
}
|
|
5181
|
+
}
|
|
5182
|
+
return {
|
|
5183
|
+
removedCount,
|
|
5184
|
+
freedBytes,
|
|
5185
|
+
keptCount
|
|
5186
|
+
};
|
|
5187
|
+
}
|
|
5188
|
+
/**
|
|
5189
|
+
* Recursive size + file count of a path (file or directory). Absent
|
|
5190
|
+
* path → `present: false` with zeros. Used by the sweep to report
|
|
5191
|
+
* reclaimed bytes and by the orchestrator to surface the legacy
|
|
5192
|
+
* staging dir's footprint without touching it.
|
|
5193
|
+
*/
|
|
5194
|
+
async function measureDirectoryFootprint(dir) {
|
|
5195
|
+
let stat;
|
|
5196
|
+
try {
|
|
5197
|
+
stat = await node_fs_promises.stat(dir);
|
|
5198
|
+
} catch {
|
|
5199
|
+
return {
|
|
5200
|
+
present: false,
|
|
5201
|
+
totalBytes: 0,
|
|
5202
|
+
fileCount: 0
|
|
5203
|
+
};
|
|
5204
|
+
}
|
|
5205
|
+
if (!stat.isDirectory()) return {
|
|
5206
|
+
present: true,
|
|
5207
|
+
totalBytes: stat.size,
|
|
5208
|
+
fileCount: 1
|
|
5209
|
+
};
|
|
5210
|
+
let totalBytes = 0;
|
|
5211
|
+
let fileCount = 0;
|
|
5212
|
+
let children;
|
|
5213
|
+
try {
|
|
5214
|
+
children = await node_fs_promises.readdir(dir);
|
|
5215
|
+
} catch {
|
|
5216
|
+
return {
|
|
5217
|
+
present: true,
|
|
5218
|
+
totalBytes: 0,
|
|
5219
|
+
fileCount: 0
|
|
5220
|
+
};
|
|
5221
|
+
}
|
|
5222
|
+
for (const child of children) {
|
|
5223
|
+
const nested = await measureDirectoryFootprint(node_path.join(dir, child));
|
|
5224
|
+
totalBytes += nested.totalBytes;
|
|
5225
|
+
fileCount += nested.fileCount;
|
|
5226
|
+
}
|
|
5227
|
+
return {
|
|
5228
|
+
present: true,
|
|
5229
|
+
totalBytes,
|
|
5230
|
+
fileCount
|
|
5231
|
+
};
|
|
5232
|
+
}
|
|
5233
|
+
//#endregion
|
|
4780
5234
|
//#region src/builtins/backup-orchestrator/write-targets.ts
|
|
4781
5235
|
/**
|
|
4782
5236
|
* Pick the destinations an archive may be fanned out to.
|
|
@@ -4821,8 +5275,9 @@ function selectBackupTargets(input) {
|
|
|
4821
5275
|
* that list with the `backup_destination_policies` table to surface
|
|
4822
5276
|
* `enabled` / `retentionCount` / `label`. The actual archive bytes
|
|
4823
5277
|
* travel through `api.storage.beginUpload` / `writeChunk` /
|
|
4824
|
-
* `finalizeUpload`, with the staging archive built into
|
|
4825
|
-
*
|
|
5278
|
+
* `finalizeUpload`, with the staging archive built into
|
|
5279
|
+
* `<nodeDataDir>/backup-staging` (outside every backed-up location —
|
|
5280
|
+
* see staging-paths.ts).
|
|
4826
5281
|
*/
|
|
4827
5282
|
/**
|
|
4828
5283
|
* Hardcoded seed for newly-discovered `backups` locations. Operators
|
|
@@ -4874,10 +5329,12 @@ var BackupOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
4874
5329
|
/**
|
|
4875
5330
|
* Transient build buffer for archive fan-out. The tar is built here
|
|
4876
5331
|
* ONCE, streamed to every destination via the storage cap's chunked
|
|
4877
|
-
* upload, then removed. Lives
|
|
4878
|
-
*
|
|
4879
|
-
*
|
|
4880
|
-
*
|
|
5332
|
+
* upload, then removed. Lives at `<nodeDataDir>/backup-staging` —
|
|
5333
|
+
* OUTSIDE every backed-up location, so a backup can never contain a
|
|
5334
|
+
* backup (the old `ctx.dataDir/staging` sat inside `addons-data` and
|
|
5335
|
+
* compounded 175M → 4.5G before Jul 28), and distinct from
|
|
5336
|
+
* `<nodeDataDir>/backups` where the default `backups` destination
|
|
5337
|
+
* seeds when `CAMSTACK_BACKUP_ROOT` is unset. See staging-paths.ts.
|
|
4881
5338
|
*/
|
|
4882
5339
|
stagingDir = "";
|
|
4883
5340
|
scheduleTimer = null;
|
|
@@ -4889,13 +5346,24 @@ var BackupOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
4889
5346
|
*/
|
|
4890
5347
|
scheduleBaselineAt = Date.now();
|
|
4891
5348
|
manifestLock = new LocationManifestLock();
|
|
5349
|
+
/**
|
|
5350
|
+
* Global execution queue — at most ONE archive build at a time,
|
|
5351
|
+
* cluster-node-wide (D342). Replaces the earlier invisible
|
|
5352
|
+
* single-flight `buildLock`: same serialisation guarantee (two
|
|
5353
|
+
* schedules due one tick apart built 8.2 GB + 3.7 GB side by side on
|
|
5354
|
+
* the hub, 2026-09-04), but every run is now observable via
|
|
5355
|
+
* `listRuns`, joinable when identical, and cancellable mid-stream.
|
|
5356
|
+
*/
|
|
5357
|
+
runQueue = null;
|
|
4892
5358
|
constructor() {
|
|
4893
5359
|
super({});
|
|
4894
5360
|
}
|
|
4895
5361
|
async onInitialize() {
|
|
4896
5362
|
this.dataDir = this.ctx.nodeDataDir;
|
|
4897
|
-
this.stagingDir =
|
|
5363
|
+
this.stagingDir = resolveStagingDir(this.ctx.nodeDataDir);
|
|
4898
5364
|
this.systemBackup = new SystemBackupService(this.dataDir, this.ctx.logger);
|
|
5365
|
+
this.runQueue = new BackupRunQueue((request, signal, progress) => this.runBackupBuild(request, signal, progress));
|
|
5366
|
+
await this.cleanStagingArea();
|
|
4899
5367
|
this.policies = this.resolvePolicyService();
|
|
4900
5368
|
if (this.policies) try {
|
|
4901
5369
|
await this.policies.initialize();
|
|
@@ -4915,6 +5383,8 @@ var BackupOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
4915
5383
|
list: () => this.aggregatedList(),
|
|
4916
5384
|
listLocations: () => this.systemBackup.statLocations(),
|
|
4917
5385
|
trigger: (input) => this.triggerBackup(input),
|
|
5386
|
+
listRuns: () => this.listRuns(),
|
|
5387
|
+
cancel: (input) => this.cancelRun(input),
|
|
4918
5388
|
getEntries: (input) => this.getEntries(input),
|
|
4919
5389
|
restore: (input) => this.scheduleRestore(input),
|
|
4920
5390
|
delete: (input) => this.deleteArchive(input),
|
|
@@ -4937,6 +5407,10 @@ var BackupOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
4937
5407
|
}
|
|
4938
5408
|
async onShutdown() {
|
|
4939
5409
|
this.stopScheduleTimer();
|
|
5410
|
+
if (this.runQueue) {
|
|
5411
|
+
for (const run of this.runQueue.listRuns()) if (run.state === "queued" || run.state === "running") this.runQueue.cancel(run.id);
|
|
5412
|
+
}
|
|
5413
|
+
this.runQueue = null;
|
|
4940
5414
|
this.systemBackup = null;
|
|
4941
5415
|
this.policies = null;
|
|
4942
5416
|
this.schedules = null;
|
|
@@ -5087,64 +5561,180 @@ var BackupOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
5087
5561
|
return node_path.join(this.stagingDir, filename);
|
|
5088
5562
|
}
|
|
5089
5563
|
/**
|
|
5090
|
-
*
|
|
5091
|
-
*
|
|
5092
|
-
*
|
|
5093
|
-
*
|
|
5094
|
-
*
|
|
5095
|
-
*
|
|
5564
|
+
* Boot-time staging hygiene. Sweeps the staging dir (sparing an
|
|
5565
|
+
* archive a pending-restore marker references — the boot hook has
|
|
5566
|
+
* not consumed it yet), then reports — without touching — the LEGACY
|
|
5567
|
+
* staging dir inside `addons-data`, where pre-move leaks live. The
|
|
5568
|
+
* legacy dir held 5.1 GB on the hub; deleting operator-visible
|
|
5569
|
+
* gigabytes silently is not this addon's call, so it warns with the
|
|
5570
|
+
* footprint instead.
|
|
5571
|
+
*/
|
|
5572
|
+
async cleanStagingArea() {
|
|
5573
|
+
const marker = this.systemBackup?.readPendingMarker() ?? null;
|
|
5574
|
+
const keepPaths = marker ? [marker.archivePath] : [];
|
|
5575
|
+
try {
|
|
5576
|
+
const swept = await sweepStagingDir({
|
|
5577
|
+
stagingDir: this.stagingDir,
|
|
5578
|
+
keepPaths,
|
|
5579
|
+
logger: this.ctx.logger
|
|
5580
|
+
});
|
|
5581
|
+
if (swept.removedCount > 0) this.ctx.logger.info("backup staging swept — leaked transient files removed", { meta: {
|
|
5582
|
+
stagingDir: this.stagingDir,
|
|
5583
|
+
removedCount: swept.removedCount,
|
|
5584
|
+
freedBytes: swept.freedBytes,
|
|
5585
|
+
keptForPendingRestore: swept.keptCount
|
|
5586
|
+
} });
|
|
5587
|
+
} catch (err) {
|
|
5588
|
+
this.ctx.logger.warn("backup staging sweep failed — continuing", { meta: {
|
|
5589
|
+
stagingDir: this.stagingDir,
|
|
5590
|
+
error: err instanceof Error ? err.message : String(err)
|
|
5591
|
+
} });
|
|
5592
|
+
}
|
|
5593
|
+
const legacyDir = node_path.join(this.ctx.dataDir, "staging");
|
|
5594
|
+
try {
|
|
5595
|
+
const legacy = await measureDirectoryFootprint(legacyDir);
|
|
5596
|
+
if (legacy.present && legacy.fileCount > 0) this.ctx.logger.warn("legacy backup staging dir still holds files inside addons-data — excluded from archives, but the disk space is only reclaimed by deleting it", { meta: {
|
|
5597
|
+
legacyDir,
|
|
5598
|
+
fileCount: legacy.fileCount,
|
|
5599
|
+
totalBytes: legacy.totalBytes
|
|
5600
|
+
} });
|
|
5601
|
+
} catch {}
|
|
5602
|
+
}
|
|
5603
|
+
/**
|
|
5604
|
+
* Trigger a backup run through the global execution queue (D342):
|
|
5605
|
+
* at most one archive build at a time, a trigger that arrives while
|
|
5606
|
+
* one is in flight WAITS (or joins an identical queued run), and the
|
|
5607
|
+
* result says which happened. The call still resolves when the run
|
|
5608
|
+
* terminates; a cancelled run resolves with `cancelled: true` and
|
|
5609
|
+
* whatever entries landed before the stop — a deliberate operator
|
|
5610
|
+
* cancel is an outcome, not an error.
|
|
5096
5611
|
*/
|
|
5097
5612
|
async triggerBackup(input) {
|
|
5613
|
+
if (!this.runQueue) throw new Error("orchestrator not initialized");
|
|
5614
|
+
const request = {
|
|
5615
|
+
...input?.destinations ? { destinations: input.destinations } : {},
|
|
5616
|
+
...input?.locations ? { locations: input.locations } : {},
|
|
5617
|
+
...input?.label !== void 0 ? { label: input.label } : {},
|
|
5618
|
+
...input?.retentionCount !== void 0 ? { retentionCount: input.retentionCount } : {}
|
|
5619
|
+
};
|
|
5620
|
+
const submitted = this.runQueue.submit(request);
|
|
5621
|
+
if (submitted.queued) this.ctx.logger.info(submitted.joined ? "backup trigger: identical run already queued — joined it" : "backup trigger: a run is in flight — enqueued behind it", { meta: { runId: submitted.runId } });
|
|
5622
|
+
let entries;
|
|
5623
|
+
try {
|
|
5624
|
+
entries = await submitted.result;
|
|
5625
|
+
} catch (err) {
|
|
5626
|
+
if (!(err instanceof BackupCancelledError)) throw err;
|
|
5627
|
+
entries = [];
|
|
5628
|
+
}
|
|
5629
|
+
const finalState = this.runQueue?.getRun(submitted.runId)?.state;
|
|
5630
|
+
return {
|
|
5631
|
+
runId: submitted.runId,
|
|
5632
|
+
queued: submitted.queued,
|
|
5633
|
+
joined: submitted.joined,
|
|
5634
|
+
cancelled: finalState === "cancelled",
|
|
5635
|
+
entries
|
|
5636
|
+
};
|
|
5637
|
+
}
|
|
5638
|
+
/** Wire `backup.listRuns` — queued + active + bounded history. */
|
|
5639
|
+
async listRuns() {
|
|
5640
|
+
return this.runQueue?.listRuns() ?? [];
|
|
5641
|
+
}
|
|
5642
|
+
/**
|
|
5643
|
+
* Wire `backup.cancel`. Delegates to the queue: a queued run is
|
|
5644
|
+
* removed before it starts; the running run's abort signal fires and
|
|
5645
|
+
* `runBackupBuild` stops the tar/upload streams and deletes its
|
|
5646
|
+
* staging archive on the way out.
|
|
5647
|
+
*/
|
|
5648
|
+
async cancelRun(input) {
|
|
5649
|
+
const cancelled = this.runQueue?.cancel(input.runId) ?? false;
|
|
5650
|
+
if (cancelled) this.ctx.logger.info("backup run cancel requested", { meta: { runId: input.runId } });
|
|
5651
|
+
else this.ctx.logger.warn("backup cancel: run unknown or already terminal", { meta: { runId: input.runId } });
|
|
5652
|
+
return { cancelled };
|
|
5653
|
+
}
|
|
5654
|
+
/**
|
|
5655
|
+
* Execute ONE backup run — only ever called by the queue, never
|
|
5656
|
+
* directly (the queue is the single serialisation authority).
|
|
5657
|
+
* Builds the archive in the private staging dir, then pushes it to
|
|
5658
|
+
* every selected destination via the storage cap's chunked upload
|
|
5659
|
+
* protocol. Per-destination errors are logged and recorded on the
|
|
5660
|
+
* run — one bad sink doesn't kill the run.
|
|
5661
|
+
*
|
|
5662
|
+
* Cleanup contract (D342): the staging file is deleted in `finally`,
|
|
5663
|
+
* which covers success, per-destination failure, build failure AND
|
|
5664
|
+
* cancellation alike. The only leak this cannot cover — the process
|
|
5665
|
+
* dying inside the run — is reclaimed by the boot sweep in
|
|
5666
|
+
* `cleanStagingArea`.
|
|
5667
|
+
*/
|
|
5668
|
+
async runBackupBuild(input, signal, progress) {
|
|
5098
5669
|
if (!this.systemBackup) throw new Error("orchestrator not initialized");
|
|
5099
|
-
const targets = await this.resolveTargets(input
|
|
5670
|
+
const targets = await this.resolveTargets(input.destinations !== void 0 ? [...input.destinations] : void 0);
|
|
5100
5671
|
if (targets.length === 0) {
|
|
5101
5672
|
this.ctx.logger.warn("backup trigger: no destinations selected");
|
|
5102
5673
|
return [];
|
|
5103
5674
|
}
|
|
5675
|
+
progress.setDestinations(targets.map((t) => t.id));
|
|
5104
5676
|
const policies = await this.loadPoliciesIndexed(targets.map((t) => t.id));
|
|
5105
5677
|
const archiveId = (0, node_crypto.randomUUID)();
|
|
5106
5678
|
const archiveFilename = `${archiveId}.tar.gz`;
|
|
5107
5679
|
const cachePath = await this.stagingPath(archiveFilename);
|
|
5108
5680
|
const result = await this.systemBackup.createArchive({
|
|
5109
5681
|
archivePath: cachePath,
|
|
5110
|
-
...input
|
|
5682
|
+
...input.locations ? { locations: [...input.locations] } : {},
|
|
5683
|
+
signal,
|
|
5684
|
+
onArchiveBytes: (bytes) => progress.setStagedBytes(bytes)
|
|
5111
5685
|
});
|
|
5686
|
+
progress.setArchiveSize(result.sizeBytes);
|
|
5687
|
+
progress.setPhase("uploading");
|
|
5112
5688
|
try {
|
|
5113
5689
|
const entries = [];
|
|
5114
5690
|
const createdAt = Date.now();
|
|
5115
|
-
for (const dest of targets)
|
|
5116
|
-
|
|
5117
|
-
|
|
5118
|
-
|
|
5119
|
-
|
|
5120
|
-
|
|
5121
|
-
|
|
5122
|
-
|
|
5123
|
-
|
|
5124
|
-
|
|
5125
|
-
|
|
5126
|
-
|
|
5127
|
-
|
|
5128
|
-
|
|
5129
|
-
|
|
5130
|
-
|
|
5131
|
-
|
|
5132
|
-
|
|
5133
|
-
|
|
5134
|
-
|
|
5135
|
-
|
|
5136
|
-
|
|
5137
|
-
|
|
5691
|
+
for (const dest of targets) {
|
|
5692
|
+
if (signal.aborted) {
|
|
5693
|
+
this.ctx.logger.info("backup run cancelled — skipping remaining destinations", { meta: {
|
|
5694
|
+
runCancelledAfter: entries.length,
|
|
5695
|
+
skippedDestinationIds: targets.slice(targets.indexOf(dest)).map((t) => t.id)
|
|
5696
|
+
} });
|
|
5697
|
+
break;
|
|
5698
|
+
}
|
|
5699
|
+
try {
|
|
5700
|
+
const policy = policies.get(dest.id) ?? this.fallbackPolicy(dest.id);
|
|
5701
|
+
const stored = await this.uploadAndIndex({
|
|
5702
|
+
locationId: dest.id,
|
|
5703
|
+
archiveId,
|
|
5704
|
+
archiveFilename,
|
|
5705
|
+
cachePath,
|
|
5706
|
+
sizeBytes: result.sizeBytes,
|
|
5707
|
+
archiveManifest: result.manifest,
|
|
5708
|
+
retentionCount: input.retentionCount ?? policy.retentionCount,
|
|
5709
|
+
createdAt,
|
|
5710
|
+
label: formatArchiveLabel(dest.id, createdAt, input.label),
|
|
5711
|
+
signal,
|
|
5712
|
+
onUploadedBytes: (bytes) => progress.setUploadedBytes(bytes)
|
|
5713
|
+
});
|
|
5714
|
+
entries.push(stored);
|
|
5715
|
+
progress.destinationCompleted(dest.id);
|
|
5716
|
+
this.ctx.eventBus.emit({
|
|
5717
|
+
id: (0, node_crypto.randomUUID)(),
|
|
5718
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
5719
|
+
source: {
|
|
5720
|
+
type: "addon",
|
|
5721
|
+
id: "backup-orchestrator"
|
|
5722
|
+
},
|
|
5723
|
+
category: require_dist.EventCategory.BackupCompleted,
|
|
5724
|
+
data: {
|
|
5725
|
+
destinationId: dest.id,
|
|
5726
|
+
backupId: stored.id,
|
|
5727
|
+
sizeMB: Math.round((stored.sizeBytes ?? 0) / (1024 * 1024))
|
|
5728
|
+
}
|
|
5729
|
+
});
|
|
5730
|
+
} catch (err) {
|
|
5731
|
+
if (err instanceof BackupCancelledError) continue;
|
|
5732
|
+
progress.destinationFailed(dest.id);
|
|
5733
|
+
this.ctx.logger.error("backup trigger: destination failed", { meta: {
|
|
5138
5734
|
destinationId: dest.id,
|
|
5139
|
-
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
});
|
|
5143
|
-
} catch (err) {
|
|
5144
|
-
this.ctx.logger.error("backup trigger: destination failed", { meta: {
|
|
5145
|
-
destinationId: dest.id,
|
|
5146
|
-
error: err instanceof Error ? err.message : String(err)
|
|
5147
|
-
} });
|
|
5735
|
+
error: err instanceof Error ? err.message : String(err)
|
|
5736
|
+
} });
|
|
5737
|
+
}
|
|
5148
5738
|
}
|
|
5149
5739
|
return entries;
|
|
5150
5740
|
} finally {
|
|
@@ -5173,6 +5763,10 @@ var BackupOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
5173
5763
|
let offset = 0;
|
|
5174
5764
|
const stream = node_fs.createReadStream(input.cachePath, { highWaterMark: UPLOAD_CHUNK_BYTES });
|
|
5175
5765
|
for await (const rawChunk of stream) {
|
|
5766
|
+
if (input.signal?.aborted) {
|
|
5767
|
+
stream.destroy();
|
|
5768
|
+
throw new BackupCancelledError(`backup upload to ${input.locationId} cancelled mid-stream`);
|
|
5769
|
+
}
|
|
5176
5770
|
const chunk = toBufferChunk(rawChunk);
|
|
5177
5771
|
const data = new Uint8Array(chunk.byteLength);
|
|
5178
5772
|
data.set(chunk);
|
|
@@ -5182,6 +5776,7 @@ var BackupOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
5182
5776
|
data
|
|
5183
5777
|
});
|
|
5184
5778
|
offset += data.byteLength;
|
|
5779
|
+
input.onUploadedBytes?.(offset);
|
|
5185
5780
|
}
|
|
5186
5781
|
await this.ctx.api.storage.finalizeUpload.mutate({ uploadId });
|
|
5187
5782
|
} catch (err) {
|