@jwilleke/ngdpbase-types 5.25.0 → 5.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/packages/ngdpbase-types/index.d.ts +4 -0
- package/lib/src/http/egressPolicy.js +61 -0
- package/lib/src/http/guardedElasticsearch.d.ts +27 -0
- package/lib/src/http/guardedElasticsearch.js +99 -0
- package/lib/src/managers/AddonsManager.d.ts +37 -1
- package/lib/src/managers/AttachmentManager.d.ts +4 -5
- package/lib/src/managers/AuthManager.d.ts +150 -32
- package/lib/src/managers/BackgroundJobManager.d.ts +22 -12
- package/lib/src/managers/BackupManager.d.ts +779 -0
- package/lib/src/managers/BackupManager.js +2042 -0
- package/lib/src/managers/BaseManager.d.ts +156 -6
- package/lib/src/managers/BaseManager.js +64 -2
- package/lib/src/managers/CommentManager.d.ts +6 -6
- package/lib/src/managers/ConsentManager.d.ts +130 -0
- package/lib/src/managers/DatabaseManager.d.ts +6 -0
- package/lib/src/managers/DatabaseManager.js +25 -15
- package/lib/src/managers/FootnoteManager.d.ts +6 -6
- package/lib/src/managers/OrganizationManager.d.ts +8 -6
- package/lib/src/managers/PageManager.d.ts +16 -0
- package/lib/src/managers/PersonManager.d.ts +8 -6
- package/lib/src/managers/RoleManager.d.ts +6 -8
- package/lib/src/providers/BaseAttachmentProvider.d.ts +4 -4
- package/lib/src/providers/BaseBackupProvider.d.ts +77 -0
- package/lib/src/providers/BaseBackupProvider.js +52 -0
- package/lib/src/providers/BasicAttachmentProvider.d.ts +4 -4
- package/lib/src/providers/FileBackupProvider.d.ts +34 -0
- package/lib/src/providers/FileBackupProvider.js +116 -0
- package/lib/src/providers/FileConsentProvider.d.ts +18 -0
- package/lib/src/providers/SqliteDatabaseProvider.d.ts +2 -2
- package/lib/src/providers/SqliteDatabaseProvider.js +12 -11
- package/lib/src/providers/attachmentFilesBackup.d.ts +5 -4
- package/lib/src/providers/databaseMigrations.js +45 -0
- package/lib/src/security/PolicyDecisionPoint.d.ts +14 -0
- package/lib/src/types/Consent.d.ts +90 -0
- package/lib/src/types/express.d.ts +6 -0
- package/lib/src/utils/addonHost.d.ts +22 -0
- package/lib/src/utils/addonHost.js +14 -1
- package/lib/src/utils/auditEventNames.generated.d.ts +20 -0
- package/lib/src/utils/auditEventNames.generated.js +20 -0
- package/lib/src/utils/backupCrypto.js +275 -0
- package/lib/src/utils/backupDocument.d.ts +36 -0
- package/lib/src/utils/backupDocument.js +99 -0
- package/lib/src/utils/browserDescription.d.ts +11 -0
- package/lib/src/utils/directoryBackup.d.ts +48 -48
- package/lib/src/utils/directoryBackup.js +291 -0
- package/lib/src/utils/instanceEnvBackup.d.ts +46 -0
- package/lib/src/utils/instanceEnvBackup.js +120 -0
- package/lib/src/utils/instanceEnvSecret.js +79 -0
- package/lib/src/utils/ipLocation.d.ts +11 -0
- package/lib/src/utils/schedule.js +171 -0
- package/lib/src/utils/secretFileMode.js +78 -0
- package/lib/src/utils/siteRestore.d.ts +13 -16
- package/lib/src/utils/siteRestore.js +33 -6
- package/lib/src/utils/treeArchive.d.ts +2 -0
- package/lib/src/utils/treeArchive.js +242 -0
- package/lib/src/utils/version.js +408 -0
- package/package.json +6 -1
- package/lib/src/utils/administrators.d.ts +0 -21
|
@@ -0,0 +1,2042 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import { create as createTar } from 'tar';
|
|
3
|
+
import BaseManager from './BaseManager.js';
|
|
4
|
+
import { actorOf } from '../context/ActorContext.js';
|
|
5
|
+
import zlib from 'zlib';
|
|
6
|
+
import { promisify } from 'util';
|
|
7
|
+
import logger from '../utils/logger.js';
|
|
8
|
+
import { parseSchedule } from '../utils/schedule.js';
|
|
9
|
+
import { recordAuditEvent } from '../utils/auditEvents.js';
|
|
10
|
+
import { AUDIT_EVENT } from '../utils/auditEventNames.js';
|
|
11
|
+
import { applyPartAtStart, beginSiteRestore, commitSiteRestore, discardSiteRestore, finishSiteRestoreAtBoot, gatePart, markPart, readSiteRestore, siteRestoreFile, stagedPathFor, stagingFor } from '../utils/siteRestore.js';
|
|
12
|
+
import { encryptBuffer, ENCRYPTED_SUFFIX } from '../utils/backupCrypto.js';
|
|
13
|
+
import { readBackupDocument, verifyBackupOffThread } from '../utils/backupDocument.js';
|
|
14
|
+
import { BACKUP_KEY_ENV, backupKeyFingerprint, currentBackupKey, isLeftOutForNoKey, LEFT_OUT_FOR_NO_KEY } from '../utils/backupKey.js';
|
|
15
|
+
import { applyDeclaredPartsAtStart, archiveReader, archiveTarget, archiveWriter, backupDeclaredParts, placePartRecords, stageDeclaredParts, stageFilePart } from '../utils/directoryBackup.js';
|
|
16
|
+
import { writeTreeArchive } from '../utils/treeArchive.js';
|
|
17
|
+
import { envForBackup, envKeyNames, envKeyNamesFingerprint, INSTANCE_ENV_PART, instanceEnvPath } from '../utils/instanceEnvBackup.js';
|
|
18
|
+
import { readEnvLine } from '../utils/instanceEnvSecret.js';
|
|
19
|
+
import { systemContext } from '../context/bootActions.js';
|
|
20
|
+
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
21
|
+
import { mkdir, readFile as readFileAsync, writeFile as writeFileAsync } from 'node:fs/promises';
|
|
22
|
+
const gzip = promisify(zlib.gzip);
|
|
23
|
+
/**
|
|
24
|
+
* A backup document's filename without `.json.gz` / `.json` (#1723). A
|
|
25
|
+
* manager's own files beside the document are named `<stem>.<name>.db` (a
|
|
26
|
+
* database, #1723) or `<stem>.<part>.tar` (a folder of files, #1876), and are
|
|
27
|
+
* pruned and downloaded with it.
|
|
28
|
+
*/
|
|
29
|
+
export function backupStem(filename) {
|
|
30
|
+
return filename.replace(/\.json(\.gz)?(\.enc)?$/, '');
|
|
31
|
+
}
|
|
32
|
+
/** Whether `filename` is one of the files a manager wrote beside the document named `documentFilename` (#1723); a file archive may be encrypted, `.tar.enc` (#1561). */
|
|
33
|
+
export function isCompanionOf(filename, documentFilename) {
|
|
34
|
+
return filename.startsWith(`${backupStem(documentFilename)}.`) && (filename.endsWith('.db') || filename.endsWith('.tar') || filename.endsWith(`.tar${ENCRYPTED_SUFFIX}`));
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* What no backup holds, so no restore brings back (#1876), in the words
|
|
38
|
+
* admins read: these stay as they are on the site. The backup-restore guide
|
|
39
|
+
* and the Backup help page list the same.
|
|
40
|
+
*/
|
|
41
|
+
export const NEVER_IN_A_BACKUP = [
|
|
42
|
+
'shares (share links stay as they are)',
|
|
43
|
+
'notifications',
|
|
44
|
+
'sign-in sessions',
|
|
45
|
+
'the backup key itself, and the database key (both stay as they are on this site)',
|
|
46
|
+
'TLS certificate files',
|
|
47
|
+
'the media library and its index'
|
|
48
|
+
];
|
|
49
|
+
/** How the "not restored" notification names a manager's data (#1876); a manager not listed is named as it is. */
|
|
50
|
+
const RESTORE_LABELS = {
|
|
51
|
+
PageManager: 'pages (with their history, trash and private stores)',
|
|
52
|
+
UserManager: 'user accounts',
|
|
53
|
+
ConfigurationManager: 'settings',
|
|
54
|
+
SearchManager: 'the search index (rebuild it from the pages)',
|
|
55
|
+
CatalogManager: 'the user keywords vocabulary',
|
|
56
|
+
AddonsManager: 'the add-on list',
|
|
57
|
+
AgentTokenManager: 'agent tokens',
|
|
58
|
+
OidcManager: 'OpenID Connect grants',
|
|
59
|
+
AuthManager: 'the credentials store (passkeys, verified email and phone, devices, linked identities, proxy grants)',
|
|
60
|
+
AuditManager: 'the audit log',
|
|
61
|
+
BackgroundJobManager: 'scheduled-job state',
|
|
62
|
+
AttachmentManager: 'attachment files',
|
|
63
|
+
RoleManager: 'role membership',
|
|
64
|
+
PersonManager: 'persons',
|
|
65
|
+
OrganizationManager: 'organizations',
|
|
66
|
+
FootnoteManager: 'footnotes on public pages',
|
|
67
|
+
CommentManager: 'comments on public pages',
|
|
68
|
+
DatabaseManager: 'the databases',
|
|
69
|
+
BackupManager: 'the instance .env (the keys and secrets in it)',
|
|
70
|
+
FormsDataManager: 'forms and their submissions',
|
|
71
|
+
CalendarDataManager: 'calendars',
|
|
72
|
+
FeedManager: 'feed records'
|
|
73
|
+
};
|
|
74
|
+
/**
|
|
75
|
+
* The entry an add-on's files go under in the backup document (#1985). The
|
|
76
|
+
* bundled add-ons keep the name their manager wrote before the declaration
|
|
77
|
+
* (v5.21 to v5.25), so an old backup restores and a v5.25.0 site can
|
|
78
|
+
* restore a new one; any other add-on's is `addon:<slug>`, which no manager
|
|
79
|
+
* name can be.
|
|
80
|
+
*/
|
|
81
|
+
const ADDON_ENTRY_NAMES = {
|
|
82
|
+
forms: 'FormsDataManager',
|
|
83
|
+
calendar: 'CalendarDataManager',
|
|
84
|
+
feeds: 'FeedManager'
|
|
85
|
+
};
|
|
86
|
+
/** The backup document entry, and the site restore part, an add-on's files are recorded under (#1985). */
|
|
87
|
+
export function addonEntryName(slug) {
|
|
88
|
+
return ADDON_ENTRY_NAMES[slug] ?? `addon:${slug}`;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Put a staged restore of a manager's declared folders and files in place,
|
|
92
|
+
* at start, before it reads them (#1985). BackupManager's step: a manager
|
|
93
|
+
* reaches it through `BaseManager.initialize()`, an add-on's folders through
|
|
94
|
+
* AddonsManager before the add-on's `register()`. A module function, not a
|
|
95
|
+
* method, because most managers start before BackupManager does; it needs
|
|
96
|
+
* only the configuration, for the site restore manifest.
|
|
97
|
+
*/
|
|
98
|
+
export function applyStagedPartsAtStart(engine, manager, declared) {
|
|
99
|
+
const configManager = engine?.getManager?.('ConfigurationManager');
|
|
100
|
+
if (!configManager)
|
|
101
|
+
return null;
|
|
102
|
+
return applyDeclaredPartsAtStart(manager, declared, configManager);
|
|
103
|
+
}
|
|
104
|
+
/** The site restore manifest's file, as configured; a module function for the start, before BackupManager exists (#1986). */
|
|
105
|
+
function manifestFileOf(engine) {
|
|
106
|
+
return siteRestoreFile(engine?.getManager?.('ConfigurationManager'));
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Put `manager`'s part of a staged site restore in place, at start, before
|
|
110
|
+
* it reads its data (#1876, #1986): when the whole restore is there to
|
|
111
|
+
* apply, each live path steps aside as `<live>.pre-restore` and the staged
|
|
112
|
+
* copy takes its name; otherwise its staged files are set aside as
|
|
113
|
+
* `*.incomplete`. The part is marked in BackupManager's record either way.
|
|
114
|
+
* For a manager whose restore is staged by its own `restore()` (the
|
|
115
|
+
* attachments, the audit log); declared folders and files go through
|
|
116
|
+
* `applyStagedPartsAtStart`. A module function: most managers start before
|
|
117
|
+
* BackupManager. Null when there was nothing to do.
|
|
118
|
+
*/
|
|
119
|
+
export function applyStagedPartAtStart(engine, manager, options) {
|
|
120
|
+
return applyPartAtStart(manifestFileOf(engine), manager, {
|
|
121
|
+
allowed: options.allowed,
|
|
122
|
+
strays: () => (options.mayBeStaged?.() ?? []).map(stagedPathFor).filter((p) => existsSync(p)),
|
|
123
|
+
...(options.put ? { put: options.put } : {})
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
/** BackupManager's start-time gate on `manager`'s part of a staged site restore (#1986): see {@link SiteRestorePartAtStart}. */
|
|
127
|
+
export function siteRestorePartAtStart(engine, manager) {
|
|
128
|
+
const file = manifestFileOf(engine);
|
|
129
|
+
const gate = gatePart(file, manager);
|
|
130
|
+
const named = gate.decision === 'apply' || (gate.decision === 'set-aside' && Boolean(gate.part));
|
|
131
|
+
return {
|
|
132
|
+
decision: gate.decision,
|
|
133
|
+
...(gate.decision === 'set-aside' ? { reason: gate.reason } : {}),
|
|
134
|
+
settle: (status, note) => {
|
|
135
|
+
if (named)
|
|
136
|
+
markPart(file, manager, status, note);
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
/** Managers whose backup entry holds nothing of the site's own: never named as not restored. */
|
|
141
|
+
const NO_SITE_DATA = new Set(['ImportManager']);
|
|
142
|
+
/** Whether a manager's backup entry holds anything: an error, or a field beyond the bookkeeping every entry has. */
|
|
143
|
+
function slotHoldsData(slot) {
|
|
144
|
+
if (!slot || typeof slot !== 'object')
|
|
145
|
+
return false;
|
|
146
|
+
return Object.entries(slot)
|
|
147
|
+
.some(([key, value]) => !['managerName', 'timestamp', 'note', 'warnings'].includes(key) && value !== null && value !== undefined);
|
|
148
|
+
}
|
|
149
|
+
/** A manager's data as the "not restored" notification names it. */
|
|
150
|
+
export function restoreLabel(manager) {
|
|
151
|
+
if (manager.startsWith('addon:'))
|
|
152
|
+
return `the ${manager.slice('addon:'.length)} add-on's files`;
|
|
153
|
+
return RESTORE_LABELS[manager] ?? manager;
|
|
154
|
+
}
|
|
155
|
+
/** BackupManager's fault, and the dashboard banner, while there is no backup key (#1561, #1912). */
|
|
156
|
+
export const BACKUP_KEY_MISSING = `${BACKUP_KEY_ENV} is not set, so backups are NOT encrypted: the document (password hashes, settings with their secrets) and the file archives beside it are in the clear, and the credentials store, the databases and the instance .env are left out`;
|
|
157
|
+
/** The title of the one notification about it. */
|
|
158
|
+
export const BACKUP_KEY_MISSING_TITLE = 'Backups are not encrypted';
|
|
159
|
+
/**
|
|
160
|
+
* The config key that records an admin's confirmation that the backup key is
|
|
161
|
+
* saved off the server (#1876): the fingerprint of the key they saved. A
|
|
162
|
+
* different key (a new one made, or one replaced) asks again.
|
|
163
|
+
*/
|
|
164
|
+
export const BACKUP_KEY_SAVED_KEY = 'ngdpbase.backup.key-saved-fingerprint';
|
|
165
|
+
/** The banner while no admin has confirmed the backup key is saved off the server (#1876). */
|
|
166
|
+
export function backupKeyNotSaved(fingerprint) {
|
|
167
|
+
return `Save the backup key off this server: Admin → Backup shows it (fingerprint ${fingerprint}). Every backup is encrypted with it, and a restore on a new server needs it. This stays until an admin confirms it is saved`;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* The config key that records the names (never the values) the instance
|
|
171
|
+
* `.env` set when an admin last confirmed it is saved off the server (#1976),
|
|
172
|
+
* sorted. Null until the first check records the baseline. The names, not
|
|
173
|
+
* only their fingerprint, so the reminder can say which keys were added or
|
|
174
|
+
* removed.
|
|
175
|
+
*/
|
|
176
|
+
export const ENV_KEYS_SAVED_KEY = 'ngdpbase.backup.env-keys-saved';
|
|
177
|
+
/** The title of the one notification per change of the `.env` key set (#1976). */
|
|
178
|
+
export const ENV_KEYS_CHANGED_TITLE = 'Save a copy of .env';
|
|
179
|
+
const ENV_KEYS_CHANGED_PREFIX = 'The keys in .env changed since an admin last saved it';
|
|
180
|
+
const KEY_NOT_SAVED_PREFIX = 'Save the backup key off this server';
|
|
181
|
+
/** The banner while the instance `.env` sets other names than the ones an admin last confirmed saved (#1976): names only, never a value. */
|
|
182
|
+
export function envKeysChanged(change) {
|
|
183
|
+
const parts = [
|
|
184
|
+
...(change.added.length > 0 ? [`added: ${change.added.join(', ')}`] : []),
|
|
185
|
+
...(change.removed.length > 0 ? [`removed: ${change.removed.join(', ')}`] : [])
|
|
186
|
+
];
|
|
187
|
+
return `${ENV_KEYS_CHANGED_PREFIX} (${parts.join('; ')}). Download .env on Admin → Backup and keep the copy off this server. This stays until an admin confirms it is saved`;
|
|
188
|
+
}
|
|
189
|
+
/** The scheduled job automatic backups run as (#1720). */
|
|
190
|
+
export const AUTO_BACKUP_JOB_ID = 'backup.auto';
|
|
191
|
+
/** The routine check that backups are still happening (#1563). */
|
|
192
|
+
export const BACKUP_HEALTH_JOB_ID = 'backup.health';
|
|
193
|
+
/** The title of the notification sent when backups stop working (#1563): a backup failed, or none is recent enough. */
|
|
194
|
+
export const BACKUPS_NOT_WORKING_TITLE = 'Backups are not working';
|
|
195
|
+
const DAY_MS = 86_400_000;
|
|
196
|
+
/** A number of bytes as admins read it. */
|
|
197
|
+
export function formatBytes(bytes) {
|
|
198
|
+
if (!Number.isFinite(bytes) || bytes < 1024)
|
|
199
|
+
return `${Math.max(0, Math.round(bytes || 0))} B`;
|
|
200
|
+
const units = ['KB', 'MB', 'GB', 'TB'];
|
|
201
|
+
let value = bytes / 1024;
|
|
202
|
+
let unit = 0;
|
|
203
|
+
while (value >= 1024 && unit < units.length - 1) {
|
|
204
|
+
value /= 1024;
|
|
205
|
+
unit += 1;
|
|
206
|
+
}
|
|
207
|
+
return `${value.toFixed(value >= 10 ? 0 : 1)} ${units[unit]}`;
|
|
208
|
+
}
|
|
209
|
+
/** Why a backup could not be written, in words an admin can act on: the system's error codes named (#1563). */
|
|
210
|
+
export function describeBackupError(err) {
|
|
211
|
+
const e = err;
|
|
212
|
+
const message = e?.message ?? String(err);
|
|
213
|
+
switch (e?.code) {
|
|
214
|
+
case 'ENOSPC': return `the backup destination is out of space (${message})`;
|
|
215
|
+
case 'EDQUOT': return `the backup destination is over its quota (${message})`;
|
|
216
|
+
case 'EACCES':
|
|
217
|
+
case 'EPERM': return `permission denied writing to the backup destination (${message})`;
|
|
218
|
+
case 'EROFS': return `the backup destination is read-only (${message})`;
|
|
219
|
+
case 'ENOENT': return `the backup destination is not there (${message})`;
|
|
220
|
+
case 'EIO': return `the backup destination reported a write error (${message})`;
|
|
221
|
+
default: return message;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* The longest gap between two runs of a schedule (#1563): what "the
|
|
226
|
+
* auto-backup interval" is for a schedule that is not evenly spaced (Monday,
|
|
227
|
+
* Wednesday and Friday is three days at its longest; monthly is 31). Counted
|
|
228
|
+
* over the next year's runs, at most 400 of them.
|
|
229
|
+
*/
|
|
230
|
+
export function longestGapMs(schedule, from) {
|
|
231
|
+
let previous = schedule.nextSlot(from);
|
|
232
|
+
let longest = 0;
|
|
233
|
+
const until = from.getTime() + 400 * DAY_MS;
|
|
234
|
+
for (let i = 0; previous && i < 400 && previous.getTime() < until; i++) {
|
|
235
|
+
const next = schedule.nextSlot(previous);
|
|
236
|
+
if (!next)
|
|
237
|
+
break;
|
|
238
|
+
longest = Math.max(longest, next.getTime() - previous.getTime());
|
|
239
|
+
previous = next;
|
|
240
|
+
}
|
|
241
|
+
return longest;
|
|
242
|
+
}
|
|
243
|
+
const BYDAY = { sun: 'SU', mon: 'MO', tue: 'TU', wed: 'WE', thu: 'TH', fri: 'FR', sat: 'SA' };
|
|
244
|
+
/**
|
|
245
|
+
* The automatic-backup settings as a schedule (#1720): `ngdpbase.backup.auto-backup-time`
|
|
246
|
+
* (HH:MM) and `ngdpbase.backup.auto-backup-days` — `daily`, `monthly` (the 1st),
|
|
247
|
+
* `weekdays` (Monday to Friday), or day names such as `Mon,Wed,Fri`.
|
|
248
|
+
*
|
|
249
|
+
* In `timeZone`: the server's own, because the timer this replaces compared
|
|
250
|
+
* the server's local clock, so an existing setting keeps its time.
|
|
251
|
+
*
|
|
252
|
+
* Throws, naming the setting, for a time or a day it does not understand. The
|
|
253
|
+
* timer this replaces matched neither, so such a setting never backed up, and
|
|
254
|
+
* said nothing.
|
|
255
|
+
*/
|
|
256
|
+
export function autoBackupSchedule(time, days, timeZone) {
|
|
257
|
+
const m = /^(\d{1,2}):(\d{2})$/.exec(String(time).trim());
|
|
258
|
+
if (!m || Number(m[1]) > 23 || Number(m[2]) > 59) {
|
|
259
|
+
throw new Error(`ngdpbase.backup.auto-backup-time '${time}' is not a time of day (HH:MM)`);
|
|
260
|
+
}
|
|
261
|
+
const at = `BYHOUR=${Number(m[1])};BYMINUTE=${Number(m[2])}`;
|
|
262
|
+
const value = String(days).trim().toLowerCase();
|
|
263
|
+
if (value === 'daily')
|
|
264
|
+
return { rrule: `FREQ=DAILY;${at}`, tz: timeZone };
|
|
265
|
+
if (value === 'monthly')
|
|
266
|
+
return { rrule: `FREQ=MONTHLY;BYMONTHDAY=1;${at}`, tz: timeZone };
|
|
267
|
+
if (value === 'weekdays')
|
|
268
|
+
return { rrule: `FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR;${at}`, tz: timeZone };
|
|
269
|
+
const names = value.split(',').map((d) => d.trim()).filter(Boolean);
|
|
270
|
+
const unknown = names.filter((d) => !(d.slice(0, 3) in BYDAY) || !'sunday monday tuesday wednesday thursday friday saturday'.split(' ').some((full) => full.startsWith(d)));
|
|
271
|
+
if (names.length === 0 || unknown.length > 0) {
|
|
272
|
+
throw new Error(`ngdpbase.backup.auto-backup-days '${days}' is not daily, monthly, weekdays or a list of day names (${unknown.join(', ') || 'empty'} not understood)`);
|
|
273
|
+
}
|
|
274
|
+
const byDay = [...new Set(names.map((d) => BYDAY[d.slice(0, 3)]))];
|
|
275
|
+
return { rrule: `FREQ=WEEKLY;BYDAY=${byDay.join(',')};${at}`, tz: timeZone };
|
|
276
|
+
}
|
|
277
|
+
/** The server's own time zone: what the replaced timer's `getHours()` read. */
|
|
278
|
+
function serverTimeZone() {
|
|
279
|
+
return Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC';
|
|
280
|
+
}
|
|
281
|
+
/** A time as admins read it: the server's local time (#1876; never UTC in a notification). */
|
|
282
|
+
function localTime(iso) {
|
|
283
|
+
const t = iso ? Date.parse(iso) : NaN;
|
|
284
|
+
return Number.isFinite(t) ? new Date(t).toLocaleString() : 'an unknown time';
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* The text of the critical "not restored" notification (#1876): what the
|
|
288
|
+
* restore put back, what it did not, and per job the runs since the backup —
|
|
289
|
+
* its name, when it last ran and how that ended — newest first.
|
|
290
|
+
*/
|
|
291
|
+
export function describeNotRestored(input) {
|
|
292
|
+
const lines = [];
|
|
293
|
+
lines.push(`The restore from ${input.source}${input.backup ? ` (backup taken ${localTime(input.backup)})` : ''} was applied: ${input.applied.join(', ')}.`);
|
|
294
|
+
const notRestored = [...input.notRestored, ...NEVER_IN_A_BACKUP];
|
|
295
|
+
lines.push(`Not restored — still as they were before the restore: ${notRestored.join('; ')}.`);
|
|
296
|
+
if (input.missingReport)
|
|
297
|
+
lines.push('The record of what this restore left out could not be read, so this list may be short.');
|
|
298
|
+
const byJob = new Map();
|
|
299
|
+
for (const r of input.jobRuns)
|
|
300
|
+
byJob.set(r.jobId, [...(byJob.get(r.jobId) ?? []), r]);
|
|
301
|
+
if (byJob.size === 0) {
|
|
302
|
+
lines.push('No scheduled job ran between the backup and the restore.');
|
|
303
|
+
}
|
|
304
|
+
else {
|
|
305
|
+
const jobs = [...byJob.entries()].map(([jobId, runs]) => {
|
|
306
|
+
const last = runs.sort((a, b) => Date.parse(b.startedAt) - Date.parse(a.startedAt))[0];
|
|
307
|
+
const result = last.status === 'completed' ? 'completed' : last.status === 'running' ? 'still running' : `${last.status}${last.error ? `: ${last.error}` : ''}`;
|
|
308
|
+
return `${input.nameOf(jobId)} — last ran ${localTime(last.completedAt ?? last.startedAt)}, ${result}${runs.length > 1 ? ` (${runs.length} runs since the backup)` : ''}`;
|
|
309
|
+
});
|
|
310
|
+
lines.push(`Jobs that ran between the backup and the restore — their runs are not in the restored job history, and they are not run again: ${jobs.join('; ')}. Check whether any of them needs running again.`);
|
|
311
|
+
}
|
|
312
|
+
return lines.join(' ');
|
|
313
|
+
}
|
|
314
|
+
class BackupManager extends BaseManager {
|
|
315
|
+
backupDirectory;
|
|
316
|
+
maxBackups;
|
|
317
|
+
autoBackupEnabled;
|
|
318
|
+
autoBackupTime; // HH:MM
|
|
319
|
+
autoBackupDays; // "daily" | "monthly" | "Mon,Wed,..."
|
|
320
|
+
// #170: storage target is delegated to a BackupProvider (default
|
|
321
|
+
// FileBackupProvider). Orchestration/serialization stays in this manager.
|
|
322
|
+
provider;
|
|
323
|
+
providerClass;
|
|
324
|
+
/** A restore started from Admin → Backup, while it runs (#1723). */
|
|
325
|
+
restoreRunning = null;
|
|
326
|
+
/** The outage this manager set because backups are not working (#1563), so only that one is cleared when they work again. */
|
|
327
|
+
healthFault = null;
|
|
328
|
+
/**
|
|
329
|
+
* The end of the queue of work that reads or changes the backup directory
|
|
330
|
+
* one at a time (#1563): a backup with its pruning, a restore's safety
|
|
331
|
+
* backup and staging, and the routine health check (`exclusive`).
|
|
332
|
+
*/
|
|
333
|
+
exclusiveTail = Promise.resolve();
|
|
334
|
+
/**
|
|
335
|
+
* Creates a new BackupManager instance
|
|
336
|
+
*
|
|
337
|
+
* @constructor
|
|
338
|
+
* @param {any} engine - The wiki engine instance
|
|
339
|
+
*/
|
|
340
|
+
constructor(engine) {
|
|
341
|
+
super(engine);
|
|
342
|
+
this.backupDirectory = null;
|
|
343
|
+
this.maxBackups = 10;
|
|
344
|
+
this.autoBackupEnabled = false;
|
|
345
|
+
this.autoBackupTime = '02:00';
|
|
346
|
+
this.autoBackupDays = 'daily';
|
|
347
|
+
this.provider = null;
|
|
348
|
+
this.providerClass = 'FileBackupProvider';
|
|
349
|
+
}
|
|
350
|
+
/** Normalize a config provider name to its PascalCase class name. */
|
|
351
|
+
normalizeProviderName(providerName) {
|
|
352
|
+
if (/^[A-Z]/.test(providerName))
|
|
353
|
+
return providerName;
|
|
354
|
+
const known = {
|
|
355
|
+
filebackupprovider: 'FileBackupProvider'
|
|
356
|
+
};
|
|
357
|
+
return known[providerName.toLowerCase()]
|
|
358
|
+
?? providerName
|
|
359
|
+
.split(/[-_]/)
|
|
360
|
+
.map(p => p.charAt(0).toUpperCase() + p.slice(1).toLowerCase())
|
|
361
|
+
.join('');
|
|
362
|
+
}
|
|
363
|
+
/**
|
|
364
|
+
* Load the configured backup storage provider (#170). Mirrors the
|
|
365
|
+
* dynamic-import + fallback pattern used by AuditManager / SearchManager.
|
|
366
|
+
* Falls back to FileBackupProvider on any load error so backups never
|
|
367
|
+
* silently lose their storage target.
|
|
368
|
+
*/
|
|
369
|
+
async loadProvider() {
|
|
370
|
+
try {
|
|
371
|
+
const mod = await import(/* @vite-ignore */ `../providers/${this.providerClass}.js`);
|
|
372
|
+
this.provider = new mod.default(this.engine);
|
|
373
|
+
await this.provider.initialize();
|
|
374
|
+
}
|
|
375
|
+
catch (error) {
|
|
376
|
+
logger.error(`Failed to load backup provider ${this.providerClass}:`, error);
|
|
377
|
+
if (this.providerClass !== 'FileBackupProvider') {
|
|
378
|
+
logger.warn('Falling back to FileBackupProvider');
|
|
379
|
+
const mod = await import('../providers/FileBackupProvider.js');
|
|
380
|
+
this.provider = new mod.default(this.engine);
|
|
381
|
+
await this.provider.initialize();
|
|
382
|
+
}
|
|
383
|
+
else {
|
|
384
|
+
throw error;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
/**
|
|
389
|
+
* Initialize BackupManager
|
|
390
|
+
*
|
|
391
|
+
* @async
|
|
392
|
+
* @param {Record<string, unknown>} [config={}] - Configuration object (unused, reads from ConfigurationManager)
|
|
393
|
+
* @returns {Promise<void>}
|
|
394
|
+
* @throws {Error} If ConfigurationManager is not available
|
|
395
|
+
*/
|
|
396
|
+
async initialize(config = {}) {
|
|
397
|
+
await super.initialize(config);
|
|
398
|
+
// Get backup directory from ConfigurationManager
|
|
399
|
+
const configManager = this.engine.getManager('ConfigurationManager');
|
|
400
|
+
if (!configManager) {
|
|
401
|
+
throw new Error('BackupManager requires ConfigurationManager to be initialized.');
|
|
402
|
+
}
|
|
403
|
+
// Uses getResolvedDataPath to support INSTANCE_DATA_FOLDER
|
|
404
|
+
this.backupDirectory = configManager.getResolvedDataPath('ngdpbase.backup.directory', './data/backups');
|
|
405
|
+
this.maxBackups = configManager.getProperty('ngdpbase.backup.max-backups');
|
|
406
|
+
this.autoBackupEnabled = configManager.getProperty('ngdpbase.backup.auto-backup') ?? false;
|
|
407
|
+
this.autoBackupTime = configManager.getProperty('ngdpbase.backup.auto-backup-time') ?? '02:00';
|
|
408
|
+
this.autoBackupDays = configManager.getProperty('ngdpbase.backup.auto-backup-days') ?? 'daily';
|
|
409
|
+
// #170: resolve the storage provider name (lowercase config -> PascalCase
|
|
410
|
+
// class), e.g. filebackupprovider -> FileBackupProvider.
|
|
411
|
+
const providerName = configManager.getProperty('ngdpbase.backup.provider', 'filebackupprovider');
|
|
412
|
+
this.providerClass = this.normalizeProviderName(providerName);
|
|
413
|
+
// Preflight: if the configured path lives on a volume that is not mounted,
|
|
414
|
+
// log a clear warning and disable backups for this session rather than
|
|
415
|
+
// letting the provider's ensureContainer() crash engine init with an
|
|
416
|
+
// opaque EACCES.
|
|
417
|
+
const preflight = this.preflightConfiguredPath('ngdpbase.backup.directory', this.backupDirectory);
|
|
418
|
+
if (!preflight.ok) {
|
|
419
|
+
logger.warn(' Backups are DISABLED for this session.');
|
|
420
|
+
this.autoBackupEnabled = false;
|
|
421
|
+
this.backupDirectory = '';
|
|
422
|
+
this.provider = null;
|
|
423
|
+
logger.info('✅ BackupManager initialized (degraded — backups disabled)');
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
// #170: load the storage provider — it owns ensuring the container exists.
|
|
427
|
+
await this.loadProvider();
|
|
428
|
+
// #1720: automatic backups are a scheduled job.
|
|
429
|
+
this.scheduleAutoBackup();
|
|
430
|
+
// #1563: whether backups work — a failure kept from before a restart, or
|
|
431
|
+
// no good backup recently enough — checked now and on a timer. #1561:
|
|
432
|
+
// without the backup key, the dashboard says so for as long as it lasts.
|
|
433
|
+
this.registerHealthCheck();
|
|
434
|
+
await this.refreshBackupHealth();
|
|
435
|
+
logger.info('✅ BackupManager initialized');
|
|
436
|
+
logger.info(`📁 Backup directory: ${this.backupDirectory}`);
|
|
437
|
+
logger.info(`📊 Max backups to retain: ${this.maxBackups}`);
|
|
438
|
+
logger.info(`🕐 Auto backup: ${this.autoBackupEnabled ? `enabled at ${this.autoBackupTime} (${this.autoBackupDays})` : 'disabled'}`);
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* Backup BackupManager's own state (conforms to BaseManager interface)
|
|
442
|
+
*
|
|
443
|
+
* @returns {Promise<BaseBackupData>} Backup data for this manager
|
|
444
|
+
*/
|
|
445
|
+
async backup() {
|
|
446
|
+
return {
|
|
447
|
+
managerName: 'BackupManager',
|
|
448
|
+
timestamp: new Date().toISOString(),
|
|
449
|
+
data: {
|
|
450
|
+
backupDirectory: this.backupDirectory,
|
|
451
|
+
maxBackups: this.maxBackups
|
|
452
|
+
}
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
/**
|
|
456
|
+
* Restore BackupManager's own state (conforms to BaseManager interface)
|
|
457
|
+
*
|
|
458
|
+
* @param {BaseBackupData} backupData - Backup data from backup()
|
|
459
|
+
* @returns {Promise<void>}
|
|
460
|
+
*/
|
|
461
|
+
async restoreState(backupData) {
|
|
462
|
+
if (backupData?.data) {
|
|
463
|
+
// BackupManager state is read from config, so nothing to restore
|
|
464
|
+
logger.info('[BackupManager] State restore not needed (config-driven)');
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* Perform a complete backup of all managers to a file
|
|
469
|
+
*
|
|
470
|
+
* Process:
|
|
471
|
+
* 1. Get all registered managers from engine
|
|
472
|
+
* 2. Call backup() on each manager
|
|
473
|
+
* 3. Aggregate all backup data
|
|
474
|
+
* 4. Compress to .gz file
|
|
475
|
+
* 5. Save with timestamp
|
|
476
|
+
* 6. Clean up old backups
|
|
477
|
+
*
|
|
478
|
+
* @param {BackupOptions} options - Backup options
|
|
479
|
+
* @param {string} options.filename - Custom filename (optional)
|
|
480
|
+
* @param {boolean} options.compress - Whether to compress (default: true)
|
|
481
|
+
* @param ctx - Who asked (#1179): the request's subject, or the schedule's JobContext. First and mandatory — a full copy of the instance is made on someone's behalf.
|
|
482
|
+
* @returns {Promise<string>} Path to created backup file
|
|
483
|
+
*/
|
|
484
|
+
async createBackup(ctx, options = {}) {
|
|
485
|
+
return this.exclusive(() => this.createBackupNow(ctx, options));
|
|
486
|
+
}
|
|
487
|
+
/**
|
|
488
|
+
* One at a time (#1563): `work` starts once everything queued before it
|
|
489
|
+
* has finished, whether that succeeded or failed, and is never run beside
|
|
490
|
+
* it. A second backup asked for while one runs waits for it and is then
|
|
491
|
+
* taken in its own right: its own document, its own options, its own
|
|
492
|
+
* result. So a backup's pruning never deletes files another backup, a
|
|
493
|
+
* restore or the health check is reading. In-process only: one server
|
|
494
|
+
* process owns a backup directory.
|
|
495
|
+
*/
|
|
496
|
+
exclusive(work) {
|
|
497
|
+
const run = this.exclusiveTail.then(() => work());
|
|
498
|
+
this.exclusiveTail = run.catch(() => undefined);
|
|
499
|
+
return run;
|
|
500
|
+
}
|
|
501
|
+
/** `createBackup` itself, for a caller already holding the queue (`exclusive`): a restore's safety backup. */
|
|
502
|
+
async createBackupNow(ctx, options = {}) {
|
|
503
|
+
const startTime = Date.now();
|
|
504
|
+
logger.info('🔄 Starting backup operation...');
|
|
505
|
+
let pending = null;
|
|
506
|
+
// #1563: every backup that fails is said, and kept, until one succeeds —
|
|
507
|
+
// except where the failure was already recorded, or is not one of backups.
|
|
508
|
+
let recorded = false;
|
|
509
|
+
try {
|
|
510
|
+
if (!this.backupDirectory || !this.provider) {
|
|
511
|
+
throw new Error('Backup directory not initialized');
|
|
512
|
+
}
|
|
513
|
+
// #1563: refused before anything is written, rather than half a backup.
|
|
514
|
+
await this.checkFreeSpace();
|
|
515
|
+
// Generate filename with timestamp
|
|
516
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
517
|
+
// #1561: with the backup key set the document is encrypted under it, and named so.
|
|
518
|
+
const backupKey = currentBackupKey();
|
|
519
|
+
const plainName = options.filename || `${timestamp}-ngdpbase-backup.json.gz`;
|
|
520
|
+
const filename = backupKey && !plainName.endsWith(ENCRYPTED_SUFFIX) ? `${plainName}${ENCRYPTED_SUFFIX}` : plainName;
|
|
521
|
+
// Collect backup data from all managers
|
|
522
|
+
const backupData = {
|
|
523
|
+
version: '1.0.0',
|
|
524
|
+
timestamp: new Date().toISOString(),
|
|
525
|
+
application: 'ngdpbase',
|
|
526
|
+
managers: {}
|
|
527
|
+
};
|
|
528
|
+
// Get all manager names from engine
|
|
529
|
+
const managerNames = this.engine.getRegisteredManagers();
|
|
530
|
+
logger.info(`📋 Found ${managerNames.length} registered managers`);
|
|
531
|
+
// #1723: a manager may write its own files (the databases) beside the
|
|
532
|
+
// document; they share its stem and are pruned with it.
|
|
533
|
+
const stem = backupStem(filename);
|
|
534
|
+
const managerOptions = {
|
|
535
|
+
ctx, directory: this.backupDirectory, stem, archive: archiveWriter(this.backupDirectory, stem),
|
|
536
|
+
...(options.safetyBackup ? { safetyBackup: true } : {})
|
|
537
|
+
};
|
|
538
|
+
const failures = [];
|
|
539
|
+
// #1912: parts left out only because there is no backup key — said by
|
|
540
|
+
// the banner and the one-time notice (checkBackupKey), not per backup.
|
|
541
|
+
const leftOutForNoKey = [];
|
|
542
|
+
pending = { filename, existing: new Set(await this.companionsOf(filename)) };
|
|
543
|
+
// #1985: the folders and files each manager and add-on declares, by entry.
|
|
544
|
+
const declared = this.declaredParts({ forBackup: true });
|
|
545
|
+
const entries = [...managerNames, ...[...declared.keys()].filter((name) => !managerNames.includes(name))];
|
|
546
|
+
// Call backup() on each manager, and write its declared parts beside the document
|
|
547
|
+
for (const managerName of entries) {
|
|
548
|
+
// Skip BackupManager itself
|
|
549
|
+
if (managerName === 'BackupManager') {
|
|
550
|
+
continue;
|
|
551
|
+
}
|
|
552
|
+
try {
|
|
553
|
+
const manager = this.engine.getManager(managerName);
|
|
554
|
+
const parts = declared.get(managerName) ?? [];
|
|
555
|
+
if (!manager && parts.length === 0) {
|
|
556
|
+
logger.warn(`⚠️ Manager not found: ${managerName}`);
|
|
557
|
+
continue;
|
|
558
|
+
}
|
|
559
|
+
// #1943: an add-on manager (engine.registerManager) has no backup();
|
|
560
|
+
// its data is in its declared databases and add-on files.
|
|
561
|
+
if (typeof manager?.backup !== 'function' && parts.length === 0) {
|
|
562
|
+
logger.debug(`${managerName} has no backup part (no backup()); skipped`);
|
|
563
|
+
continue;
|
|
564
|
+
}
|
|
565
|
+
logger.info(`📦 Backing up ${managerName}...`);
|
|
566
|
+
let managerBackup = typeof manager?.backup === 'function'
|
|
567
|
+
? await manager.backup(managerOptions)
|
|
568
|
+
: { managerName, timestamp: new Date().toISOString(), data: null };
|
|
569
|
+
if (parts.length > 0)
|
|
570
|
+
managerBackup = placePartRecords(managerBackup, parts, await backupDeclaredParts(managerName, parts, managerOptions));
|
|
571
|
+
backupData.managers[managerName] = managerBackup;
|
|
572
|
+
for (const warning of managerBackup?.warnings ?? []) {
|
|
573
|
+
if (!backupKey && isLeftOutForNoKey(warning))
|
|
574
|
+
leftOutForNoKey.push(`${managerName}: ${warning}`);
|
|
575
|
+
else
|
|
576
|
+
failures.push(`${managerName}: ${warning}`);
|
|
577
|
+
}
|
|
578
|
+
logger.info(`✅ ${managerName} backed up${managerBackup?.warnings?.length ? ' in part' : ' successfully'}`);
|
|
579
|
+
}
|
|
580
|
+
catch (error) {
|
|
581
|
+
logger.error(`❌ Failed to backup ${managerName}:`, error);
|
|
582
|
+
const reason = error.message;
|
|
583
|
+
if (!backupKey && isLeftOutForNoKey(reason))
|
|
584
|
+
leftOutForNoKey.push(`${managerName}: ${reason}`);
|
|
585
|
+
else
|
|
586
|
+
failures.push(`${managerName}: ${reason}`);
|
|
587
|
+
// Continue with other managers even if one fails
|
|
588
|
+
backupData.managers[managerName] = {
|
|
589
|
+
error: error.message,
|
|
590
|
+
timestamp: new Date().toISOString()
|
|
591
|
+
};
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
// #1876: BackupManager's own part, the instance .env without the backup key's line.
|
|
595
|
+
try {
|
|
596
|
+
const env = await this.backupInstanceEnv(backupData, filename, backupKey);
|
|
597
|
+
if (env.warning) {
|
|
598
|
+
if (isLeftOutForNoKey(env.warning))
|
|
599
|
+
leftOutForNoKey.push(`BackupManager: ${env.warning}`);
|
|
600
|
+
else
|
|
601
|
+
failures.push(`BackupManager: ${env.warning}`);
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
catch (error) {
|
|
605
|
+
logger.error('❌ Failed to back up the instance .env:', error);
|
|
606
|
+
failures.push(`BackupManager: the instance .env: ${error.message}`);
|
|
607
|
+
}
|
|
608
|
+
// Serialize to JSON
|
|
609
|
+
const jsonData = JSON.stringify(backupData, null, 2);
|
|
610
|
+
logger.info(`📊 Backup data size: ${(jsonData.length / 1024).toFixed(2)} KB`);
|
|
611
|
+
// Compress with gzip
|
|
612
|
+
let finalData = jsonData;
|
|
613
|
+
if (options.compress !== false) {
|
|
614
|
+
const compressed = await gzip(jsonData);
|
|
615
|
+
finalData = compressed;
|
|
616
|
+
logger.info(`🗜️ Compressed size: ${(compressed.length / 1024).toFixed(2)} KB (${((compressed.length / jsonData.length) * 100).toFixed(1)}% of original)`);
|
|
617
|
+
}
|
|
618
|
+
// #1561: the document holds password hashes and the custom configuration with its secrets.
|
|
619
|
+
if (backupKey)
|
|
620
|
+
finalData = await encryptBuffer(Buffer.isBuffer(finalData) ? finalData : Buffer.from(finalData, 'utf8'), backupKey);
|
|
621
|
+
// Write via the storage provider (#170)
|
|
622
|
+
let backupPath;
|
|
623
|
+
try {
|
|
624
|
+
backupPath = await this.provider.writeBackup(filename, finalData);
|
|
625
|
+
}
|
|
626
|
+
catch (err) {
|
|
627
|
+
// #1563: half a file (out of space mid-write) is not left to look like a backup.
|
|
628
|
+
await this.provider.removeBackup(filename).catch(() => undefined);
|
|
629
|
+
throw err;
|
|
630
|
+
}
|
|
631
|
+
// #1563: a file is not a backup until it reads back.
|
|
632
|
+
await this.verifyWrittenBackup(backupPath, filename, backupKey);
|
|
633
|
+
pending = null;
|
|
634
|
+
const duration = Date.now() - startTime;
|
|
635
|
+
logger.info(`✅ Backup completed successfully in ${duration}ms`);
|
|
636
|
+
logger.info(`📁 Backup saved to: ${backupPath}`);
|
|
637
|
+
// #1215: a full copy of the instance now exists somewhere; say who asked
|
|
638
|
+
// and where it went. on-failure: continue — the backup is already written and a
|
|
639
|
+
// slow sink must not fail it.
|
|
640
|
+
const who = actorOf(ctx);
|
|
641
|
+
await recordAuditEvent(this.engine.getManager('AuditManager'), {
|
|
642
|
+
eventType: AUDIT_EVENT.BACKUP_CREATE,
|
|
643
|
+
...who,
|
|
644
|
+
action: 'backup-create',
|
|
645
|
+
result: 'success',
|
|
646
|
+
severity: 'medium',
|
|
647
|
+
resource: backupPath,
|
|
648
|
+
resourceType: 'backup',
|
|
649
|
+
metadata: { ...who.metadata, filename, bytes: finalData.length, managers: managerNames.length }
|
|
650
|
+
}, (err) => logger.warn('[BackupManager] Audit record failed for backup-create:', err));
|
|
651
|
+
// #1723, #1563: a backup with a part missing is said, not just logged,
|
|
652
|
+
// and it is a failed backup: the outage stays until one is whole.
|
|
653
|
+
// #1561: a backup that is not encrypted is said by the dashboard banner
|
|
654
|
+
// and one notification (checkBackupKey), never once per backup.
|
|
655
|
+
const incomplete = failures.length > 0 ? `Backup ${filename} was written, but without: ${failures.join('; ')}` : null;
|
|
656
|
+
if (incomplete) {
|
|
657
|
+
logger.error(`❌ ${incomplete}`);
|
|
658
|
+
await this.notifyAdmins('error', 'A backup is incomplete', incomplete);
|
|
659
|
+
await this.recordBackupFailure(`a backup is incomplete — ${failures.join('; ')}`, { filename, notify: false });
|
|
660
|
+
}
|
|
661
|
+
else {
|
|
662
|
+
await this.clearBackupFailure();
|
|
663
|
+
}
|
|
664
|
+
await this.refreshBackupHealthNow();
|
|
665
|
+
recorded = true;
|
|
666
|
+
if (incomplete && options.requireComplete)
|
|
667
|
+
throw new Error(incomplete);
|
|
668
|
+
if (!backupKey) {
|
|
669
|
+
logger.warn(`⚠️ Backup ${filename} was written NOT encrypted (${BACKUP_KEY_ENV} is not set)${leftOutForNoKey.length > 0 ? `, without: ${leftOutForNoKey.join('; ')}` : ''}`);
|
|
670
|
+
}
|
|
671
|
+
// A restore's safety backup must be whole, whatever left a part out.
|
|
672
|
+
if (options.requireComplete && leftOutForNoKey.length > 0)
|
|
673
|
+
throw new Error(`Backup ${filename} was written, but without: ${leftOutForNoKey.join('; ')}`);
|
|
674
|
+
// Clean up old backups
|
|
675
|
+
if (!options.skipCleanup)
|
|
676
|
+
await this.cleanupOldBackups();
|
|
677
|
+
return backupPath;
|
|
678
|
+
}
|
|
679
|
+
catch (error) {
|
|
680
|
+
logger.error('❌ Backup operation failed:', error);
|
|
681
|
+
// #1723: files the managers wrote for a document that was never written are removed.
|
|
682
|
+
if (pending)
|
|
683
|
+
await this.removeCompanions(pending.filename, pending.existing);
|
|
684
|
+
if (!recorded) {
|
|
685
|
+
await this.recordBackupFailure(describeBackupError(error));
|
|
686
|
+
await this.refreshBackupHealthNow();
|
|
687
|
+
}
|
|
688
|
+
throw error;
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
/**
|
|
692
|
+
* The instance `.env` beside the document (#1876): `<stem>.env.tar.enc`,
|
|
693
|
+
* only ever encrypted, and never with the backup key's own line. Without
|
|
694
|
+
* the key it is left out (`LEFT_OUT_FOR_NO_KEY`). BackupManager owns this
|
|
695
|
+
* part: the file belongs to no manager, and the backup key is this one's.
|
|
696
|
+
*/
|
|
697
|
+
async backupInstanceEnv(backupData, filename, backupKey) {
|
|
698
|
+
const slot = (env) => {
|
|
699
|
+
backupData.managers.BackupManager = { managerName: 'BackupManager', timestamp: new Date().toISOString(), data: { env } };
|
|
700
|
+
};
|
|
701
|
+
if (!backupKey) {
|
|
702
|
+
slot(null);
|
|
703
|
+
return { warning: `${LEFT_OUT_FOR_NO_KEY}the instance .env was left out: it is only ever backed up encrypted under the backup key (#1876)` };
|
|
704
|
+
}
|
|
705
|
+
const file = this.instanceEnvFile();
|
|
706
|
+
const target = archiveTarget(backupStem(filename), 'env');
|
|
707
|
+
const scratch = mkdtempSync(path.join(this.backupDirectory, '.env-part-'));
|
|
708
|
+
try {
|
|
709
|
+
const entries = file && existsSync(file) ? ['.env'] : [];
|
|
710
|
+
if (file && entries.length > 0)
|
|
711
|
+
writeFileSync(path.join(scratch, '.env'), envForBackup(readFileSync(file, 'utf8')), { mode: 0o600 });
|
|
712
|
+
const outcome = await writeTreeArchive(path.join(this.backupDirectory, target.archive), scratch, { entries, backupKey: target.backupKey });
|
|
713
|
+
slot({ format: 1, archive: target.archive, files: outcome.files, bytes: outcome.bytes, ...(target.keyFingerprint ? { keyFingerprint: target.keyFingerprint } : {}) });
|
|
714
|
+
return {};
|
|
715
|
+
}
|
|
716
|
+
finally {
|
|
717
|
+
rmSync(scratch, { recursive: true, force: true });
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
/** `<FAST_STORAGE>/.env`, or null when the instance data folder is not known. */
|
|
721
|
+
instanceEnvFile() {
|
|
722
|
+
const dir = this.engine.getManager('ConfigurationManager')?.getInstanceDataFolder?.();
|
|
723
|
+
return dir ? instanceEnvPath(dir) : null;
|
|
724
|
+
}
|
|
725
|
+
/** The files the managers wrote beside one backup document (#1723). */
|
|
726
|
+
async companionsOf(documentFilename) {
|
|
727
|
+
return (await this.provider?.listBackupObjects() ?? []).filter((o) => isCompanionOf(o.filename, documentFilename)).map((o) => o.filename);
|
|
728
|
+
}
|
|
729
|
+
/** Remove the files the managers wrote beside one backup document (#1723), except those in `keep`. */
|
|
730
|
+
async removeCompanions(documentFilename, keep = new Set()) {
|
|
731
|
+
if (!this.provider)
|
|
732
|
+
return;
|
|
733
|
+
try {
|
|
734
|
+
for (const filename of await this.companionsOf(documentFilename)) {
|
|
735
|
+
if (!keep.has(filename))
|
|
736
|
+
await this.provider.removeBackup(filename);
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
catch (err) {
|
|
740
|
+
logger.warn(`⚠️ Failed to remove the files beside ${documentFilename}:`, err);
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
/**
|
|
744
|
+
* Without `NGDPBASE_BACKUP_KEY` backups are still taken, unencrypted, with
|
|
745
|
+
* the credentials store and the databases left out (#1561, #1876). That is
|
|
746
|
+
* said twice, never once per backup (operator, #1912): BackupManager is
|
|
747
|
+
* degraded for as long as the key is missing — the persistent banner on the
|
|
748
|
+
* dashboard — and admins get ONE notification the first time. It is
|
|
749
|
+
* deduplicated the way other one-time notices are (`pageMetadataMissing`):
|
|
750
|
+
* by its title among the stored notifications, which persist across
|
|
751
|
+
* restarts and keep a dismissed one. With the key set the fault clears.
|
|
752
|
+
* Read at start and at each backup.
|
|
753
|
+
*/
|
|
754
|
+
async checkBackupKey() {
|
|
755
|
+
// #1976: the .env key set is checked whatever the key, so its one notice is never lost behind another banner.
|
|
756
|
+
const envReminder = await this.checkInstanceEnvKeys();
|
|
757
|
+
const key = currentBackupKey();
|
|
758
|
+
if (key) {
|
|
759
|
+
this.markReadyIfDegradedBy(BACKUP_KEY_MISSING);
|
|
760
|
+
// #1876: the banner asking an admin to save the key off the server, until one confirms;
|
|
761
|
+
// then #1976's, while the .env key set differs from the one last confirmed. One at a time,
|
|
762
|
+
// and one confirmation ("I have saved .env", which holds the key) can clear both.
|
|
763
|
+
const reminder = this.backupKeyInfo().saved ? envReminder : backupKeyNotSaved(backupKeyFingerprint(key));
|
|
764
|
+
const status = this.getManagerStatus();
|
|
765
|
+
const ours = status.state === 'degraded' && (status.reason?.startsWith(KEY_NOT_SAVED_PREFIX) || status.reason?.startsWith(ENV_KEYS_CHANGED_PREFIX));
|
|
766
|
+
if (!reminder) {
|
|
767
|
+
if (ours)
|
|
768
|
+
this.markReady();
|
|
769
|
+
}
|
|
770
|
+
else if (status.state !== 'degraded' || ours) {
|
|
771
|
+
this.markNeedsAttention(reminder, reminder === envReminder ? ENV_KEYS_SAVED_KEY : BACKUP_KEY_SAVED_KEY);
|
|
772
|
+
}
|
|
773
|
+
return;
|
|
774
|
+
}
|
|
775
|
+
// A degraded backup directory (preflight) is the worse fault, and keeps the banner.
|
|
776
|
+
const status = this.getManagerStatus();
|
|
777
|
+
if (status.state !== 'degraded' || status.reason === BACKUP_KEY_MISSING)
|
|
778
|
+
this.markNeedsAttention(BACKUP_KEY_MISSING, BACKUP_KEY_ENV);
|
|
779
|
+
if (this.alreadyNotified(BACKUP_KEY_MISSING_TITLE))
|
|
780
|
+
return;
|
|
781
|
+
await this.notifyAdmins('error', BACKUP_KEY_MISSING_TITLE, `${BACKUP_KEY_MISSING}. Set ${BACKUP_KEY_ENV} in the environment or the instance .env, and restart. Until then the dashboard lists Backup under "Working, but needs your attention"; this is the only notification about it.`);
|
|
782
|
+
}
|
|
783
|
+
/**
|
|
784
|
+
* Whether a notification with this title (and message, when given) is
|
|
785
|
+
* among the stored ones, which persist across restarts and keep a dismissed
|
|
786
|
+
* one: how a one-time notice is deduplicated (`pageMetadataMissing`).
|
|
787
|
+
*/
|
|
788
|
+
alreadyNotified(title, message) {
|
|
789
|
+
try {
|
|
790
|
+
const notifications = this.engine.getManager('NotificationManager');
|
|
791
|
+
return Boolean(notifications?.getAllNotifications?.(true).some((n) => n.title === title && (message === undefined || n.message === message)));
|
|
792
|
+
}
|
|
793
|
+
catch (err) {
|
|
794
|
+
logger.warn(`[BackupManager] Could not check for an earlier '${title}' notice:`, err);
|
|
795
|
+
return false;
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
/**
|
|
799
|
+
* The instance `.env` key set against the one an admin last confirmed saved
|
|
800
|
+
* off the server (#1976): the reminder while they differ, else null. Names
|
|
801
|
+
* only — a value is never kept, logged or shown.
|
|
802
|
+
*
|
|
803
|
+
* The first check on a site with nothing recorded (a new site, or one
|
|
804
|
+
* upgraded to this release) records the names it finds as the baseline,
|
|
805
|
+
* silently: an upgraded site's keys are already in every encrypted backup,
|
|
806
|
+
* and a new site is already asked to save its backup key, so asking again at
|
|
807
|
+
* that moment would be noise. Every change after it asks. One notification
|
|
808
|
+
* per change, deduplicated by its title and words among the stored
|
|
809
|
+
* notifications, so a restart never repeats it.
|
|
810
|
+
*/
|
|
811
|
+
async checkInstanceEnvKeys() {
|
|
812
|
+
const info = this.envKeysInfo();
|
|
813
|
+
if (!info)
|
|
814
|
+
return null;
|
|
815
|
+
if (!info.recorded) {
|
|
816
|
+
// A site not installed yet has no custom configuration to record into: the first check after the install does it.
|
|
817
|
+
const customFile = this.configManager()?.getCustomConfigPath?.();
|
|
818
|
+
if (customFile && !existsSync(customFile))
|
|
819
|
+
return null;
|
|
820
|
+
try {
|
|
821
|
+
await this.configManager()?.setProperty(ENV_KEYS_SAVED_KEY, info.names, systemContext(this.engine, 'the names in the instance .env recorded as the baseline for the reminder to save it (#1976)'));
|
|
822
|
+
}
|
|
823
|
+
catch (err) {
|
|
824
|
+
logger.warn('[BackupManager] The names in the instance .env could not be recorded:', err);
|
|
825
|
+
}
|
|
826
|
+
return null;
|
|
827
|
+
}
|
|
828
|
+
if (info.added.length === 0 && info.removed.length === 0)
|
|
829
|
+
return null;
|
|
830
|
+
const reminder = envKeysChanged(info);
|
|
831
|
+
const message = `${reminder}. The /admin dashboard lists Backup under "Working, but needs your attention" until then.`;
|
|
832
|
+
if (!this.alreadyNotified(ENV_KEYS_CHANGED_TITLE, message))
|
|
833
|
+
await this.notifyAdmins('warning', ENV_KEYS_CHANGED_TITLE, message);
|
|
834
|
+
return reminder;
|
|
835
|
+
}
|
|
836
|
+
/** The instance `.env` key set as Admin → Backup shows it (#1976), or null when the site has no instance data folder or the file can't be read. Never a value. */
|
|
837
|
+
envKeysInfo() {
|
|
838
|
+
const file = this.instanceEnvFile();
|
|
839
|
+
if (!file)
|
|
840
|
+
return null;
|
|
841
|
+
let content = '';
|
|
842
|
+
try {
|
|
843
|
+
if (existsSync(file))
|
|
844
|
+
content = readFileSync(file, 'utf8');
|
|
845
|
+
}
|
|
846
|
+
catch (err) {
|
|
847
|
+
logger.warn(`[BackupManager] The instance .env could not be read for its key names: ${err.code ?? 'error'}`);
|
|
848
|
+
return null;
|
|
849
|
+
}
|
|
850
|
+
const names = envKeyNames(content);
|
|
851
|
+
const stored = this.setting(ENV_KEYS_SAVED_KEY, null);
|
|
852
|
+
const recorded = Array.isArray(stored) ? stored.filter((n) => typeof n === 'string') : null;
|
|
853
|
+
const key = currentBackupKey();
|
|
854
|
+
return {
|
|
855
|
+
names,
|
|
856
|
+
fingerprint: envKeyNamesFingerprint(names),
|
|
857
|
+
recorded: recorded !== null,
|
|
858
|
+
added: recorded ? names.filter((n) => !recorded.includes(n)) : [],
|
|
859
|
+
removed: recorded ? recorded.filter((n) => !names.includes(n)) : [],
|
|
860
|
+
holdsBackupKey: Boolean(key) && readEnvLine(content, BACKUP_KEY_ENV) === key
|
|
861
|
+
};
|
|
862
|
+
}
|
|
863
|
+
/** The backup key as Admin → Backup shows it: set or not, its fingerprint, and whether an admin confirmed it is saved. Never the key. */
|
|
864
|
+
backupKeyInfo() {
|
|
865
|
+
const key = currentBackupKey();
|
|
866
|
+
if (!key)
|
|
867
|
+
return { set: false, fingerprint: null, saved: false };
|
|
868
|
+
const fingerprint = backupKeyFingerprint(key);
|
|
869
|
+
const saved = this.engine.getManager('ConfigurationManager')?.getProperty(BACKUP_KEY_SAVED_KEY, '') === fingerprint;
|
|
870
|
+
return { set: true, fingerprint, saved };
|
|
871
|
+
}
|
|
872
|
+
/**
|
|
873
|
+
* The backup key itself, for the one page that asks an admin to copy it
|
|
874
|
+
* off the server (#1876). The route asks `config-manage`, with its fresh
|
|
875
|
+
* sign-in; nothing else shows, logs or returns it.
|
|
876
|
+
*/
|
|
877
|
+
revealBackupKey() {
|
|
878
|
+
return currentBackupKey() || null;
|
|
879
|
+
}
|
|
880
|
+
/**
|
|
881
|
+
* The instance `.env` exactly as it is on disk, backup key line included,
|
|
882
|
+
* for an admin to keep a copy off the server (#1938). The route asks
|
|
883
|
+
* `config-manage`, with its fresh sign-in. Recorded as `secret-reveal`:
|
|
884
|
+
* the file, never its contents. Null when the site has no instance `.env`.
|
|
885
|
+
*/
|
|
886
|
+
async revealInstanceEnv(ctx) {
|
|
887
|
+
const file = this.instanceEnvFile();
|
|
888
|
+
if (!file || !existsSync(file))
|
|
889
|
+
return null;
|
|
890
|
+
const content = readFileSync(file);
|
|
891
|
+
const who = actorOf(ctx);
|
|
892
|
+
await recordAuditEvent(this.engine.getManager('AuditManager'), {
|
|
893
|
+
eventType: AUDIT_EVENT.SECRET_REVEAL,
|
|
894
|
+
...who,
|
|
895
|
+
action: 'secret-reveal',
|
|
896
|
+
result: 'success',
|
|
897
|
+
severity: 'high',
|
|
898
|
+
resource: '.env',
|
|
899
|
+
resourceType: 'instance-env',
|
|
900
|
+
metadata: { ...who.metadata, file: '.env', bytes: content.length }
|
|
901
|
+
}, (err) => logger.warn('[BackupManager] Audit record failed for the .env download:', err));
|
|
902
|
+
return content;
|
|
903
|
+
}
|
|
904
|
+
/**
|
|
905
|
+
* An admin confirms the backup key with this fingerprint is saved off the
|
|
906
|
+
* server (#1876). Recorded in the configuration (`ngdpbase.backup.key-saved-fingerprint`,
|
|
907
|
+
* written through ConfigurationManager, which records the change), so it
|
|
908
|
+
* survives restarts; a different key asks again. Refused when the
|
|
909
|
+
* fingerprint is not the current key's: the page they read was stale.
|
|
910
|
+
*
|
|
911
|
+
* With `envFingerprint` (#1976) the admin confirms the whole `.env` is
|
|
912
|
+
* saved, the one confirmation for both reminders: the names it sets are
|
|
913
|
+
* recorded (`ngdpbase.backup.env-keys-saved`), and so is the backup key
|
|
914
|
+
* when the file holds it. Refused when the names changed since the page
|
|
915
|
+
* was loaded.
|
|
916
|
+
*/
|
|
917
|
+
async confirmBackupKeySaved(ctx, fingerprint, envFingerprint) {
|
|
918
|
+
const STALE_KEY = 'That is not the fingerprint of the current backup key; reload Admin → Backup and check the key you saved';
|
|
919
|
+
const info = this.backupKeyInfo();
|
|
920
|
+
const env = envFingerprint === undefined ? null : this.envKeysInfo();
|
|
921
|
+
if (envFingerprint !== undefined && env?.fingerprint !== envFingerprint)
|
|
922
|
+
throw new Error('The keys in .env changed since that page was loaded; reload Admin → Backup, download .env again and confirm');
|
|
923
|
+
// A saved .env is a saved backup key only when the file holds the key this site uses.
|
|
924
|
+
const recordsKey = info.set && (env ? env.holdsBackupKey : true);
|
|
925
|
+
if (recordsKey ? info.fingerprint !== fingerprint : !env)
|
|
926
|
+
throw new Error(STALE_KEY);
|
|
927
|
+
const configManager = this.engine.getManager('ConfigurationManager');
|
|
928
|
+
if (!configManager)
|
|
929
|
+
throw new Error('ConfigurationManager not available');
|
|
930
|
+
if (recordsKey)
|
|
931
|
+
await configManager.setProperty(BACKUP_KEY_SAVED_KEY, fingerprint, ctx);
|
|
932
|
+
if (env)
|
|
933
|
+
await configManager.setProperty(ENV_KEYS_SAVED_KEY, env.names, ctx);
|
|
934
|
+
await this.checkBackupKey();
|
|
935
|
+
}
|
|
936
|
+
// ---------------------------------------------------------------------------
|
|
937
|
+
// Backups never fail silently (#1563). A failed backup, or no good backup
|
|
938
|
+
// recently enough, is an outage on the /admin dashboard until a backup
|
|
939
|
+
// succeeds. BackupManager decides; the pages only show what it says.
|
|
940
|
+
// ---------------------------------------------------------------------------
|
|
941
|
+
configManager() {
|
|
942
|
+
return this.engine.getManager('ConfigurationManager') ?? null;
|
|
943
|
+
}
|
|
944
|
+
setting(key, fallback) {
|
|
945
|
+
return (this.configManager()?.getProperty(key, fallback) ?? fallback);
|
|
946
|
+
}
|
|
947
|
+
/** `ngdpbase.backup.health-file`: the failure kept across restarts. */
|
|
948
|
+
healthFile() {
|
|
949
|
+
return this.configManager()?.getResolvedDataPath?.('ngdpbase.backup.health-file', './data/backup-health.json') || null;
|
|
950
|
+
}
|
|
951
|
+
async readHealthRecord() {
|
|
952
|
+
const file = this.healthFile();
|
|
953
|
+
if (!file)
|
|
954
|
+
return { version: 1 };
|
|
955
|
+
try {
|
|
956
|
+
const record = JSON.parse(await readFileAsync(file, 'utf8'));
|
|
957
|
+
return record && typeof record === 'object' ? record : { version: 1 };
|
|
958
|
+
}
|
|
959
|
+
catch (err) {
|
|
960
|
+
if (err.code !== 'ENOENT')
|
|
961
|
+
logger.error(`❌ The backup health record ${file} could not be read:`, err);
|
|
962
|
+
return { version: 1 };
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
async writeHealthRecord(record) {
|
|
966
|
+
const file = this.healthFile();
|
|
967
|
+
if (!file)
|
|
968
|
+
return;
|
|
969
|
+
try {
|
|
970
|
+
await mkdir(path.dirname(file), { recursive: true });
|
|
971
|
+
await writeFileAsync(file, JSON.stringify(record, null, 2) + '\n', { mode: 0o600 });
|
|
972
|
+
}
|
|
973
|
+
catch (err) {
|
|
974
|
+
// The outage still shows until the restart; it just isn't kept across one.
|
|
975
|
+
logger.error(`❌ The backup health record ${file} could not be written:`, err);
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
/**
|
|
979
|
+
* A backup failed (#1563): kept until one succeeds, across restarts, and
|
|
980
|
+
* said once — admins get one "Backups are not working" notification per
|
|
981
|
+
* reason, not one per failed attempt. `notify: false` where another
|
|
982
|
+
* notification already said it ("A backup is incomplete").
|
|
983
|
+
*/
|
|
984
|
+
async recordBackupFailure(reason, options = {}) {
|
|
985
|
+
const record = await this.readHealthRecord();
|
|
986
|
+
const repeated = record.failure?.reason === reason;
|
|
987
|
+
const failure = { at: repeated ? record.failure.at : new Date().toISOString(), reason, ...(options.filename ? { filename: options.filename } : {}) };
|
|
988
|
+
await this.writeHealthRecord({ ...record, failure });
|
|
989
|
+
logger.error(`❌ Backups are not working: ${reason}`);
|
|
990
|
+
if (options.notify !== false && !repeated) {
|
|
991
|
+
const lastGood = (await this.lastGoodBackup(failure))?.at;
|
|
992
|
+
await this.notifyAdmins('error', BACKUPS_NOT_WORKING_TITLE, `A backup failed: ${reason}. Last good backup: ${lastGood ? localTime(lastGood) : 'never'}. The /admin dashboard lists Backup as not working until a backup succeeds.`);
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
/** A backup succeeded (#1563): the failure kept since the last one is cleared. */
|
|
996
|
+
async clearBackupFailure() {
|
|
997
|
+
const record = await this.readHealthRecord();
|
|
998
|
+
if (!record.failure && !record.staleNotified)
|
|
999
|
+
return;
|
|
1000
|
+
await this.writeHealthRecord({ version: 1 });
|
|
1001
|
+
}
|
|
1002
|
+
/** The newest backup in the backup directory that is not the failed one: read from the directory, never remembered. */
|
|
1003
|
+
async lastGoodBackup(failure) {
|
|
1004
|
+
const good = (await this.listStoredBackups()).find((b) => b.filename !== failure?.filename);
|
|
1005
|
+
if (!good)
|
|
1006
|
+
return null;
|
|
1007
|
+
const at = good.created.getTime() > 0 ? good.created : good.modified;
|
|
1008
|
+
return { filename: good.filename, at: at.toISOString(), size: good.totalSize, location: this.backupDirectory ?? '' };
|
|
1009
|
+
}
|
|
1010
|
+
/** Automatic backups as they run (#1563): on only when scheduled and not paused, with the schedule in force and the next run. */
|
|
1011
|
+
async autoBackupRun(now) {
|
|
1012
|
+
if (!this.autoBackupEnabled || !this.provider)
|
|
1013
|
+
return null;
|
|
1014
|
+
const jobs = this.engine.getManager('BackgroundJobManager');
|
|
1015
|
+
if (!jobs)
|
|
1016
|
+
return null;
|
|
1017
|
+
const view = typeof jobs.scheduledJobView === 'function' ? await jobs.scheduledJobView(AUTO_BACKUP_JOB_ID) : undefined;
|
|
1018
|
+
if (view === null || view?.enabled === false)
|
|
1019
|
+
return null;
|
|
1020
|
+
try {
|
|
1021
|
+
const timeZone = serverTimeZone();
|
|
1022
|
+
const schedule = parseSchedule(view ? { rrule: view.rrule, tz: view.timeZone } : autoBackupSchedule(this.autoBackupTime, this.autoBackupDays, timeZone), { defaultTimeZone: timeZone });
|
|
1023
|
+
return { schedule, nextRun: view?.nextSlot ?? schedule.nextSlot(now)?.toISOString() ?? null };
|
|
1024
|
+
}
|
|
1025
|
+
catch {
|
|
1026
|
+
return null;
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
/**
|
|
1030
|
+
* Whether backups are working (#1563), from what is on disk: the newest
|
|
1031
|
+
* good backup in the backup directory, the failure kept in the health
|
|
1032
|
+
* record, the automatic-backup schedule, and free space at the destination.
|
|
1033
|
+
*
|
|
1034
|
+
* No good backup within the window is an outage. The window is
|
|
1035
|
+
* `ngdpbase.backup.stale-interval-multiplier` (default 2) times the longest
|
|
1036
|
+
* gap between automatic backups when they are on, and
|
|
1037
|
+
* `ngdpbase.backup.stale-fallback-days` (default 7) when they are off.
|
|
1038
|
+
*/
|
|
1039
|
+
async getBackupHealth() {
|
|
1040
|
+
const now = new Date();
|
|
1041
|
+
const record = await this.readHealthRecord();
|
|
1042
|
+
const failure = record.failure ?? null;
|
|
1043
|
+
const lastGood = await this.lastGoodBackup(failure ?? undefined);
|
|
1044
|
+
const auto = await this.autoBackupRun(now);
|
|
1045
|
+
const fallbackMs = Math.max(0, Number(this.setting('ngdpbase.backup.stale-fallback-days', 7))) * DAY_MS;
|
|
1046
|
+
const gap = auto ? longestGapMs(auto.schedule, now) : 0;
|
|
1047
|
+
const staleAfterMs = gap > 0 ? Math.max(1, Number(this.setting('ngdpbase.backup.stale-interval-multiplier', 2))) * gap : fallbackMs;
|
|
1048
|
+
const stale = !lastGood || now.getTime() - Date.parse(lastGood.at) > staleAfterMs;
|
|
1049
|
+
let freeBytes = null;
|
|
1050
|
+
try {
|
|
1051
|
+
freeBytes = await this.provider?.freeSpace?.() ?? null;
|
|
1052
|
+
}
|
|
1053
|
+
catch (err) {
|
|
1054
|
+
logger.warn('[BackupManager] Free space at the backup destination could not be read:', err);
|
|
1055
|
+
}
|
|
1056
|
+
const since = lastGood ? localTime(lastGood.at) : null;
|
|
1057
|
+
let fault = null;
|
|
1058
|
+
if (failure) {
|
|
1059
|
+
fault = `Backups are not working: ${failure.reason} (${localTime(failure.at)}). Last good backup: ${since ?? 'never'}`;
|
|
1060
|
+
}
|
|
1061
|
+
else if (stale) {
|
|
1062
|
+
const why = auto ? 'automatic backups are on, but none has succeeded' : 'automatic backups are off';
|
|
1063
|
+
fault = `${since ? `No good backup since ${since}` : 'No good backup since: never — this site has no backup'} (${why}). Make one from Admin → Backup`;
|
|
1064
|
+
}
|
|
1065
|
+
return { lastGood, autoBackup: Boolean(auto), nextRun: auto?.nextRun ?? null, freeBytes, failure, staleAfterMs, stale, fault };
|
|
1066
|
+
}
|
|
1067
|
+
/**
|
|
1068
|
+
* Bring the dashboard in line with whether backups work (#1563): an outage
|
|
1069
|
+
* (`markDegraded`) while a backup has failed or none is recent enough,
|
|
1070
|
+
* cleared — only that one — once they work, then the backup key's reminder
|
|
1071
|
+
* (`checkBackupKey`), which never hides an outage. A stretch with no recent
|
|
1072
|
+
* backup notifies admins once, kept across restarts. Run at start, on the
|
|
1073
|
+
* `backup.health` timer, and after every backup.
|
|
1074
|
+
*/
|
|
1075
|
+
async refreshBackupHealth() {
|
|
1076
|
+
return this.exclusive(() => this.refreshBackupHealthNow());
|
|
1077
|
+
}
|
|
1078
|
+
/** `refreshBackupHealth` itself, for a caller already holding the queue: a backup that just ended. */
|
|
1079
|
+
async refreshBackupHealthNow() {
|
|
1080
|
+
let health = null;
|
|
1081
|
+
try {
|
|
1082
|
+
// A backup directory that could not be used at start is the outage already shown.
|
|
1083
|
+
if (this.provider) {
|
|
1084
|
+
health = await this.getBackupHealth();
|
|
1085
|
+
if (health.fault) {
|
|
1086
|
+
this.markDegraded(health.fault, health.failure ? undefined : 'ngdpbase.backup.auto-backup');
|
|
1087
|
+
this.healthFault = health.fault;
|
|
1088
|
+
if (!health.failure)
|
|
1089
|
+
await this.notifyStaleOnce(health);
|
|
1090
|
+
}
|
|
1091
|
+
else if (this.healthFault) {
|
|
1092
|
+
this.markReadyIfDegradedBy(this.healthFault);
|
|
1093
|
+
this.healthFault = null;
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
catch (err) {
|
|
1098
|
+
logger.error('❌ Whether backups are working could not be checked:', err);
|
|
1099
|
+
}
|
|
1100
|
+
await this.checkBackupKey();
|
|
1101
|
+
return health;
|
|
1102
|
+
}
|
|
1103
|
+
/** One notification per stretch without a recent good backup (#1563), kept across restarts by the health record. */
|
|
1104
|
+
async notifyStaleOnce(health) {
|
|
1105
|
+
const marker = health.lastGood?.at ?? 'never';
|
|
1106
|
+
const record = await this.readHealthRecord();
|
|
1107
|
+
if (record.staleNotified === marker)
|
|
1108
|
+
return;
|
|
1109
|
+
await this.writeHealthRecord({ ...record, staleNotified: marker });
|
|
1110
|
+
logger.error(`❌ ${health.fault}`);
|
|
1111
|
+
await this.notifyAdmins('error', BACKUPS_NOT_WORKING_TITLE, `${health.fault}. The /admin dashboard lists Backup as not working until a backup succeeds.`);
|
|
1112
|
+
}
|
|
1113
|
+
/**
|
|
1114
|
+
* The routine check (#1563): `backup.health`, every
|
|
1115
|
+
* `ngdpbase.backup.health-check-interval` ms (default one hour), as the
|
|
1116
|
+
* system principal (BackgroundJobManager's routine maintenance). It is how
|
|
1117
|
+
* a backup that never comes is noticed.
|
|
1118
|
+
*/
|
|
1119
|
+
registerHealthCheck() {
|
|
1120
|
+
const jobs = this.engine.getManager('BackgroundJobManager');
|
|
1121
|
+
if (typeof jobs?.registerMaintenance !== 'function')
|
|
1122
|
+
return;
|
|
1123
|
+
jobs.registerMaintenance({
|
|
1124
|
+
id: BACKUP_HEALTH_JOB_ID,
|
|
1125
|
+
displayName: 'Check that backups are working',
|
|
1126
|
+
everyMs: Number(this.setting('ngdpbase.backup.health-check-interval', 3_600_000)),
|
|
1127
|
+
run: async () => {
|
|
1128
|
+
const health = await this.refreshBackupHealth();
|
|
1129
|
+
return health?.fault ?? undefined;
|
|
1130
|
+
}
|
|
1131
|
+
});
|
|
1132
|
+
}
|
|
1133
|
+
/**
|
|
1134
|
+
* Refuse a backup there is no room for (#1563), before anything is
|
|
1135
|
+
* written: free space at the destination against the last backup's size
|
|
1136
|
+
* (the document and every file beside it) times
|
|
1137
|
+
* `ngdpbase.backup.space-margin` (default 1.5). Skipped when there is no
|
|
1138
|
+
* earlier backup or the free space can't be told; a write that then runs
|
|
1139
|
+
* out still fails, and says so.
|
|
1140
|
+
*/
|
|
1141
|
+
async checkFreeSpace() {
|
|
1142
|
+
const last = (await this.listStoredBackups())[0];
|
|
1143
|
+
if (!last || !this.provider)
|
|
1144
|
+
return;
|
|
1145
|
+
let free;
|
|
1146
|
+
try {
|
|
1147
|
+
free = await this.provider.freeSpace();
|
|
1148
|
+
}
|
|
1149
|
+
catch (err) {
|
|
1150
|
+
logger.warn('[BackupManager] Free space at the backup destination could not be read; the backup goes ahead:', err);
|
|
1151
|
+
return;
|
|
1152
|
+
}
|
|
1153
|
+
if (free === null)
|
|
1154
|
+
return;
|
|
1155
|
+
const needed = Math.ceil(last.totalSize * Math.max(1, Number(this.setting('ngdpbase.backup.space-margin', 1.5))));
|
|
1156
|
+
if (free < needed) {
|
|
1157
|
+
throw new Error(`not enough free space at the backup destination ${this.backupDirectory}: ${formatBytes(free)} free, and the last backup took ${formatBytes(last.totalSize)} (about ${formatBytes(needed)} is needed). Nothing was written`);
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
/**
|
|
1161
|
+
* Read a backup just written back (#1563): decrypt it (every record's GCM
|
|
1162
|
+
* tag), decompress it, parse it and check its shape, on a worker thread.
|
|
1163
|
+
* A document that does not read back is removed — it is not a backup — and
|
|
1164
|
+
* the backup fails, naming why.
|
|
1165
|
+
*/
|
|
1166
|
+
async verifyWrittenBackup(backupPath, filename, backupKey) {
|
|
1167
|
+
try {
|
|
1168
|
+
const bytes = await this.provider.readBackup(backupPath);
|
|
1169
|
+
await verifyBackupOffThread(bytes, backupKey, filename, Number(this.setting('ngdpbase.backup.verify-timeout-ms', 600_000)));
|
|
1170
|
+
}
|
|
1171
|
+
catch (err) {
|
|
1172
|
+
try {
|
|
1173
|
+
await this.provider?.removeBackup(backupPath);
|
|
1174
|
+
}
|
|
1175
|
+
catch (removeErr) {
|
|
1176
|
+
logger.error(`❌ The unreadable backup ${filename} could not be removed:`, removeErr);
|
|
1177
|
+
}
|
|
1178
|
+
throw new Error(`the backup ${filename} was written but does not read back, so it was removed: ${err.message}`, { cause: err });
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
async notifyAdmins(level, title, message) {
|
|
1182
|
+
try {
|
|
1183
|
+
await this.engine.getManager('NotificationManager')
|
|
1184
|
+
?.addNotification({ type: 'system', level, title, message });
|
|
1185
|
+
}
|
|
1186
|
+
catch (err) {
|
|
1187
|
+
logger.error(`❌ The notification '${title}' could not be sent:`, err);
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
/**
|
|
1191
|
+
* Restore all managers from a backup file
|
|
1192
|
+
*
|
|
1193
|
+
* Process:
|
|
1194
|
+
* 1. Read and decompress backup file
|
|
1195
|
+
* 2. Parse JSON data
|
|
1196
|
+
* 3. Validate backup structure
|
|
1197
|
+
* 4. Call restore() on each manager with its data
|
|
1198
|
+
*
|
|
1199
|
+
* The databases (#1723) go first, and are staged for the next start rather
|
|
1200
|
+
* than restored in place: a fresh backup of the live data is taken, then
|
|
1201
|
+
* DatabaseManager stages every database in the backup. If either fails,
|
|
1202
|
+
* nothing at all is restored.
|
|
1203
|
+
*
|
|
1204
|
+
* @param ctx - Who asked (#1179): a restore takes a backup and replaces the instance's data on someone's behalf.
|
|
1205
|
+
* @param {string} backupPath - Path to backup file
|
|
1206
|
+
* @param {RestoreOptions} options - Restore options
|
|
1207
|
+
* @param {boolean} options.skipValidation - Skip validation (default: false)
|
|
1208
|
+
* @param {string[]} options.managerFilter - Only restore specific managers
|
|
1209
|
+
* @returns {Promise<RestoreResults>} Restore results
|
|
1210
|
+
*/
|
|
1211
|
+
async restoreFromFile(ctx, backupPath, options = {}) {
|
|
1212
|
+
// #1563: its safety backup and staging run in the queue, never beside a backup's pruning.
|
|
1213
|
+
return this.exclusive(() => this.restoreFromFileNow(ctx, backupPath, options));
|
|
1214
|
+
}
|
|
1215
|
+
async restoreFromFileNow(ctx, backupPath, options = {}) {
|
|
1216
|
+
const startTime = Date.now();
|
|
1217
|
+
logger.info(`🔄 Starting restore operation from: ${backupPath}`);
|
|
1218
|
+
try {
|
|
1219
|
+
if (!this.provider) {
|
|
1220
|
+
throw new Error('Backup provider not initialized');
|
|
1221
|
+
}
|
|
1222
|
+
// Verify backup file exists
|
|
1223
|
+
if (!(await this.provider.backupExists(backupPath))) {
|
|
1224
|
+
throw new Error(`Backup file not found: ${backupPath}`);
|
|
1225
|
+
}
|
|
1226
|
+
// Read backup file via the storage provider (#170)
|
|
1227
|
+
const fileData = await this.provider.readBackup(backupPath);
|
|
1228
|
+
logger.info(`📁 Read backup file: ${(fileData.length / 1024).toFixed(2)} KB`);
|
|
1229
|
+
// #1561: an encrypted document is read only under the backup key it was
|
|
1230
|
+
// made with; #1563: read the one way every backup is read back.
|
|
1231
|
+
let backupData;
|
|
1232
|
+
try {
|
|
1233
|
+
backupData = await readBackupDocument(fileData, currentBackupKey(), path.basename(backupPath), { validate: !options.skipValidation });
|
|
1234
|
+
}
|
|
1235
|
+
catch (err) {
|
|
1236
|
+
const message = err.message;
|
|
1237
|
+
throw new Error(message.startsWith('Invalid backup') ? message : `${message}; nothing was restored`, { cause: err });
|
|
1238
|
+
}
|
|
1239
|
+
logger.info(`📊 Backup version: ${backupData.version}`);
|
|
1240
|
+
logger.info(`📅 Backup timestamp: ${backupData.timestamp}`);
|
|
1241
|
+
// Restore results
|
|
1242
|
+
const results = {
|
|
1243
|
+
success: [],
|
|
1244
|
+
failed: [],
|
|
1245
|
+
skipped: []
|
|
1246
|
+
};
|
|
1247
|
+
// Get manager names from backup
|
|
1248
|
+
const managerNames = Object.keys(backupData.managers);
|
|
1249
|
+
logger.info(`📋 Found ${managerNames.length} managers in backup`);
|
|
1250
|
+
// Where the document is, so a manager finds the files it wrote beside it (#1723).
|
|
1251
|
+
const directory = path.dirname(path.isAbsolute(backupPath) ? backupPath : path.join(this.backupDirectory ?? '', backupPath));
|
|
1252
|
+
// #1723: the databases first, all or nothing — if they cannot be staged,
|
|
1253
|
+
// nothing else is restored either. A fresh backup of the live data is
|
|
1254
|
+
// taken before anything is touched.
|
|
1255
|
+
const databases = backupData.managers.DatabaseManager;
|
|
1256
|
+
const databaseList = databases?.data?.databases;
|
|
1257
|
+
const hasDatabases = Array.isArray(databaseList) && databaseList.length > 0;
|
|
1258
|
+
// #1876: every manager whose restore is staged goes in one site restore,
|
|
1259
|
+
// all or nothing, when any besides the databases is asked for.
|
|
1260
|
+
const filtered = (name) => !options.managerFilter || options.managerFilter.includes(name);
|
|
1261
|
+
const stagedNames = managerNames.filter((name) => filtered(name) && this.restoresStaged(name));
|
|
1262
|
+
const siteRestore = stagedNames.some((name) => name !== 'DatabaseManager');
|
|
1263
|
+
if (siteRestore) {
|
|
1264
|
+
await this.stageSiteRestore(ctx, backupPath, backupData, directory, stagedNames, options);
|
|
1265
|
+
results.success.push(...stagedNames);
|
|
1266
|
+
}
|
|
1267
|
+
else if (hasDatabases && filtered('DatabaseManager')) {
|
|
1268
|
+
const databaseManager = this.engine.getManager('DatabaseManager');
|
|
1269
|
+
if (!databaseManager)
|
|
1270
|
+
throw new Error('The backup holds databases, but DatabaseManager is not available; nothing was restored');
|
|
1271
|
+
// D5: never replaced silently — refused before the fresh backup is taken.
|
|
1272
|
+
this.refuseIfSiteRestoreStaged();
|
|
1273
|
+
const staged = databaseManager.getStagedRestore?.();
|
|
1274
|
+
if (staged && !options.replaceStaged) {
|
|
1275
|
+
throw new Error(`A restore from ${staged.source || staged.backup} is already staged; cancel it or replace it. Nothing was restored`);
|
|
1276
|
+
}
|
|
1277
|
+
logger.info('📦 Taking a backup of the live data before staging the databases...');
|
|
1278
|
+
const before = await this.createBackupNow(ctx, { skipCleanup: true, requireComplete: true, safetyBackup: true });
|
|
1279
|
+
logger.info(`✅ Live data backed up to ${before}`);
|
|
1280
|
+
await databaseManager.restore(databases, { ctx, directory, source: path.basename(backupPath), replaceStaged: options.replaceStaged, safetyBackup: path.basename(before) });
|
|
1281
|
+
results.success.push('DatabaseManager');
|
|
1282
|
+
// #1876: what this databases-only restore leaves as it is, for the notification once it is applied.
|
|
1283
|
+
await this.writeRestoreReport(backupPath, backupData, managerNames.filter((n) => n !== 'DatabaseManager'), actorOf(ctx).user);
|
|
1284
|
+
}
|
|
1285
|
+
// #1876: while a site restore is staged, nothing is restored live: it
|
|
1286
|
+
// would be overwritten, or contradicted, by the parts the restart puts
|
|
1287
|
+
// in place. A site restore just staged leaves them out and says so.
|
|
1288
|
+
const live = managerNames.filter((name) => !(siteRestore && stagedNames.includes(name)) && !(name === 'DatabaseManager' && hasDatabases) && filtered(name));
|
|
1289
|
+
if (live.length > 0 && !siteRestore) {
|
|
1290
|
+
let staged;
|
|
1291
|
+
try {
|
|
1292
|
+
staged = this.getStagedSiteRestore();
|
|
1293
|
+
}
|
|
1294
|
+
catch (err) {
|
|
1295
|
+
throw new Error(`A site restore record is there but can't be read (${err.message}); nothing was restored`, { cause: err });
|
|
1296
|
+
}
|
|
1297
|
+
if (staged)
|
|
1298
|
+
throw new Error(`A site restore from ${staged.source || staged.backup} is staged, so ${live.join(', ')} can't be restored now; cancel it first. Nothing was restored`);
|
|
1299
|
+
}
|
|
1300
|
+
// Restore each manager
|
|
1301
|
+
for (const managerName of managerNames) {
|
|
1302
|
+
if (siteRestore && stagedNames.includes(managerName))
|
|
1303
|
+
continue;
|
|
1304
|
+
if (siteRestore && live.includes(managerName)) {
|
|
1305
|
+
logger.warn(`⏭️ ${managerName} not restored: it has no staged restore, and a site restore is staged (#1876)`);
|
|
1306
|
+
results.skipped.push(managerName);
|
|
1307
|
+
continue;
|
|
1308
|
+
}
|
|
1309
|
+
if (managerName === 'DatabaseManager' && hasDatabases) {
|
|
1310
|
+
if (options.managerFilter && !options.managerFilter.includes(managerName))
|
|
1311
|
+
results.skipped.push(managerName);
|
|
1312
|
+
continue;
|
|
1313
|
+
}
|
|
1314
|
+
// Check if manager filter is applied
|
|
1315
|
+
if (options.managerFilter && !options.managerFilter.includes(managerName)) {
|
|
1316
|
+
logger.info(`⏭️ Skipping ${managerName} (filtered)`);
|
|
1317
|
+
results.skipped.push(managerName);
|
|
1318
|
+
continue;
|
|
1319
|
+
}
|
|
1320
|
+
try {
|
|
1321
|
+
const manager = this.engine.getManager(managerName);
|
|
1322
|
+
if (!manager) {
|
|
1323
|
+
logger.warn(`⚠️ Manager not found: ${managerName}, skipping`);
|
|
1324
|
+
results.skipped.push(managerName);
|
|
1325
|
+
continue;
|
|
1326
|
+
}
|
|
1327
|
+
// #1943, #1813: an add-on manager (engine.registerManager) has no
|
|
1328
|
+
// restore(), as it has no backup(); its data comes back through its
|
|
1329
|
+
// declared databases and add-on files. An older backup may still name
|
|
1330
|
+
// it (calendar and journal managers once extended BaseManager, whose
|
|
1331
|
+
// backup() wrote an empty entry), so it is skipped, not failed.
|
|
1332
|
+
if (typeof manager.restore !== 'function') {
|
|
1333
|
+
logger.debug(`${managerName} has no restore(); skipped`);
|
|
1334
|
+
results.skipped.push(managerName);
|
|
1335
|
+
continue;
|
|
1336
|
+
}
|
|
1337
|
+
const managerBackupData = backupData.managers[managerName];
|
|
1338
|
+
// Skip if manager backup had an error
|
|
1339
|
+
if ('error' in managerBackupData && typeof managerBackupData.error === 'string') {
|
|
1340
|
+
logger.warn(`⚠️ Skipping ${managerName} (backup had error: ${managerBackupData.error})`);
|
|
1341
|
+
results.skipped.push(managerName);
|
|
1342
|
+
continue;
|
|
1343
|
+
}
|
|
1344
|
+
logger.info(`📦 Restoring ${managerName}...`);
|
|
1345
|
+
await manager.restore(managerBackupData, { ctx, directory });
|
|
1346
|
+
results.success.push(managerName);
|
|
1347
|
+
logger.info(`✅ ${managerName} restored successfully`);
|
|
1348
|
+
}
|
|
1349
|
+
catch (error) {
|
|
1350
|
+
logger.error(`❌ Failed to restore ${managerName}:`, error);
|
|
1351
|
+
results.failed.push({ manager: managerName, error: error.message });
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
const duration = Date.now() - startTime;
|
|
1355
|
+
logger.info(`✅ Restore completed in ${duration}ms`);
|
|
1356
|
+
logger.info(`✅ Success: ${results.success.length}, ❌ Failed: ${results.failed.length}, ⏭️ Skipped: ${results.skipped.length}`);
|
|
1357
|
+
return results;
|
|
1358
|
+
}
|
|
1359
|
+
catch (error) {
|
|
1360
|
+
logger.error('❌ Restore operation failed:', error);
|
|
1361
|
+
throw error;
|
|
1362
|
+
}
|
|
1363
|
+
}
|
|
1364
|
+
/** Whether this site's manager of that name stages its restore (#1876), or its entry is declared folders (#1985). */
|
|
1365
|
+
restoresStaged(name) {
|
|
1366
|
+
if (this.declaredFor(name).length > 0)
|
|
1367
|
+
return true;
|
|
1368
|
+
const manager = this.engine.getManager(name);
|
|
1369
|
+
return typeof manager?.restoreIsStaged === 'function' && manager.restoreIsStaged();
|
|
1370
|
+
}
|
|
1371
|
+
/**
|
|
1372
|
+
* Each loaded add-on's folders (#1985): its `dataPath`, which counts for it
|
|
1373
|
+
* whatever it declares, and any further folders its module declares; under
|
|
1374
|
+
* the entry its files have always had (`addonEntryName`). For a backup, a
|
|
1375
|
+
* `dataPath` that does not exist is left out: that add-on keeps no files.
|
|
1376
|
+
*/
|
|
1377
|
+
addonParts(forBackup = false) {
|
|
1378
|
+
const addons = this.engine.getManager('AddonsManager');
|
|
1379
|
+
const declared = typeof addons?.addonBackupFolders === 'function' ? addons.addonBackupFolders() : [];
|
|
1380
|
+
return declared.map(({ slug, dataPath, folders }) => ({
|
|
1381
|
+
entry: addonEntryName(slug),
|
|
1382
|
+
folders: forBackup ? folders.filter((f) => !('folder' in f) || f.folder !== dataPath || (dataPath !== null && existsSync(dataPath))) : folders
|
|
1383
|
+
}));
|
|
1384
|
+
}
|
|
1385
|
+
/** What a manager declares, and an add-on's folders recorded under the same entry (#1985). */
|
|
1386
|
+
declaredFor(name, forBackup = false) {
|
|
1387
|
+
const manager = this.engine.getManager(name);
|
|
1388
|
+
let own = [];
|
|
1389
|
+
try {
|
|
1390
|
+
own = typeof manager?.backupFolders === 'function' ? manager.backupFolders() : [];
|
|
1391
|
+
}
|
|
1392
|
+
catch (err) {
|
|
1393
|
+
logger.error(`[BackupManager] ${name}'s backup folders could not be read: ${err.message}`);
|
|
1394
|
+
}
|
|
1395
|
+
return [...own, ...this.addonParts(forBackup).filter((a) => a.entry === name).flatMap((a) => a.folders)];
|
|
1396
|
+
}
|
|
1397
|
+
/** Every entry with declared folders or files, in registration order, add-ons' after (#1985). */
|
|
1398
|
+
declaredParts(options = {}) {
|
|
1399
|
+
const names = [...this.engine.getRegisteredManagers(), ...this.addonParts().map((a) => a.entry)];
|
|
1400
|
+
const out = new Map();
|
|
1401
|
+
for (const name of names) {
|
|
1402
|
+
if (out.has(name) || name === 'BackupManager')
|
|
1403
|
+
continue;
|
|
1404
|
+
const declared = this.declaredFor(name, options.forBackup);
|
|
1405
|
+
if (declared.length > 0)
|
|
1406
|
+
out.set(name, declared);
|
|
1407
|
+
}
|
|
1408
|
+
return out;
|
|
1409
|
+
}
|
|
1410
|
+
/**
|
|
1411
|
+
* Put a staged restore of an add-on's folders in place before its
|
|
1412
|
+
* `register()` runs, so before any of its managers reads them (#1985).
|
|
1413
|
+
* AddonsManager calls it as it loads each add-on.
|
|
1414
|
+
*/
|
|
1415
|
+
applyAddonPartsAtStart(slug, folders) {
|
|
1416
|
+
return applyStagedPartsAtStart(this.engine, addonEntryName(slug), folders);
|
|
1417
|
+
}
|
|
1418
|
+
/** The site restore manifest, as configured (#1876). */
|
|
1419
|
+
siteRestorePath() {
|
|
1420
|
+
return siteRestoreFile(this.engine.getManager('ConfigurationManager'));
|
|
1421
|
+
}
|
|
1422
|
+
/** The site restore staged and waiting for the next start, or null (#1876). Throws for a record that can't be read. */
|
|
1423
|
+
getStagedSiteRestore() {
|
|
1424
|
+
return readSiteRestore(this.siteRestorePath());
|
|
1425
|
+
}
|
|
1426
|
+
/** A databases-only restore is refused while a site restore is staged: it would take the site restore's databases away from it (#1876). */
|
|
1427
|
+
refuseIfSiteRestoreStaged() {
|
|
1428
|
+
let staged;
|
|
1429
|
+
try {
|
|
1430
|
+
staged = this.getStagedSiteRestore();
|
|
1431
|
+
}
|
|
1432
|
+
catch (err) {
|
|
1433
|
+
throw new Error(`A site restore record is there but can't be read (${err.message}); nothing was restored`, { cause: err });
|
|
1434
|
+
}
|
|
1435
|
+
if (staged)
|
|
1436
|
+
throw new Error(`A site restore from ${staged.source || staged.backup} is staged; it includes the databases. Cancel it first; nothing was restored`);
|
|
1437
|
+
}
|
|
1438
|
+
/**
|
|
1439
|
+
* Stage a whole-site restore (#1876): every manager in `names` stages its
|
|
1440
|
+
* part for the next start, after a complete backup of the live data, under
|
|
1441
|
+
* one site restore manifest. All or nothing: a part that can't be staged
|
|
1442
|
+
* discards every part staged before it, and the live data is never
|
|
1443
|
+
* touched. The next start applies every part or none
|
|
1444
|
+
* (`src/utils/siteRestore.ts`).
|
|
1445
|
+
*/
|
|
1446
|
+
async stageSiteRestore(ctx, backupPath, backupData, directory, names, options) {
|
|
1447
|
+
const siteFile = this.siteRestorePath();
|
|
1448
|
+
let existing;
|
|
1449
|
+
try {
|
|
1450
|
+
existing = readSiteRestore(siteFile);
|
|
1451
|
+
}
|
|
1452
|
+
catch (err) {
|
|
1453
|
+
if (!options.replaceStaged)
|
|
1454
|
+
throw new Error(`A site restore record is there but can't be read (${err.message}); replace it to go on. Nothing was restored`, { cause: err });
|
|
1455
|
+
existing = null;
|
|
1456
|
+
}
|
|
1457
|
+
const databaseManager = this.engine.getManager('DatabaseManager');
|
|
1458
|
+
const databaseStaged = databaseManager?.getStagedRestore?.() ?? null;
|
|
1459
|
+
if ((existing || databaseStaged) && !options.replaceStaged) {
|
|
1460
|
+
const what = existing ? `A site restore from ${existing.source || existing.backup}` : `A database restore from ${databaseStaged.source || databaseStaged.backup}`;
|
|
1461
|
+
throw new Error(`${what} is already staged; cancel it or replace it. Nothing was restored`);
|
|
1462
|
+
}
|
|
1463
|
+
// A part whose backup failed can't be restored, so none is.
|
|
1464
|
+
for (const name of names) {
|
|
1465
|
+
const slot = backupData.managers[name];
|
|
1466
|
+
if (slot && typeof slot.error === 'string')
|
|
1467
|
+
throw new Error(`The backup has no ${name} part (its backup failed: ${slot.error}); nothing was restored`);
|
|
1468
|
+
}
|
|
1469
|
+
logger.info('📦 Taking a backup of the live data before staging the site restore...');
|
|
1470
|
+
const before = path.basename(await this.createBackupNow(ctx, { skipCleanup: true, requireComplete: true, safetyBackup: true }));
|
|
1471
|
+
logger.info(`✅ Live data backed up to ${before}`);
|
|
1472
|
+
discardSiteRestore(siteFile);
|
|
1473
|
+
beginSiteRestore(siteFile, {
|
|
1474
|
+
stagedAt: new Date().toISOString(),
|
|
1475
|
+
backup: String(backupData.timestamp ?? ''),
|
|
1476
|
+
source: path.basename(backupPath),
|
|
1477
|
+
stagedBy: actorOf(ctx).user,
|
|
1478
|
+
safetyBackup: before
|
|
1479
|
+
});
|
|
1480
|
+
// The databases first, as before (#1723); then the rest in registration order.
|
|
1481
|
+
const ordered = [...names].sort((a, b) => Number(b === 'DatabaseManager') - Number(a === 'DatabaseManager'));
|
|
1482
|
+
try {
|
|
1483
|
+
for (const name of ordered) {
|
|
1484
|
+
logger.info(`📦 Staging ${name}...`);
|
|
1485
|
+
const restoreOptions = {
|
|
1486
|
+
ctx, directory, source: path.basename(backupPath), safetyBackup: before, siteRestore: stagingFor(siteFile, name),
|
|
1487
|
+
replaceStaged: options.replaceStaged || Boolean(databaseStaged), readArchive: archiveReader(directory)
|
|
1488
|
+
};
|
|
1489
|
+
// #1985: declared folders and files are staged here; the manager writes no restore code for them.
|
|
1490
|
+
const declared = this.declaredFor(name);
|
|
1491
|
+
if (declared.length > 0) {
|
|
1492
|
+
await stageDeclaredParts(name, declared, backupData.managers[name], restoreOptions);
|
|
1493
|
+
continue;
|
|
1494
|
+
}
|
|
1495
|
+
await this.engine.getManager(name).restore(backupData.managers[name], restoreOptions);
|
|
1496
|
+
}
|
|
1497
|
+
// #1876: the instance .env, BackupManager's own part; applied at the very start of the process.
|
|
1498
|
+
const env = backupData.managers.BackupManager?.data?.env;
|
|
1499
|
+
const envFile = this.instanceEnvFile();
|
|
1500
|
+
if (env && envFile) {
|
|
1501
|
+
logger.info('📦 Staging the instance .env...');
|
|
1502
|
+
await stageFilePart(INSTANCE_ENV_PART, envFile, env, { ctx, directory, siteRestore: stagingFor(siteFile, INSTANCE_ENV_PART) }, { whenAbsent: '' });
|
|
1503
|
+
}
|
|
1504
|
+
const committed = commitSiteRestore(siteFile);
|
|
1505
|
+
// #1876: every part of the backup that is not in the restore — no staged
|
|
1506
|
+
// restore here, or a staged one that found nothing to stage.
|
|
1507
|
+
const recorded = new Set(committed.parts.map((p) => p.manager));
|
|
1508
|
+
await this.writeRestoreReport(backupPath, backupData, Object.keys(backupData.managers).filter((n) => !recorded.has(n)), actorOf(ctx).user);
|
|
1509
|
+
}
|
|
1510
|
+
catch (err) {
|
|
1511
|
+
discardSiteRestore(siteFile);
|
|
1512
|
+
this.removeRestoreReport();
|
|
1513
|
+
throw new Error(`The site restore was not staged, and nothing was changed: ${err.message}`, { cause: err });
|
|
1514
|
+
}
|
|
1515
|
+
const message = `${ordered.join(', ')} from the backup taken ${String(backupData.timestamp)} are staged. Restart the site to put them in place, all together; the current data of each is kept beside it as *.pre-restore. The current data was backed up first, as ${before}.`;
|
|
1516
|
+
logger.warn(`[BackupManager] Site restore staged: ${message} (#1876)`);
|
|
1517
|
+
await this.notifyAdmins('warning', 'A site restore is staged', message);
|
|
1518
|
+
}
|
|
1519
|
+
/**
|
|
1520
|
+
* Throw away a staged site restore before the restart applies it (#1876):
|
|
1521
|
+
* every staged file and the record are removed, the live data is
|
|
1522
|
+
* untouched. Recorded first (`backup-restore-cancel`, refuse).
|
|
1523
|
+
*/
|
|
1524
|
+
async cancelStagedSiteRestore(ctx) {
|
|
1525
|
+
const staged = this.getStagedSiteRestore();
|
|
1526
|
+
if (!staged)
|
|
1527
|
+
throw new Error('No site restore is staged');
|
|
1528
|
+
const who = actorOf(ctx);
|
|
1529
|
+
await recordAuditEvent(this.engine.getManager('AuditManager'), {
|
|
1530
|
+
eventType: AUDIT_EVENT.BACKUP_RESTORE_CANCEL,
|
|
1531
|
+
...who,
|
|
1532
|
+
action: 'backup-restore-cancel',
|
|
1533
|
+
result: 'success',
|
|
1534
|
+
severity: 'high',
|
|
1535
|
+
resource: staged.source || staged.backup,
|
|
1536
|
+
resourceType: 'backup',
|
|
1537
|
+
metadata: { ...who.metadata, stagedAt: staged.stagedAt, stagedBy: staged.stagedBy, parts: staged.parts.map((p) => p.manager) }
|
|
1538
|
+
});
|
|
1539
|
+
discardSiteRestore(this.siteRestorePath());
|
|
1540
|
+
this.removeRestoreReport();
|
|
1541
|
+
logger.warn(`[BackupManager] Staged site restore from ${staged.source || staged.backup} cancelled by ${who.user} (#1876)`);
|
|
1542
|
+
return staged;
|
|
1543
|
+
}
|
|
1544
|
+
/** Where this manager keeps the report of a staged restore: beside the site restore manifest (#1876). */
|
|
1545
|
+
restoreReportPath() {
|
|
1546
|
+
return `${this.siteRestorePath()}.report.json`;
|
|
1547
|
+
}
|
|
1548
|
+
removeRestoreReport() {
|
|
1549
|
+
rmSync(this.restoreReportPath(), { force: true });
|
|
1550
|
+
}
|
|
1551
|
+
/**
|
|
1552
|
+
* Record, when a restore is staged, what it will not bring back (#1876):
|
|
1553
|
+
* the parts of the backup not in it, and the job runs since the backup,
|
|
1554
|
+
* read from the live job state now, before the restore replaces it.
|
|
1555
|
+
*/
|
|
1556
|
+
async writeRestoreReport(backupPath, backupData, notStaged, stagedBy) {
|
|
1557
|
+
const since = String(backupData.timestamp ?? '');
|
|
1558
|
+
let jobRuns = [];
|
|
1559
|
+
try {
|
|
1560
|
+
jobRuns = await this.engine.getManager('BackgroundJobManager')?.runsSince?.(since) ?? [];
|
|
1561
|
+
}
|
|
1562
|
+
catch (err) {
|
|
1563
|
+
logger.warn('[BackupManager] The job runs since the backup could not be read for the restore report:', err);
|
|
1564
|
+
}
|
|
1565
|
+
const notRestored = notStaged.filter((name) => !NO_SITE_DATA.has(name) && slotHoldsData(backupData.managers[name]));
|
|
1566
|
+
const report = { version: 1, source: path.basename(backupPath), backup: since, notRestored, jobRuns, stagedBy };
|
|
1567
|
+
writeFileSync(this.restoreReportPath(), JSON.stringify(report, null, 2) + '\n', { mode: 0o600 });
|
|
1568
|
+
}
|
|
1569
|
+
/**
|
|
1570
|
+
* Tell admins what this start did with a staged restore (#1876). Called
|
|
1571
|
+
* once every manager and add-on has started (`WikiEngine.initializeAddons`):
|
|
1572
|
+
* each owner applied (or set aside) its part as it started, add-ons last.
|
|
1573
|
+
* After any restore that was applied, a critical notification lists what it
|
|
1574
|
+
* did not bring back, and the job runs between the backup and the restore.
|
|
1575
|
+
*/
|
|
1576
|
+
async reportRestoreAtBoot() {
|
|
1577
|
+
const siteFile = this.siteRestorePath();
|
|
1578
|
+
let finished;
|
|
1579
|
+
try {
|
|
1580
|
+
finished = finishSiteRestoreAtBoot(siteFile);
|
|
1581
|
+
}
|
|
1582
|
+
catch (err) {
|
|
1583
|
+
const message = `The site restore record ${path.basename(siteFile)} could not be read at start: ${err.message}`;
|
|
1584
|
+
logger.error(`❌ ${message}`);
|
|
1585
|
+
await this.notifyAdmins('error', 'A site restore could not be checked', message);
|
|
1586
|
+
return;
|
|
1587
|
+
}
|
|
1588
|
+
const databasesOnly = finished ? [] : this.engine.getManager('DatabaseManager')?.getRestoredAtStart?.() ?? [];
|
|
1589
|
+
const applied = finished ? finished.parts.filter((p) => p.status === 'applied').map((p) => p.manager) : databasesOnly.length > 0 ? ['DatabaseManager'] : [];
|
|
1590
|
+
if (applied.length > 0) {
|
|
1591
|
+
const report = this.readRestoreReport();
|
|
1592
|
+
await this.recordRestoreApplied(applied, finished, report);
|
|
1593
|
+
await this.reportNotRestored(applied, finished ? finished.parts.filter((p) => p.status !== 'applied').map((p) => p.manager) : [], report);
|
|
1594
|
+
}
|
|
1595
|
+
this.removeRestoreReport();
|
|
1596
|
+
if (finished)
|
|
1597
|
+
await this.reportSiteRestoreOutcome(finished);
|
|
1598
|
+
}
|
|
1599
|
+
/**
|
|
1600
|
+
* The critical notification after a restore was applied (#1876): every
|
|
1601
|
+
* part of the site it did not bring back, and the job runs between the
|
|
1602
|
+
* backup and the restore (name, last run, result), which are not in the
|
|
1603
|
+
* restored job history. Jobs are never run again because of a restore.
|
|
1604
|
+
*/
|
|
1605
|
+
async reportNotRestored(applied, notApplied, report) {
|
|
1606
|
+
const jobs = this.engine.getManager('BackgroundJobManager');
|
|
1607
|
+
const runs = new Map();
|
|
1608
|
+
for (const r of [...(report?.jobRuns ?? []), ...(jobs?.getRunsReplacedAtStart?.() ?? [])])
|
|
1609
|
+
runs.set(`${r.jobId}@${r.startedAt}`, r);
|
|
1610
|
+
const notRestored = [...new Set([...(report?.notRestored ?? []), ...notApplied])].map(restoreLabel);
|
|
1611
|
+
const message = describeNotRestored({
|
|
1612
|
+
source: report?.source ?? 'a backup', backup: report?.backup ?? '', applied: applied.map(restoreLabel), notRestored,
|
|
1613
|
+
jobRuns: [...runs.values()], nameOf: (id) => jobs?.displayNameOf?.(id) ?? id, missingReport: !report
|
|
1614
|
+
});
|
|
1615
|
+
logger.error(`[BackupManager] ${message} (#1876)`);
|
|
1616
|
+
await this.notifyAdmins('error', 'Critical: a restore did not bring everything back', message);
|
|
1617
|
+
}
|
|
1618
|
+
/** The report written when the restore was staged, or null. */
|
|
1619
|
+
readRestoreReport() {
|
|
1620
|
+
try {
|
|
1621
|
+
const file = this.restoreReportPath();
|
|
1622
|
+
return existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')) : null;
|
|
1623
|
+
}
|
|
1624
|
+
catch (err) {
|
|
1625
|
+
logger.error('[BackupManager] The restore report could not be read:', err);
|
|
1626
|
+
return null;
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
/**
|
|
1630
|
+
* `backup-restore-applied` (#1876, operator-approved): written at the start
|
|
1631
|
+
* that applied a restore, so into the restored audit log — which otherwise
|
|
1632
|
+
* would not say a restore happened (the `backup-restore` record of the
|
|
1633
|
+
* staging is in the log it replaced). Names the backup, when it was taken,
|
|
1634
|
+
* who started the restore and what was put back. on-failure: continue — the
|
|
1635
|
+
* restore is already in place.
|
|
1636
|
+
*/
|
|
1637
|
+
async recordRestoreApplied(applied, finished, report) {
|
|
1638
|
+
const who = actorOf(systemContext(this.engine, 'a staged restore was applied at start'));
|
|
1639
|
+
const source = finished?.source || report?.source || 'unknown';
|
|
1640
|
+
await recordAuditEvent(this.engine.getManager('AuditManager'), {
|
|
1641
|
+
eventType: AUDIT_EVENT.BACKUP_RESTORE_APPLIED,
|
|
1642
|
+
...who,
|
|
1643
|
+
action: 'backup-restore-applied',
|
|
1644
|
+
result: 'success',
|
|
1645
|
+
severity: 'high',
|
|
1646
|
+
resource: source,
|
|
1647
|
+
resourceType: 'backup',
|
|
1648
|
+
metadata: {
|
|
1649
|
+
...who.metadata,
|
|
1650
|
+
source,
|
|
1651
|
+
backupTakenAt: finished?.backup || report?.backup || null,
|
|
1652
|
+
startedBy: finished?.stagedBy || report?.stagedBy || null,
|
|
1653
|
+
kind: finished ? 'site' : 'databases',
|
|
1654
|
+
applied
|
|
1655
|
+
}
|
|
1656
|
+
}, (err) => logger.warn('[BackupManager] Audit record failed for backup-restore-applied:', err));
|
|
1657
|
+
}
|
|
1658
|
+
/** Tell admins what the start did with a staged site restore (#1876). */
|
|
1659
|
+
async reportSiteRestoreOutcome(finished) {
|
|
1660
|
+
const applied = finished.parts.filter((p) => p.status === 'applied').map((p) => p.manager);
|
|
1661
|
+
const notApplied = finished.parts.filter((p) => p.status !== 'applied').map((p) => p.manager);
|
|
1662
|
+
const from = `the backup ${finished.source || finished.backup}`;
|
|
1663
|
+
const undo = finished.safetyBackup ? ` The data from before was backed up as ${finished.safetyBackup}.` : '';
|
|
1664
|
+
if (notApplied.length === 0) {
|
|
1665
|
+
const message = `Restored at start from ${from}: ${applied.join(', ')}. The previous data of each is kept beside it as *.pre-restore.${undo}`;
|
|
1666
|
+
logger.warn(`[BackupManager] ${message} (#1876)`);
|
|
1667
|
+
await this.notifyAdmins('success', 'A site restore was applied', message);
|
|
1668
|
+
}
|
|
1669
|
+
else if (applied.length === 0) {
|
|
1670
|
+
const message = `The restore from ${from} was not applied: ${finished.reason ?? 'it was refused'}. Nothing was restored; the staged files were set aside as *.incomplete, and the site started on its current data.`;
|
|
1671
|
+
logger.error(`❌ ${message} (#1876)`);
|
|
1672
|
+
await this.notifyAdmins('error', 'A site restore was not applied', message);
|
|
1673
|
+
}
|
|
1674
|
+
else {
|
|
1675
|
+
const message = `The restore from ${from} was applied only in part: ${applied.join(', ')} restored; ${notApplied.join(', ')} not (${finished.reason ?? 'refused'}). The previous data of each restored part is kept as *.pre-restore.${undo}`;
|
|
1676
|
+
logger.error(`❌ ${message} (#1876)`);
|
|
1677
|
+
await this.notifyAdmins('error', 'A site restore was applied only in part', message);
|
|
1678
|
+
}
|
|
1679
|
+
}
|
|
1680
|
+
/**
|
|
1681
|
+
* The backups in the backup directory with the database files beside each
|
|
1682
|
+
* (#1723): what Admin → Backup offers to download and restore. Read from
|
|
1683
|
+
* the file names, so no document is opened to list them.
|
|
1684
|
+
*/
|
|
1685
|
+
async listStoredBackups() {
|
|
1686
|
+
const backups = await this.listBackups();
|
|
1687
|
+
if (!this.provider || backups.length === 0)
|
|
1688
|
+
return [];
|
|
1689
|
+
const objects = await this.provider.listBackupObjects();
|
|
1690
|
+
// #1563: one pruned between the two listings is gone, not listed without its files.
|
|
1691
|
+
const present = new Set(objects.map((o) => o.filename));
|
|
1692
|
+
return backups.filter((b) => present.has(b.filename)).map((b) => {
|
|
1693
|
+
const companions = objects.filter((o) => isCompanionOf(o.filename, b.filename));
|
|
1694
|
+
const stem = backupStem(b.filename);
|
|
1695
|
+
return {
|
|
1696
|
+
...b,
|
|
1697
|
+
databases: companions.filter((o) => o.filename.endsWith('.db')).map((o) => o.filename.slice(stem.length + 1, -'.db'.length)).sort(),
|
|
1698
|
+
parts: companions.filter((o) => !o.filename.endsWith('.db')).map((o) => o.filename.slice(stem.length + 1).replace(/\.tar(\.enc)?$/, '')).sort(),
|
|
1699
|
+
files: companions.map((o) => o.filename).sort(),
|
|
1700
|
+
encrypted: b.filename.endsWith(ENCRYPTED_SUFFIX),
|
|
1701
|
+
totalSize: b.size + companions.reduce((n, o) => n + o.size, 0)
|
|
1702
|
+
};
|
|
1703
|
+
});
|
|
1704
|
+
}
|
|
1705
|
+
/** A backup in the backup directory by its bare filename, or null: never a path the caller made up. */
|
|
1706
|
+
async findStoredBackup(filename) {
|
|
1707
|
+
return (await this.listStoredBackups()).find((b) => b.filename === filename) ?? null;
|
|
1708
|
+
}
|
|
1709
|
+
/**
|
|
1710
|
+
* One backup as a `.tar` stream (#1723, D2): the document and every file
|
|
1711
|
+
* beside it — the databases, and the archives of attachment files, role
|
|
1712
|
+
* membership, persons, organizations, footnotes and comments (#1876) —
|
|
1713
|
+
* read from disk as the stream is consumed, never built in memory. With
|
|
1714
|
+
* the backup key set every file in it is encrypted under that key, which
|
|
1715
|
+
* is never in the archive (#1723, #1561).
|
|
1716
|
+
*/
|
|
1717
|
+
async backupArchive(filename) {
|
|
1718
|
+
const backup = await this.findStoredBackup(filename);
|
|
1719
|
+
if (!backup || !this.backupDirectory)
|
|
1720
|
+
throw new Error(`No backup named ${filename} in the backup directory`);
|
|
1721
|
+
const stem = backupStem(backup.filename);
|
|
1722
|
+
const files = [backup.filename, ...backup.files];
|
|
1723
|
+
const stream = createTar({ cwd: this.backupDirectory, portable: true, noDirRecurse: true }, files);
|
|
1724
|
+
return { name: `${stem}.tar`, stream };
|
|
1725
|
+
}
|
|
1726
|
+
/** Whether a restore started from Admin → Backup is still taking its fresh backup or staging. */
|
|
1727
|
+
isRestoreRunning() {
|
|
1728
|
+
return this.restoreRunning !== null;
|
|
1729
|
+
}
|
|
1730
|
+
/** The managers on this site whose restore is staged (#1876), in registration order: what a site restore stages. */
|
|
1731
|
+
stagedManagerNames() {
|
|
1732
|
+
const names = this.engine.getRegisteredManagers().filter((name) => this.restoresStaged(name));
|
|
1733
|
+
// #1985: an add-on's files under an entry no manager of that name has (`addon:<slug>`).
|
|
1734
|
+
for (const { entry } of this.addonParts())
|
|
1735
|
+
if (!names.includes(entry))
|
|
1736
|
+
names.push(entry);
|
|
1737
|
+
return names;
|
|
1738
|
+
}
|
|
1739
|
+
/**
|
|
1740
|
+
* What a whole-site restore on this site puts back and what it does not
|
|
1741
|
+
* (#1945), in the words admins read: the labels of the "not restored"
|
|
1742
|
+
* notification (`restoreLabel`, `NEVER_IN_A_BACKUP`), so the restore card
|
|
1743
|
+
* and the notification after the restart say the same. Read from the
|
|
1744
|
+
* managers on this site, not from any backup: a backup can hold less (the
|
|
1745
|
+
* credentials store of one taken without the backup key), never more.
|
|
1746
|
+
*/
|
|
1747
|
+
siteRestoreScope() {
|
|
1748
|
+
const databases = this.engine.getManager('DatabaseManager');
|
|
1749
|
+
const restored = this.stagedManagerNames()
|
|
1750
|
+
.filter((name) => name !== 'DatabaseManager' || databases?.isEnabled?.() !== false)
|
|
1751
|
+
.map(restoreLabel);
|
|
1752
|
+
const notRestored = Object.keys(RESTORE_LABELS)
|
|
1753
|
+
.filter((name) => name !== 'BackupManager' && this.engine.getManager(name) && !this.restoresStaged(name))
|
|
1754
|
+
.map(restoreLabel);
|
|
1755
|
+
return { restored, notRestored: [...notRestored, ...NEVER_IN_A_BACKUP] };
|
|
1756
|
+
}
|
|
1757
|
+
/**
|
|
1758
|
+
* The restore waiting for the next start, whichever it is (#1945): a site
|
|
1759
|
+
* restore, or a databases-only one. A site restore stages the databases
|
|
1760
|
+
* too, so it is the one named while both records are there.
|
|
1761
|
+
*/
|
|
1762
|
+
getStagedRestore() {
|
|
1763
|
+
let site;
|
|
1764
|
+
try {
|
|
1765
|
+
site = this.getStagedSiteRestore();
|
|
1766
|
+
}
|
|
1767
|
+
catch (err) {
|
|
1768
|
+
throw new Error(`A site restore record is there but can't be read (${err.message})`, { cause: err });
|
|
1769
|
+
}
|
|
1770
|
+
if (site)
|
|
1771
|
+
return { scope: 'site', source: site.source || site.backup, stagedBy: site.stagedBy };
|
|
1772
|
+
const databases = this.engine.getManager('DatabaseManager')?.getStagedRestore?.() ?? null;
|
|
1773
|
+
return databases ? { scope: 'databases', source: databases.source || databases.backup, stagedBy: databases.stagedBy } : null;
|
|
1774
|
+
}
|
|
1775
|
+
/**
|
|
1776
|
+
* Throw away the restore waiting for the next start, whichever it is
|
|
1777
|
+
* (#1945): a staged site restore through `cancelStagedSiteRestore`, a
|
|
1778
|
+
* databases-only one through `DatabaseManager.cancelStagedRestore`. Each
|
|
1779
|
+
* records `backup-restore-cancel` first (refuse).
|
|
1780
|
+
*/
|
|
1781
|
+
async cancelStagedRestore(ctx) {
|
|
1782
|
+
const staged = this.getStagedRestore();
|
|
1783
|
+
if (!staged)
|
|
1784
|
+
throw new Error('No restore is staged');
|
|
1785
|
+
if (staged.scope === 'site') {
|
|
1786
|
+
await this.cancelStagedSiteRestore(ctx);
|
|
1787
|
+
}
|
|
1788
|
+
else {
|
|
1789
|
+
const databaseManager = this.engine.getManager('DatabaseManager');
|
|
1790
|
+
if (!databaseManager)
|
|
1791
|
+
throw new Error('DatabaseManager is not available');
|
|
1792
|
+
await databaseManager.cancelStagedRestore(ctx);
|
|
1793
|
+
}
|
|
1794
|
+
return { scope: staged.scope, source: staged.source };
|
|
1795
|
+
}
|
|
1796
|
+
/**
|
|
1797
|
+
* Start a restore of a stored backup from Admin → Backup: recorded, then a
|
|
1798
|
+
* fresh backup of the live data and the staging run in the background, and
|
|
1799
|
+
* admins are told when it is staged or why it was not. Nothing changes
|
|
1800
|
+
* until the next restart, which the site never does by itself. Refused
|
|
1801
|
+
* while another is being prepared, and when a restore is already staged
|
|
1802
|
+
* unless `replace` says to replace it.
|
|
1803
|
+
*
|
|
1804
|
+
* - `scope: 'databases'` (the default; #1723, D1): the databases only.
|
|
1805
|
+
* Refused while a site restore is staged, since that includes them.
|
|
1806
|
+
* - `scope: 'site'` (#1945): a whole-site restore, every manager whose
|
|
1807
|
+
* restore is staged and the instance .env, all or nothing
|
|
1808
|
+
* (`stageSiteRestore`), from any stored backup. It replaces a staged
|
|
1809
|
+
* restore of either kind only with `replace`.
|
|
1810
|
+
*
|
|
1811
|
+
* @returns the running restore, for a caller (a test) that waits on it.
|
|
1812
|
+
*/
|
|
1813
|
+
async startRestore(ctx, filename, options = {}) {
|
|
1814
|
+
const scope = options.scope ?? 'databases';
|
|
1815
|
+
if (scope !== 'databases' && scope !== 'site')
|
|
1816
|
+
throw new Error(`Unknown restore scope ${String(scope)}`);
|
|
1817
|
+
if (this.restoreRunning)
|
|
1818
|
+
throw new Error('A restore is already being prepared; wait for it to finish');
|
|
1819
|
+
const backup = await this.findStoredBackup(filename);
|
|
1820
|
+
if (!backup)
|
|
1821
|
+
throw new Error(`No backup named ${filename} in the backup directory`);
|
|
1822
|
+
let staged;
|
|
1823
|
+
if (scope === 'databases') {
|
|
1824
|
+
if (backup.databases.length === 0)
|
|
1825
|
+
throw new Error(`Backup ${filename} holds no database to restore`);
|
|
1826
|
+
const databaseManager = this.engine.getManager('DatabaseManager');
|
|
1827
|
+
if (!databaseManager)
|
|
1828
|
+
throw new Error('DatabaseManager is not available');
|
|
1829
|
+
this.refuseIfSiteRestoreStaged();
|
|
1830
|
+
const manifest = databaseManager.getStagedRestore();
|
|
1831
|
+
staged = manifest ? { source: manifest.source || manifest.backup, stagedBy: manifest.stagedBy } : null;
|
|
1832
|
+
}
|
|
1833
|
+
else {
|
|
1834
|
+
staged = this.getStagedRestore();
|
|
1835
|
+
}
|
|
1836
|
+
if (staged && !options.replace)
|
|
1837
|
+
throw new Error(`A restore from ${staged.source} is already staged; cancel it or replace it`);
|
|
1838
|
+
if (!staged && options.replace)
|
|
1839
|
+
throw new Error('There is no staged restore to replace');
|
|
1840
|
+
// Recorded before anything happens (refuse): a restore that cannot be recorded does not start.
|
|
1841
|
+
const who = actorOf(ctx);
|
|
1842
|
+
await recordAuditEvent(this.engine.getManager('AuditManager'), {
|
|
1843
|
+
eventType: staged ? AUDIT_EVENT.BACKUP_RESTORE_REPLACE : AUDIT_EVENT.BACKUP_RESTORE,
|
|
1844
|
+
...who,
|
|
1845
|
+
action: staged ? 'backup-restore-replace' : 'backup-restore',
|
|
1846
|
+
result: 'success',
|
|
1847
|
+
severity: 'high',
|
|
1848
|
+
resource: backup.filename,
|
|
1849
|
+
resourceType: 'backup',
|
|
1850
|
+
metadata: {
|
|
1851
|
+
...who.metadata, scope, databases: backup.databases,
|
|
1852
|
+
...(scope === 'site' ? { parts: backup.parts } : {}),
|
|
1853
|
+
...(staged ? { replaces: staged.source, replacesStagedBy: staged.stagedBy } : {})
|
|
1854
|
+
}
|
|
1855
|
+
});
|
|
1856
|
+
// A site restore names only the managers whose restore is staged, so
|
|
1857
|
+
// nothing is ever restored live from here (#1945).
|
|
1858
|
+
const managerFilter = scope === 'site' ? this.stagedManagerNames() : ['DatabaseManager'];
|
|
1859
|
+
const unchanged = scope === 'site' ? "The site's data stays as it is." : 'The current databases stay as they are.';
|
|
1860
|
+
const done = this.restoreFromFile(ctx, backup.path, { managerFilter, replaceStaged: Boolean(staged) })
|
|
1861
|
+
.catch(async (err) => {
|
|
1862
|
+
logger.error(`❌ Restore of ${filename} was not staged:`, err);
|
|
1863
|
+
await this.notifyAdmins('error', `Restore of ${filename} was not staged`, `${err.message}. ${unchanged}`);
|
|
1864
|
+
return null;
|
|
1865
|
+
})
|
|
1866
|
+
.finally(() => { this.restoreRunning = null; });
|
|
1867
|
+
this.restoreRunning = done;
|
|
1868
|
+
return { done };
|
|
1869
|
+
}
|
|
1870
|
+
/**
|
|
1871
|
+
* List all available backups
|
|
1872
|
+
* @returns {Promise<BackupFileInfo[]>} List of backup files with metadata
|
|
1873
|
+
*/
|
|
1874
|
+
async listBackups() {
|
|
1875
|
+
try {
|
|
1876
|
+
if (!this.backupDirectory || !this.provider) {
|
|
1877
|
+
return [];
|
|
1878
|
+
}
|
|
1879
|
+
// Storage listing comes from the provider (#170); the backup-naming
|
|
1880
|
+
// convention stays this manager's concern.
|
|
1881
|
+
const objects = await this.provider.listBackupObjects();
|
|
1882
|
+
const backups = objects
|
|
1883
|
+
.filter(o =>
|
|
1884
|
+
// Support both old format (ngdpbase-backup-{timestamp}) and new format ({timestamp}-ngdpbase-backup)
|
|
1885
|
+
(o.filename.startsWith('ngdpbase-backup-') || /-ngdpbase-backup\.json\.gz(\.enc)?$/.test(o.filename))
|
|
1886
|
+
&& /\.json\.gz(\.enc)?$/.test(o.filename))
|
|
1887
|
+
.map(o => ({
|
|
1888
|
+
filename: o.filename,
|
|
1889
|
+
path: o.path,
|
|
1890
|
+
size: o.size,
|
|
1891
|
+
created: o.created,
|
|
1892
|
+
modified: o.modified
|
|
1893
|
+
}));
|
|
1894
|
+
// Sort by creation time, newest first
|
|
1895
|
+
backups.sort((a, b) => b.created.getTime() - a.created.getTime());
|
|
1896
|
+
return backups;
|
|
1897
|
+
}
|
|
1898
|
+
catch (error) {
|
|
1899
|
+
logger.error('❌ Failed to list backups:', error);
|
|
1900
|
+
return [];
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
/**
|
|
1904
|
+
* Clean up old backups, keeping only maxBackups most recent
|
|
1905
|
+
* @private
|
|
1906
|
+
*/
|
|
1907
|
+
async cleanupOldBackups() {
|
|
1908
|
+
try {
|
|
1909
|
+
const backups = await this.listBackups();
|
|
1910
|
+
if (backups.length <= this.maxBackups) {
|
|
1911
|
+
logger.info(`📊 ${backups.length} backups exist, no cleanup needed`);
|
|
1912
|
+
return;
|
|
1913
|
+
}
|
|
1914
|
+
// Delete oldest backups
|
|
1915
|
+
const toDelete = backups.slice(this.maxBackups);
|
|
1916
|
+
logger.info(`🗑️ Cleaning up ${toDelete.length} old backups`);
|
|
1917
|
+
for (const backup of toDelete) {
|
|
1918
|
+
await this.provider?.removeBackup(backup.path);
|
|
1919
|
+
await this.removeCompanions(backup.filename);
|
|
1920
|
+
logger.info(`🗑️ Deleted: ${backup.filename}`);
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
catch (error) {
|
|
1924
|
+
logger.warn('⚠️ Failed to cleanup old backups:', error);
|
|
1925
|
+
// Don't throw - cleanup failure shouldn't fail the backup
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
/**
|
|
1929
|
+
* Get the most recent backup file path
|
|
1930
|
+
* @returns {Promise<string | null>} Path to most recent backup, or null if none exist
|
|
1931
|
+
*/
|
|
1932
|
+
async getLatestBackup() {
|
|
1933
|
+
const backups = await this.listBackups();
|
|
1934
|
+
return backups.length > 0 ? backups[0].path : null;
|
|
1935
|
+
}
|
|
1936
|
+
/**
|
|
1937
|
+
* Return current auto-backup configuration and last backup info for the admin UI.
|
|
1938
|
+
*/
|
|
1939
|
+
async getAutoBackupStatus() {
|
|
1940
|
+
const backups = await this.listBackups();
|
|
1941
|
+
return {
|
|
1942
|
+
config: {
|
|
1943
|
+
enabled: this.autoBackupEnabled,
|
|
1944
|
+
time: this.autoBackupTime,
|
|
1945
|
+
days: this.autoBackupDays,
|
|
1946
|
+
maxBackups: this.maxBackups,
|
|
1947
|
+
directory: this.backupDirectory ?? ''
|
|
1948
|
+
},
|
|
1949
|
+
lastBackup: backups.length > 0 ? backups[0].created : null
|
|
1950
|
+
};
|
|
1951
|
+
}
|
|
1952
|
+
/**
|
|
1953
|
+
* Update auto-backup configuration and persist to config store.
|
|
1954
|
+
* Restarts the scheduler with the new settings.
|
|
1955
|
+
*/
|
|
1956
|
+
async updateAutoBackupConfig(config, ctx) {
|
|
1957
|
+
const configManager = this.engine.getManager('ConfigurationManager');
|
|
1958
|
+
if (!configManager)
|
|
1959
|
+
throw new Error('ConfigurationManager not available');
|
|
1960
|
+
if (config.enabled !== undefined) {
|
|
1961
|
+
await configManager.setProperty('ngdpbase.backup.auto-backup', config.enabled, ctx);
|
|
1962
|
+
this.autoBackupEnabled = config.enabled;
|
|
1963
|
+
}
|
|
1964
|
+
if (config.time !== undefined) {
|
|
1965
|
+
await configManager.setProperty('ngdpbase.backup.auto-backup-time', config.time, ctx);
|
|
1966
|
+
this.autoBackupTime = config.time;
|
|
1967
|
+
}
|
|
1968
|
+
if (config.days !== undefined) {
|
|
1969
|
+
await configManager.setProperty('ngdpbase.backup.auto-backup-days', config.days, ctx);
|
|
1970
|
+
this.autoBackupDays = config.days;
|
|
1971
|
+
}
|
|
1972
|
+
if (config.maxBackups !== undefined) {
|
|
1973
|
+
await configManager.setProperty('ngdpbase.backup.max-backups', config.maxBackups, ctx);
|
|
1974
|
+
this.maxBackups = config.maxBackups;
|
|
1975
|
+
}
|
|
1976
|
+
if (config.directory !== undefined) {
|
|
1977
|
+
await configManager.setProperty('ngdpbase.backup.directory', config.directory, ctx);
|
|
1978
|
+
this.backupDirectory = config.directory;
|
|
1979
|
+
// #170: re-point the storage provider at the new directory.
|
|
1980
|
+
await this.provider?.setBackupDirectory(config.directory);
|
|
1981
|
+
}
|
|
1982
|
+
// The job takes the new settings; a changed rule counts its slots from now (#1720).
|
|
1983
|
+
this.scheduleAutoBackup();
|
|
1984
|
+
logger.info(`✅ Auto-backup config updated: ${this.autoBackupEnabled ? `enabled at ${this.autoBackupTime} (${this.autoBackupDays})` : 'disabled'}`);
|
|
1985
|
+
}
|
|
1986
|
+
/**
|
|
1987
|
+
* Automatic backups as a scheduled job (#1720, epic #1611).
|
|
1988
|
+
*
|
|
1989
|
+
* They ran from a timer that checked every minute for an exact match of the
|
|
1990
|
+
* clock, so a server that was down at the backup time — or a tick that
|
|
1991
|
+
* landed a minute late — skipped that backup and left no record that it had
|
|
1992
|
+
* (operator: backups are never silent, #1563). As a scheduled job a backup
|
|
1993
|
+
* missed while the server was down runs once when it is back (`catchUp:
|
|
1994
|
+
* 'latest'`), a failed one is tried again and then reported, and the run
|
|
1995
|
+
* survives a restart. A backup still running when the next one comes due is
|
|
1996
|
+
* not doubled (`overlap: 'skip'`).
|
|
1997
|
+
*
|
|
1998
|
+
* Called at start and whenever the settings change. With automatic backups
|
|
1999
|
+
* off, or a setting that cannot be read, the job is registered without a
|
|
2000
|
+
* schedule: it can still be run by hand, and nothing runs by itself.
|
|
2001
|
+
*/
|
|
2002
|
+
scheduleAutoBackup() {
|
|
2003
|
+
const jobs = this.engine.getManager('BackgroundJobManager');
|
|
2004
|
+
if (!jobs) {
|
|
2005
|
+
if (this.autoBackupEnabled)
|
|
2006
|
+
logger.error('❌ Automatic backups are ON but BackgroundJobManager is not available — NO automatic backup will run');
|
|
2007
|
+
return;
|
|
2008
|
+
}
|
|
2009
|
+
const def = {
|
|
2010
|
+
id: AUTO_BACKUP_JOB_ID,
|
|
2011
|
+
displayName: 'Automatic backup',
|
|
2012
|
+
run: async (_progress, ctx) => {
|
|
2013
|
+
// #1196: the run's context names the system principal on a schedule, or the admin who ran it.
|
|
2014
|
+
const file = await this.createBackup(ctx);
|
|
2015
|
+
return { success: true, summary: `Backup written: ${file}` };
|
|
2016
|
+
}
|
|
2017
|
+
};
|
|
2018
|
+
if (!this.autoBackupEnabled || !this.provider) {
|
|
2019
|
+
jobs.registerJob(def);
|
|
2020
|
+
return;
|
|
2021
|
+
}
|
|
2022
|
+
try {
|
|
2023
|
+
jobs.registerJob({
|
|
2024
|
+
...def,
|
|
2025
|
+
schedule: autoBackupSchedule(this.autoBackupTime, this.autoBackupDays, serverTimeZone()),
|
|
2026
|
+
catchUp: 'latest',
|
|
2027
|
+
overlap: 'skip',
|
|
2028
|
+
persist: true
|
|
2029
|
+
});
|
|
2030
|
+
logger.info(`🕐 Automatic backup scheduled: ${this.autoBackupTime} (${this.autoBackupDays}), ${serverTimeZone()}`);
|
|
2031
|
+
}
|
|
2032
|
+
catch (err) {
|
|
2033
|
+
jobs.registerJob(def);
|
|
2034
|
+
logger.error(`❌ Automatic backups are ON but NOT scheduled: ${err.message}. Fix the setting in Admin → Backup.`);
|
|
2035
|
+
void this.engine.getManager('NotificationManager')?.addNotification({
|
|
2036
|
+
type: 'system', level: 'error', title: 'Automatic backups are not running',
|
|
2037
|
+
message: `${err.message}. Fix the setting in Admin → Backup.`
|
|
2038
|
+
}).catch(() => undefined);
|
|
2039
|
+
}
|
|
2040
|
+
}
|
|
2041
|
+
}
|
|
2042
|
+
export default BackupManager;
|