@jwilleke/ngdpbase-types 5.24.0 → 5.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/lib/packages/ngdpbase-types/index.d.ts +6 -2
- package/lib/packages/ngdpbase-types/testing.d.ts +7 -2
- package/lib/packages/ngdpbase-types/testing.js +7 -2
- package/lib/src/http/egressPolicy.js +61 -0
- package/lib/src/http/guardedElasticsearch.d.ts +27 -0
- package/lib/src/http/guardedElasticsearch.js +99 -0
- package/lib/src/managers/AddonsManager.d.ts +119 -2
- package/lib/src/managers/AttachmentManager.d.ts +4 -5
- package/lib/src/managers/AuthManager.d.ts +150 -32
- package/lib/src/managers/BackgroundJobManager.d.ts +22 -12
- package/lib/src/managers/BackupManager.d.ts +779 -0
- package/lib/src/managers/BackupManager.js +2042 -0
- package/lib/src/managers/BaseManager.d.ts +156 -6
- package/lib/src/managers/BaseManager.js +64 -2
- package/lib/src/managers/CommentManager.d.ts +6 -6
- package/lib/src/managers/ConsentManager.d.ts +130 -0
- package/lib/src/managers/DatabaseManager.d.ts +6 -0
- package/lib/src/managers/DatabaseManager.js +25 -15
- package/lib/src/managers/FootnoteManager.d.ts +6 -6
- package/lib/src/managers/OrganizationManager.d.ts +8 -6
- package/lib/src/managers/PageManager.d.ts +16 -0
- package/lib/src/managers/PersonManager.d.ts +8 -6
- package/lib/src/managers/RoleManager.d.ts +6 -8
- package/lib/src/providers/BaseAttachmentProvider.d.ts +4 -4
- package/lib/src/providers/BaseBackupProvider.d.ts +77 -0
- package/lib/src/providers/BaseBackupProvider.js +52 -0
- package/lib/src/providers/BasicAttachmentProvider.d.ts +4 -4
- package/lib/src/providers/FileBackupProvider.d.ts +34 -0
- package/lib/src/providers/FileBackupProvider.js +116 -0
- package/lib/src/providers/FileConsentProvider.d.ts +18 -0
- package/lib/src/providers/SqliteDatabaseProvider.d.ts +2 -2
- package/lib/src/providers/SqliteDatabaseProvider.js +12 -11
- package/lib/src/providers/attachmentFilesBackup.d.ts +5 -4
- package/lib/src/providers/databaseMigrations.js +45 -0
- package/lib/src/security/PolicyDecisionPoint.d.ts +14 -0
- package/lib/src/security/PolicyInformationPoint.d.ts +8 -0
- package/lib/src/types/Consent.d.ts +90 -0
- package/lib/src/types/express.d.ts +6 -0
- package/lib/src/utils/addonHost.d.ts +51 -3
- package/lib/src/utils/addonHost.js +29 -2
- package/lib/src/utils/addonPageSync.d.ts +2 -2
- package/lib/src/utils/auditEventNames.generated.d.ts +20 -0
- package/lib/src/utils/auditEventNames.generated.js +20 -0
- package/lib/src/utils/backupCrypto.js +275 -0
- package/lib/src/utils/backupDocument.d.ts +36 -0
- package/lib/src/utils/backupDocument.js +99 -0
- package/lib/src/utils/browserDescription.d.ts +11 -0
- package/lib/src/utils/directoryBackup.d.ts +48 -48
- package/lib/src/utils/directoryBackup.js +291 -0
- package/lib/src/utils/instanceEnvBackup.d.ts +46 -0
- package/lib/src/utils/instanceEnvBackup.js +120 -0
- package/lib/src/utils/instanceEnvSecret.js +79 -0
- package/lib/src/utils/ipLocation.d.ts +11 -0
- package/lib/src/utils/schedule.js +171 -0
- package/lib/src/utils/secretFileMode.js +78 -0
- package/lib/src/utils/siteRestore.d.ts +13 -16
- package/lib/src/utils/siteRestore.js +33 -6
- package/lib/src/utils/treeArchive.d.ts +2 -0
- package/lib/src/utils/treeArchive.js +242 -0
- package/lib/src/utils/version.js +408 -0
- package/package.json +6 -1
- package/lib/src/utils/administrators.d.ts +0 -21
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #1532 — a browser and the kind of device it runs on, in words a person
|
|
3
|
+
* recognises ("Chrome on Windows"), from a User-Agent header. Used where a
|
|
4
|
+
* person is asked to judge a request someone made: the sign-in approval
|
|
5
|
+
* message and page. The rules are the ones `suggestedName()` in
|
|
6
|
+
* `public/js/passkey.js` applies in the browser, so a passkey's suggested name
|
|
7
|
+
* and an approval request describe the same device the same way.
|
|
8
|
+
*
|
|
9
|
+
* Null when the header names neither a browser nor a device we know.
|
|
10
|
+
*/
|
|
11
|
+
export declare function describeBrowser(userAgent: string | undefined): string | null;
|
|
@@ -1,15 +1,7 @@
|
|
|
1
|
-
import type { BackupData, ManagerBackupOptions, ManagerRestoreOptions } from '../managers/BaseManager.js';
|
|
2
|
-
import { siteRestoreFile
|
|
3
|
-
/** A folder part's
|
|
4
|
-
export
|
|
5
|
-
format: 1;
|
|
6
|
-
/** The archive beside the document: `<stem>.<part>.tar`, or `.tar.enc` when encrypted. */
|
|
7
|
-
archive: string;
|
|
8
|
-
files: number;
|
|
9
|
-
bytes: number;
|
|
10
|
-
/** `backupKeyFingerprint()` of the key the archive is encrypted under; absent when it is not encrypted. */
|
|
11
|
-
keyFingerprint?: string;
|
|
12
|
-
}
|
|
1
|
+
import type { BackupArchiveTarget, BackupData, BackupFolder, BackupPartRecord, ManagerBackupOptions, ManagerRestoreOptions, StagedRestoreAtStart } from '../managers/BaseManager.js';
|
|
2
|
+
import { siteRestoreFile } from './siteRestore.js';
|
|
3
|
+
/** A folder or file part's record in the backup document (the name every backup since v5.21 used). */
|
|
4
|
+
export type DirectoryBackupData = BackupPartRecord;
|
|
13
5
|
/** Whether `name` is a bare archive name in the backup directory, never a path. */
|
|
14
6
|
export declare function isArchiveName(name: unknown): name is string;
|
|
15
7
|
/**
|
|
@@ -31,50 +23,58 @@ export declare function archiveTarget(stem: string, part: string): {
|
|
|
31
23
|
export declare function archiveReadKey(manager: string, data: {
|
|
32
24
|
keyFingerprint?: string;
|
|
33
25
|
}): string | undefined;
|
|
26
|
+
/** `ManagerBackupOptions.archive`: one more archive beside the document, for a manager whose files are not a declared part. */
|
|
27
|
+
export declare function archiveWriter(directory: string, stem: string): (part: string) => BackupArchiveTarget;
|
|
28
|
+
/** `ManagerRestoreOptions.readArchive`: an archive the backup names, checked, and the key to read it under. */
|
|
29
|
+
export declare function archiveReader(directory: string): NonNullable<ManagerRestoreOptions['readArchive']>;
|
|
30
|
+
/** The live path a declared part names: its folder or its file. */
|
|
31
|
+
export declare function livePathOf(part: BackupFolder): string | null;
|
|
32
|
+
/** What backing up a manager's declared parts wrote: each part's record (null when left out), and what to tell admins. */
|
|
33
|
+
export interface DeclaredPartsBackup {
|
|
34
|
+
records: Record<string, BackupPartRecord | null>;
|
|
35
|
+
warnings: string[];
|
|
36
|
+
notes: string[];
|
|
37
|
+
}
|
|
34
38
|
/**
|
|
35
|
-
* Back
|
|
36
|
-
* outside BackupManager: the files go beside a document,
|
|
37
|
-
*
|
|
38
|
-
|
|
39
|
-
export declare function backupDirectoryPart(manager: string, part: string, folder: string | null, options?: ManagerBackupOptions, extra?: {
|
|
40
|
-
exclude?: (relative: string) => boolean;
|
|
41
|
-
}): Promise<BackupData>;
|
|
42
|
-
/** The log's note of entries deleted while they were backed up (#1563): skipped, not a failure. */
|
|
43
|
-
export declare function vanishedNote(vanished: string[]): string;
|
|
44
|
-
/**
|
|
45
|
-
* Stage the folder from a backup for the next start: the archive is
|
|
46
|
-
* extracted to `<folder>.staged`, and the swap recorded in the site restore.
|
|
47
|
-
* Throws — staging nothing — outside a site restore, for a backup without
|
|
48
|
-
* this part, or for an archive that is missing, unsafe, or under another key.
|
|
39
|
+
* Back up a manager's declared parts beside the document BackupManager is
|
|
40
|
+
* writing. Throws outside BackupManager: the files go beside a document,
|
|
41
|
+
* never into it. A part that fails throws, and the manager's entry is then
|
|
42
|
+
* an error, as when its own `backup()` throws.
|
|
49
43
|
*/
|
|
50
|
-
export declare function
|
|
44
|
+
export declare function backupDeclaredParts(manager: string, declared: BackupFolder[], options: ManagerBackupOptions | undefined): Promise<DeclaredPartsBackup>;
|
|
51
45
|
/**
|
|
52
|
-
* Put
|
|
53
|
-
*
|
|
46
|
+
* Put the declared parts' records into a manager's entry, in the shape
|
|
47
|
+
* every backup has had: the entry's `data` is the record when the manager
|
|
48
|
+
* has no data of its own and declares one part (roles, persons, …, an
|
|
49
|
+
* add-on's folder); otherwise each record is `data[<part>]` beside the
|
|
50
|
+
* manager's own data (the credentials store beside the provider list).
|
|
54
51
|
*/
|
|
55
|
-
export declare function
|
|
52
|
+
export declare function placePartRecords(entry: BackupData, declared: BackupFolder[], parts: DeclaredPartsBackup): BackupData;
|
|
53
|
+
/** A declared part's record in a manager's entry, read the way `placePartRecords` wrote it. */
|
|
54
|
+
export declare function partRecordIn(entry: BackupData | undefined, declared: BackupFolder[], part: string): unknown;
|
|
56
55
|
/**
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
56
|
+
* Stage a manager's declared parts from its entry in a backup, for the next
|
|
57
|
+
* start: each archive is extracted to `<path>.staged`, and the swaps are
|
|
58
|
+
* recorded in the site restore together. Throws — staging nothing, and
|
|
59
|
+
* removing what it extracted — outside a site restore, for a backup without
|
|
60
|
+
* a part that is not optional, or for an archive that is missing, unsafe,
|
|
61
|
+
* or under another key. Returns the parts the backup left out.
|
|
61
62
|
*/
|
|
62
|
-
export declare function
|
|
63
|
-
|
|
64
|
-
what: string;
|
|
65
|
-
}): Promise<{
|
|
66
|
-
data: DirectoryBackupData | null;
|
|
67
|
-
warnings: string[];
|
|
68
|
-
note?: string;
|
|
63
|
+
export declare function stageDeclaredParts(manager: string, declared: BackupFolder[], entry: BackupData | undefined, options: ManagerRestoreOptions | undefined): Promise<{
|
|
64
|
+
leftOut: string[];
|
|
69
65
|
}>;
|
|
70
66
|
/**
|
|
71
|
-
* Stage one file from a backup
|
|
72
|
-
*
|
|
73
|
-
* `whenAbsent` is staged in its place (an empty store), so the restore still
|
|
74
|
-
* puts the site back as it was.
|
|
67
|
+
* Stage one file from a backup for the next start and record the swap: the
|
|
68
|
+
* instance `.env`, BackupManager's own part.
|
|
75
69
|
*/
|
|
76
|
-
export declare function stageFilePart(manager: string, file: string, data:
|
|
70
|
+
export declare function stageFilePart(manager: string, file: string, data: BackupPartRecord, options: ManagerRestoreOptions | undefined, { whenAbsent }: {
|
|
77
71
|
whenAbsent: string;
|
|
78
72
|
}): Promise<void>;
|
|
79
|
-
/**
|
|
80
|
-
|
|
73
|
+
/**
|
|
74
|
+
* Put a staged restore of a manager's declared parts in place, at start,
|
|
75
|
+
* before the manager reads them. Only the paths it declares may be
|
|
76
|
+
* replaced, whatever the manifest says; a staged copy no committed restore
|
|
77
|
+
* names (an interrupted staging) is set aside. Loud when something was
|
|
78
|
+
* staged but not applied. Null when there was nothing to do.
|
|
79
|
+
*/
|
|
80
|
+
export declare function applyDeclaredPartsAtStart(manager: string, declared: BackupFolder[], configManager: Parameters<typeof siteRestoreFile>[0]): StagedRestoreAtStart | null;
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BackupManager's folder and file parts (#1876, #1985). Imported by
|
|
3
|
+
* BackupManager alone (`scripts/check-backup-boundary.ts`): a manager
|
|
4
|
+
* declares where its data lives (`BaseManager.backupFolders()`), an add-on's
|
|
5
|
+
* `dataPath` counts for it, and BackupManager does the rest through here.
|
|
6
|
+
*
|
|
7
|
+
* - __Backup__: each declared folder is written as `<stem>.<part>.tar` beside
|
|
8
|
+
* the backup document (`treeArchive.ts`), streamed, bytes as stored, and
|
|
9
|
+
* the manager's entry records it. With `NGDPBASE_BACKUP_KEY` set it is
|
|
10
|
+
* encrypted under it (#1561) and named `<stem>.<part>.tar.enc`. A declared
|
|
11
|
+
* file is the same with the one file in the archive; one that requires the
|
|
12
|
+
* key is left out without it (the credentials store).
|
|
13
|
+
* - __Restore__: each part's archive is extracted beside the live path, as
|
|
14
|
+
* `<path>.staged`, and the swaps are recorded in the site restore
|
|
15
|
+
* manifest. Nothing is written into the live data.
|
|
16
|
+
* - __At start__: the staged copy is put in place before the owner reads it,
|
|
17
|
+
* when the whole site restore is there to apply (`siteRestore.ts`), and
|
|
18
|
+
* only for the paths the owner declares.
|
|
19
|
+
*
|
|
20
|
+
* The archive names and the entry shapes are those every backup since
|
|
21
|
+
* v5.21 has (#1985 keeps them), so a backup made before the declarations
|
|
22
|
+
* restores, and one made after can be restored by v5.25.0.
|
|
23
|
+
*/
|
|
24
|
+
import { existsSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
25
|
+
import path from 'node:path';
|
|
26
|
+
import logger from './logger.js';
|
|
27
|
+
import { extractTreeArchive, vanishedNote, writeTreeArchive } from './treeArchive.js';
|
|
28
|
+
import { applyPartAtStart, readSiteRestore, siteRestoreFile, stagedPathFor } from './siteRestore.js';
|
|
29
|
+
import { BACKUP_KEY_ENV, backupKeyFingerprint, currentBackupKey, LEFT_OUT_FOR_NO_KEY } from './backupKey.js';
|
|
30
|
+
import { ENCRYPTED_SUFFIX } from './backupCrypto.js';
|
|
31
|
+
/** Whether `name` is a bare archive name in the backup directory, never a path. */
|
|
32
|
+
export function isArchiveName(name) {
|
|
33
|
+
return typeof name === 'string' && (name.endsWith('.tar') || name.endsWith(`.tar${ENCRYPTED_SUFFIX}`)) && path.basename(name) === name && !name.startsWith('.');
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* The name and key for one archive of a backup: encrypted under the backup
|
|
37
|
+
* key when it is set (#1561), plain otherwise. BackupManager says, once per
|
|
38
|
+
* backup, that a backup without the key is not encrypted.
|
|
39
|
+
*/
|
|
40
|
+
export function archiveTarget(stem, part) {
|
|
41
|
+
const key = currentBackupKey();
|
|
42
|
+
if (!key)
|
|
43
|
+
return { archive: `${stem}.${part}.tar` };
|
|
44
|
+
return { archive: `${stem}.${part}.tar${ENCRYPTED_SUFFIX}`, backupKey: key, keyFingerprint: backupKeyFingerprint(key) };
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The key to read an archive of a backup with, checked against the
|
|
48
|
+
* fingerprint the backup recorded: a backup made under another key is
|
|
49
|
+
* refused before anything is staged. None for a backup that was not
|
|
50
|
+
* encrypted, which is then read as the plain archive it says it is.
|
|
51
|
+
*/
|
|
52
|
+
export function archiveReadKey(manager, data) {
|
|
53
|
+
if (!data.keyFingerprint)
|
|
54
|
+
return undefined;
|
|
55
|
+
const key = currentBackupKey();
|
|
56
|
+
if (!key)
|
|
57
|
+
throw new Error(`${manager}'s files in this backup are encrypted under the backup key, and ${BACKUP_KEY_ENV} is not set; nothing was staged`);
|
|
58
|
+
const now = backupKeyFingerprint(key);
|
|
59
|
+
if (now !== data.keyFingerprint) {
|
|
60
|
+
throw new Error(`${manager}'s files in this backup were made with a different backup key (fingerprint ${data.keyFingerprint}; ${BACKUP_KEY_ENV} now has ${now}). Set ${BACKUP_KEY_ENV} to the key it was made with; nothing was staged`);
|
|
61
|
+
}
|
|
62
|
+
return key;
|
|
63
|
+
}
|
|
64
|
+
/** `ManagerBackupOptions.archive`: one more archive beside the document, for a manager whose files are not a declared part. */
|
|
65
|
+
export function archiveWriter(directory, stem) {
|
|
66
|
+
return (part) => {
|
|
67
|
+
const target = archiveTarget(stem, part);
|
|
68
|
+
return { ...target, path: path.join(directory, target.archive) };
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
/** `ManagerRestoreOptions.readArchive`: an archive the backup names, checked, and the key to read it under. */
|
|
72
|
+
export function archiveReader(directory) {
|
|
73
|
+
return (manager, archive, keyFingerprint) => {
|
|
74
|
+
if (!isArchiveName(archive)) {
|
|
75
|
+
throw new Error(`The backup names ${manager}'s archive '${String(archive)}', which is not a file in the backup directory; nothing was staged`);
|
|
76
|
+
}
|
|
77
|
+
return { path: path.join(directory, archive), backupKey: archiveReadKey(manager, { keyFingerprint }) };
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
function requireBackupManager(manager, options) {
|
|
81
|
+
if (!options?.directory || !options.stem) {
|
|
82
|
+
throw new Error(`${manager} writes its files beside the backup document, so it is backed up through BackupManager (#1876)`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
function isFilePart(part) {
|
|
86
|
+
return 'file' in part;
|
|
87
|
+
}
|
|
88
|
+
/** The live path a declared part names: its folder or its file. */
|
|
89
|
+
export function livePathOf(part) {
|
|
90
|
+
return isFilePart(part) ? part.file : part.folder;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Back up a manager's declared parts beside the document BackupManager is
|
|
94
|
+
* writing. Throws outside BackupManager: the files go beside a document,
|
|
95
|
+
* never into it. A part that fails throws, and the manager's entry is then
|
|
96
|
+
* an error, as when its own `backup()` throws.
|
|
97
|
+
*/
|
|
98
|
+
export async function backupDeclaredParts(manager, declared, options) {
|
|
99
|
+
requireBackupManager(manager, options);
|
|
100
|
+
const out = { records: {}, warnings: [], notes: [] };
|
|
101
|
+
for (const part of declared) {
|
|
102
|
+
if (isFilePart(part))
|
|
103
|
+
await backupFilePart(manager, part, options, out);
|
|
104
|
+
else
|
|
105
|
+
await backupFolderPart(manager, part, options, out);
|
|
106
|
+
}
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
async function backupFolderPart(manager, part, options, out) {
|
|
110
|
+
if (!part.folder) {
|
|
111
|
+
out.records[part.part] = null;
|
|
112
|
+
out.notes.push(`${manager} has no folder configured`);
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
const target = archiveTarget(options.stem, part.part);
|
|
116
|
+
const outcome = await writeTreeArchive(path.join(options.directory, target.archive), part.folder, { backupKey: target.backupKey, exclude: part.exclude });
|
|
117
|
+
out.records[part.part] = { format: 1, archive: target.archive, files: outcome.files, bytes: outcome.bytes, ...(target.keyFingerprint ? { keyFingerprint: target.keyFingerprint } : {}) };
|
|
118
|
+
const warnings = outcome.skipped.map((s) => `${s} in ${part.folder} is not a file or a folder, so it was left out`);
|
|
119
|
+
for (const w of warnings)
|
|
120
|
+
logger.error(`[${manager}] ${w}`);
|
|
121
|
+
out.warnings.push(...warnings);
|
|
122
|
+
logger.info(`[${manager}] Backed up ${outcome.files} file(s), ${(outcome.bytes / 1024).toFixed(1)} KB, as ${target.archive} (#1876)${vanishedNote(outcome.vanished)}`);
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* One file, encrypted under the backup key, as `<stem>.<part>.tar.enc`. With
|
|
126
|
+
* `requireKey` and no key the file is left out: its record is null and a
|
|
127
|
+
* warning says why (#1876: the credentials store is never in a backup
|
|
128
|
+
* unencrypted).
|
|
129
|
+
*/
|
|
130
|
+
async function backupFilePart(manager, part, options, out) {
|
|
131
|
+
if (!part.file) {
|
|
132
|
+
out.records[part.part] = null;
|
|
133
|
+
out.notes.push(`${manager} has no ${part.what} configured`);
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
const target = archiveTarget(options.stem, part.part);
|
|
137
|
+
if (part.requireKey && !target.backupKey) {
|
|
138
|
+
const warning = `${LEFT_OUT_FOR_NO_KEY}the ${part.what} was left out: it is only ever backed up encrypted under the backup key (#1876)`;
|
|
139
|
+
logger.error(`[${manager}] ${warning}`);
|
|
140
|
+
out.records[part.part] = null;
|
|
141
|
+
out.warnings.push(warning);
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
const entries = existsSync(part.file) ? [path.basename(part.file)] : [];
|
|
145
|
+
const outcome = await writeTreeArchive(path.join(options.directory, target.archive), path.dirname(part.file), { backupKey: target.backupKey, entries });
|
|
146
|
+
logger.info(`[${manager}] Backed up the ${part.what} (${outcome.files} file, ${(outcome.bytes / 1024).toFixed(1)} KB) as ${target.archive} (#1876)${vanishedNote(outcome.vanished)}`);
|
|
147
|
+
out.records[part.part] = { format: 1, archive: target.archive, files: outcome.files, bytes: outcome.bytes, ...(target.keyFingerprint ? { keyFingerprint: target.keyFingerprint } : {}) };
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Put the declared parts' records into a manager's entry, in the shape
|
|
151
|
+
* every backup has had: the entry's `data` is the record when the manager
|
|
152
|
+
* has no data of its own and declares one part (roles, persons, …, an
|
|
153
|
+
* add-on's folder); otherwise each record is `data[<part>]` beside the
|
|
154
|
+
* manager's own data (the credentials store beside the provider list).
|
|
155
|
+
*/
|
|
156
|
+
export function placePartRecords(entry, declared, parts) {
|
|
157
|
+
const own = entry.data;
|
|
158
|
+
const data = declared.length === 1 && (own === null || own === undefined)
|
|
159
|
+
? parts.records[declared[0].part] ?? null
|
|
160
|
+
: { ...(own && typeof own === 'object' ? own : {}), ...parts.records };
|
|
161
|
+
const warnings = [...(entry.warnings ?? []), ...parts.warnings];
|
|
162
|
+
const note = [entry.note, ...parts.notes].filter(Boolean).join('; ');
|
|
163
|
+
return { ...entry, data, ...(warnings.length > 0 ? { warnings } : {}), ...(note ? { note } : {}) };
|
|
164
|
+
}
|
|
165
|
+
/** A declared part's record in a manager's entry, read the way `placePartRecords` wrote it. */
|
|
166
|
+
export function partRecordIn(entry, declared, part) {
|
|
167
|
+
const data = entry?.data;
|
|
168
|
+
if (declared.length === 1 && isPartRecord(data))
|
|
169
|
+
return data;
|
|
170
|
+
return data && typeof data === 'object' ? data[part] : undefined;
|
|
171
|
+
}
|
|
172
|
+
function isPartRecord(value) {
|
|
173
|
+
return Boolean(value) && typeof value === 'object' && value.format === 1 && 'archive' in value;
|
|
174
|
+
}
|
|
175
|
+
function requireSiteRestore(manager, options) {
|
|
176
|
+
if (!options?.siteRestore || !options.directory) {
|
|
177
|
+
throw new Error(`${manager}'s restore is staged with the rest of the site, so it is restored through BackupManager (#1876)`);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
/** The archive a part's record names, checked: format, a bare name, and the key it needs. */
|
|
181
|
+
function readablePart(manager, record) {
|
|
182
|
+
if (!isPartRecord(record)) {
|
|
183
|
+
throw new Error(`The backup holds no ${manager} files (it was taken before they were backed up, #1876); nothing was staged`);
|
|
184
|
+
}
|
|
185
|
+
if (!isArchiveName(record.archive)) {
|
|
186
|
+
throw new Error(`The backup names ${manager}'s archive '${String(record.archive)}', which is not a file in the backup directory; nothing was staged`);
|
|
187
|
+
}
|
|
188
|
+
return { data: record, backupKey: archiveReadKey(manager, record) };
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Stage a manager's declared parts from its entry in a backup, for the next
|
|
192
|
+
* start: each archive is extracted to `<path>.staged`, and the swaps are
|
|
193
|
+
* recorded in the site restore together. Throws — staging nothing, and
|
|
194
|
+
* removing what it extracted — outside a site restore, for a backup without
|
|
195
|
+
* a part that is not optional, or for an archive that is missing, unsafe,
|
|
196
|
+
* or under another key. Returns the parts the backup left out.
|
|
197
|
+
*/
|
|
198
|
+
export async function stageDeclaredParts(manager, declared, entry, options) {
|
|
199
|
+
requireSiteRestore(manager, options);
|
|
200
|
+
const swaps = [];
|
|
201
|
+
const leftOut = [];
|
|
202
|
+
try {
|
|
203
|
+
for (const part of declared) {
|
|
204
|
+
const record = partRecordIn(entry, declared, part.part);
|
|
205
|
+
const live = livePathOf(part);
|
|
206
|
+
if (!isPartRecord(record) && (part.optional || isFilePart(part))) {
|
|
207
|
+
logger.warn(`[${manager}] The backup holds no ${part.part} part (left out when it was taken); this site's is kept, and named as not restored (#1876)`);
|
|
208
|
+
leftOut.push(part.part);
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
if (!live)
|
|
212
|
+
throw new Error(`${manager} has no ${isFilePart(part) ? part.what : 'folder'} configured; nothing was staged`);
|
|
213
|
+
const { data, backupKey } = readablePart(manager, record);
|
|
214
|
+
const staged = stagedPathFor(live);
|
|
215
|
+
swaps.push({ live, staged });
|
|
216
|
+
if (isFilePart(part))
|
|
217
|
+
await stageOneFile(live, path.join(options.directory, data.archive), backupKey, part.whenAbsent);
|
|
218
|
+
else
|
|
219
|
+
await extractTreeArchive(path.join(options.directory, data.archive), staged, { backupKey });
|
|
220
|
+
}
|
|
221
|
+
if (swaps.length > 0)
|
|
222
|
+
options.siteRestore.record(swaps);
|
|
223
|
+
}
|
|
224
|
+
catch (err) {
|
|
225
|
+
for (const s of swaps)
|
|
226
|
+
rmSync(s.staged, { recursive: true, force: true });
|
|
227
|
+
throw err;
|
|
228
|
+
}
|
|
229
|
+
for (const s of swaps)
|
|
230
|
+
logger.warn(`[${manager}] ${path.basename(s.live)} staged as ${path.basename(s.staged)} — applied at the next start with the rest of the site restore (#1876)`);
|
|
231
|
+
return { leftOut };
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* One file from its archive as `<file>.staged`. When the file was absent at
|
|
235
|
+
* backup time the archive holds nothing, and `whenAbsent` is staged in its
|
|
236
|
+
* place (an empty store), so the restore still puts the site back as it was.
|
|
237
|
+
*/
|
|
238
|
+
async function stageOneFile(file, archive, backupKey, whenAbsent) {
|
|
239
|
+
const staged = stagedPathFor(file);
|
|
240
|
+
const tmp = `${staged}.extract`;
|
|
241
|
+
try {
|
|
242
|
+
const name = path.basename(file);
|
|
243
|
+
const { files } = await extractTreeArchive(archive, tmp, { backupKey, select: (rel) => rel === name });
|
|
244
|
+
rmSync(staged, { force: true });
|
|
245
|
+
if (files === 1)
|
|
246
|
+
renameSync(path.join(tmp, name), staged);
|
|
247
|
+
else
|
|
248
|
+
writeFileSync(staged, whenAbsent, { mode: 0o600 });
|
|
249
|
+
}
|
|
250
|
+
finally {
|
|
251
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Stage one file from a backup for the next start and record the swap: the
|
|
256
|
+
* instance `.env`, BackupManager's own part.
|
|
257
|
+
*/
|
|
258
|
+
export async function stageFilePart(manager, file, data, options, { whenAbsent }) {
|
|
259
|
+
await stageDeclaredParts(manager, [{ part: 'file', file, what: path.basename(file), whenAbsent }], { managerName: manager, timestamp: '', data }, options);
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Put a staged restore of a manager's declared parts in place, at start,
|
|
263
|
+
* before the manager reads them. Only the paths it declares may be
|
|
264
|
+
* replaced, whatever the manifest says; a staged copy no committed restore
|
|
265
|
+
* names (an interrupted staging) is set aside. Loud when something was
|
|
266
|
+
* staged but not applied. Null when there was nothing to do.
|
|
267
|
+
*/
|
|
268
|
+
export function applyDeclaredPartsAtStart(manager, declared, configManager) {
|
|
269
|
+
const lives = declared.map(livePathOf).filter((p) => Boolean(p)).map((p) => path.resolve(p));
|
|
270
|
+
if (lives.length === 0)
|
|
271
|
+
return null;
|
|
272
|
+
const file = siteRestoreFile(configManager);
|
|
273
|
+
let backupTakenAt;
|
|
274
|
+
try {
|
|
275
|
+
const manifest = readSiteRestore(file);
|
|
276
|
+
if (manifest?.state === 'staged' && manifest.parts.some((p) => p.manager === manager && !p.status))
|
|
277
|
+
backupTakenAt = manifest.backup;
|
|
278
|
+
}
|
|
279
|
+
catch { /* the gate below says so */ }
|
|
280
|
+
const outcome = applyPartAtStart(file, manager, {
|
|
281
|
+
allowed: (swap) => lives.includes(path.resolve(swap.live)),
|
|
282
|
+
strays: () => lives.map(stagedPathFor).filter((p) => existsSync(p))
|
|
283
|
+
});
|
|
284
|
+
if (!outcome)
|
|
285
|
+
return null;
|
|
286
|
+
if (outcome.refused)
|
|
287
|
+
logger.error(`[${manager}] A staged restore was NOT applied: ${outcome.refused} (#1876)`);
|
|
288
|
+
for (const a of outcome.applied)
|
|
289
|
+
logger.warn(`[${manager}] Restore applied: ${a.live} (previous kept as ${path.basename(a.kept)}) (#1876)`);
|
|
290
|
+
return { ...outcome, ...(backupTakenAt ? { backupTakenAt } : {}) };
|
|
291
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/** The part's name in the site restore manifest: BackupManager owns the `.env` part. */
|
|
2
|
+
export declare const INSTANCE_ENV_PART = "BackupManager";
|
|
3
|
+
/** The lines a restore keeps from the live file, never from the backup. */
|
|
4
|
+
export declare const KEPT_ON_RESTORE: readonly string[];
|
|
5
|
+
/** The instance `.env`: the one file ngdpbase writes generated secrets into (#1194). */
|
|
6
|
+
export declare function instanceEnvPath(instanceDataDir: string): string;
|
|
7
|
+
/**
|
|
8
|
+
* The names the instance `.env` sets, sorted, read the way the boot loads it
|
|
9
|
+
* (`dotenv.parse`, `bootstrap-env.ts`): never a value (#1976).
|
|
10
|
+
*/
|
|
11
|
+
export declare function envKeyNames(content: string): string[];
|
|
12
|
+
/** A short fingerprint of a set of `.env` names (#1976): the first 16 hex digits of a SHA-256 of the sorted names, one per line. */
|
|
13
|
+
export declare function envKeyNamesFingerprint(names: readonly string[]): string;
|
|
14
|
+
/** The file without any line assigning one of `names`. */
|
|
15
|
+
export declare function withoutAssignments(content: string, names: readonly string[]): string;
|
|
16
|
+
/** What goes into a backup: the file without the backup key's line. */
|
|
17
|
+
export declare function envForBackup(content: string): string;
|
|
18
|
+
/**
|
|
19
|
+
* The file a restore puts in place: the backup's, without the kept names,
|
|
20
|
+
* then the live file's lines for them (a name the live file does not set
|
|
21
|
+
* stays unset).
|
|
22
|
+
*/
|
|
23
|
+
export declare function mergeRestoredEnv(restored: string, live: string): string;
|
|
24
|
+
/**
|
|
25
|
+
* The site restore manifest's path, read before the configuration exists:
|
|
26
|
+
* `ngdpbase.backup.restore-manifest` from the custom file, else the shipped
|
|
27
|
+
* default, with `${FAST_STORAGE}` (and `./data`) resolved to the instance data
|
|
28
|
+
* folder and `${SLOW_STORAGE}` to its variable — as `getResolvedDataPath`
|
|
29
|
+
* resolves it once the configuration is loaded.
|
|
30
|
+
*/
|
|
31
|
+
export declare function siteRestoreFileAtBoot(instanceDataDir: string, env?: Record<string, string | undefined>, cwd?: string): string;
|
|
32
|
+
export interface EnvRestoreOutcome {
|
|
33
|
+
/** The file before and after, when the restore was applied: the boot reloads the environment from them. */
|
|
34
|
+
applied?: {
|
|
35
|
+
before: string;
|
|
36
|
+
after: string;
|
|
37
|
+
};
|
|
38
|
+
refused?: string;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Put a staged `.env` in place at the start of the process, when the whole
|
|
42
|
+
* site restore is there to apply (`siteRestore.ts`). The live file is kept as
|
|
43
|
+
* `.env.pre-restore`; the kept lines are carried over first. Null when there
|
|
44
|
+
* is nothing to do.
|
|
45
|
+
*/
|
|
46
|
+
export declare function applyInstanceEnvAtStart(instanceDataDir: string, siteFile: string): EnvRestoreOutcome | null;
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The instance `.env` in a backup, and its staged restore (#1876, operator
|
|
3
|
+
* plan "one thing to save", 2026-10-20).
|
|
4
|
+
*
|
|
5
|
+
* `<FAST_STORAGE>/.env` holds the keys the site made for itself and the ones
|
|
6
|
+
* the operator put there: the database key, the credentials key, the session
|
|
7
|
+
* secret, the OpenID Connect keys, mail and add-on secrets. A backup carries
|
|
8
|
+
* it, encrypted under the backup key, __without the backup key's own line__:
|
|
9
|
+
* that one key is what the operator saves off the server, and a backup that
|
|
10
|
+
* held it would hold the key to itself.
|
|
11
|
+
*
|
|
12
|
+
* A restore stages it as `<FAST_STORAGE>/.env.staged` with the rest of the
|
|
13
|
+
* site restore. It is applied at the very start of the process
|
|
14
|
+
* (`bootstrap-env.ts`), before the file is loaded, because every key in it is
|
|
15
|
+
* read from the environment from then on. Two lines are always kept from the
|
|
16
|
+
* live file, never taken from the backup:
|
|
17
|
+
*
|
|
18
|
+
* - `NGDPBASE_BACKUP_KEY` — the key this site's backups are made under;
|
|
19
|
+
* - `NGDPBASE_DATABASE_KEY` — the restored databases are staged under the
|
|
20
|
+
* site's own database key (`DatabaseManager.restore`), so the key they open
|
|
21
|
+
* with is the live one.
|
|
22
|
+
*
|
|
23
|
+
* Everything here is synchronous and depends on nothing but the filesystem:
|
|
24
|
+
* it runs before the logger, the configuration or any manager exists.
|
|
25
|
+
*/
|
|
26
|
+
import { createHash } from 'node:crypto';
|
|
27
|
+
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
28
|
+
import path from 'node:path';
|
|
29
|
+
import dotenv from 'dotenv';
|
|
30
|
+
import { BACKUP_KEY_ENV } from './backupKey.js';
|
|
31
|
+
import { DATABASE_KEY_ENV } from './databaseKey.js';
|
|
32
|
+
import { configFilePaths, readConfigFilesSync } from './configFiles.js';
|
|
33
|
+
import { applyPartAtStart, gatePart, SITE_RESTORE_MANIFEST_KEY, stagedPathFor } from './siteRestore.js';
|
|
34
|
+
/** The part's name in the site restore manifest: BackupManager owns the `.env` part. */
|
|
35
|
+
export const INSTANCE_ENV_PART = 'BackupManager';
|
|
36
|
+
/** The lines a restore keeps from the live file, never from the backup. */
|
|
37
|
+
export const KEPT_ON_RESTORE = [BACKUP_KEY_ENV, DATABASE_KEY_ENV];
|
|
38
|
+
/** The instance `.env`: the one file ngdpbase writes generated secrets into (#1194). */
|
|
39
|
+
export function instanceEnvPath(instanceDataDir) {
|
|
40
|
+
return path.join(instanceDataDir, '.env');
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The names the instance `.env` sets, sorted, read the way the boot loads it
|
|
44
|
+
* (`dotenv.parse`, `bootstrap-env.ts`): never a value (#1976).
|
|
45
|
+
*/
|
|
46
|
+
export function envKeyNames(content) {
|
|
47
|
+
return Object.keys(dotenv.parse(content)).sort();
|
|
48
|
+
}
|
|
49
|
+
/** A short fingerprint of a set of `.env` names (#1976): the first 16 hex digits of a SHA-256 of the sorted names, one per line. */
|
|
50
|
+
export function envKeyNamesFingerprint(names) {
|
|
51
|
+
return createHash('sha256').update([...names].sort().join('\n')).digest('hex').slice(0, 16);
|
|
52
|
+
}
|
|
53
|
+
function assigns(line, names) {
|
|
54
|
+
const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/.exec(line);
|
|
55
|
+
return m !== null && names.includes(m[1]);
|
|
56
|
+
}
|
|
57
|
+
/** The file without any line assigning one of `names`. */
|
|
58
|
+
export function withoutAssignments(content, names) {
|
|
59
|
+
return content.split('\n').filter((line) => !assigns(line, names)).join('\n');
|
|
60
|
+
}
|
|
61
|
+
/** What goes into a backup: the file without the backup key's line. */
|
|
62
|
+
export function envForBackup(content) {
|
|
63
|
+
return withoutAssignments(content, [BACKUP_KEY_ENV]);
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* The file a restore puts in place: the backup's, without the kept names,
|
|
67
|
+
* then the live file's lines for them (a name the live file does not set
|
|
68
|
+
* stays unset).
|
|
69
|
+
*/
|
|
70
|
+
export function mergeRestoredEnv(restored, live) {
|
|
71
|
+
const kept = live.split('\n').filter((line) => assigns(line, KEPT_ON_RESTORE));
|
|
72
|
+
const body = withoutAssignments(restored, KEPT_ON_RESTORE).replace(/\n*$/, '\n');
|
|
73
|
+
return kept.length === 0 ? body : `${body}# Kept from this site when a backup was restored (#1876): the backup key, and the database key the restored databases were staged under.\n${kept.join('\n')}\n`;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* The site restore manifest's path, read before the configuration exists:
|
|
77
|
+
* `ngdpbase.backup.restore-manifest` from the custom file, else the shipped
|
|
78
|
+
* default, with `${FAST_STORAGE}` (and `./data`) resolved to the instance data
|
|
79
|
+
* folder and `${SLOW_STORAGE}` to its variable — as `getResolvedDataPath`
|
|
80
|
+
* resolves it once the configuration is loaded.
|
|
81
|
+
*/
|
|
82
|
+
export function siteRestoreFileAtBoot(instanceDataDir, env = process.env, cwd = process.cwd()) {
|
|
83
|
+
let value = '${FAST_STORAGE}/site-restore.json';
|
|
84
|
+
try {
|
|
85
|
+
const files = readConfigFilesSync(configFilePaths({ ...env, FAST_STORAGE: instanceDataDir }, cwd));
|
|
86
|
+
const configured = files.customConfig[SITE_RESTORE_MANIFEST_KEY] ?? files.defaultConfig?.[SITE_RESTORE_MANIFEST_KEY];
|
|
87
|
+
if (typeof configured === 'string' && configured.trim())
|
|
88
|
+
value = configured.trim();
|
|
89
|
+
}
|
|
90
|
+
catch { /* a configuration that can't be read: the default, as the boot will find out itself */ }
|
|
91
|
+
const resolved = value
|
|
92
|
+
.replace(/^\$\{(FAST_STORAGE|INSTANCE_DATA_FOLDER)\}/, instanceDataDir)
|
|
93
|
+
.replace(/^\$\{SLOW_STORAGE\}/, env.SLOW_STORAGE ?? instanceDataDir)
|
|
94
|
+
.replace(/^\.\/data(?=\/|$)/, instanceDataDir);
|
|
95
|
+
return resolved.includes('${') ? path.join(instanceDataDir, path.basename(resolved)) : resolved;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Put a staged `.env` in place at the start of the process, when the whole
|
|
99
|
+
* site restore is there to apply (`siteRestore.ts`). The live file is kept as
|
|
100
|
+
* `.env.pre-restore`; the kept lines are carried over first. Null when there
|
|
101
|
+
* is nothing to do.
|
|
102
|
+
*/
|
|
103
|
+
export function applyInstanceEnvAtStart(instanceDataDir, siteFile) {
|
|
104
|
+
const live = instanceEnvPath(instanceDataDir);
|
|
105
|
+
const staged = stagedPathFor(live);
|
|
106
|
+
const gate = gatePart(siteFile, INSTANCE_ENV_PART);
|
|
107
|
+
const before = existsSync(live) ? readFileSync(live, 'utf8') : '';
|
|
108
|
+
if (gate.decision === 'apply' && gate.part.swaps.length === 1 && path.resolve(gate.part.swaps[0].live) === path.resolve(live) && existsSync(staged)) {
|
|
109
|
+
writeFileSync(staged, mergeRestoredEnv(readFileSync(staged, 'utf8'), before), { mode: 0o600 });
|
|
110
|
+
}
|
|
111
|
+
const outcome = applyPartAtStart(siteFile, INSTANCE_ENV_PART, {
|
|
112
|
+
allowed: (swap) => path.resolve(swap.live) === path.resolve(live),
|
|
113
|
+
strays: () => (existsSync(staged) ? [staged] : [])
|
|
114
|
+
});
|
|
115
|
+
if (!outcome)
|
|
116
|
+
return null;
|
|
117
|
+
if (outcome.refused)
|
|
118
|
+
return { refused: outcome.refused };
|
|
119
|
+
return { applied: { before, after: existsSync(live) ? readFileSync(live, 'utf8') : '' } };
|
|
120
|
+
}
|