@jwilleke/ngdpbase-types 5.20.0 → 5.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/packages/ngdpbase-types/index.d.ts +5 -0
- package/lib/src/mail/MailProvider.d.ts +2 -0
- package/lib/src/managers/AddonsManager.d.ts +3 -8
- package/lib/src/managers/AttachmentManager.d.ts +64 -12
- package/lib/src/managers/AuthManager.d.ts +24 -4
- package/lib/src/managers/BackgroundJobManager.d.ts +71 -1
- package/lib/src/managers/BaseManager.d.ts +20 -0
- package/lib/src/managers/BaseManager.js +9 -0
- package/lib/src/managers/CommentManager.d.ts +10 -1
- package/lib/src/managers/ConfigurationManager.d.ts +16 -1
- package/lib/src/managers/DatabaseManager.d.ts +53 -16
- package/lib/src/managers/DatabaseManager.js +172 -38
- package/lib/src/managers/EmailManager.d.ts +42 -0
- package/lib/src/managers/FootnoteManager.d.ts +10 -1
- package/lib/src/managers/NotificationManager.d.ts +13 -0
- package/lib/src/managers/OrganizationManager.d.ts +40 -6
- package/lib/src/managers/PersonManager.d.ts +106 -9
- package/lib/src/managers/RoleManager.d.ts +19 -1
- package/lib/src/managers/UserManager.d.ts +15 -0
- package/lib/src/providers/BaseAttachmentProvider.d.ts +14 -0
- package/lib/src/providers/BasicAttachmentProvider.d.ts +26 -2
- package/lib/src/providers/SqliteDatabaseProvider.d.ts +12 -0
- package/lib/src/providers/SqliteDatabaseProvider.js +50 -7
- package/lib/src/providers/SqliteJobStateProvider.d.ts +10 -0
- package/lib/src/providers/attachmentFilesBackup.d.ts +52 -0
- package/lib/src/providers/sqliteBackup.d.ts +6 -0
- package/lib/src/providers/sqliteBackup.js +16 -6
- package/lib/src/security/permissions.generated.d.ts +4 -0
- package/lib/src/types/Asset.d.ts +10 -0
- package/lib/src/types/Config.d.ts +11 -0
- package/lib/src/types/Organization.d.ts +7 -0
- package/lib/src/types/Person.d.ts +27 -3
- package/lib/src/types/PersonProvider.d.ts +6 -0
- package/lib/src/types/WikiEngine.d.ts +9 -21
- package/lib/src/utils/addonConfigLayer.d.ts +19 -0
- package/lib/src/utils/addonConfigLayer.js +45 -3
- package/lib/src/utils/addonPageSync.d.ts +24 -0
- package/lib/src/utils/administrators.d.ts +21 -0
- package/lib/src/utils/auditEventNames.generated.d.ts +4 -0
- package/lib/src/utils/auditEventNames.generated.js +4 -0
- package/lib/src/utils/backupCrypto.d.ts +28 -0
- package/lib/src/utils/backupKey.d.ts +23 -0
- package/lib/src/utils/backupKey.js +37 -0
- package/lib/src/utils/configPaths.d.ts +55 -0
- package/lib/src/utils/configPaths.js +133 -0
- package/lib/src/utils/contactAccess.d.ts +58 -0
- package/lib/src/utils/databaseKey.d.ts +7 -0
- package/lib/src/utils/databaseKey.js +7 -0
- package/lib/src/utils/directoryBackup.d.ts +78 -0
- package/lib/src/utils/emailForLog.d.ts +20 -0
- package/lib/src/utils/maintenanceState.d.ts +153 -0
- package/lib/src/utils/pluginFormatters.d.ts +6 -1
- package/lib/src/utils/pluginFormatters.js +5 -2
- package/lib/src/utils/privateStoreTakeout.d.ts +2 -2
- package/lib/src/utils/redactSecrets.js +31 -11
- package/lib/src/utils/sensitiveValues.d.ts +49 -3
- package/lib/src/utils/sensitiveValues.js +139 -11
- package/lib/src/utils/siteRestore.d.ts +110 -0
- package/lib/src/utils/siteRestore.js +309 -0
- package/lib/src/utils/treeArchive.d.ts +44 -0
- package/lib/src/utils/username.d.ts +7 -1
- package/package.json +1 -1
|
@@ -15,12 +15,20 @@
|
|
|
15
15
|
* hides secret-looking fields nested inside an object value (an OIDC client's
|
|
16
16
|
* `client_secret` inside `oidc-auth-server.clients`).
|
|
17
17
|
*
|
|
18
|
+
* An entry may also name something inside a key's value by its path (#1889):
|
|
19
|
+
* `ngdpbase.addons.accounting.accounts.*.account-number`, where `*` stands for
|
|
20
|
+
* any one map key (or list index). It hides what sits there, at the entry's
|
|
21
|
+
* level, wherever a value is shown, except a bare `$NAME` reference, which is
|
|
22
|
+
* a pointer to the environment, not the secret. The paths are walked by
|
|
23
|
+
* `utils/configPaths.ts`, the same walk that resolves `$NAME` inside a value.
|
|
24
|
+
*
|
|
18
25
|
* Hidden values are replaced on the server, before any HTML is built: they
|
|
19
26
|
* never reach a page, its source or the parse cache.
|
|
20
27
|
*
|
|
21
28
|
* @module utils/sensitiveValues
|
|
22
29
|
*/
|
|
23
30
|
import { enabledEntries } from './configFiles.js';
|
|
31
|
+
import { envRefName, mapConfigValue, parsePathEntry, pathMatches, pathString, patternBelow, valueAt } from './configPaths.js';
|
|
24
32
|
export const SENSITIVE_VALUES_KEY = 'ngdpbase.config.sensitive-values';
|
|
25
33
|
/** The list's name before #1750; still read, with a boot warning to rename it. */
|
|
26
34
|
export const LEGACY_SECRET_KEYS_KEY = 'ngdpbase.config.secret-keys';
|
|
@@ -30,34 +38,83 @@ const SECRET_NAME_PATTERN = /secret|passw(or)?d|token|credential|api[-_]?key|pri
|
|
|
30
38
|
export function looksSecret(name) {
|
|
31
39
|
return SECRET_NAME_PATTERN.test(name);
|
|
32
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* What an administrator's screen, a log line and an audit record may hold:
|
|
43
|
+
* sensitive values, never secrets. Secrets there are revealed one key at a
|
|
44
|
+
* time, on demand (`secret-reveal`).
|
|
45
|
+
*/
|
|
46
|
+
export const NO_SECRETS = Object.freeze({ sensitive: true, secret: false });
|
|
33
47
|
/** What a hidden value is shown as. */
|
|
34
48
|
export const HIDDEN = '(hidden)';
|
|
35
49
|
/**
|
|
36
50
|
* The declared levels: the legacy list's keys as `secret`, then the new map
|
|
37
51
|
* (`"secret"`, `"sensitive"`, `true` = secret; `false` / `null` removes an
|
|
38
|
-
* entry, so an instance can override a shipped one).
|
|
52
|
+
* entry, so an instance can override a shipped one). An entry that is not a
|
|
53
|
+
* well-formed key or path (#1889: an empty segment, a `*` inside a segment, or
|
|
54
|
+
* a `*` before `ngdpbase.<category>`) is no entry, as a level that is none of
|
|
55
|
+
* these is no entry; {@link sensitiveValueProblems} names both.
|
|
39
56
|
*/
|
|
40
57
|
export function valueLevels(read) {
|
|
41
58
|
const levels = new Map();
|
|
42
|
-
|
|
43
|
-
|
|
59
|
+
const entry = (key) => (parsePathEntry(key.trim()) ? key.trim() : null);
|
|
60
|
+
for (const key of enabledEntries(read(LEGACY_SECRET_KEYS_KEY, {}))) {
|
|
61
|
+
const k = entry(key);
|
|
62
|
+
if (k)
|
|
63
|
+
levels.set(k, 'secret');
|
|
64
|
+
}
|
|
44
65
|
const declared = read(SENSITIVE_VALUES_KEY, {});
|
|
45
66
|
if (declared && typeof declared === 'object' && !Array.isArray(declared)) {
|
|
46
67
|
for (const [key, level] of Object.entries(declared)) {
|
|
68
|
+
const k = entry(key);
|
|
69
|
+
if (!k)
|
|
70
|
+
continue;
|
|
47
71
|
if (level === 'secret' || level === true)
|
|
48
|
-
levels.set(
|
|
72
|
+
levels.set(k, 'secret');
|
|
49
73
|
else if (level === 'sensitive')
|
|
50
|
-
levels.set(
|
|
74
|
+
levels.set(k, 'sensitive');
|
|
51
75
|
else
|
|
52
|
-
levels.delete(
|
|
76
|
+
levels.delete(k);
|
|
53
77
|
}
|
|
54
78
|
}
|
|
55
79
|
else {
|
|
56
|
-
for (const key of enabledEntries(declared))
|
|
57
|
-
|
|
80
|
+
for (const key of enabledEntries(declared)) {
|
|
81
|
+
const k = entry(key);
|
|
82
|
+
if (k)
|
|
83
|
+
levels.set(k, 'secret');
|
|
84
|
+
}
|
|
58
85
|
}
|
|
59
86
|
return levels;
|
|
60
87
|
}
|
|
88
|
+
/**
|
|
89
|
+
* What is wrong with the declared entries, one line each (#1889): an entry
|
|
90
|
+
* that is not a well-formed key or path, or a level that is none of
|
|
91
|
+
* `"secret"`, `"sensitive"`, `true`, `false`, `null`. {@link valueLevels}
|
|
92
|
+
* drops each such entry; the engine warns at boot.
|
|
93
|
+
*/
|
|
94
|
+
export function sensitiveValueProblems(read) {
|
|
95
|
+
const problems = [];
|
|
96
|
+
const named = (list, key) => {
|
|
97
|
+
if (!parsePathEntry(key.trim())) {
|
|
98
|
+
problems.push(`${list}: '${key}' is not a configuration key or path (an empty segment, or a * that is not a whole segment after ngdpbase.<category>); it is ignored`);
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
for (const key of enabledEntries(read(LEGACY_SECRET_KEYS_KEY, {})))
|
|
102
|
+
named(LEGACY_SECRET_KEYS_KEY, key);
|
|
103
|
+
const declared = read(SENSITIVE_VALUES_KEY, {});
|
|
104
|
+
if (declared && typeof declared === 'object' && !Array.isArray(declared)) {
|
|
105
|
+
for (const [key, level] of Object.entries(declared)) {
|
|
106
|
+
named(SENSITIVE_VALUES_KEY, key);
|
|
107
|
+
if (!(level === 'secret' || level === 'sensitive' || level === true || level === false || level === null)) {
|
|
108
|
+
problems.push(`${SENSITIVE_VALUES_KEY}: '${key}' has level ${JSON.stringify(level)}, which is not "secret" or "sensitive"; the entry is ignored`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
for (const key of enabledEntries(declared))
|
|
114
|
+
named(SENSITIVE_VALUES_KEY, key);
|
|
115
|
+
}
|
|
116
|
+
return problems;
|
|
117
|
+
}
|
|
61
118
|
/** The keys declared `secret`: what logs, the audit log and the admin screen never print. */
|
|
62
119
|
export function secretKeys(read) {
|
|
63
120
|
return [...valueLevels(read)].filter(([, level]) => level === 'secret').map(([key]) => key);
|
|
@@ -91,9 +148,79 @@ function hideNestedSecrets(value) {
|
|
|
91
148
|
}
|
|
92
149
|
return value;
|
|
93
150
|
}
|
|
151
|
+
/** The path entries inside a key's value, as patterns below the key, with their levels. */
|
|
152
|
+
function entriesBelow(key, levels) {
|
|
153
|
+
const out = [];
|
|
154
|
+
for (const [entry, level] of levels) {
|
|
155
|
+
const pattern = patternBelow(entry, key);
|
|
156
|
+
if (pattern && pattern.length > 0)
|
|
157
|
+
out.push([pattern, level]);
|
|
158
|
+
}
|
|
159
|
+
return out;
|
|
160
|
+
}
|
|
161
|
+
/** Whether a viewer may see a value at this level. */
|
|
162
|
+
function maySee(level, viewer) {
|
|
163
|
+
return level === 'secret' ? viewer.secret : viewer.sensitive;
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* A key's value with what its path entries name hidden from this viewer
|
|
167
|
+
* (#1889). An empty value, and a bare `$NAME` reference, are kept: neither is
|
|
168
|
+
* the secret. The key's own level is {@link shownValue}'s business.
|
|
169
|
+
*/
|
|
170
|
+
export function hideDeclaredPaths(key, value, levels, viewer) {
|
|
171
|
+
const hidden = entriesBelow(key, levels).filter(([, level]) => !maySee(level, viewer)).map(([pattern]) => pattern);
|
|
172
|
+
if (hidden.length === 0)
|
|
173
|
+
return value;
|
|
174
|
+
return mapConfigValue(value, (node, below) => {
|
|
175
|
+
if (below.length === 0 || !hidden.some((pattern) => pathMatches(pattern, below)))
|
|
176
|
+
return undefined;
|
|
177
|
+
return node === undefined || node === null || node === '' || envRefName(node) ? node : HIDDEN;
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* The strings (and numbers, as text) a key's secret path entries name inside
|
|
182
|
+
* its value, each with its full path (#1889): what log redaction strikes.
|
|
183
|
+
* Everything under a named map or list counts.
|
|
184
|
+
*/
|
|
185
|
+
export function secretValuesBelow(key, value, levels) {
|
|
186
|
+
const secret = entriesBelow(key, levels).filter(([, level]) => level === 'secret').map(([pattern]) => pattern);
|
|
187
|
+
const found = [];
|
|
188
|
+
if (secret.length === 0)
|
|
189
|
+
return found;
|
|
190
|
+
mapConfigValue(value, (node, below) => {
|
|
191
|
+
if (below.length === 0 || !secret.some((pattern) => pathMatches(pattern, below)))
|
|
192
|
+
return undefined;
|
|
193
|
+
const at = [...below];
|
|
194
|
+
mapConfigValue(node, (leaf, under) => {
|
|
195
|
+
if (typeof leaf === 'string' || typeof leaf === 'number')
|
|
196
|
+
found.push({ path: pathString(key, [...at, ...under]), value: String(leaf) });
|
|
197
|
+
return undefined;
|
|
198
|
+
});
|
|
199
|
+
return node;
|
|
200
|
+
});
|
|
201
|
+
return found;
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* A value submitted from a screen that showed it with paths hidden, with each
|
|
205
|
+
* `HIDDEN` placeholder at a declared path put back to the stored value there
|
|
206
|
+
* (#1889). Without this, saving an edit to an accounts map would write the
|
|
207
|
+
* placeholder over every account number.
|
|
208
|
+
*/
|
|
209
|
+
export function keepHiddenValues(key, submitted, stored, levels) {
|
|
210
|
+
const patterns = entriesBelow(key, levels).map(([pattern]) => pattern);
|
|
211
|
+
if (patterns.length === 0)
|
|
212
|
+
return submitted;
|
|
213
|
+
return mapConfigValue(submitted, (node, below) => {
|
|
214
|
+
if (node !== HIDDEN || !patterns.some((pattern) => pathMatches(pattern, below)))
|
|
215
|
+
return undefined;
|
|
216
|
+
const kept = valueAt(stored, below);
|
|
217
|
+
return kept === undefined ? node : kept;
|
|
218
|
+
});
|
|
219
|
+
}
|
|
94
220
|
/**
|
|
95
|
-
* The value a viewer may see for a key: itself, or `HIDDEN
|
|
96
|
-
* is
|
|
221
|
+
* The value a viewer may see for a key: itself, or `HIDDEN`; inside it, what
|
|
222
|
+
* its path entries name is hidden as well (#1889). An empty value is shown as
|
|
223
|
+
* it is: "not set" reveals nothing.
|
|
97
224
|
*/
|
|
98
225
|
export function shownValue(key, value, levels, viewer) {
|
|
99
226
|
const empty = value === undefined || value === null || value === '';
|
|
@@ -102,5 +229,6 @@ export function shownValue(key, value, levels, viewer) {
|
|
|
102
229
|
return empty ? value : HIDDEN;
|
|
103
230
|
if (level === 'sensitive' && !viewer.sensitive)
|
|
104
231
|
return empty ? value : HIDDEN;
|
|
105
|
-
|
|
232
|
+
const shown = hideDeclaredPaths(key, value, levels, viewer);
|
|
233
|
+
return viewer.secret ? shown : hideNestedSecrets(shown);
|
|
106
234
|
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/** The config key naming the site restore manifest. */
|
|
2
|
+
export declare const SITE_RESTORE_MANIFEST_KEY = "ngdpbase.backup.restore-manifest";
|
|
3
|
+
/** Its default: beside the other instance data in FAST_STORAGE. */
|
|
4
|
+
export declare const SITE_RESTORE_MANIFEST_DEFAULT = "./data/site-restore.json";
|
|
5
|
+
/** One live path and the staged copy that replaces it at the next start. */
|
|
6
|
+
export interface StagedSwap {
|
|
7
|
+
live: string;
|
|
8
|
+
staged: string;
|
|
9
|
+
}
|
|
10
|
+
/** One manager's share of a site restore. */
|
|
11
|
+
export interface SiteRestorePart {
|
|
12
|
+
manager: string;
|
|
13
|
+
swaps: StagedSwap[];
|
|
14
|
+
/** Further files to remove when the restore is discarded (DatabaseManager's own manifest). */
|
|
15
|
+
discard?: string[];
|
|
16
|
+
status?: 'applied' | 'set-aside';
|
|
17
|
+
note?: string;
|
|
18
|
+
}
|
|
19
|
+
export interface SiteRestoreManifest {
|
|
20
|
+
version: 1;
|
|
21
|
+
state: 'staging' | 'staged' | 'refused';
|
|
22
|
+
reason?: string;
|
|
23
|
+
/** When staging began. */
|
|
24
|
+
stagedAt: string;
|
|
25
|
+
/** When the backup it came from was taken. */
|
|
26
|
+
backup: string;
|
|
27
|
+
/** The backup document's filename. */
|
|
28
|
+
source: string;
|
|
29
|
+
stagedBy: string;
|
|
30
|
+
/** The backup of the live data taken just before staging: restoring it undoes this one. */
|
|
31
|
+
safetyBackup?: string;
|
|
32
|
+
parts: SiteRestorePart[];
|
|
33
|
+
}
|
|
34
|
+
/** The staged copy of a live path: beside it, so staging and applying name the same path. */
|
|
35
|
+
export declare const stagedPathFor: (live: string) => string;
|
|
36
|
+
/** Where a live path is kept once a restore has replaced it. */
|
|
37
|
+
export declare const keptPathFor: (live: string) => string;
|
|
38
|
+
/** The manifest file for this site, read through the configuration (never a constant). */
|
|
39
|
+
export declare function siteRestoreFile(configManager: {
|
|
40
|
+
getResolvedDataPath(key: string, def: string): string;
|
|
41
|
+
} | null | undefined): string;
|
|
42
|
+
/**
|
|
43
|
+
* The manifest, or null when there is none. Throws when a file is there but
|
|
44
|
+
* is not a manifest: a restore record that can't be read is never ignored.
|
|
45
|
+
*/
|
|
46
|
+
export declare function readSiteRestore(file: string): SiteRestoreManifest | null;
|
|
47
|
+
/** Start staging a site restore: the record exists before any part is staged, so an interrupted staging is recognised. */
|
|
48
|
+
export declare function beginSiteRestore(file: string, meta: Omit<SiteRestoreManifest, 'version' | 'state' | 'parts'>): void;
|
|
49
|
+
/** Record one owner's staged part. Only while staging. */
|
|
50
|
+
export declare function recordStagedPart(file: string, part: SiteRestorePart): void;
|
|
51
|
+
/** The commit: every part is staged. */
|
|
52
|
+
export declare function commitSiteRestore(file: string): SiteRestoreManifest;
|
|
53
|
+
/**
|
|
54
|
+
* Throw a staged (or half-staged) site restore away: every part's staged
|
|
55
|
+
* files and the manifest are removed. Live data is never touched.
|
|
56
|
+
*/
|
|
57
|
+
export declare function discardSiteRestore(file: string): SiteRestoreManifest | null;
|
|
58
|
+
/** Move a staged path aside as `*.incomplete`, so it is neither applied nor retried. */
|
|
59
|
+
export declare function setAsidePath(staged: string): void;
|
|
60
|
+
export type PartGate = {
|
|
61
|
+
decision: 'none';
|
|
62
|
+
} | {
|
|
63
|
+
decision: 'apply';
|
|
64
|
+
part: SiteRestorePart;
|
|
65
|
+
manifest: SiteRestoreManifest;
|
|
66
|
+
} | {
|
|
67
|
+
decision: 'set-aside';
|
|
68
|
+
reason: string;
|
|
69
|
+
part?: SiteRestorePart;
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* Whether `manager`'s part of a site restore is to be applied now. `none`:
|
|
73
|
+
* there is no site restore, or it has no part for this manager, or that part
|
|
74
|
+
* is done. `apply`: the restore is committed and every part still pending has
|
|
75
|
+
* its staged files. `set-aside`: it isn't, and the set is marked refused.
|
|
76
|
+
*/
|
|
77
|
+
export declare function gatePart(file: string, manager: string): PartGate;
|
|
78
|
+
/**
|
|
79
|
+
* Record what became of `manager`'s part. A part set aside refuses the rest
|
|
80
|
+
* of the set. When every part is done the manifest becomes `<file>.done`.
|
|
81
|
+
*/
|
|
82
|
+
export declare function markPart(file: string, manager: string, status: 'applied' | 'set-aside', note?: string): void;
|
|
83
|
+
export interface PartOutcome {
|
|
84
|
+
/** Each live path now restored, with where its previous content was kept. */
|
|
85
|
+
applied: Array<{
|
|
86
|
+
live: string;
|
|
87
|
+
kept: string;
|
|
88
|
+
}>;
|
|
89
|
+
/** Why nothing (or, after a failed rename, not everything) was applied. */
|
|
90
|
+
refused?: string;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Apply `manager`'s part of a staged site restore, at start, before the
|
|
94
|
+
* manager reads its data. `allowed` says which live paths this manager may
|
|
95
|
+
* replace, so a tampered manifest cannot rename an arbitrary file.
|
|
96
|
+
* `strays` lists staged paths of this manager's that exist on disk, found by
|
|
97
|
+
* the manager: with no committed restore naming them they are set aside.
|
|
98
|
+
* Returns null when there was nothing to do.
|
|
99
|
+
*/
|
|
100
|
+
export declare function applyPartAtStart(file: string, manager: string, options: {
|
|
101
|
+
allowed: (swap: StagedSwap) => boolean;
|
|
102
|
+
strays?: () => string[];
|
|
103
|
+
}): PartOutcome | null;
|
|
104
|
+
/**
|
|
105
|
+
* What a start did with a site restore, for BackupManager to tell admins at
|
|
106
|
+
* the end of the start: the finished record (`<file>.done`, then kept as
|
|
107
|
+
* `<file>.last`), or a record with parts still pending — their managers did
|
|
108
|
+
* not start — which is then refused and its pending parts set aside.
|
|
109
|
+
*/
|
|
110
|
+
export declare function finishSiteRestoreAtBoot(file: string): SiteRestoreManifest | null;
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A whole-site restore, staged for the next start, all or nothing (#1876).
|
|
3
|
+
*
|
|
4
|
+
* The databases already restore this way (#1723): each is copied to
|
|
5
|
+
* `<live>.staged`, and the next start renames it into place before anything
|
|
6
|
+
* opens it. This extends the same shape to every part of the site whose
|
|
7
|
+
* owning manager stages its restore (attachments, role membership, persons,
|
|
8
|
+
* organizations, footnotes and comments). Each owner does its own work, as
|
|
9
|
+
* Manager-SOT asks; this module only holds the one record that binds the
|
|
10
|
+
* parts together, so they go in together or not at all.
|
|
11
|
+
*
|
|
12
|
+
* The record is the __site restore manifest__ (`ngdpbase.backup.restore-manifest`,
|
|
13
|
+
* default `${FAST_STORAGE}/site-restore.json`):
|
|
14
|
+
*
|
|
15
|
+
* - __Staging__ (BackupManager.restoreFromFile): the manifest is written in
|
|
16
|
+
* state `staging` before any part is staged. Each owner stages its files
|
|
17
|
+
* beside its live ones and records its part — the pairs `{ live, staged }`
|
|
18
|
+
* it will swap. Any failure discards every part already recorded and the
|
|
19
|
+
* manifest. Only when every part is staged is the state set to `staged`:
|
|
20
|
+
* that write is the commit.
|
|
21
|
+
* - __Applying__ (each owner's `initialize`, before it reads its data): the
|
|
22
|
+
* owner asks {@link gatePart}. A part is applied only when the manifest is
|
|
23
|
+
* committed and every part that is still pending has all of its staged
|
|
24
|
+
* files in place, so a set that lost a part applies none of it. Each live
|
|
25
|
+
* path steps aside as `<live>.pre-restore`, and the staged one takes its
|
|
26
|
+
* name: a rename, never a write into live data. The part is then marked
|
|
27
|
+
* applied in the manifest, so a start that stops half-way picks up where it
|
|
28
|
+
* stopped instead of seeing a "missing" part.
|
|
29
|
+
* - __Refused__: an interrupted staging, a part whose staged files are gone,
|
|
30
|
+
* or a tampered manifest (an owner checks every live path it is asked to
|
|
31
|
+
* replace) sets the manifest to `refused`, and each part's staged files are
|
|
32
|
+
* set aside as `*.incomplete` — never applied, never retried.
|
|
33
|
+
* - __Done__: once every part is applied or set aside, the manifest is
|
|
34
|
+
* renamed `<manifest>.done`, which BackupManager reports to admins at the
|
|
35
|
+
* end of the start and keeps as `<manifest>.last`.
|
|
36
|
+
*
|
|
37
|
+
* Everything here is synchronous file renames on the owner's own paths: it
|
|
38
|
+
* runs once at start, before the owner serves anything.
|
|
39
|
+
*/
|
|
40
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
41
|
+
import path from 'node:path';
|
|
42
|
+
/** The config key naming the site restore manifest. */
|
|
43
|
+
export const SITE_RESTORE_MANIFEST_KEY = 'ngdpbase.backup.restore-manifest';
|
|
44
|
+
/** Its default: beside the other instance data in FAST_STORAGE. */
|
|
45
|
+
export const SITE_RESTORE_MANIFEST_DEFAULT = './data/site-restore.json';
|
|
46
|
+
/** The staged copy of a live path: beside it, so staging and applying name the same path. */
|
|
47
|
+
export const stagedPathFor = (live) => `${live}.staged`;
|
|
48
|
+
/** Where a live path is kept once a restore has replaced it. */
|
|
49
|
+
export const keptPathFor = (live) => `${live}.pre-restore`;
|
|
50
|
+
/** The manifest file for this site, read through the configuration (never a constant). */
|
|
51
|
+
export function siteRestoreFile(configManager) {
|
|
52
|
+
return configManager?.getResolvedDataPath(SITE_RESTORE_MANIFEST_KEY, SITE_RESTORE_MANIFEST_DEFAULT) ?? SITE_RESTORE_MANIFEST_DEFAULT;
|
|
53
|
+
}
|
|
54
|
+
function isFile(file) {
|
|
55
|
+
try {
|
|
56
|
+
return statSync(file).isFile();
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* The manifest, or null when there is none. Throws when a file is there but
|
|
64
|
+
* is not a manifest: a restore record that can't be read is never ignored.
|
|
65
|
+
*/
|
|
66
|
+
export function readSiteRestore(file) {
|
|
67
|
+
if (!isFile(file))
|
|
68
|
+
return null;
|
|
69
|
+
const parsed = JSON.parse(readFileSync(file, 'utf8'));
|
|
70
|
+
if (parsed?.version !== 1 || !Array.isArray(parsed.parts) || !['staging', 'staged', 'refused'].includes(parsed.state)) {
|
|
71
|
+
throw new Error(`${path.basename(file)} is not a site restore manifest`);
|
|
72
|
+
}
|
|
73
|
+
return parsed;
|
|
74
|
+
}
|
|
75
|
+
function writeSiteRestore(file, manifest) {
|
|
76
|
+
mkdirSync(path.dirname(file), { recursive: true });
|
|
77
|
+
const tmp = `${file}.tmp`;
|
|
78
|
+
writeFileSync(tmp, JSON.stringify(manifest, null, 2) + '\n', { mode: 0o600 });
|
|
79
|
+
renameSync(tmp, file);
|
|
80
|
+
}
|
|
81
|
+
/** Start staging a site restore: the record exists before any part is staged, so an interrupted staging is recognised. */
|
|
82
|
+
export function beginSiteRestore(file, meta) {
|
|
83
|
+
if (isFile(file))
|
|
84
|
+
throw new Error('A site restore is already staged; cancel it or replace it first');
|
|
85
|
+
writeSiteRestore(file, { version: 1, state: 'staging', ...meta, parts: [] });
|
|
86
|
+
}
|
|
87
|
+
/** Record one owner's staged part. Only while staging. */
|
|
88
|
+
export function recordStagedPart(file, part) {
|
|
89
|
+
const manifest = readSiteRestore(file);
|
|
90
|
+
if (!manifest || manifest.state !== 'staging')
|
|
91
|
+
throw new Error('No site restore is being staged');
|
|
92
|
+
manifest.parts = [...manifest.parts.filter((p) => p.manager !== part.manager), { manager: part.manager, swaps: part.swaps, ...(part.discard ? { discard: part.discard } : {}) }];
|
|
93
|
+
writeSiteRestore(file, manifest);
|
|
94
|
+
}
|
|
95
|
+
/** The commit: every part is staged. */
|
|
96
|
+
export function commitSiteRestore(file) {
|
|
97
|
+
const manifest = readSiteRestore(file);
|
|
98
|
+
if (!manifest || manifest.state !== 'staging')
|
|
99
|
+
throw new Error('No site restore is being staged');
|
|
100
|
+
for (const part of manifest.parts) {
|
|
101
|
+
const missing = part.swaps.filter((s) => !existsSync(s.staged));
|
|
102
|
+
if (missing.length > 0)
|
|
103
|
+
throw new Error(`${part.manager} recorded staged files that are not there: ${missing.map((s) => path.basename(s.staged)).join(', ')}`);
|
|
104
|
+
}
|
|
105
|
+
manifest.state = 'staged';
|
|
106
|
+
writeSiteRestore(file, manifest);
|
|
107
|
+
return manifest;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Throw a staged (or half-staged) site restore away: every part's staged
|
|
111
|
+
* files and the manifest are removed. Live data is never touched.
|
|
112
|
+
*/
|
|
113
|
+
export function discardSiteRestore(file) {
|
|
114
|
+
let manifest;
|
|
115
|
+
try {
|
|
116
|
+
manifest = readSiteRestore(file);
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
manifest = null;
|
|
120
|
+
}
|
|
121
|
+
for (const part of manifest?.parts ?? []) {
|
|
122
|
+
for (const swap of part.swaps)
|
|
123
|
+
rmSync(swap.staged, { recursive: true, force: true });
|
|
124
|
+
for (const extra of part.discard ?? [])
|
|
125
|
+
rmSync(extra, { recursive: true, force: true });
|
|
126
|
+
}
|
|
127
|
+
rmSync(file, { force: true });
|
|
128
|
+
return manifest;
|
|
129
|
+
}
|
|
130
|
+
/** Move a staged path aside as `*.incomplete`, so it is neither applied nor retried. */
|
|
131
|
+
export function setAsidePath(staged) {
|
|
132
|
+
if (!existsSync(staged))
|
|
133
|
+
return;
|
|
134
|
+
const target = `${staged}.incomplete`;
|
|
135
|
+
rmSync(target, { recursive: true, force: true });
|
|
136
|
+
renameSync(staged, target);
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Whether `manager`'s part of a site restore is to be applied now. `none`:
|
|
140
|
+
* there is no site restore, or it has no part for this manager, or that part
|
|
141
|
+
* is done. `apply`: the restore is committed and every part still pending has
|
|
142
|
+
* its staged files. `set-aside`: it isn't, and the set is marked refused.
|
|
143
|
+
*/
|
|
144
|
+
export function gatePart(file, manager) {
|
|
145
|
+
let manifest;
|
|
146
|
+
try {
|
|
147
|
+
manifest = readSiteRestore(file);
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
// A record that can't be read applies nothing. It is left where it is,
|
|
151
|
+
// untouched — the path may be misconfigured to name some other file —
|
|
152
|
+
// and every start says so until it is fixed.
|
|
153
|
+
return { decision: 'set-aside', reason: `the site restore manifest ${file} cannot be read: ${err.message}` };
|
|
154
|
+
}
|
|
155
|
+
if (!manifest)
|
|
156
|
+
return { decision: 'none' };
|
|
157
|
+
const part = manifest.parts.find((p) => p.manager === manager);
|
|
158
|
+
if (!part || part.status)
|
|
159
|
+
return { decision: 'none' };
|
|
160
|
+
if (manifest.state === 'staging') {
|
|
161
|
+
return refuse(file, manifest, part, 'the site restore was still being staged when the server stopped (interrupted staging)');
|
|
162
|
+
}
|
|
163
|
+
if (manifest.state === 'refused') {
|
|
164
|
+
return { decision: 'set-aside', reason: manifest.reason ?? 'the site restore was refused', part };
|
|
165
|
+
}
|
|
166
|
+
const setAsideAlready = manifest.parts.filter((p) => p.status === 'set-aside').map((p) => p.manager);
|
|
167
|
+
if (setAsideAlready.length > 0) {
|
|
168
|
+
return refuse(file, manifest, part, `part of it (${setAsideAlready.join(', ')}) was not applied, and the parts belong together`);
|
|
169
|
+
}
|
|
170
|
+
const missing = manifest.parts
|
|
171
|
+
.filter((p) => !p.status)
|
|
172
|
+
.flatMap((p) => p.swaps.filter((s) => !existsSync(s.staged)).map((s) => `${p.manager}: ${path.basename(s.staged)}`));
|
|
173
|
+
if (missing.length > 0) {
|
|
174
|
+
return refuse(file, manifest, part, `staged files are missing (${missing.join(', ')}), and the parts belong together`);
|
|
175
|
+
}
|
|
176
|
+
return { decision: 'apply', part, manifest };
|
|
177
|
+
}
|
|
178
|
+
function refuse(file, manifest, part, reason) {
|
|
179
|
+
manifest.state = 'refused';
|
|
180
|
+
manifest.reason = reason;
|
|
181
|
+
writeSiteRestore(file, manifest);
|
|
182
|
+
return { decision: 'set-aside', reason, part };
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Record what became of `manager`'s part. A part set aside refuses the rest
|
|
186
|
+
* of the set. When every part is done the manifest becomes `<file>.done`.
|
|
187
|
+
*/
|
|
188
|
+
export function markPart(file, manager, status, note) {
|
|
189
|
+
let manifest;
|
|
190
|
+
try {
|
|
191
|
+
manifest = readSiteRestore(file);
|
|
192
|
+
}
|
|
193
|
+
catch {
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
if (!manifest)
|
|
197
|
+
return;
|
|
198
|
+
const part = manifest.parts.find((p) => p.manager === manager);
|
|
199
|
+
if (!part)
|
|
200
|
+
return;
|
|
201
|
+
part.status = status;
|
|
202
|
+
if (note)
|
|
203
|
+
part.note = note;
|
|
204
|
+
if (status === 'set-aside' && manifest.state !== 'refused') {
|
|
205
|
+
manifest.state = 'refused';
|
|
206
|
+
manifest.reason = note ?? `${manager}'s part was not applied`;
|
|
207
|
+
}
|
|
208
|
+
writeSiteRestore(file, manifest);
|
|
209
|
+
if (manifest.parts.every((p) => p.status)) {
|
|
210
|
+
const done = `${file}.done`;
|
|
211
|
+
rmSync(done, { force: true });
|
|
212
|
+
renameSync(file, done);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Apply `manager`'s part of a staged site restore, at start, before the
|
|
217
|
+
* manager reads its data. `allowed` says which live paths this manager may
|
|
218
|
+
* replace, so a tampered manifest cannot rename an arbitrary file.
|
|
219
|
+
* `strays` lists staged paths of this manager's that exist on disk, found by
|
|
220
|
+
* the manager: with no committed restore naming them they are set aside.
|
|
221
|
+
* Returns null when there was nothing to do.
|
|
222
|
+
*/
|
|
223
|
+
export function applyPartAtStart(file, manager, options) {
|
|
224
|
+
const gate = gatePart(file, manager);
|
|
225
|
+
if (gate.decision === 'none') {
|
|
226
|
+
const strays = options.strays?.() ?? [];
|
|
227
|
+
if (strays.length === 0)
|
|
228
|
+
return null;
|
|
229
|
+
for (const s of strays)
|
|
230
|
+
setAsidePath(s);
|
|
231
|
+
return { applied: [], refused: `staged files with no site restore naming them (an interrupted staging): ${strays.map((s) => path.basename(s)).join(', ')} — set aside as *.incomplete` };
|
|
232
|
+
}
|
|
233
|
+
if (gate.decision === 'set-aside') {
|
|
234
|
+
for (const swap of gate.part?.swaps ?? [])
|
|
235
|
+
setAsidePath(swap.staged);
|
|
236
|
+
for (const s of options.strays?.() ?? [])
|
|
237
|
+
setAsidePath(s);
|
|
238
|
+
if (gate.part)
|
|
239
|
+
markPart(file, manager, 'set-aside', gate.reason);
|
|
240
|
+
return { applied: [], refused: gate.reason };
|
|
241
|
+
}
|
|
242
|
+
const bad = gate.part.swaps.find((s) => !options.allowed(s) || s.staged !== stagedPathFor(s.live));
|
|
243
|
+
if (bad) {
|
|
244
|
+
const reason = `the site restore manifest names ${bad.live}, which is not ${manager}'s to replace`;
|
|
245
|
+
for (const swap of gate.part.swaps)
|
|
246
|
+
if (swap.staged === stagedPathFor(swap.live))
|
|
247
|
+
setAsidePath(swap.staged);
|
|
248
|
+
markPart(file, manager, 'set-aside', reason);
|
|
249
|
+
return { applied: [], refused: reason };
|
|
250
|
+
}
|
|
251
|
+
const applied = [];
|
|
252
|
+
try {
|
|
253
|
+
for (const swap of gate.part.swaps) {
|
|
254
|
+
const kept = keptPathFor(swap.live);
|
|
255
|
+
rmSync(kept, { recursive: true, force: true });
|
|
256
|
+
if (existsSync(swap.live))
|
|
257
|
+
renameSync(swap.live, kept);
|
|
258
|
+
mkdirSync(path.dirname(swap.live), { recursive: true });
|
|
259
|
+
renameSync(swap.staged, swap.live);
|
|
260
|
+
applied.push({ live: swap.live, kept });
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
catch (err) {
|
|
264
|
+
const reason = `applying ${manager}'s part failed after ${applied.length} of ${gate.part.swaps.length} paths: ${err.message}`;
|
|
265
|
+
markPart(file, manager, 'set-aside', reason);
|
|
266
|
+
return { applied, refused: reason };
|
|
267
|
+
}
|
|
268
|
+
markPart(file, manager, 'applied');
|
|
269
|
+
return { applied };
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* What a start did with a site restore, for BackupManager to tell admins at
|
|
273
|
+
* the end of the start: the finished record (`<file>.done`, then kept as
|
|
274
|
+
* `<file>.last`), or a record with parts still pending — their managers did
|
|
275
|
+
* not start — which is then refused and its pending parts set aside.
|
|
276
|
+
*/
|
|
277
|
+
export function finishSiteRestoreAtBoot(file) {
|
|
278
|
+
const done = `${file}.done`;
|
|
279
|
+
let manifest;
|
|
280
|
+
try {
|
|
281
|
+
manifest = readSiteRestore(file);
|
|
282
|
+
}
|
|
283
|
+
catch {
|
|
284
|
+
manifest = null;
|
|
285
|
+
}
|
|
286
|
+
if (manifest) {
|
|
287
|
+
if (manifest.state === 'staging')
|
|
288
|
+
refuse(file, manifest, manifest.parts[0], 'the site restore was still being staged when the server stopped (interrupted staging)');
|
|
289
|
+
const pending = manifest.parts.filter((p) => !p.status);
|
|
290
|
+
for (const part of pending) {
|
|
291
|
+
for (const swap of part.swaps)
|
|
292
|
+
setAsidePath(swap.staged);
|
|
293
|
+
for (const extra of part.discard ?? [])
|
|
294
|
+
setAsidePath(extra);
|
|
295
|
+
markPart(file, part.manager, 'set-aside', `${part.manager} did not apply its part at start, so it was set aside`);
|
|
296
|
+
}
|
|
297
|
+
if (manifest.parts.length === 0 && isFile(file)) {
|
|
298
|
+
rmSync(done, { force: true });
|
|
299
|
+
renameSync(file, done);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
if (!isFile(done))
|
|
303
|
+
return null;
|
|
304
|
+
const finished = JSON.parse(readFileSync(done, 'utf8'));
|
|
305
|
+
const last = `${file}.last`;
|
|
306
|
+
rmSync(last, { force: true });
|
|
307
|
+
renameSync(done, last);
|
|
308
|
+
return finished;
|
|
309
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
export interface TreeArchiveOutcome {
|
|
2
|
+
/** Regular files written. */
|
|
3
|
+
files: number;
|
|
4
|
+
/** Their bytes. */
|
|
5
|
+
bytes: number;
|
|
6
|
+
/** Entries left out because they are not a file or a folder (a symbolic link), relative to the root. */
|
|
7
|
+
skipped: string[];
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Write `entries` (paths relative to `root`; default: everything in it) into
|
|
11
|
+
* `target`, streamed. `exclude` leaves out a path relative to `root`, in
|
|
12
|
+
* posix form. A missing `root` writes an empty archive. Never overwrites.
|
|
13
|
+
*/
|
|
14
|
+
export declare function writeTreeArchive(target: string, root: string, options?: {
|
|
15
|
+
entries?: string[];
|
|
16
|
+
exclude?: (relative: string) => boolean;
|
|
17
|
+
backupKey?: string;
|
|
18
|
+
}): Promise<TreeArchiveOutcome>;
|
|
19
|
+
/**
|
|
20
|
+
* Extract `archive` into `dest`, which is emptied first, streamed.
|
|
21
|
+
* `select(relative)` picks the entries to extract (default: all). Refuses —
|
|
22
|
+
* removing `dest` — an entry that is not a file or a folder, an absolute
|
|
23
|
+
* path, or one that climbs out (`..`).
|
|
24
|
+
*/
|
|
25
|
+
export declare function extractTreeArchive(archive: string, dest: string, options?: {
|
|
26
|
+
select?: (relative: string) => boolean;
|
|
27
|
+
strip?: number;
|
|
28
|
+
backupKey?: string;
|
|
29
|
+
}): Promise<{
|
|
30
|
+
files: number;
|
|
31
|
+
bytes: number;
|
|
32
|
+
}>;
|
|
33
|
+
/**
|
|
34
|
+
* Every entry in an archive, plain or encrypted, read as a stream: what a
|
|
35
|
+
* caller checks before extracting. `onEntry` gets each entry's path and type.
|
|
36
|
+
*/
|
|
37
|
+
export declare function listTreeArchive(archive: string, onEntry: (entry: {
|
|
38
|
+
path: string;
|
|
39
|
+
type: string;
|
|
40
|
+
}) => void, options?: {
|
|
41
|
+
backupKey?: string;
|
|
42
|
+
}): Promise<void>;
|
|
43
|
+
/** The first `n` bytes of a file, read without reading the rest. */
|
|
44
|
+
export declare function readHead(file: string, n: number): Promise<Buffer>;
|
|
@@ -49,5 +49,11 @@ export declare function sameUsername(a: unknown, b: unknown): boolean;
|
|
|
49
49
|
* checked separately by `isSystemPrincipal`, because an operator chooses it.
|
|
50
50
|
*/
|
|
51
51
|
export declare const RESERVED_USERNAMES: readonly string[];
|
|
52
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* The prefix of a person's identifier when the person has no account (#1930):
|
|
54
|
+
* `person:<uuid>`. No username may start with it, so an account-less person's
|
|
55
|
+
* identifier can never be, or later become, someone's username.
|
|
56
|
+
*/
|
|
57
|
+
export declare const ACCOUNTLESS_PERSON_PREFIX = "person:";
|
|
58
|
+
/** True when the name belongs to a principal rather than a person, or has the account-less person prefix. */
|
|
53
59
|
export declare function isReservedUsername(username: unknown): boolean;
|