@jwilleke/ngdpbase-types 5.25.0 → 5.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/packages/ngdpbase-types/index.d.ts +9 -2
- package/lib/packages/ngdpbase-types/testing.d.ts +7 -3
- package/lib/packages/ngdpbase-types/testing.js +7 -3
- package/lib/src/converters/DocxConverter.d.ts +30 -0
- package/lib/src/converters/MarkdownConverter.d.ts +31 -0
- package/lib/src/export/ChromeServicePdfProvider.d.ts +52 -0
- package/lib/src/export/ChromeServicePdfProvider.js +153 -0
- package/lib/src/export/LocalChromePdfProvider.d.ts +25 -0
- package/lib/src/export/LocalChromePdfProvider.js +33 -0
- package/lib/src/export/PdfProvider.d.ts +51 -0
- package/lib/src/export/PdfProvider.js +37 -0
- package/lib/src/export/pdfEngine.d.ts +113 -0
- package/lib/src/export/pdfEngine.js +195 -0
- package/lib/src/http/egressPolicy.js +61 -0
- package/lib/src/http/guardedElasticsearch.d.ts +27 -0
- package/lib/src/http/guardedElasticsearch.js +99 -0
- package/lib/src/ice/converterShim.d.ts +5 -0
- package/lib/src/ice/fileHandling.d.ts +30 -0
- package/lib/src/ice/ledger.d.ts +37 -0
- package/lib/src/ice/runImport.d.ts +51 -0
- package/lib/src/ice/types.d.ts +262 -0
- package/lib/src/mail/MailProvider.d.ts +15 -0
- package/lib/src/mail/NodemailerMailProvider.d.ts +1 -1
- package/lib/src/managers/AddonsManager.d.ts +55 -3
- package/lib/src/managers/AttachmentManager.d.ts +4 -5
- package/lib/src/managers/AuthManager.d.ts +150 -32
- package/lib/src/managers/BackgroundJobManager.d.ts +22 -12
- package/lib/src/managers/BackupManager.d.ts +779 -0
- package/lib/src/managers/BackupManager.js +2042 -0
- package/lib/src/managers/BaseManager.d.ts +156 -6
- package/lib/src/managers/BaseManager.js +64 -2
- package/lib/src/managers/CommentManager.d.ts +6 -6
- package/lib/src/managers/ConsentManager.d.ts +130 -0
- package/lib/src/managers/DatabaseManager.d.ts +6 -0
- package/lib/src/managers/DatabaseManager.js +25 -15
- package/lib/src/managers/EmailManager.d.ts +32 -5
- package/lib/src/managers/ExportManager.d.ts +62 -0
- package/lib/src/managers/ExportManager.js +147 -2
- package/lib/src/managers/FootnoteManager.d.ts +6 -6
- package/lib/src/managers/ImportManager.d.ts +590 -0
- package/lib/src/managers/OrganizationManager.d.ts +8 -6
- package/lib/src/managers/PageManager.d.ts +61 -1
- package/lib/src/managers/PersonManager.d.ts +8 -6
- package/lib/src/managers/RoleManager.d.ts +6 -8
- package/lib/src/providers/BaseAttachmentProvider.d.ts +4 -4
- package/lib/src/providers/BaseBackupProvider.d.ts +77 -0
- package/lib/src/providers/BaseBackupProvider.js +52 -0
- package/lib/src/providers/BasicAttachmentProvider.d.ts +4 -4
- package/lib/src/providers/FileBackupProvider.d.ts +34 -0
- package/lib/src/providers/FileBackupProvider.js +116 -0
- package/lib/src/providers/FileConsentProvider.d.ts +18 -0
- package/lib/src/providers/SqliteDatabaseProvider.d.ts +2 -2
- package/lib/src/providers/SqliteDatabaseProvider.js +12 -11
- package/lib/src/providers/attachmentFilesBackup.d.ts +5 -4
- package/lib/src/providers/databaseMigrations.js +45 -0
- package/lib/src/security/PolicyDecisionPoint.d.ts +14 -0
- package/lib/src/security/PolicyInformationPoint.d.ts +25 -2
- package/lib/src/types/Config.d.ts +2 -0
- package/lib/src/types/Consent.d.ts +90 -0
- package/lib/src/types/express.d.ts +6 -0
- package/lib/src/utils/addonHost.d.ts +45 -2
- package/lib/src/utils/addonHost.js +39 -3
- package/lib/src/utils/auditEventNames.generated.d.ts +20 -0
- package/lib/src/utils/auditEventNames.generated.js +20 -0
- package/lib/src/utils/backupCrypto.js +275 -0
- package/lib/src/utils/backupDocument.d.ts +36 -0
- package/lib/src/utils/backupDocument.js +99 -0
- package/lib/src/utils/browserDescription.d.ts +11 -0
- package/lib/src/utils/directoryBackup.d.ts +48 -48
- package/lib/src/utils/directoryBackup.js +291 -0
- package/lib/src/utils/frontmatterAccess.d.ts +54 -11
- package/lib/src/utils/importRunSummary.d.ts +64 -0
- package/lib/src/utils/instanceEnvBackup.d.ts +46 -0
- package/lib/src/utils/instanceEnvBackup.js +120 -0
- package/lib/src/utils/instanceEnvSecret.js +79 -0
- package/lib/src/utils/ipLocation.d.ts +11 -0
- package/lib/src/utils/ncmNotify.d.ts +30 -0
- package/lib/src/utils/privateStoreImport.d.ts +58 -0
- package/lib/src/utils/schedule.js +171 -0
- package/lib/src/utils/secretFileMode.js +78 -0
- package/lib/src/utils/siteRestore.d.ts +13 -16
- package/lib/src/utils/siteRestore.js +33 -6
- package/lib/src/utils/treeArchive.d.ts +2 -0
- package/lib/src/utils/treeArchive.js +242 -0
- package/lib/src/utils/version.js +408 -0
- package/lib/src/utils/zipArchive.d.ts +71 -0
- package/package.json +6 -1
- package/lib/src/utils/administrators.d.ts +0 -21
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Encrypted backup files (#1561, #1876): the backup document, every file
|
|
3
|
+
* archive beside it, and the credentials store, under `NGDPBASE_BACKUP_KEY`.
|
|
4
|
+
* The database files are encrypted by SQLCipher itself (`sqliteBackup.ts`);
|
|
5
|
+
* everything else a backup writes goes through here.
|
|
6
|
+
*
|
|
7
|
+
* Format, version 1:
|
|
8
|
+
*
|
|
9
|
+
* - `NGDPBK01` (8 bytes), then a big-endian u32 length and that many bytes of
|
|
10
|
+
* JSON header: the cipher, the key derivation (scrypt, its cost and a fresh
|
|
11
|
+
* random salt), the backup key's fingerprint (`backupKeyFingerprint`, never
|
|
12
|
+
* the key), an 8-byte random nonce prefix and the chunk size.
|
|
13
|
+
* - Then records: u32 plaintext length, one byte (1 on the last record, else
|
|
14
|
+
* 0), the ciphertext, and its 16-byte AES-256-GCM tag. Each record's nonce
|
|
15
|
+
* is the prefix and its u32 index; its additional data is the SHA-256 of
|
|
16
|
+
* the header, the index and the last-record byte. So a record moved,
|
|
17
|
+
* dropped, swapped from another file or cut off at the end fails, and so
|
|
18
|
+
* does any change to the header.
|
|
19
|
+
*
|
|
20
|
+
* Streamed both ways, a chunk at a time: a file's size never becomes memory.
|
|
21
|
+
* Decrypting checks the fingerprint first, so a backup made under another key
|
|
22
|
+
* is refused by name ({@link BackupKeyMismatchError}) before any work is done.
|
|
23
|
+
*/
|
|
24
|
+
import { createCipheriv, createDecipheriv, createHash, randomBytes, scrypt } from 'node:crypto';
|
|
25
|
+
import { Readable, Transform, Writable } from 'node:stream';
|
|
26
|
+
import { pipeline } from 'node:stream/promises';
|
|
27
|
+
import { backupKeyFingerprint, BACKUP_KEY_ENV } from './backupKey.js';
|
|
28
|
+
import { open } from 'node:fs/promises';
|
|
29
|
+
/** The first bytes of every encrypted backup file. */
|
|
30
|
+
export const BACKUP_FILE_MAGIC = Buffer.from('NGDPBK01', 'ascii');
|
|
31
|
+
/** What an encrypted backup file's name ends with. */
|
|
32
|
+
export const ENCRYPTED_SUFFIX = '.enc';
|
|
33
|
+
/** The key derivation's cost, written into each file's header so a reader uses the file's own. A format constant, not a tunable: changing it changes no file already written. */
|
|
34
|
+
const KDF = { N: 16384, r: 8, p: 1 };
|
|
35
|
+
const CHUNK_BYTES = 64 * 1024;
|
|
36
|
+
const SALT_BYTES = 16;
|
|
37
|
+
const NONCE_PREFIX_BYTES = 8;
|
|
38
|
+
const TAG_BYTES = 16;
|
|
39
|
+
const RECORD_HEAD = 5;
|
|
40
|
+
/** The largest header a reader accepts: a header is a few hundred bytes. */
|
|
41
|
+
const MAX_HEADER = 4096;
|
|
42
|
+
/** A backup file made under a different backup key than the one set now. */
|
|
43
|
+
export class BackupKeyMismatchError extends Error {
|
|
44
|
+
fileFingerprint;
|
|
45
|
+
keyFingerprint;
|
|
46
|
+
constructor(fileFingerprint, keyFingerprint) {
|
|
47
|
+
super(`it was made with a different backup key (fingerprint ${fileFingerprint}; ${BACKUP_KEY_ENV} now has ${keyFingerprint}). Set ${BACKUP_KEY_ENV} to the key it was made with`);
|
|
48
|
+
this.fileFingerprint = fileFingerprint;
|
|
49
|
+
this.keyFingerprint = keyFingerprint;
|
|
50
|
+
this.name = 'BackupKeyMismatchError';
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function deriveKey(backupKey, salt, kdf) {
|
|
54
|
+
const options = { N: kdf.N, r: kdf.r, p: kdf.p, maxmem: 256 * kdf.N * kdf.r };
|
|
55
|
+
return new Promise((resolve, reject) => scrypt(backupKey, salt, 32, options, (err, key) => (err ? reject(err) : resolve(key))));
|
|
56
|
+
}
|
|
57
|
+
function nonceFor(prefix, index) {
|
|
58
|
+
const nonce = Buffer.alloc(12);
|
|
59
|
+
prefix.copy(nonce, 0, 0, 8);
|
|
60
|
+
nonce.writeUInt32BE(index, 8);
|
|
61
|
+
return nonce;
|
|
62
|
+
}
|
|
63
|
+
function aadFor(headerHash, index, last) {
|
|
64
|
+
const tail = Buffer.alloc(5);
|
|
65
|
+
tail.writeUInt32BE(index, 0);
|
|
66
|
+
tail.writeUInt8(last ? 1 : 0, 4);
|
|
67
|
+
return Buffer.concat([headerHash, tail]);
|
|
68
|
+
}
|
|
69
|
+
function requireKey(backupKey) {
|
|
70
|
+
if (!backupKey.trim())
|
|
71
|
+
throw new Error(`a backup key is required: ${BACKUP_KEY_ENV} is not set`);
|
|
72
|
+
}
|
|
73
|
+
/** Whether these bytes (a file's first) are an encrypted backup file. */
|
|
74
|
+
export function isEncryptedBackup(head) {
|
|
75
|
+
return head.length >= BACKUP_FILE_MAGIC.length && head.subarray(0, BACKUP_FILE_MAGIC.length).equals(BACKUP_FILE_MAGIC);
|
|
76
|
+
}
|
|
77
|
+
/** Whether the file on disk is an encrypted backup file, read from its first bytes only. */
|
|
78
|
+
export async function fileIsEncrypted(file) {
|
|
79
|
+
const handle = await open(file, 'r');
|
|
80
|
+
try {
|
|
81
|
+
const head = Buffer.alloc(BACKUP_FILE_MAGIC.length);
|
|
82
|
+
const { bytesRead } = await handle.read(head, 0, head.length, 0);
|
|
83
|
+
return isEncryptedBackup(head.subarray(0, bytesRead));
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
await handle.close();
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/** A stream that encrypts what is written to it under `backupKey`. */
|
|
90
|
+
export function encryptStream(backupKey) {
|
|
91
|
+
requireKey(backupKey);
|
|
92
|
+
let key = null;
|
|
93
|
+
let headerHash;
|
|
94
|
+
let prefix;
|
|
95
|
+
let index = 0;
|
|
96
|
+
let pending = Buffer.alloc(0);
|
|
97
|
+
const record = (plain, last) => {
|
|
98
|
+
const cipher = createCipheriv('aes-256-gcm', key, nonceFor(prefix, index));
|
|
99
|
+
cipher.setAAD(aadFor(headerHash, index, last));
|
|
100
|
+
const ct = Buffer.concat([cipher.update(plain), cipher.final()]);
|
|
101
|
+
const head = Buffer.alloc(RECORD_HEAD);
|
|
102
|
+
head.writeUInt32BE(plain.length, 0);
|
|
103
|
+
head.writeUInt8(last ? 1 : 0, 4);
|
|
104
|
+
index++;
|
|
105
|
+
return Buffer.concat([head, ct, cipher.getAuthTag()]);
|
|
106
|
+
};
|
|
107
|
+
const start = async (stream) => {
|
|
108
|
+
if (key)
|
|
109
|
+
return;
|
|
110
|
+
const salt = randomBytes(SALT_BYTES);
|
|
111
|
+
prefix = randomBytes(NONCE_PREFIX_BYTES);
|
|
112
|
+
const header = {
|
|
113
|
+
format: 1,
|
|
114
|
+
cipher: 'aes-256-gcm',
|
|
115
|
+
kdf: { alg: 'scrypt', ...KDF, salt: salt.toString('base64') },
|
|
116
|
+
keyFingerprint: backupKeyFingerprint(backupKey),
|
|
117
|
+
noncePrefix: prefix.toString('base64'),
|
|
118
|
+
chunkBytes: CHUNK_BYTES
|
|
119
|
+
};
|
|
120
|
+
const json = Buffer.from(JSON.stringify(header), 'utf8');
|
|
121
|
+
const len = Buffer.alloc(4);
|
|
122
|
+
len.writeUInt32BE(json.length, 0);
|
|
123
|
+
const headerBytes = Buffer.concat([BACKUP_FILE_MAGIC, len, json]);
|
|
124
|
+
headerHash = createHash('sha256').update(headerBytes).digest();
|
|
125
|
+
key = await deriveKey(backupKey, salt, KDF);
|
|
126
|
+
stream.push(headerBytes);
|
|
127
|
+
};
|
|
128
|
+
return new Transform({
|
|
129
|
+
transform(chunk, _enc, done) {
|
|
130
|
+
start(this).then(() => {
|
|
131
|
+
pending = pending.length === 0 ? chunk : Buffer.concat([pending, chunk]);
|
|
132
|
+
// The last record is written at the end, so a full chunk is held back
|
|
133
|
+
// until more follows: the last-record byte is then always right.
|
|
134
|
+
while (pending.length > CHUNK_BYTES) {
|
|
135
|
+
this.push(record(pending.subarray(0, CHUNK_BYTES), false));
|
|
136
|
+
pending = pending.subarray(CHUNK_BYTES);
|
|
137
|
+
}
|
|
138
|
+
done();
|
|
139
|
+
}, done);
|
|
140
|
+
},
|
|
141
|
+
flush(done) {
|
|
142
|
+
start(this).then(() => {
|
|
143
|
+
this.push(record(pending, true));
|
|
144
|
+
done();
|
|
145
|
+
}, done);
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* A stream that decrypts an encrypted backup file written to it. Fails —
|
|
151
|
+
* naming why — on a file that is not one, a different backup key
|
|
152
|
+
* ({@link BackupKeyMismatchError}), any record that does not verify, and a
|
|
153
|
+
* file cut short or with bytes after its last record.
|
|
154
|
+
*/
|
|
155
|
+
export function decryptStream(backupKey) {
|
|
156
|
+
requireKey(backupKey);
|
|
157
|
+
let buffer = Buffer.alloc(0);
|
|
158
|
+
let header = null;
|
|
159
|
+
let headerHash;
|
|
160
|
+
let prefix;
|
|
161
|
+
let key;
|
|
162
|
+
let index = 0;
|
|
163
|
+
let ended = false;
|
|
164
|
+
const parseHeader = async () => {
|
|
165
|
+
if (buffer.length < BACKUP_FILE_MAGIC.length + 4)
|
|
166
|
+
return false;
|
|
167
|
+
if (!isEncryptedBackup(buffer))
|
|
168
|
+
throw new Error('not an encrypted backup file');
|
|
169
|
+
const len = buffer.readUInt32BE(BACKUP_FILE_MAGIC.length);
|
|
170
|
+
if (len === 0 || len > MAX_HEADER)
|
|
171
|
+
throw new Error('the encrypted backup file has a damaged header');
|
|
172
|
+
const total = BACKUP_FILE_MAGIC.length + 4 + len;
|
|
173
|
+
if (buffer.length < total)
|
|
174
|
+
return false;
|
|
175
|
+
const headerBytes = buffer.subarray(0, total);
|
|
176
|
+
let parsed;
|
|
177
|
+
try {
|
|
178
|
+
parsed = JSON.parse(headerBytes.subarray(BACKUP_FILE_MAGIC.length + 4).toString('utf8'));
|
|
179
|
+
}
|
|
180
|
+
catch {
|
|
181
|
+
throw new Error('the encrypted backup file has a damaged header');
|
|
182
|
+
}
|
|
183
|
+
// The header is not authenticated until the first record verifies, so
|
|
184
|
+
// nothing in it may set the cost of the work before that: format 1 has
|
|
185
|
+
// one key derivation cost, and a header asking for another is refused
|
|
186
|
+
// (a crafted file must not make the server run a huge scrypt).
|
|
187
|
+
const salt = typeof parsed.kdf?.salt === 'string' ? Buffer.from(parsed.kdf.salt, 'base64') : Buffer.alloc(0);
|
|
188
|
+
const noncePrefix = typeof parsed.noncePrefix === 'string' ? Buffer.from(parsed.noncePrefix, 'base64') : Buffer.alloc(0);
|
|
189
|
+
if (parsed.format !== 1 || parsed.cipher !== 'aes-256-gcm' || parsed.kdf?.alg !== 'scrypt'
|
|
190
|
+
|| parsed.kdf.N !== KDF.N || parsed.kdf.r !== KDF.r || parsed.kdf.p !== KDF.p
|
|
191
|
+
|| salt.length !== SALT_BYTES || noncePrefix.length !== NONCE_PREFIX_BYTES
|
|
192
|
+
|| !(parsed.chunkBytes > 0 && parsed.chunkBytes <= 16 * CHUNK_BYTES)) {
|
|
193
|
+
throw new Error('the encrypted backup file is in a format this build does not read');
|
|
194
|
+
}
|
|
195
|
+
const fingerprint = backupKeyFingerprint(backupKey);
|
|
196
|
+
if (parsed.keyFingerprint !== fingerprint)
|
|
197
|
+
throw new BackupKeyMismatchError(String(parsed.keyFingerprint), fingerprint);
|
|
198
|
+
headerHash = createHash('sha256').update(headerBytes).digest();
|
|
199
|
+
prefix = noncePrefix;
|
|
200
|
+
key = await deriveKey(backupKey, salt, KDF);
|
|
201
|
+
header = parsed;
|
|
202
|
+
buffer = buffer.subarray(total);
|
|
203
|
+
return true;
|
|
204
|
+
};
|
|
205
|
+
const drain = (stream) => {
|
|
206
|
+
while (buffer.length >= RECORD_HEAD) {
|
|
207
|
+
if (ended)
|
|
208
|
+
throw new Error('the encrypted backup file has bytes after its last record');
|
|
209
|
+
const len = buffer.readUInt32BE(0);
|
|
210
|
+
const last = buffer.readUInt8(4);
|
|
211
|
+
if (len > header.chunkBytes || last > 1)
|
|
212
|
+
throw new Error('the encrypted backup file is damaged');
|
|
213
|
+
const size = RECORD_HEAD + len + TAG_BYTES;
|
|
214
|
+
if (buffer.length < size)
|
|
215
|
+
return;
|
|
216
|
+
const decipher = createDecipheriv('aes-256-gcm', key, nonceFor(prefix, index));
|
|
217
|
+
decipher.setAAD(aadFor(headerHash, index, last === 1));
|
|
218
|
+
decipher.setAuthTag(buffer.subarray(RECORD_HEAD + len, size));
|
|
219
|
+
let plain;
|
|
220
|
+
try {
|
|
221
|
+
plain = Buffer.concat([decipher.update(buffer.subarray(RECORD_HEAD, RECORD_HEAD + len)), decipher.final()]);
|
|
222
|
+
}
|
|
223
|
+
catch {
|
|
224
|
+
throw new Error('the encrypted backup file does not verify: it is damaged or was changed');
|
|
225
|
+
}
|
|
226
|
+
index++;
|
|
227
|
+
buffer = buffer.subarray(size);
|
|
228
|
+
if (last === 1)
|
|
229
|
+
ended = true;
|
|
230
|
+
if (plain.length > 0)
|
|
231
|
+
stream.push(plain);
|
|
232
|
+
}
|
|
233
|
+
};
|
|
234
|
+
return new Transform({
|
|
235
|
+
transform(chunk, _enc, done) {
|
|
236
|
+
buffer = buffer.length === 0 ? chunk : Buffer.concat([buffer, chunk]);
|
|
237
|
+
(header ? Promise.resolve(true) : parseHeader()).then((ready) => {
|
|
238
|
+
if (ready)
|
|
239
|
+
drain(this);
|
|
240
|
+
done();
|
|
241
|
+
}).catch(done);
|
|
242
|
+
},
|
|
243
|
+
flush(done) {
|
|
244
|
+
try {
|
|
245
|
+
if (!header)
|
|
246
|
+
throw new Error(buffer.length === 0 ? 'the encrypted backup file is empty' : 'the encrypted backup file has a damaged header');
|
|
247
|
+
drain(this);
|
|
248
|
+
if (!ended || buffer.length > 0)
|
|
249
|
+
throw new Error('the encrypted backup file is cut short');
|
|
250
|
+
done();
|
|
251
|
+
}
|
|
252
|
+
catch (err) {
|
|
253
|
+
done(err);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
async function collect(source, transform) {
|
|
259
|
+
const out = [];
|
|
260
|
+
await pipeline(Readable.from([source]), transform, new Writable({
|
|
261
|
+
write(chunk, _enc, done) {
|
|
262
|
+
out.push(chunk);
|
|
263
|
+
done();
|
|
264
|
+
}
|
|
265
|
+
}));
|
|
266
|
+
return Buffer.concat(out);
|
|
267
|
+
}
|
|
268
|
+
/** Encrypt a whole buffer (the backup document, already in memory). */
|
|
269
|
+
export function encryptBuffer(data, backupKey) {
|
|
270
|
+
return collect(data, encryptStream(backupKey));
|
|
271
|
+
}
|
|
272
|
+
/** Decrypt a whole buffer. */
|
|
273
|
+
export function decryptBuffer(data, backupKey) {
|
|
274
|
+
return collect(data, decryptStream(backupKey));
|
|
275
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/** A backup document's shape: what every backup holds at its top level. */
|
|
2
|
+
export interface BackupDocument {
|
|
3
|
+
version: string;
|
|
4
|
+
timestamp: string;
|
|
5
|
+
application: string;
|
|
6
|
+
managers: Record<string, unknown>;
|
|
7
|
+
}
|
|
8
|
+
/** Throws, naming what is missing, when `doc` is not a backup document. */
|
|
9
|
+
export declare function validateBackupDocument(doc: BackupDocument): void;
|
|
10
|
+
/**
|
|
11
|
+
* The document in `bytes`, read as a restore reads it: an encrypted one only
|
|
12
|
+
* under `backupKey`, gzip by its first bytes whatever the name says.
|
|
13
|
+
*
|
|
14
|
+
* @param name - the file's name, for the messages
|
|
15
|
+
* @param options.validate - check the shape (default true)
|
|
16
|
+
*/
|
|
17
|
+
export declare function readBackupDocument(bytes: Buffer, backupKey: string, name: string, options?: {
|
|
18
|
+
validate?: boolean;
|
|
19
|
+
}): Promise<BackupDocument>;
|
|
20
|
+
/** What a read-back found: the document's timestamp and how many manager parts it holds. */
|
|
21
|
+
export interface VerifiedBackup {
|
|
22
|
+
timestamp: string;
|
|
23
|
+
managers: number;
|
|
24
|
+
}
|
|
25
|
+
/** The worker's job: the document's bytes, the key and the name. */
|
|
26
|
+
export interface VerifyJob {
|
|
27
|
+
bytes: Uint8Array;
|
|
28
|
+
backupKey: string;
|
|
29
|
+
name: string;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Read a backup document back on a worker thread (#1563): rejects, naming
|
|
33
|
+
* why, when it does not decrypt (a wrong key, a changed byte, a bad tag), does
|
|
34
|
+
* not decompress, does not parse, or is not a backup document.
|
|
35
|
+
*/
|
|
36
|
+
export declare function verifyBackupOffThread(bytes: Buffer, backupKey: string, name: string, timeoutMs: number): Promise<VerifiedBackup>;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading a backup document back (#1563): decrypt it under the backup key
|
|
3
|
+
* (the GCM tag of every record is checked), decompress it, parse it, and
|
|
4
|
+
* check its shape. The one place a document is read: a restore reads it this
|
|
5
|
+
* way, and every backup is read back this way before it counts as one.
|
|
6
|
+
*
|
|
7
|
+
* `verifyBackupOffThread` does it on a worker thread, so a large document is
|
|
8
|
+
* never decrypted, unzipped and parsed on the thread that serves requests.
|
|
9
|
+
*/
|
|
10
|
+
import { promisify } from 'node:util';
|
|
11
|
+
import zlib from 'node:zlib';
|
|
12
|
+
import { Worker } from 'node:worker_threads';
|
|
13
|
+
import { decryptBuffer, isEncryptedBackup } from './backupCrypto.js';
|
|
14
|
+
import { BACKUP_KEY_ENV } from './backupKey.js';
|
|
15
|
+
const gunzip = promisify(zlib.gunzip);
|
|
16
|
+
/** Throws, naming what is missing, when `doc` is not a backup document. */
|
|
17
|
+
export function validateBackupDocument(doc) {
|
|
18
|
+
if (!doc || typeof doc !== 'object')
|
|
19
|
+
throw new Error('Invalid backup: not a backup document');
|
|
20
|
+
if (!doc.version)
|
|
21
|
+
throw new Error('Invalid backup: missing version');
|
|
22
|
+
if (!doc.timestamp)
|
|
23
|
+
throw new Error('Invalid backup: missing timestamp');
|
|
24
|
+
if (!doc.managers || typeof doc.managers !== 'object')
|
|
25
|
+
throw new Error('Invalid backup: missing or invalid managers data');
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* The document in `bytes`, read as a restore reads it: an encrypted one only
|
|
29
|
+
* under `backupKey`, gzip by its first bytes whatever the name says.
|
|
30
|
+
*
|
|
31
|
+
* @param name - the file's name, for the messages
|
|
32
|
+
* @param options.validate - check the shape (default true)
|
|
33
|
+
*/
|
|
34
|
+
export async function readBackupDocument(bytes, backupKey, name, options = {}) {
|
|
35
|
+
let plain = bytes;
|
|
36
|
+
if (isEncryptedBackup(bytes)) {
|
|
37
|
+
if (!backupKey)
|
|
38
|
+
throw new Error(`The backup ${name} is encrypted under the backup key, and ${BACKUP_KEY_ENV} is not set`);
|
|
39
|
+
try {
|
|
40
|
+
plain = await decryptBuffer(bytes, backupKey);
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
throw new Error(`The backup ${name} could not be read: ${err.message}`, { cause: err });
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
let json;
|
|
47
|
+
if (plain[0] === 0x1f && plain[1] === 0x8b) {
|
|
48
|
+
try {
|
|
49
|
+
json = (await gunzip(plain)).toString('utf8');
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
throw new Error(`The backup ${name} does not decompress: ${err.message}`, { cause: err });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
json = plain.toString('utf8');
|
|
57
|
+
}
|
|
58
|
+
let doc;
|
|
59
|
+
try {
|
|
60
|
+
doc = JSON.parse(json);
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
throw new Error(`The backup ${name} is not a readable document: ${err.message}`, { cause: err });
|
|
64
|
+
}
|
|
65
|
+
if (options.validate !== false)
|
|
66
|
+
validateBackupDocument(doc);
|
|
67
|
+
return doc;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Read a backup document back on a worker thread (#1563): rejects, naming
|
|
71
|
+
* why, when it does not decrypt (a wrong key, a changed byte, a bad tag), does
|
|
72
|
+
* not decompress, does not parse, or is not a backup document.
|
|
73
|
+
*/
|
|
74
|
+
export function verifyBackupOffThread(bytes, backupKey, name, timeoutMs) {
|
|
75
|
+
// Compiled, the worker is the .js beside this file. From source (vitest, tsx) it is the .ts, which needs tsx's loader.
|
|
76
|
+
const fromSource = import.meta.url.endsWith('.ts');
|
|
77
|
+
const entry = new URL(fromSource ? './backupVerifyWorker.ts' : './backupVerifyWorker.js', import.meta.url);
|
|
78
|
+
return new Promise((resolve, reject) => {
|
|
79
|
+
const workerData = { bytes, backupKey, name };
|
|
80
|
+
const worker = new Worker(entry, { workerData, ...(fromSource ? { execArgv: ['--import', 'tsx'] } : {}) });
|
|
81
|
+
let settled = false;
|
|
82
|
+
const settle = (fn) => {
|
|
83
|
+
if (settled)
|
|
84
|
+
return;
|
|
85
|
+
settled = true;
|
|
86
|
+
clearTimeout(timer);
|
|
87
|
+
fn();
|
|
88
|
+
};
|
|
89
|
+
const timer = setTimeout(() => {
|
|
90
|
+
settle(() => reject(new Error(`reading the backup ${name} back did not finish within ${timeoutMs} ms (ngdpbase.backup.verify-timeout-ms)`)));
|
|
91
|
+
void worker.terminate();
|
|
92
|
+
}, timeoutMs);
|
|
93
|
+
worker.once('message', (m) => {
|
|
94
|
+
settle(() => (m.ok ? resolve(m.result) : reject(new Error(m.error))));
|
|
95
|
+
});
|
|
96
|
+
worker.once('error', (err) => settle(() => reject(err)));
|
|
97
|
+
worker.once('exit', (code) => settle(() => reject(new Error(`the backup read-back worker exited (code ${code}) without a result`))));
|
|
98
|
+
});
|
|
99
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #1532 — a browser and the kind of device it runs on, in words a person
|
|
3
|
+
* recognises ("Chrome on Windows"), from a User-Agent header. Used where a
|
|
4
|
+
* person is asked to judge a request someone made: the sign-in approval
|
|
5
|
+
* message and page. The rules are the ones `suggestedName()` in
|
|
6
|
+
* `public/js/passkey.js` applies in the browser, so a passkey's suggested name
|
|
7
|
+
* and an approval request describe the same device the same way.
|
|
8
|
+
*
|
|
9
|
+
* Null when the header names neither a browser nor a device we know.
|
|
10
|
+
*/
|
|
11
|
+
export declare function describeBrowser(userAgent: string | undefined): string | null;
|
|
@@ -1,15 +1,7 @@
|
|
|
1
|
-
import type { BackupData, ManagerBackupOptions, ManagerRestoreOptions } from '../managers/BaseManager.js';
|
|
2
|
-
import { siteRestoreFile
|
|
3
|
-
/** A folder part's
|
|
4
|
-
export
|
|
5
|
-
format: 1;
|
|
6
|
-
/** The archive beside the document: `<stem>.<part>.tar`, or `.tar.enc` when encrypted. */
|
|
7
|
-
archive: string;
|
|
8
|
-
files: number;
|
|
9
|
-
bytes: number;
|
|
10
|
-
/** `backupKeyFingerprint()` of the key the archive is encrypted under; absent when it is not encrypted. */
|
|
11
|
-
keyFingerprint?: string;
|
|
12
|
-
}
|
|
1
|
+
import type { BackupArchiveTarget, BackupData, BackupFolder, BackupPartRecord, ManagerBackupOptions, ManagerRestoreOptions, StagedRestoreAtStart } from '../managers/BaseManager.js';
|
|
2
|
+
import { siteRestoreFile } from './siteRestore.js';
|
|
3
|
+
/** A folder or file part's record in the backup document (the name every backup since v5.21 used). */
|
|
4
|
+
export type DirectoryBackupData = BackupPartRecord;
|
|
13
5
|
/** Whether `name` is a bare archive name in the backup directory, never a path. */
|
|
14
6
|
export declare function isArchiveName(name: unknown): name is string;
|
|
15
7
|
/**
|
|
@@ -31,50 +23,58 @@ export declare function archiveTarget(stem: string, part: string): {
|
|
|
31
23
|
export declare function archiveReadKey(manager: string, data: {
|
|
32
24
|
keyFingerprint?: string;
|
|
33
25
|
}): string | undefined;
|
|
26
|
+
/** `ManagerBackupOptions.archive`: one more archive beside the document, for a manager whose files are not a declared part. */
|
|
27
|
+
export declare function archiveWriter(directory: string, stem: string): (part: string) => BackupArchiveTarget;
|
|
28
|
+
/** `ManagerRestoreOptions.readArchive`: an archive the backup names, checked, and the key to read it under. */
|
|
29
|
+
export declare function archiveReader(directory: string): NonNullable<ManagerRestoreOptions['readArchive']>;
|
|
30
|
+
/** The live path a declared part names: its folder or its file. */
|
|
31
|
+
export declare function livePathOf(part: BackupFolder): string | null;
|
|
32
|
+
/** What backing up a manager's declared parts wrote: each part's record (null when left out), and what to tell admins. */
|
|
33
|
+
export interface DeclaredPartsBackup {
|
|
34
|
+
records: Record<string, BackupPartRecord | null>;
|
|
35
|
+
warnings: string[];
|
|
36
|
+
notes: string[];
|
|
37
|
+
}
|
|
34
38
|
/**
|
|
35
|
-
* Back
|
|
36
|
-
* outside BackupManager: the files go beside a document,
|
|
37
|
-
*
|
|
38
|
-
|
|
39
|
-
export declare function backupDirectoryPart(manager: string, part: string, folder: string | null, options?: ManagerBackupOptions, extra?: {
|
|
40
|
-
exclude?: (relative: string) => boolean;
|
|
41
|
-
}): Promise<BackupData>;
|
|
42
|
-
/** The log's note of entries deleted while they were backed up (#1563): skipped, not a failure. */
|
|
43
|
-
export declare function vanishedNote(vanished: string[]): string;
|
|
44
|
-
/**
|
|
45
|
-
* Stage the folder from a backup for the next start: the archive is
|
|
46
|
-
* extracted to `<folder>.staged`, and the swap recorded in the site restore.
|
|
47
|
-
* Throws — staging nothing — outside a site restore, for a backup without
|
|
48
|
-
* this part, or for an archive that is missing, unsafe, or under another key.
|
|
39
|
+
* Back up a manager's declared parts beside the document BackupManager is
|
|
40
|
+
* writing. Throws outside BackupManager: the files go beside a document,
|
|
41
|
+
* never into it. A part that fails throws, and the manager's entry is then
|
|
42
|
+
* an error, as when its own `backup()` throws.
|
|
49
43
|
*/
|
|
50
|
-
export declare function
|
|
44
|
+
export declare function backupDeclaredParts(manager: string, declared: BackupFolder[], options: ManagerBackupOptions | undefined): Promise<DeclaredPartsBackup>;
|
|
51
45
|
/**
|
|
52
|
-
* Put
|
|
53
|
-
*
|
|
46
|
+
* Put the declared parts' records into a manager's entry, in the shape
|
|
47
|
+
* every backup has had: the entry's `data` is the record when the manager
|
|
48
|
+
* has no data of its own and declares one part (roles, persons, …, an
|
|
49
|
+
* add-on's folder); otherwise each record is `data[<part>]` beside the
|
|
50
|
+
* manager's own data (the credentials store beside the provider list).
|
|
54
51
|
*/
|
|
55
|
-
export declare function
|
|
52
|
+
export declare function placePartRecords(entry: BackupData, declared: BackupFolder[], parts: DeclaredPartsBackup): BackupData;
|
|
53
|
+
/** A declared part's record in a manager's entry, read the way `placePartRecords` wrote it. */
|
|
54
|
+
export declare function partRecordIn(entry: BackupData | undefined, declared: BackupFolder[], part: string): unknown;
|
|
56
55
|
/**
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
56
|
+
* Stage a manager's declared parts from its entry in a backup, for the next
|
|
57
|
+
* start: each archive is extracted to `<path>.staged`, and the swaps are
|
|
58
|
+
* recorded in the site restore together. Throws — staging nothing, and
|
|
59
|
+
* removing what it extracted — outside a site restore, for a backup without
|
|
60
|
+
* a part that is not optional, or for an archive that is missing, unsafe,
|
|
61
|
+
* or under another key. Returns the parts the backup left out.
|
|
61
62
|
*/
|
|
62
|
-
export declare function
|
|
63
|
-
|
|
64
|
-
what: string;
|
|
65
|
-
}): Promise<{
|
|
66
|
-
data: DirectoryBackupData | null;
|
|
67
|
-
warnings: string[];
|
|
68
|
-
note?: string;
|
|
63
|
+
export declare function stageDeclaredParts(manager: string, declared: BackupFolder[], entry: BackupData | undefined, options: ManagerRestoreOptions | undefined): Promise<{
|
|
64
|
+
leftOut: string[];
|
|
69
65
|
}>;
|
|
70
66
|
/**
|
|
71
|
-
* Stage one file from a backup
|
|
72
|
-
*
|
|
73
|
-
* `whenAbsent` is staged in its place (an empty store), so the restore still
|
|
74
|
-
* puts the site back as it was.
|
|
67
|
+
* Stage one file from a backup for the next start and record the swap: the
|
|
68
|
+
* instance `.env`, BackupManager's own part.
|
|
75
69
|
*/
|
|
76
|
-
export declare function stageFilePart(manager: string, file: string, data:
|
|
70
|
+
export declare function stageFilePart(manager: string, file: string, data: BackupPartRecord, options: ManagerRestoreOptions | undefined, { whenAbsent }: {
|
|
77
71
|
whenAbsent: string;
|
|
78
72
|
}): Promise<void>;
|
|
79
|
-
/**
|
|
80
|
-
|
|
73
|
+
/**
|
|
74
|
+
* Put a staged restore of a manager's declared parts in place, at start,
|
|
75
|
+
* before the manager reads them. Only the paths it declares may be
|
|
76
|
+
* replaced, whatever the manifest says; a staged copy no committed restore
|
|
77
|
+
* names (an interrupted staging) is set aside. Loud when something was
|
|
78
|
+
* staged but not applied. Null when there was nothing to do.
|
|
79
|
+
*/
|
|
80
|
+
export declare function applyDeclaredPartsAtStart(manager: string, declared: BackupFolder[], configManager: Parameters<typeof siteRestoreFile>[0]): StagedRestoreAtStart | null;
|