@le-space/orbitdb-storage-bridge 0.28.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,507 @@
1
+ /**
2
+ * @fileoverview The envelope in frames (`OSBE` version 2), so a sealed file can
3
+ * be opened as it arrives (#187).
4
+ *
5
+ * Version 1 (./encryption.js) seals the whole blob under one AES-GCM nonce:
6
+ * nothing can be read before the last byte is there, and opening holds the
7
+ * ciphertext and the plaintext at once. A backup of several hundred MB is more
8
+ * than a phone's browser tab may use. Version 2 seals the same bytes in frames
9
+ * of a fixed size, each one opened by itself, so a reader holds one frame.
10
+ *
11
+ * ## The file
12
+ *
13
+ * ```
14
+ * "OSBE" | 2 | frame size | salt | key check | frame 0 | frame 1 | … | last frame
15
+ * 4 1 4 (BE) 32 16
16
+ * ```
17
+ *
18
+ * - **A key per file.** `HKDF-SHA256(key, salt, info)` with `info` =
19
+ * `"OSBE v2"` ‖ frame size ‖ SHA-256(context). The caller's key seals many
20
+ * files; the key that seals frames seals one, so a counter is a nonce that
21
+ * cannot repeat under it. `context` is the caller's (an application backup
22
+ * passes its header): a file opened with another context does not open.
23
+ * - **The nonce** of a frame is its index (11 bytes, big-endian) and a mark:
24
+ * `0` a frame with more behind it, `1` the last frame, `2` the key check.
25
+ * - **The additional data** of a frame is the 41 bytes in front of the key
26
+ * check, the index (8 bytes) and the mark. The index is bound twice, the
27
+ * header once: a frame of another file, or of another place in this one,
28
+ * does not open.
29
+ * - **The key check** is the tag of an empty plaintext. A wrong key is known
30
+ * there, before a frame is opened, and a frame that does not open later is
31
+ * damage and never the key.
32
+ * - **Frames** hold `frame size` bytes of plaintext and a 16-byte tag; only
33
+ * the last may hold fewer (none at all only in an empty file). There are no
34
+ * lengths in the file: the reader cuts at the frame size and opens what is
35
+ * left at the end as the last frame.
36
+ *
37
+ * ## What a reader may rely on
38
+ *
39
+ * Every chunk {@link openFrames} yields is authentic and in its place. A frame
40
+ * is opened as "not the last" only once a byte behind it has arrived, and the
41
+ * last one only once the source has ended, so a file cut short, at a frame's
42
+ * edge or inside one, fails at the frame where it was cut, and so does one
43
+ * with anything appended. **What it cannot say before the end is that the end
44
+ * will come**: a reader that acts on chunks as they arrive (blocks into a
45
+ * blockstore) must keep what cannot be taken back (joining heads) for after
46
+ * the iteration finished without an error.
47
+ *
48
+ * ## The key
49
+ *
50
+ * Version 1 takes the caller's `encrypt` and `decrypt`. Frames cannot: the
51
+ * nonce and the additional data are the format's to choose, not the caller's.
52
+ * So this takes the key itself, 32 bytes or a `CryptoKey` for HKDF, for the
53
+ * length of the call. It derives from it, and keeps nothing.
54
+ *
55
+ * @module backends/frames
56
+ */
57
+
58
+ import { BackendError } from "./types.js";
59
+ import { ENVELOPE_MAGIC } from "./encryption.js";
60
+
61
+ /** The envelope version this module writes and reads. */
62
+ export const FRAMED_VERSION = 2;
63
+
64
+ /** Plaintext bytes in a frame, unless the writer says otherwise. */
65
+ export const DEFAULT_FRAME_SIZE = 1024 * 1024;
66
+
67
+ /** The largest frame a reader allocates for: a header may not ask for more. */
68
+ export const MAX_FRAME_SIZE = 16 * 1024 * 1024;
69
+
70
+ const SALT_BYTES = 32;
71
+ const TAG_BYTES = 16;
72
+ /** magic ‖ version ‖ frame size ‖ salt. */
73
+ export const FRAMED_HEADER_BYTES = ENVELOPE_MAGIC.length + 1 + 4 + SALT_BYTES;
74
+ /** What stands in front of the first frame. */
75
+ export const FRAMED_PREAMBLE_BYTES = FRAMED_HEADER_BYTES + TAG_BYTES;
76
+
77
+ const MORE = 0;
78
+ const LAST = 1;
79
+ const KEY_CHECK = 2;
80
+
81
+ const INFO_PREFIX = new TextEncoder().encode("OSBE v2");
82
+
83
+ /**
84
+ * A sealed file that could not be opened, with the reason as `reason` and, for
85
+ * a frame, its index as `frame`.
86
+ *
87
+ * - `wrong-key`: the key check failed. Another key, or another context.
88
+ * - `frame`: frame `frame` does not open. It was altered, moved, belongs to
89
+ * another file, or the file ends where it should not.
90
+ * - `cut-short`: the file ends before frame `frame` is complete.
91
+ * - `header`: not a framed envelope this build reads.
92
+ */
93
+ export class FrameError extends BackendError {
94
+ /**
95
+ * @param {"wrong-key"|"frame"|"cut-short"|"header"} reason
96
+ * @param {string} message
97
+ * @param {{ frame?: number, cause?: unknown }} [details]
98
+ */
99
+ constructor(reason, message, { frame, cause } = {}) {
100
+ super("INVALID_BACKEND", message);
101
+ this.name = "FrameError";
102
+ if (cause !== undefined) this.cause = cause;
103
+ this.reason = reason;
104
+ if (frame !== undefined) this.frame = frame;
105
+ }
106
+ }
107
+
108
+ /** The envelope version these bytes start with, or null when they are none. */
109
+ export function envelopeVersion(bytes) {
110
+ if (
111
+ !(bytes instanceof Uint8Array) ||
112
+ bytes.length < ENVELOPE_MAGIC.length + 1
113
+ )
114
+ return null;
115
+ if (!ENVELOPE_MAGIC.every((byte, index) => bytes[index] === byte))
116
+ return null;
117
+ return bytes[ENVELOPE_MAGIC.length];
118
+ }
119
+
120
+ /** Do these bytes start a framed envelope? */
121
+ export function isFramed(bytes) {
122
+ return envelopeVersion(bytes) === FRAMED_VERSION;
123
+ }
124
+
125
+ /**
126
+ * Bytes as they come, taken out in pieces of a chosen length, without copying
127
+ * what is already there each time more arrives.
128
+ */
129
+ export class ByteQueue {
130
+ constructor() {
131
+ /** @type {Uint8Array[]} */
132
+ this.chunks = [];
133
+ this.length = 0;
134
+ }
135
+
136
+ /** @param {Uint8Array} chunk */
137
+ push(chunk) {
138
+ if (!chunk?.length) return;
139
+ this.chunks.push(chunk);
140
+ this.length += chunk.length;
141
+ }
142
+
143
+ /**
144
+ * The first `n` bytes, removed from the queue.
145
+ *
146
+ * @param {number} n
147
+ * @returns {Uint8Array}
148
+ */
149
+ take(n) {
150
+ if (n > this.length) throw new RangeError("ByteQueue: not that many bytes");
151
+ const first = this.chunks[0];
152
+ // The common case with large chunks: no copy at all.
153
+ if (first && first.length >= n) {
154
+ this.length -= n;
155
+ if (first.length === n) {
156
+ this.chunks.shift();
157
+ return first;
158
+ }
159
+ this.chunks[0] = first.subarray(n);
160
+ return first.subarray(0, n);
161
+ }
162
+ const out = new Uint8Array(n);
163
+ let at = 0;
164
+ while (at < n) {
165
+ const chunk = this.chunks[0];
166
+ const want = n - at;
167
+ if (chunk.length <= want) {
168
+ out.set(chunk, at);
169
+ at += chunk.length;
170
+ this.chunks.shift();
171
+ } else {
172
+ out.set(chunk.subarray(0, want), at);
173
+ this.chunks[0] = chunk.subarray(want);
174
+ at += want;
175
+ }
176
+ }
177
+ this.length -= n;
178
+ return out;
179
+ }
180
+ }
181
+
182
+ /** @param {Uint8Array | AsyncIterable<Uint8Array> | Iterable<Uint8Array>} source */
183
+ async function* chunksOf(source) {
184
+ if (source instanceof Uint8Array) {
185
+ yield source;
186
+ return;
187
+ }
188
+ yield* source;
189
+ }
190
+
191
+ async function hkdfKey(key) {
192
+ if (key instanceof Uint8Array) {
193
+ if (key.length !== 32) {
194
+ throw new BackendError(
195
+ "INVALID_BACKEND",
196
+ `A framed envelope is sealed with a 32-byte key; this one has ${key.length}`,
197
+ );
198
+ }
199
+ return globalThis.crypto.subtle.importKey("raw", key, "HKDF", false, [
200
+ "deriveKey",
201
+ ]);
202
+ }
203
+ if (key && typeof key === "object" && key.algorithm?.name === "HKDF")
204
+ return key;
205
+ throw new BackendError(
206
+ "INVALID_BACKEND",
207
+ "A framed envelope needs its key: 32 bytes, or a CryptoKey for HKDF. This package holds no keys of its own",
208
+ );
209
+ }
210
+
211
+ /**
212
+ * The key that seals this one file's frames.
213
+ *
214
+ * @param {Uint8Array | CryptoKey} key
215
+ * @param {Uint8Array} salt
216
+ * @param {number} frameSize
217
+ * @param {Uint8Array | undefined} context
218
+ */
219
+ async function fileKey(key, salt, frameSize, context) {
220
+ const about = new Uint8Array(
221
+ await globalThis.crypto.subtle.digest(
222
+ "SHA-256",
223
+ context ?? new Uint8Array(0),
224
+ ),
225
+ );
226
+ const info = new Uint8Array(INFO_PREFIX.length + 4 + about.length);
227
+ info.set(INFO_PREFIX, 0);
228
+ new DataView(info.buffer).setUint32(INFO_PREFIX.length, frameSize);
229
+ info.set(about, INFO_PREFIX.length + 4);
230
+ return globalThis.crypto.subtle.deriveKey(
231
+ { name: "HKDF", hash: "SHA-256", salt, info },
232
+ await hkdfKey(key),
233
+ { name: "AES-GCM", length: 256 },
234
+ false,
235
+ ["encrypt", "decrypt"],
236
+ );
237
+ }
238
+
239
+ /** Index (11 bytes, big-endian) ‖ mark. @param {number} index @param {number} mark */
240
+ function nonceOf(index, mark) {
241
+ const nonce = new Uint8Array(12);
242
+ // The upper 3 bytes stay zero: an index is a count of frames a file can hold.
243
+ new DataView(nonce.buffer).setBigUint64(3, BigInt(index));
244
+ nonce[11] = mark;
245
+ return nonce;
246
+ }
247
+
248
+ /** Header ‖ index (8 bytes) ‖ mark. */
249
+ function aadOf(header, index, mark) {
250
+ const aad = new Uint8Array(header.length + 9);
251
+ aad.set(header, 0);
252
+ new DataView(aad.buffer).setBigUint64(header.length, BigInt(index));
253
+ aad[header.length + 8] = mark;
254
+ return aad;
255
+ }
256
+
257
+ const algorithm = (header, index, mark) => ({
258
+ name: "AES-GCM",
259
+ iv: nonceOf(index, mark),
260
+ additionalData: aadOf(header, index, mark),
261
+ tagLength: TAG_BYTES * 8,
262
+ });
263
+
264
+ function assertFrameSize(frameSize) {
265
+ if (
266
+ !Number.isInteger(frameSize) ||
267
+ frameSize < 1 ||
268
+ frameSize > MAX_FRAME_SIZE
269
+ ) {
270
+ throw new BackendError(
271
+ "INVALID_BACKEND",
272
+ `A frame holds between 1 and ${MAX_FRAME_SIZE} bytes, not ${frameSize}`,
273
+ );
274
+ }
275
+ }
276
+
277
+ /**
278
+ * Seal bytes in frames, as they come.
279
+ *
280
+ * @param {Uint8Array | AsyncIterable<Uint8Array> | Iterable<Uint8Array>} source the plaintext
281
+ * @param {object} options
282
+ * @param {Uint8Array | CryptoKey} options.key 32 bytes, or a `CryptoKey` for HKDF
283
+ * @param {Uint8Array} [options.context] bound into the file's key; the same
284
+ * bytes open it again
285
+ * @param {number} [options.frameSize] plaintext bytes in a frame
286
+ * @param {Uint8Array} [options.salt] for tests only: a salt used twice with
287
+ * one key repeats every nonce
288
+ * @returns {AsyncGenerator<Uint8Array>} the sealed file, in pieces
289
+ */
290
+ export async function* sealFrames(
291
+ source,
292
+ { key, context, frameSize = DEFAULT_FRAME_SIZE, salt } = {},
293
+ ) {
294
+ assertFrameSize(frameSize);
295
+ const fresh =
296
+ salt ?? globalThis.crypto.getRandomValues(new Uint8Array(SALT_BYTES));
297
+ if (fresh.length !== SALT_BYTES) {
298
+ throw new BackendError("INVALID_BACKEND", `A salt has ${SALT_BYTES} bytes`);
299
+ }
300
+ const header = new Uint8Array(FRAMED_HEADER_BYTES);
301
+ header.set(ENVELOPE_MAGIC, 0);
302
+ header[ENVELOPE_MAGIC.length] = FRAMED_VERSION;
303
+ new DataView(header.buffer).setUint32(ENVELOPE_MAGIC.length + 1, frameSize);
304
+ header.set(fresh, ENVELOPE_MAGIC.length + 5);
305
+
306
+ const sealing = await fileKey(key, fresh, frameSize, context);
307
+ const { subtle } = globalThis.crypto;
308
+ const seal = async (plaintext, index, mark) =>
309
+ new Uint8Array(
310
+ await subtle.encrypt(algorithm(header, index, mark), sealing, plaintext),
311
+ );
312
+
313
+ const preamble = new Uint8Array(FRAMED_PREAMBLE_BYTES);
314
+ preamble.set(header, 0);
315
+ preamble.set(await seal(new Uint8Array(0), 0, KEY_CHECK), header.length);
316
+ yield preamble;
317
+
318
+ const queue = new ByteQueue();
319
+ let index = 0;
320
+ for await (const chunk of chunksOf(source)) {
321
+ queue.push(chunk);
322
+ // A frame is "not the last" only once a byte behind it is in hand.
323
+ while (queue.length > frameSize) {
324
+ yield seal(queue.take(frameSize), index++, MORE);
325
+ }
326
+ }
327
+ yield seal(queue.take(queue.length), index, LAST);
328
+ }
329
+
330
+ /**
331
+ * Open a framed envelope as it arrives: each frame authenticated and in its
332
+ * place before its plaintext is handed on.
333
+ *
334
+ * @param {Uint8Array | AsyncIterable<Uint8Array> | Iterable<Uint8Array>} source the sealed file
335
+ * @param {object} options
336
+ * @param {Uint8Array | CryptoKey} options.key
337
+ * @param {Uint8Array} [options.context] as it was sealed with
338
+ * @param {(progress: { stage: "opened", frames: number, bytes: number }) => void} [options.onProgress]
339
+ * after each frame: how many are open, and their plaintext bytes
340
+ * @returns {AsyncGenerator<Uint8Array>} the plaintext, a frame at a time
341
+ * @throws {FrameError}
342
+ */
343
+ export async function* openFrames(source, { key, context, onProgress } = {}) {
344
+ const queue = new ByteQueue();
345
+ const chunks = chunksOf(source)[Symbol.asyncIterator]();
346
+ let ended = false;
347
+ /** Read until `n` bytes are queued or the source ends. */
348
+ const fill = async (n) => {
349
+ while (!ended && queue.length < n) {
350
+ const { done, value } = await chunks.next();
351
+ if (done) ended = true;
352
+ else queue.push(value);
353
+ }
354
+ return queue.length >= n;
355
+ };
356
+
357
+ try {
358
+ if (!(await fill(FRAMED_PREAMBLE_BYTES))) {
359
+ const start = queue.take(queue.length);
360
+ if (envelopeVersion(start) === null) {
361
+ throw new FrameError(
362
+ "header",
363
+ "These bytes are not an encrypted backup",
364
+ );
365
+ }
366
+ throw new FrameError("cut-short", "This backup ends inside its header", {
367
+ frame: 0,
368
+ });
369
+ }
370
+ const header = queue.take(FRAMED_HEADER_BYTES);
371
+ const version = envelopeVersion(header);
372
+ if (version === null) {
373
+ throw new FrameError("header", "These bytes are not an encrypted backup");
374
+ }
375
+ if (version !== FRAMED_VERSION) {
376
+ throw new FrameError(
377
+ "header",
378
+ `This backup was written with envelope version ${version}, and frames are version ${FRAMED_VERSION}.` +
379
+ (version === 1
380
+ ? " It is sealed in one piece: open it with decrypt, whole."
381
+ : " Upgrade @le-space/orbitdb-storage-bridge to restore it."),
382
+ );
383
+ }
384
+ const frameSize = new DataView(
385
+ header.buffer,
386
+ header.byteOffset,
387
+ header.byteLength,
388
+ ).getUint32(ENVELOPE_MAGIC.length + 1);
389
+ if (frameSize < 1 || frameSize > MAX_FRAME_SIZE) {
390
+ throw new FrameError(
391
+ "header",
392
+ `This backup names frames of ${frameSize} bytes; this build reads up to ${MAX_FRAME_SIZE}`,
393
+ );
394
+ }
395
+ // Its own copy: the header is every frame's additional data, long after
396
+ // the chunk it arrived in is gone.
397
+ const bound = new Uint8Array(header);
398
+ const salt = bound.subarray(ENVELOPE_MAGIC.length + 5);
399
+ const opening = await fileKey(key, salt, frameSize, context);
400
+ const { subtle } = globalThis.crypto;
401
+
402
+ try {
403
+ await subtle.decrypt(
404
+ algorithm(bound, 0, KEY_CHECK),
405
+ opening,
406
+ queue.take(TAG_BYTES),
407
+ );
408
+ } catch (cause) {
409
+ throw new FrameError(
410
+ "wrong-key",
411
+ "This backup does not open with this key: it was sealed with another one, or for another purpose.",
412
+ { cause },
413
+ );
414
+ }
415
+
416
+ const sealedFrame = frameSize + TAG_BYTES;
417
+ let index = 0;
418
+ let bytes = 0;
419
+ for (;;) {
420
+ // One byte more than a frame says this frame is not the last.
421
+ const more = await fill(sealedFrame + 1);
422
+ if (!more && queue.length < TAG_BYTES) {
423
+ throw new FrameError(
424
+ "cut-short",
425
+ `This backup is cut short: it ends before frame ${index} is complete.`,
426
+ { frame: index },
427
+ );
428
+ }
429
+ const mark = more ? MORE : LAST;
430
+ const sealed = queue.take(more ? sealedFrame : queue.length);
431
+ let plaintext;
432
+ try {
433
+ plaintext = new Uint8Array(
434
+ await subtle.decrypt(algorithm(bound, index, mark), opening, sealed),
435
+ );
436
+ } catch (cause) {
437
+ throw new FrameError(
438
+ "frame",
439
+ `Frame ${index} of this backup does not open: ` +
440
+ (more
441
+ ? "it was altered, or frames were moved or left out."
442
+ : "the backup is cut short, has something appended, or its last frame was altered."),
443
+ { frame: index, cause },
444
+ );
445
+ }
446
+ // Only the one frame of an empty file holds nothing.
447
+ if (plaintext.length === 0 && index > 0) {
448
+ throw new FrameError(
449
+ "frame",
450
+ `Frame ${index} of this backup is empty, which no writer makes.`,
451
+ { frame: index },
452
+ );
453
+ }
454
+ bytes += plaintext.length;
455
+ index++;
456
+ onProgress?.({ stage: "opened", frames: index, bytes });
457
+ if (plaintext.length) yield plaintext;
458
+ if (!more) return;
459
+ }
460
+ } finally {
461
+ // A reader that stops early, or a frame that failed: let the source go.
462
+ await chunks.return?.();
463
+ }
464
+ }
465
+
466
+ /** @param {AsyncIterable<Uint8Array>} chunks @returns {Promise<Uint8Array>} */
467
+ export async function collectBytes(chunks) {
468
+ const parts = [];
469
+ let total = 0;
470
+ for await (const chunk of chunks) {
471
+ parts.push(chunk);
472
+ total += chunk.length;
473
+ }
474
+ if (parts.length === 1) return parts[0];
475
+ const out = new Uint8Array(total);
476
+ let at = 0;
477
+ for (const part of parts) {
478
+ out.set(part, at);
479
+ at += part.length;
480
+ }
481
+ return out;
482
+ }
483
+
484
+ /**
485
+ * Seal bytes in frames, in one piece: for a writer that has the whole
486
+ * plaintext in memory anyway and an upload that wants the whole body.
487
+ *
488
+ * @param {Uint8Array} bytes
489
+ * @param {Parameters<typeof sealFrames>[1]} options
490
+ * @returns {Promise<Uint8Array>}
491
+ */
492
+ export function sealFramed(bytes, options) {
493
+ return collectBytes(sealFrames(bytes, options));
494
+ }
495
+
496
+ /**
497
+ * Open a framed envelope in one piece. For small files and for readers that
498
+ * need all of it at once; a restore that must stay small uses {@link openFrames}.
499
+ *
500
+ * @param {Uint8Array} bytes
501
+ * @param {Parameters<typeof openFrames>[1]} options
502
+ * @returns {Promise<Uint8Array>}
503
+ */
504
+ export async function openFramed(bytes, options) {
505
+ const plaintext = await collectBytes(openFrames(bytes, options));
506
+ return plaintext ?? new Uint8Array(0);
507
+ }
@@ -56,6 +56,9 @@
56
56
  * @property {(handle: BackendHandle|string, options?: { onProgress?: (received: number, total: number) => void }) => Promise<Uint8Array>} getBlob
57
57
  * `onProgress`: as the bytes arrive, where the driver can tell (the Aleph
58
58
  * driver can; others may ignore it)
59
+ * @property {(handle: BackendHandle|string, options?: { size?: number, offset?: number, timeout?: number, signal?: AbortSignal, onProgress?: (received: number, total: number) => void }) => AsyncIterable<Uint8Array>} [getBlobStream]
60
+ * the blob as it arrives, never whole, for a file too large to hold; a
61
+ * driver that has it checks at the end what `getBlob` checks (aleph)
59
62
  * @property {((cid: string, meta?: object) => Promise<BackendHandle>)} [pinCid]
60
63
  * @property {((options?: object) => Promise<BackendHandle[]>)} [list]
61
64
  * @property {((handle: BackendHandle|string) => Promise<void>)} [remove]