@le-space/orbitdb-storage-bridge 0.28.1 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -59,8 +59,8 @@ entry is the Storacha-era class and the helpers it grew.
59
59
  | `/live-backup` | increments: `createLiveBackup`, `collectIncrement`, `readIncrement`, `mergeIncrements`, `restoreFromIncrements`, `joinIncrements`, and `advanceConfirmation` / `reconcileConfirmation` for a store that refuses later |
60
60
  | `/dehydrate` | `dehydrate`, `hydrate`: a database where a second device finds it from a secret alone |
61
61
  | `/pointer-ipns` | `publishPointer`, `resolvePointer`: the IPNS pointer `dehydrate` publishes, under a name derived from a seed |
62
- | `/gateway-fetch` | `fetchFromGateways`: bytes by CID over HTTP, verified against the CID |
63
- | `/peer-fetch` | `createPeerFetch`, `createGatewayFirstFetch`: bytes by CID from the peers that hold them, over libp2p |
62
+ | `/gateway-fetch` | `fetchFromGateways`: bytes by CID over HTTP, verified against the CID; `streamFromGateways`: the same as they arrive, picked up where a connection dropped |
63
+ | `/peer-fetch` | `createPeerFetch`, `createGatewayFirstFetch`: bytes by CID from the peers that hold them, over libp2p; `.stream` and `createGatewayFirstStream` for a file too large to hold |
64
64
  | `/courier-sync` | `createCourierSync`: replication over any byte courier — a LoRa mesh, a QR relay, a file |
65
65
  | `/memory-courier` | `createMemoryCourierPair`: the in-process courier for tests and demos |
66
66
  | `/backends/types` | the contract: `defineBackend`, `BackendError`, the capability flags |
@@ -73,6 +73,7 @@ entry is the Storacha-era class and the helpers it grew.
73
73
  | `/backends/storacha` | the Storacha driver; needs the optional peer `@storacha/client`, and a service that no longer answers |
74
74
  | `/backends/mirror` | one backup to several backends |
75
75
  | `/backends/encryption` | `withEncryption`: the service holds ciphertext |
76
+ | `/backends/frames` | `sealFrames`, `openFrames`: the envelope in frames, opened as it arrives, for a backup a phone restores |
76
77
  | `/backends/choose` | `createBackendFromChoice`: a backend from a name and a key, for a page's form |
77
78
  | `/backends/resolve` | `resolveBackend`: the options-to-backend step the older functions share |
78
79
  | `/storacha-backup`, `/components/*` | the JavaScript of the old browser widget: `storacha-backup.js`, `WebAuthnDIDProvider.js`, `theme.js`. The two `.svelte` components were dropped — they imported four files no release ever contained ([docs/STORACHA-UI-HISTORY.md](docs/STORACHA-UI-HISTORY.md)) |
package/lib/app-backup.js CHANGED
@@ -18,16 +18,20 @@
18
18
  * finds the key to the rest. This module does not interpret it. Its SHA-256
19
19
  * is in the manifest, so a header swapped after the backup was made is
20
20
  * refused once the body is open.
21
- * - **The envelope** is the package's own (`OSBE`, see ./backends/encryption.js)
22
- * around the CAR, encrypted by the caller's `encrypt`. The package holds no
23
- * keys, here as everywhere.
21
+ * - **The envelope** is the package's own (`OSBE`) around the CAR. Given a
22
+ * `key`, it is the envelope in frames (version 2, ./backends/frames.js): the
23
+ * file can then be restored as it arrives ({@link restoreAppBackupStream}),
24
+ * in the memory of one frame and one block, and the header is bound into
25
+ * the frames' key. Given the caller's `encrypt`, it is the envelope in one
26
+ * piece (version 1, ./backends/encryption.js), which can only be opened
27
+ * whole. The package keeps no keys, here as everywhere.
24
28
  * - **The CAR's root** is a dag-cbor manifest: `kind` and `v` say what this is,
25
29
  * `app` whose it is, and `metadata` is `bundleDatabases`'s, naming every
26
30
  * database with its heads. `extra` is the application's own (belege lists its
27
31
  * receipt files there).
28
32
  *
29
- * Everything is built in memory, as the rest of the package does; a backup is
30
- * as large as the books it holds.
33
+ * A backup is built in memory, as the rest of the package does, and an upload
34
+ * wants its whole body; it is the restore that has to fit a phone.
31
35
  *
32
36
  * @requires ./extract-blocks.js - bundleDatabases
33
37
  * @requires ./backup-car.js - createCARFromBlocks
@@ -39,8 +43,9 @@ import { CID } from "multiformats/cid";
39
43
  import { sha256 } from "multiformats/hashes/sha2";
40
44
  import { bundleDatabases } from "./extract-blocks.js";
41
45
  import { createCARFromBlocks } from "./backup-car.js";
42
- import { readBlocksFromCAR, restoreFromBlocks } from "./restore-cid.js";
46
+ import { readBlocksFromCAR, restoreFromBlocks, restoreFromStoredBlocks, storeBlocksFromCAR } from "./restore-cid.js";
43
47
  import { wrapEnvelope, readEnvelope } from "./backends/encryption.js";
48
+ import { ByteQueue, collectBytes, envelopeVersion, isFramed, openFramed, openFrames, sealFramed } from "./backends/frames.js";
44
49
 
45
50
  /** `OSBA` — orbitdb-storage-bridge, application backup. */
46
51
  export const APP_BACKUP_MAGIC = new Uint8Array([0x4f, 0x53, 0x42, 0x41]);
@@ -85,8 +90,12 @@ async function sha256Hex(bytes) {
85
90
  * @param {string} params.app - whose backup this is; {@link openAppBackup} can insist on it
86
91
  * @param {Object[]|Object<string, Object>} params.databases - open OrbitDB
87
92
  * databases. Given by name, the names go into the metadata as `collection`.
88
- * @param {(plaintext: Uint8Array) => Promise<{ ciphertext: Uint8Array, iv: Uint8Array }>} params.encrypt -
89
- * the caller's, with the caller's key
93
+ * @param {Uint8Array | CryptoKey} [params.key] - seal in frames with this key
94
+ * (32 bytes, or a `CryptoKey` for HKDF): the form to prefer, the only one a
95
+ * restore can stream
96
+ * @param {number} [params.frameSize] - with `key`: plaintext bytes in a frame
97
+ * @param {(plaintext: Uint8Array) => Promise<{ ciphertext: Uint8Array, iv: Uint8Array }>} [params.encrypt] -
98
+ * instead of `key`: the caller's, with the caller's key, in one piece
90
99
  * @param {Map<string, { bytes: Uint8Array }>} [params.blocks] - blocks of the
91
100
  * application's own, by CID string: they travel in the CAR and come back into
92
101
  * the blockstore on restore
@@ -101,6 +110,8 @@ export async function buildAppBackup({
101
110
  app,
102
111
  databases,
103
112
  encrypt,
113
+ key,
114
+ frameSize,
104
115
  blocks: own,
105
116
  header,
106
117
  appVersion,
@@ -109,7 +120,10 @@ export async function buildAppBackup({
109
120
  onProgress,
110
121
  }) {
111
122
  if (typeof app !== "string" || !app) throw new Error("buildAppBackup needs the application's name as `app`");
112
- if (typeof encrypt !== "function") throw new Error("buildAppBackup needs an `encrypt` function: the package holds no keys");
123
+ if (key === undefined && typeof encrypt !== "function") {
124
+ throw new Error("buildAppBackup needs a `key` (sealed in frames) or an `encrypt` function: the package holds no keys");
125
+ }
126
+ if (key !== undefined && encrypt !== undefined) throw new Error("buildAppBackup takes a `key` or `encrypt`, not both");
113
127
  if (header !== undefined && !(header instanceof Uint8Array)) throw new Error("A backup's header is bytes");
114
128
  if (own !== undefined && !(own instanceof Map)) throw new Error("An application's own blocks come as a Map, by CID string");
115
129
 
@@ -142,21 +156,29 @@ export async function buildAppBackup({
142
156
  };
143
157
  const root = dagCbor.encode(manifest);
144
158
  const rootCid = CID.createV1(dagCbor.code, await sha256.digest(root));
145
- blocks.set(rootCid.toString(), { cid: rootCid, bytes: root });
159
+ // The manifest first: a restore that reads the file as it arrives knows
160
+ // whose backup this is before it stores the rest.
161
+ const ordered = new Map([[rootCid.toString(), { cid: rootCid, bytes: root }]]);
162
+ for (const [name, block] of blocks) if (!ordered.has(name)) ordered.set(name, block);
146
163
 
147
- const car = await createCARFromBlocks(blocks, rootCid.toString());
164
+ const car = await createCARFromBlocks(ordered, rootCid.toString());
148
165
  onProgress?.({ stage: "sealing", bytes: car.length });
149
- const { ciphertext, iv } = await encrypt(car);
150
- const envelope = wrapEnvelope(ciphertext, iv);
151
-
152
166
  const head = header ?? new Uint8Array(0);
167
+ let envelope;
168
+ if (key !== undefined) {
169
+ envelope = await sealFramed(car, { key, context: head, ...(frameSize ? { frameSize } : {}) });
170
+ } else {
171
+ const { ciphertext, iv } = await encrypt(car);
172
+ envelope = wrapEnvelope(ciphertext, iv);
173
+ }
174
+
153
175
  const bytes = new Uint8Array(PREAMBLE + head.length + envelope.length);
154
176
  bytes.set(APP_BACKUP_MAGIC, 0);
155
177
  bytes[APP_BACKUP_MAGIC.length] = APP_BACKUP_VERSION;
156
178
  new DataView(bytes.buffer).setUint32(APP_BACKUP_MAGIC.length + 1, head.length);
157
179
  bytes.set(head, PREAMBLE);
158
180
  bytes.set(envelope, PREAMBLE + head.length);
159
- return { bytes, manifest, blocks: blocks.size, carBytes: car.length };
181
+ return { bytes, manifest, blocks: ordered.size, carBytes: car.length };
160
182
  }
161
183
 
162
184
  /** Is this an application backup this module wrote, of any version? */
@@ -201,37 +223,33 @@ export function readAppBackupHeader(bytes) {
201
223
  *
202
224
  * @param {Uint8Array} bytes
203
225
  * @param {object} options
204
- * @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} options.decrypt
226
+ * @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} [options.decrypt] -
227
+ * for a backup sealed in one piece
228
+ * @param {Uint8Array | CryptoKey} [options.key] - for a backup sealed in frames
205
229
  * @param {string} [options.app] - refuse a backup of another application
206
230
  * @returns {Promise<{ header: Uint8Array, manifest: AppBackupManifest, blocks: Map<string, { cid: any, bytes: Uint8Array }> }>}
207
231
  */
208
- export async function openAppBackup(bytes, { decrypt, app } = {}) {
209
- if (typeof decrypt !== "function") throw new Error("openAppBackup needs a `decrypt` function");
232
+ export async function openAppBackup(bytes, { decrypt, key, app } = {}) {
210
233
  const { header, body } = readAppBackupHeader(bytes);
211
- const { ciphertext, iv } = readEnvelope(body);
212
- const car = await decrypt(ciphertext, iv);
234
+ let car;
235
+ if (isFramed(body)) {
236
+ if (key === undefined) throw new Error("This backup is sealed in frames: openAppBackup needs its `key`");
237
+ car = await openFramed(body, { key, context: header });
238
+ } else {
239
+ if (typeof decrypt !== "function") throw new Error("openAppBackup needs a `decrypt` function");
240
+ const { ciphertext, iv } = readEnvelope(body);
241
+ car = await decrypt(ciphertext, iv);
242
+ }
213
243
  const blocks = await readBlocksFromCAR(car, { verify: true });
214
244
 
215
245
  let manifest;
216
246
  for (const [name, { bytes: block }] of blocks) {
217
247
  if (CID.parse(name).code !== dagCbor.code) continue;
218
- let value;
219
- try {
220
- value = dagCbor.decode(block);
221
- } catch {
222
- continue;
223
- }
224
- if (value?.kind === APP_BACKUP_KIND && value?.v === APP_BACKUP_VERSION) {
225
- manifest = value;
226
- break;
227
- }
248
+ manifest = manifestIn(block);
249
+ if (manifest) break;
228
250
  }
229
251
  if (!manifest) throw new Error("This backup holds no manifest.");
230
- if (app && manifest.app !== app) throw new Error(`This is a backup of ${manifest.app}, not of ${app}.`);
231
- // The header is outside the seal; the manifest inside it says which header belongs.
232
- const expected = manifest.header ?? null;
233
- const actual = header.length ? await sha256Hex(header) : null;
234
- if (expected !== actual) throw new Error("This backup's header is not the one it was made with.");
252
+ await assertManifest(manifest, header, app);
235
253
  return { header, manifest, blocks };
236
254
  }
237
255
 
@@ -263,20 +281,35 @@ export async function openAppBackup(bytes, { decrypt, app } = {}) {
263
281
  * @returns {Promise<{ databases: { address: string, collection?: string, joined: number, entries: number | null }[] }>}
264
282
  */
265
283
  export async function restoreAppBackup({ orbitdb, opened, addresses, open, onProgress }) {
266
- const metadata = opened?.manifest?.metadata;
267
- if (!metadata?.databases?.length) throw new Error("This backup names no databases.");
268
- const byAddress = Array.isArray(addresses)
284
+ const byAddress = collectionsByAddress(addresses);
285
+ assertTheseBooks(opened?.manifest, byAddress);
286
+ return putBack(byAddress, onProgress, (options) =>
287
+ restoreFromBlocks(orbitdb, opened.blocks, opened.manifest.metadata, { open, ...options }),
288
+ );
289
+ }
290
+
291
+ /** These books' database addresses, each with its collection when they came by name. */
292
+ function collectionsByAddress(addresses) {
293
+ return Array.isArray(addresses)
269
294
  ? Object.fromEntries(addresses.map((address) => [String(address), undefined]))
270
295
  : Object.fromEntries(Object.entries(addresses ?? {}).map(([collection, address]) => [String(address), collection]));
296
+ }
297
+
298
+ /** Every database the backup names is one of these books, or it is refused. */
299
+ function assertTheseBooks(manifest, byAddress) {
300
+ const metadata = manifest?.metadata;
301
+ if (!metadata?.databases?.length) throw new Error("This backup names no databases.");
271
302
  for (const d of metadata.databases) {
272
303
  if (!(String(d.address) in byAddress)) {
273
304
  throw new Error("This backup holds databases these books do not have.");
274
305
  }
275
306
  }
307
+ }
276
308
 
309
+ /** The join, with a head that could not be joined as a failure and the answer in the application's words. */
310
+ async function putBack(byAddress, onProgress, restore) {
277
311
  const warnings = [];
278
- const restored = await restoreFromBlocks(orbitdb, opened.blocks, metadata, {
279
- open,
312
+ const restored = await restore({
280
313
  log: { info() {}, debug() {}, warn: (message) => warnings.push(String(message)) },
281
314
  onProgress: (p) =>
282
315
  onProgress?.({
@@ -298,3 +331,220 @@ export async function restoreAppBackup({ orbitdb, opened, addresses, open, onPro
298
331
  })),
299
332
  };
300
333
  }
334
+
335
+ /**
336
+ * The header of a backup that arrives in pieces, without a key, and the rest
337
+ * of the stream behind it: what a keyring needs before anything can be opened,
338
+ * read from the first bytes and not from the whole file.
339
+ *
340
+ * @param {AsyncIterable<Uint8Array> | Iterable<Uint8Array> | Uint8Array} source
341
+ * @returns {Promise<{ version: number, header: Uint8Array, body: AsyncGenerator<Uint8Array> }>}
342
+ * `body` is the envelope; it must be read, or returned, for the source to be let go
343
+ */
344
+ export async function readAppBackupHeaderStream(source) {
345
+ const chunks = (
346
+ source instanceof Uint8Array
347
+ ? (async function* () {
348
+ yield source;
349
+ })()
350
+ : source
351
+ )[Symbol.asyncIterator]?.() ?? source[Symbol.iterator]();
352
+ const queue = new ByteQueue();
353
+ let ended = false;
354
+ const fill = async (n) => {
355
+ while (!ended && queue.length < n) {
356
+ const { done, value } = await chunks.next();
357
+ if (done) ended = true;
358
+ else queue.push(value);
359
+ }
360
+ return queue.length >= n;
361
+ };
362
+ try {
363
+ if (!(await fill(PREAMBLE))) throw new Error("This is not an application backup.");
364
+ const preamble = queue.take(PREAMBLE);
365
+ if (!isAppBackup(preamble)) throw new Error("This is not an application backup.");
366
+ const version = preamble[APP_BACKUP_MAGIC.length];
367
+ if (version !== APP_BACKUP_VERSION) {
368
+ throw new Error(
369
+ `This backup was written in format ${version}, and this build reads ${APP_BACKUP_VERSION}. ` +
370
+ "Upgrade @le-space/orbitdb-storage-bridge to open it.",
371
+ );
372
+ }
373
+ const length = new DataView(preamble.buffer, preamble.byteOffset, preamble.byteLength).getUint32(
374
+ APP_BACKUP_MAGIC.length + 1,
375
+ );
376
+ if (length > MAX_HEADER_BYTES) throw new Error(`This backup names a header of ${length} bytes; a header has at most ${MAX_HEADER_BYTES}.`);
377
+ if (!(await fill(length))) throw new Error("This backup is cut short.");
378
+ // Its own copy: it outlives the chunk it arrived in.
379
+ const header = new Uint8Array(queue.take(length));
380
+ return {
381
+ version,
382
+ header,
383
+ body: (async function* () {
384
+ try {
385
+ while (queue.length) yield queue.take(queue.chunks[0].length);
386
+ while (!ended) {
387
+ const { done, value } = await chunks.next();
388
+ if (done) ended = true;
389
+ else if (value?.length) yield value;
390
+ }
391
+ } finally {
392
+ await chunks.return?.();
393
+ }
394
+ })(),
395
+ };
396
+ } catch (error) {
397
+ await chunks.return?.();
398
+ throw error;
399
+ }
400
+ }
401
+
402
+ /** The longest header a streamed reader holds: a keyring, not a payload. */
403
+ export const MAX_HEADER_BYTES = 16 * 1024 * 1024;
404
+
405
+ /**
406
+ * @typedef {{ stage: "opened", frames: number, bytes: number }
407
+ * | { stage: "stored", blocks: number, bytes: number }
408
+ * | AppRestoreProgress} AppRestoreStreamProgress
409
+ */
410
+
411
+ /**
412
+ * Restore a backup as it arrives (#187): {@link openAppBackup} and
413
+ * {@link restoreAppBackup} in one pass, for a file too large to hold. Each
414
+ * frame is opened as it comes, each block checked and put into the blockstore
415
+ * and let go, so the memory this needs is a frame's and a block's, whatever
416
+ * the backup's size.
417
+ *
418
+ * The same refusals as there, at the earliest moment each can be known: a
419
+ * wrong key before the first frame; another application's backup, or one of
420
+ * other books, at the manifest, which a backup made here carries first, so
421
+ * before any block is stored. **The databases are joined only after the last
422
+ * frame opened**: a file that is cut short, or damaged anywhere, throws with
423
+ * the books untouched. The blocks it stored until then stay in the blockstore,
424
+ * sealed and pointed at by nothing.
425
+ *
426
+ * A backup sealed in one piece (with `encrypt`) cannot be opened as it
427
+ * arrives. It is restored all the same when `decrypt` is given, held whole as
428
+ * {@link openAppBackup} holds it.
429
+ *
430
+ * @param {object} params
431
+ * @param {any} params.orbitdb
432
+ * @param {AsyncIterable<Uint8Array> | Iterable<Uint8Array> | Uint8Array} params.source -
433
+ * the file, e.g. a backend's `getBlobStream`
434
+ * @param {Uint8Array | CryptoKey} [params.key] - opens a backup sealed in frames
435
+ * @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} [params.decrypt] -
436
+ * opens a backup sealed in one piece
437
+ * @param {string} [params.app] - refuse a backup of another application
438
+ * @param {Record<string, string> | string[]} params.addresses - as {@link restoreAppBackup}
439
+ * @param {Record<string, any>} [params.open]
440
+ * @param {(header: Uint8Array) => Promise<{ key?: Uint8Array | CryptoKey, decrypt?: Function } | void> | { key?: Uint8Array | CryptoKey, decrypt?: Function } | void} [params.onHeader] -
441
+ * sees the header before anything is opened, and may answer with the key it
442
+ * leads to: how a keyring in the header is used
443
+ * @param {(progress: AppRestoreStreamProgress) => void} [params.onProgress] -
444
+ * frames opened, blocks stored, databases joined; bytes fetched are the
445
+ * source's to report (`getBlobStream`'s `onProgress`)
446
+ * @returns {Promise<{ header: Uint8Array, manifest: AppBackupManifest, blocks: number, streamed: boolean,
447
+ * databases: { address: string, collection?: string, joined: number, entries: number | null }[] }>}
448
+ */
449
+ export async function restoreAppBackupStream({
450
+ orbitdb,
451
+ source,
452
+ key,
453
+ decrypt,
454
+ app,
455
+ addresses,
456
+ open,
457
+ onHeader,
458
+ onProgress,
459
+ }) {
460
+ if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
461
+ const byAddress = collectionsByAddress(addresses);
462
+ const { header, body } = await readAppBackupHeaderStream(source);
463
+ const chunks = body[Symbol.asyncIterator]();
464
+ try {
465
+ const found = (await onHeader?.(header)) ?? {};
466
+ key = found.key ?? key;
467
+ decrypt = found.decrypt ?? decrypt;
468
+
469
+ // The envelope's version is in its first five bytes.
470
+ const start = new ByteQueue();
471
+ let ended = false;
472
+ while (!ended && start.length < 5) {
473
+ const { done, value } = await chunks.next();
474
+ if (done) ended = true;
475
+ else start.push(value);
476
+ }
477
+ const first = start.take(start.length);
478
+ const rest = (async function* () {
479
+ if (first.length) yield first;
480
+ if (!ended) yield* { [Symbol.asyncIterator]: () => chunks };
481
+ })();
482
+
483
+ if (envelopeVersion(first) !== 2) {
484
+ // In one piece: there is no opening it before all of it is here.
485
+ const opened = await openEnvelope(await collectBytes(rest), header, { decrypt, app });
486
+ const restored = await restoreAppBackup({ orbitdb, opened, addresses, open, onProgress });
487
+ return { header, manifest: opened.manifest, blocks: opened.blocks.size, streamed: false, ...restored };
488
+ }
489
+ if (key === undefined) throw new Error("This backup is sealed in frames: restoring it needs its `key`");
490
+
491
+ let manifest;
492
+ const stored = await storeBlocksFromCAR(orbitdb.ipfs.blockstore, openFrames(rest, { key, context: header, onProgress }), {
493
+ onProgress,
494
+ onBlock: async ({ cid, bytes }, { roots }) => {
495
+ if (manifest || !roots.some((root) => root.equals(cid))) return;
496
+ const value = manifestIn(bytes);
497
+ if (!value) return;
498
+ await assertManifest(value, header, app);
499
+ assertTheseBooks(value, byAddress);
500
+ manifest = value;
501
+ },
502
+ });
503
+ if (!manifest) throw new Error("This backup holds no manifest.");
504
+ const restored = await putBack(byAddress, onProgress, (options) =>
505
+ restoreFromStoredBlocks(orbitdb, stored, manifest.metadata, { open, ...options }),
506
+ );
507
+ return { header, manifest, blocks: stored.blocks, streamed: true, ...restored };
508
+ } finally {
509
+ await chunks.return?.();
510
+ }
511
+ }
512
+
513
+ /** A block's value, when it is an application backup's manifest. */
514
+ function manifestIn(bytes) {
515
+ let value;
516
+ try {
517
+ value = dagCbor.decode(bytes);
518
+ } catch {
519
+ return null;
520
+ }
521
+ return value?.kind === APP_BACKUP_KIND && value?.v === APP_BACKUP_VERSION ? value : null;
522
+ }
523
+
524
+ /** The manifest is of this application, and names the header the file has. */
525
+ async function assertManifest(manifest, header, app) {
526
+ if (app && manifest.app !== app) throw new Error(`This is a backup of ${manifest.app}, not of ${app}.`);
527
+ // The header is outside the seal; the manifest inside it says which header belongs.
528
+ const expected = manifest.header ?? null;
529
+ const actual = header.length ? await sha256Hex(header) : null;
530
+ if (expected !== actual) throw new Error("This backup's header is not the one it was made with.");
531
+ }
532
+
533
+ /** An envelope in one piece, opened: {@link openAppBackup} behind the header. */
534
+ async function openEnvelope(body, header, { decrypt, app }) {
535
+ if (typeof decrypt !== "function") {
536
+ throw new Error("This backup is sealed in one piece: restoring it needs a `decrypt` function");
537
+ }
538
+ const { ciphertext, iv } = readEnvelope(body);
539
+ const car = await decrypt(ciphertext, iv);
540
+ const blocks = await readBlocksFromCAR(car, { verify: true });
541
+ let manifest;
542
+ for (const [name, { bytes: block }] of blocks) {
543
+ if (CID.parse(name).code !== dagCbor.code) continue;
544
+ manifest = manifestIn(block);
545
+ if (manifest) break;
546
+ }
547
+ if (!manifest) throw new Error("This backup holds no manifest.");
548
+ await assertManifest(manifest, header, app);
549
+ return { header, manifest, blocks };
550
+ }
@@ -96,6 +96,101 @@ export async function unixfsCid(bytes) {
96
96
  return last.cid.toString();
97
97
  }
98
98
 
99
+ const IMPORT_AS_IPFS_ADD = {
100
+ cidVersion: 0,
101
+ rawLeaves: false,
102
+ reduceSingleLeafToSelf: true,
103
+ };
104
+
105
+ /**
106
+ * {@link matchesUnixfsCid} for a file that arrives in pieces (#187): the DAG
107
+ * is built again as the chunks come, each one let go once it is hashed, and
108
+ * `done()` says whether its root is the CID. `null` for a CID this check
109
+ * cannot prove, as there.
110
+ *
111
+ * `write` resolves once the importer has taken the chunk, so a fast source
112
+ * does not pile chunks up in front of a slow hash.
113
+ *
114
+ * @param {string} cid
115
+ * @returns {{ write(chunk: Uint8Array): Promise<void>, done(): Promise<boolean> } | null}
116
+ */
117
+ export function unixfsCidVerifier(cid) {
118
+ let parsed;
119
+ try {
120
+ parsed = CID.parse(String(cid));
121
+ } catch {
122
+ return null;
123
+ }
124
+ if (parsed.code !== 0x70 || parsed.multihash.code !== 0x12) return null;
125
+
126
+ /** @type {Uint8Array[]} */
127
+ const waiting = [];
128
+ let ended = false;
129
+ let wake = () => {};
130
+ let taken = () => {};
131
+ async function* content() {
132
+ for (;;) {
133
+ if (waiting.length) {
134
+ const chunk = waiting.shift();
135
+ taken();
136
+ yield chunk;
137
+ } else if (ended) {
138
+ return;
139
+ } else {
140
+ await new Promise((resolve) => (wake = resolve));
141
+ }
142
+ }
143
+ }
144
+ const sink = {
145
+ put: async (key) => key,
146
+ get: async () => {
147
+ throw new Error("unixfsCidVerifier keeps no blocks");
148
+ },
149
+ has: async () => false,
150
+ };
151
+ let failure = null;
152
+ const root = (async () => {
153
+ let last;
154
+ for await (const entry of importer(
155
+ [{ content: content() }],
156
+ sink,
157
+ IMPORT_AS_IPFS_ADD,
158
+ )) {
159
+ last = entry;
160
+ }
161
+ return last?.cid ?? null;
162
+ })().catch((error) => {
163
+ failure = error;
164
+ taken();
165
+ return null;
166
+ });
167
+
168
+ return {
169
+ async write(chunk) {
170
+ if (failure) throw failure;
171
+ if (!chunk?.length) return;
172
+ waiting.push(chunk);
173
+ const handed = new Promise((resolve) => (taken = resolve));
174
+ wake();
175
+ await handed;
176
+ if (failure) throw failure;
177
+ },
178
+ async done() {
179
+ ended = true;
180
+ wake();
181
+ const made = await root;
182
+ if (failure) throw failure;
183
+ if (!made) return false;
184
+ const digest = made.multihash.digest;
185
+ const expected = parsed.multihash.digest;
186
+ if (digest.length !== expected.length) return false;
187
+ for (let i = 0; i < digest.length; i++)
188
+ if (digest[i] !== expected[i]) return false;
189
+ return true;
190
+ },
191
+ };
192
+ }
193
+
99
194
  /**
100
195
  * Whether `bytes` are the file a UnixFS CID names – one made with `ipfs add`'s
101
196
  * defaults, as Aleph's hosts and `unixfsCid` make it. A gateway assembles the
@@ -64,7 +64,7 @@
64
64
  */
65
65
 
66
66
  import { defineBackend, handleId, BackendError } from "./types.js";
67
- import { fetchFromGateways } from "../gateway-fetch.js";
67
+ import { fetchFromGateways, streamFromGateways } from "../gateway-fetch.js";
68
68
  import {
69
69
  ALEPH_API_FORM_HEADROOM,
70
70
  ALEPH_API_SIGNED_LIMIT,
@@ -75,6 +75,7 @@ import {
75
75
  postFile,
76
76
  unixfsCid,
77
77
  matchesUnixfsCid,
78
+ unixfsCidVerifier,
78
79
  uploadTimeout,
79
80
  } from "./aleph-upload.js";
80
81
 
@@ -349,6 +350,49 @@ export function createAlephBackend(options = {}) {
349
350
  );
350
351
  }
351
352
  },
353
+
354
+ /**
355
+ * The blob as it arrives, never all of it at once (#187): for a file of
356
+ * several hundred MB on a device that cannot hold it. Checked like
357
+ * `getBlob` – the size the STORE names, the UnixFS CID built again from
358
+ * the bytes – but **at the end**, after the chunks were handed on: see
359
+ * `streamFromGateways` for what a caller may rely on meanwhile. A dropped
360
+ * connection picks up where it was.
361
+ *
362
+ * @param {import("./types.js").BackendHandle|string} handle
363
+ * @param {{ size?: number, offset?: number, timeout?: number, signal?: AbortSignal,
364
+ * onProgress?: (received: number, total: number) => void }} [options]
365
+ * @returns {AsyncGenerator<Uint8Array>}
366
+ */
367
+ async *getBlobStream(handle, options = {}) {
368
+ const id = handleId(handle);
369
+ const size = Number.isInteger(options?.size)
370
+ ? options.size
371
+ : Number.isInteger(handle?.size)
372
+ ? handle.size
373
+ : null;
374
+ try {
375
+ yield* streamFromGateways(id, {
376
+ gateways,
377
+ timeout: options?.timeout ?? timeout,
378
+ verifier: unixfsCidVerifier,
379
+ fetchImpl: doFetch,
380
+ ...(options?.signal ? { signal: options.signal } : {}),
381
+ ...(options?.offset ? { offset: options.offset } : {}),
382
+ ...(size != null && size > 0 ? { expectedSize: size } : {}),
383
+ ...(typeof options?.onProgress === "function"
384
+ ? { onProgress: options.onProgress }
385
+ : {}),
386
+ });
387
+ } catch (error) {
388
+ const failed = new BackendError(
389
+ "NOT_FOUND",
390
+ `No blob for ${id}: ${error.message}`,
391
+ );
392
+ if (Number.isInteger(error?.received)) failed.received = error.received;
393
+ throw failed;
394
+ }
395
+ },
352
396
  };
353
397
 
354
398
  if (pin) {
@@ -46,7 +46,11 @@ import { defineBackend, BackendError, handleId } from "./types.js";
46
46
  // buffer the engine will not seal.
47
47
  export const ENVELOPE_MAGIC = new Uint8Array([0x4f, 0x53, 0x42, 0x45]);
48
48
 
49
- /** Bumped when the envelope or the algorithm changes in a way readers must notice. */
49
+ /**
50
+ * The envelope this module writes: one ciphertext under one IV, by the
51
+ * caller's `encrypt`. Version 2 is the envelope in frames, sealed with the key
52
+ * itself and opened as it arrives: ./frames.js.
53
+ */
50
54
  export const ENVELOPE_VERSION = 1;
51
55
 
52
56
  const HEADER_LENGTH = ENVELOPE_MAGIC.length + 2;
@@ -101,6 +105,15 @@ export function readEnvelope(bytes) {
101
105
  );
102
106
  }
103
107
  const version = bytes[ENVELOPE_MAGIC.length];
108
+ if (version === 2) {
109
+ // Sealed in frames (./frames.js): there is no one ciphertext and no one
110
+ // IV to hand to a caller's `decrypt`.
111
+ throw new BackendError(
112
+ "INVALID_BACKEND",
113
+ "This backup is sealed in frames (envelope version 2). It opens with the key itself: " +
114
+ "openFrames or openFramed from backends/frames.",
115
+ );
116
+ }
104
117
  if (version !== ENVELOPE_VERSION) {
105
118
  throw new BackendError(
106
119
  "INVALID_BACKEND",