@le-space/orbitdb-storage-bridge 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/lib/app-backup.js +290 -40
- package/lib/backends/aleph-pin.js +14 -3
- package/lib/backends/aleph-upload.js +95 -0
- package/lib/backends/aleph.js +45 -1
- package/lib/backends/encryption.js +14 -1
- package/lib/backends/frames.js +507 -0
- package/lib/backends/types.js +3 -0
- package/lib/gateway-fetch.js +221 -0
- package/lib/live-backup.js +101 -41
- package/lib/peer-fetch.js +115 -9
- package/lib/restore-cid.js +257 -22
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -59,8 +59,8 @@ entry is the Storacha-era class and the helpers it grew.
|
|
|
59
59
|
| `/live-backup` | increments: `createLiveBackup`, `collectIncrement`, `readIncrement`, `mergeIncrements`, `restoreFromIncrements`, `joinIncrements`, and `advanceConfirmation` / `reconcileConfirmation` for a store that refuses later |
|
|
60
60
|
| `/dehydrate` | `dehydrate`, `hydrate`: a database where a second device finds it from a secret alone |
|
|
61
61
|
| `/pointer-ipns` | `publishPointer`, `resolvePointer`: the IPNS pointer `dehydrate` publishes, under a name derived from a seed |
|
|
62
|
-
| `/gateway-fetch` | `fetchFromGateways`: bytes by CID over HTTP, verified against the CID |
|
|
63
|
-
| `/peer-fetch` | `createPeerFetch`, `createGatewayFirstFetch`: bytes by CID from the peers that hold them, over libp2p |
|
|
62
|
+
| `/gateway-fetch` | `fetchFromGateways`: bytes by CID over HTTP, verified against the CID; `streamFromGateways`: the same as they arrive, picked up where a connection dropped |
|
|
63
|
+
| `/peer-fetch` | `createPeerFetch`, `createGatewayFirstFetch`: bytes by CID from the peers that hold them, over libp2p; `.stream` and `createGatewayFirstStream` for a file too large to hold |
|
|
64
64
|
| `/courier-sync` | `createCourierSync`: replication over any byte courier — a LoRa mesh, a QR relay, a file |
|
|
65
65
|
| `/memory-courier` | `createMemoryCourierPair`: the in-process courier for tests and demos |
|
|
66
66
|
| `/backends/types` | the contract: `defineBackend`, `BackendError`, the capability flags |
|
|
@@ -73,6 +73,7 @@ entry is the Storacha-era class and the helpers it grew.
|
|
|
73
73
|
| `/backends/storacha` | the Storacha driver; needs the optional peer `@storacha/client`, and a service that no longer answers |
|
|
74
74
|
| `/backends/mirror` | one backup to several backends |
|
|
75
75
|
| `/backends/encryption` | `withEncryption`: the service holds ciphertext |
|
|
76
|
+
| `/backends/frames` | `sealFrames`, `openFrames`: the envelope in frames, opened as it arrives, for a backup a phone restores |
|
|
76
77
|
| `/backends/choose` | `createBackendFromChoice`: a backend from a name and a key, for a page's form |
|
|
77
78
|
| `/backends/resolve` | `resolveBackend`: the options-to-backend step the older functions share |
|
|
78
79
|
| `/storacha-backup`, `/components/*` | the JavaScript of the old browser widget: `storacha-backup.js`, `WebAuthnDIDProvider.js`, `theme.js`. The two `.svelte` components were dropped — they imported four files no release ever contained ([docs/STORACHA-UI-HISTORY.md](docs/STORACHA-UI-HISTORY.md)) |
|
package/lib/app-backup.js
CHANGED
|
@@ -18,16 +18,20 @@
|
|
|
18
18
|
* finds the key to the rest. This module does not interpret it. Its SHA-256
|
|
19
19
|
* is in the manifest, so a header swapped after the backup was made is
|
|
20
20
|
* refused once the body is open.
|
|
21
|
-
* - **The envelope** is the package's own (`OSBE
|
|
22
|
-
*
|
|
23
|
-
*
|
|
21
|
+
* - **The envelope** is the package's own (`OSBE`) around the CAR. Given a
|
|
22
|
+
* `key`, it is the envelope in frames (version 2, ./backends/frames.js): the
|
|
23
|
+
* file can then be restored as it arrives ({@link restoreAppBackupStream}),
|
|
24
|
+
* in the memory of one frame and one block, and the header is bound into
|
|
25
|
+
* the frames' key. Given the caller's `encrypt`, it is the envelope in one
|
|
26
|
+
* piece (version 1, ./backends/encryption.js), which can only be opened
|
|
27
|
+
* whole. The package keeps no keys, here as everywhere.
|
|
24
28
|
* - **The CAR's root** is a dag-cbor manifest: `kind` and `v` say what this is,
|
|
25
29
|
* `app` whose it is, and `metadata` is `bundleDatabases`'s, naming every
|
|
26
30
|
* database with its heads. `extra` is the application's own (belege lists its
|
|
27
31
|
* receipt files there).
|
|
28
32
|
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
33
|
+
* A backup is built in memory, as the rest of the package does, and an upload
|
|
34
|
+
* wants its whole body; it is the restore that has to fit a phone.
|
|
31
35
|
*
|
|
32
36
|
* @requires ./extract-blocks.js - bundleDatabases
|
|
33
37
|
* @requires ./backup-car.js - createCARFromBlocks
|
|
@@ -39,8 +43,9 @@ import { CID } from "multiformats/cid";
|
|
|
39
43
|
import { sha256 } from "multiformats/hashes/sha2";
|
|
40
44
|
import { bundleDatabases } from "./extract-blocks.js";
|
|
41
45
|
import { createCARFromBlocks } from "./backup-car.js";
|
|
42
|
-
import { readBlocksFromCAR, restoreFromBlocks } from "./restore-cid.js";
|
|
46
|
+
import { readBlocksFromCAR, restoreFromBlocks, restoreFromStoredBlocks, storeBlocksFromCAR } from "./restore-cid.js";
|
|
43
47
|
import { wrapEnvelope, readEnvelope } from "./backends/encryption.js";
|
|
48
|
+
import { ByteQueue, collectBytes, envelopeVersion, isFramed, openFramed, openFrames, sealFramed } from "./backends/frames.js";
|
|
44
49
|
|
|
45
50
|
/** `OSBA` — orbitdb-storage-bridge, application backup. */
|
|
46
51
|
export const APP_BACKUP_MAGIC = new Uint8Array([0x4f, 0x53, 0x42, 0x41]);
|
|
@@ -85,8 +90,12 @@ async function sha256Hex(bytes) {
|
|
|
85
90
|
* @param {string} params.app - whose backup this is; {@link openAppBackup} can insist on it
|
|
86
91
|
* @param {Object[]|Object<string, Object>} params.databases - open OrbitDB
|
|
87
92
|
* databases. Given by name, the names go into the metadata as `collection`.
|
|
88
|
-
* @param {
|
|
89
|
-
*
|
|
93
|
+
* @param {Uint8Array | CryptoKey} [params.key] - seal in frames with this key
|
|
94
|
+
* (32 bytes, or a `CryptoKey` for HKDF): the form to prefer, the only one a
|
|
95
|
+
* restore can stream
|
|
96
|
+
* @param {number} [params.frameSize] - with `key`: plaintext bytes in a frame
|
|
97
|
+
* @param {(plaintext: Uint8Array) => Promise<{ ciphertext: Uint8Array, iv: Uint8Array }>} [params.encrypt] -
|
|
98
|
+
* instead of `key`: the caller's, with the caller's key, in one piece
|
|
90
99
|
* @param {Map<string, { bytes: Uint8Array }>} [params.blocks] - blocks of the
|
|
91
100
|
* application's own, by CID string: they travel in the CAR and come back into
|
|
92
101
|
* the blockstore on restore
|
|
@@ -101,6 +110,8 @@ export async function buildAppBackup({
|
|
|
101
110
|
app,
|
|
102
111
|
databases,
|
|
103
112
|
encrypt,
|
|
113
|
+
key,
|
|
114
|
+
frameSize,
|
|
104
115
|
blocks: own,
|
|
105
116
|
header,
|
|
106
117
|
appVersion,
|
|
@@ -109,7 +120,10 @@ export async function buildAppBackup({
|
|
|
109
120
|
onProgress,
|
|
110
121
|
}) {
|
|
111
122
|
if (typeof app !== "string" || !app) throw new Error("buildAppBackup needs the application's name as `app`");
|
|
112
|
-
if (typeof encrypt !== "function")
|
|
123
|
+
if (key === undefined && typeof encrypt !== "function") {
|
|
124
|
+
throw new Error("buildAppBackup needs a `key` (sealed in frames) or an `encrypt` function: the package holds no keys");
|
|
125
|
+
}
|
|
126
|
+
if (key !== undefined && encrypt !== undefined) throw new Error("buildAppBackup takes a `key` or `encrypt`, not both");
|
|
113
127
|
if (header !== undefined && !(header instanceof Uint8Array)) throw new Error("A backup's header is bytes");
|
|
114
128
|
if (own !== undefined && !(own instanceof Map)) throw new Error("An application's own blocks come as a Map, by CID string");
|
|
115
129
|
|
|
@@ -142,21 +156,29 @@ export async function buildAppBackup({
|
|
|
142
156
|
};
|
|
143
157
|
const root = dagCbor.encode(manifest);
|
|
144
158
|
const rootCid = CID.createV1(dagCbor.code, await sha256.digest(root));
|
|
145
|
-
|
|
159
|
+
// The manifest first: a restore that reads the file as it arrives knows
|
|
160
|
+
// whose backup this is before it stores the rest.
|
|
161
|
+
const ordered = new Map([[rootCid.toString(), { cid: rootCid, bytes: root }]]);
|
|
162
|
+
for (const [name, block] of blocks) if (!ordered.has(name)) ordered.set(name, block);
|
|
146
163
|
|
|
147
|
-
const car = await createCARFromBlocks(
|
|
164
|
+
const car = await createCARFromBlocks(ordered, rootCid.toString());
|
|
148
165
|
onProgress?.({ stage: "sealing", bytes: car.length });
|
|
149
|
-
const { ciphertext, iv } = await encrypt(car);
|
|
150
|
-
const envelope = wrapEnvelope(ciphertext, iv);
|
|
151
|
-
|
|
152
166
|
const head = header ?? new Uint8Array(0);
|
|
167
|
+
let envelope;
|
|
168
|
+
if (key !== undefined) {
|
|
169
|
+
envelope = await sealFramed(car, { key, context: head, ...(frameSize ? { frameSize } : {}) });
|
|
170
|
+
} else {
|
|
171
|
+
const { ciphertext, iv } = await encrypt(car);
|
|
172
|
+
envelope = wrapEnvelope(ciphertext, iv);
|
|
173
|
+
}
|
|
174
|
+
|
|
153
175
|
const bytes = new Uint8Array(PREAMBLE + head.length + envelope.length);
|
|
154
176
|
bytes.set(APP_BACKUP_MAGIC, 0);
|
|
155
177
|
bytes[APP_BACKUP_MAGIC.length] = APP_BACKUP_VERSION;
|
|
156
178
|
new DataView(bytes.buffer).setUint32(APP_BACKUP_MAGIC.length + 1, head.length);
|
|
157
179
|
bytes.set(head, PREAMBLE);
|
|
158
180
|
bytes.set(envelope, PREAMBLE + head.length);
|
|
159
|
-
return { bytes, manifest, blocks:
|
|
181
|
+
return { bytes, manifest, blocks: ordered.size, carBytes: car.length };
|
|
160
182
|
}
|
|
161
183
|
|
|
162
184
|
/** Is this an application backup this module wrote, of any version? */
|
|
@@ -201,37 +223,33 @@ export function readAppBackupHeader(bytes) {
|
|
|
201
223
|
*
|
|
202
224
|
* @param {Uint8Array} bytes
|
|
203
225
|
* @param {object} options
|
|
204
|
-
* @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} options.decrypt
|
|
226
|
+
* @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} [options.decrypt] -
|
|
227
|
+
* for a backup sealed in one piece
|
|
228
|
+
* @param {Uint8Array | CryptoKey} [options.key] - for a backup sealed in frames
|
|
205
229
|
* @param {string} [options.app] - refuse a backup of another application
|
|
206
230
|
* @returns {Promise<{ header: Uint8Array, manifest: AppBackupManifest, blocks: Map<string, { cid: any, bytes: Uint8Array }> }>}
|
|
207
231
|
*/
|
|
208
|
-
export async function openAppBackup(bytes, { decrypt, app } = {}) {
|
|
209
|
-
if (typeof decrypt !== "function") throw new Error("openAppBackup needs a `decrypt` function");
|
|
232
|
+
export async function openAppBackup(bytes, { decrypt, key, app } = {}) {
|
|
210
233
|
const { header, body } = readAppBackupHeader(bytes);
|
|
211
|
-
|
|
212
|
-
|
|
234
|
+
let car;
|
|
235
|
+
if (isFramed(body)) {
|
|
236
|
+
if (key === undefined) throw new Error("This backup is sealed in frames: openAppBackup needs its `key`");
|
|
237
|
+
car = await openFramed(body, { key, context: header });
|
|
238
|
+
} else {
|
|
239
|
+
if (typeof decrypt !== "function") throw new Error("openAppBackup needs a `decrypt` function");
|
|
240
|
+
const { ciphertext, iv } = readEnvelope(body);
|
|
241
|
+
car = await decrypt(ciphertext, iv);
|
|
242
|
+
}
|
|
213
243
|
const blocks = await readBlocksFromCAR(car, { verify: true });
|
|
214
244
|
|
|
215
245
|
let manifest;
|
|
216
246
|
for (const [name, { bytes: block }] of blocks) {
|
|
217
247
|
if (CID.parse(name).code !== dagCbor.code) continue;
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
value = dagCbor.decode(block);
|
|
221
|
-
} catch {
|
|
222
|
-
continue;
|
|
223
|
-
}
|
|
224
|
-
if (value?.kind === APP_BACKUP_KIND && value?.v === APP_BACKUP_VERSION) {
|
|
225
|
-
manifest = value;
|
|
226
|
-
break;
|
|
227
|
-
}
|
|
248
|
+
manifest = manifestIn(block);
|
|
249
|
+
if (manifest) break;
|
|
228
250
|
}
|
|
229
251
|
if (!manifest) throw new Error("This backup holds no manifest.");
|
|
230
|
-
|
|
231
|
-
// The header is outside the seal; the manifest inside it says which header belongs.
|
|
232
|
-
const expected = manifest.header ?? null;
|
|
233
|
-
const actual = header.length ? await sha256Hex(header) : null;
|
|
234
|
-
if (expected !== actual) throw new Error("This backup's header is not the one it was made with.");
|
|
252
|
+
await assertManifest(manifest, header, app);
|
|
235
253
|
return { header, manifest, blocks };
|
|
236
254
|
}
|
|
237
255
|
|
|
@@ -263,20 +281,35 @@ export async function openAppBackup(bytes, { decrypt, app } = {}) {
|
|
|
263
281
|
* @returns {Promise<{ databases: { address: string, collection?: string, joined: number, entries: number | null }[] }>}
|
|
264
282
|
*/
|
|
265
283
|
export async function restoreAppBackup({ orbitdb, opened, addresses, open, onProgress }) {
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
284
|
+
const byAddress = collectionsByAddress(addresses);
|
|
285
|
+
assertTheseBooks(opened?.manifest, byAddress);
|
|
286
|
+
return putBack(byAddress, onProgress, (options) =>
|
|
287
|
+
restoreFromBlocks(orbitdb, opened.blocks, opened.manifest.metadata, { open, ...options }),
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** These books' database addresses, each with its collection when they came by name. */
|
|
292
|
+
function collectionsByAddress(addresses) {
|
|
293
|
+
return Array.isArray(addresses)
|
|
269
294
|
? Object.fromEntries(addresses.map((address) => [String(address), undefined]))
|
|
270
295
|
: Object.fromEntries(Object.entries(addresses ?? {}).map(([collection, address]) => [String(address), collection]));
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/** Every database the backup names is one of these books, or it is refused. */
|
|
299
|
+
function assertTheseBooks(manifest, byAddress) {
|
|
300
|
+
const metadata = manifest?.metadata;
|
|
301
|
+
if (!metadata?.databases?.length) throw new Error("This backup names no databases.");
|
|
271
302
|
for (const d of metadata.databases) {
|
|
272
303
|
if (!(String(d.address) in byAddress)) {
|
|
273
304
|
throw new Error("This backup holds databases these books do not have.");
|
|
274
305
|
}
|
|
275
306
|
}
|
|
307
|
+
}
|
|
276
308
|
|
|
309
|
+
/** The join, with a head that could not be joined as a failure and the answer in the application's words. */
|
|
310
|
+
async function putBack(byAddress, onProgress, restore) {
|
|
277
311
|
const warnings = [];
|
|
278
|
-
const restored = await
|
|
279
|
-
open,
|
|
312
|
+
const restored = await restore({
|
|
280
313
|
log: { info() {}, debug() {}, warn: (message) => warnings.push(String(message)) },
|
|
281
314
|
onProgress: (p) =>
|
|
282
315
|
onProgress?.({
|
|
@@ -298,3 +331,220 @@ export async function restoreAppBackup({ orbitdb, opened, addresses, open, onPro
|
|
|
298
331
|
})),
|
|
299
332
|
};
|
|
300
333
|
}
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* The header of a backup that arrives in pieces, without a key, and the rest
|
|
337
|
+
* of the stream behind it: what a keyring needs before anything can be opened,
|
|
338
|
+
* read from the first bytes and not from the whole file.
|
|
339
|
+
*
|
|
340
|
+
* @param {AsyncIterable<Uint8Array> | Iterable<Uint8Array> | Uint8Array} source
|
|
341
|
+
* @returns {Promise<{ version: number, header: Uint8Array, body: AsyncGenerator<Uint8Array> }>}
|
|
342
|
+
* `body` is the envelope; it must be read, or returned, for the source to be let go
|
|
343
|
+
*/
|
|
344
|
+
export async function readAppBackupHeaderStream(source) {
|
|
345
|
+
const chunks = (
|
|
346
|
+
source instanceof Uint8Array
|
|
347
|
+
? (async function* () {
|
|
348
|
+
yield source;
|
|
349
|
+
})()
|
|
350
|
+
: source
|
|
351
|
+
)[Symbol.asyncIterator]?.() ?? source[Symbol.iterator]();
|
|
352
|
+
const queue = new ByteQueue();
|
|
353
|
+
let ended = false;
|
|
354
|
+
const fill = async (n) => {
|
|
355
|
+
while (!ended && queue.length < n) {
|
|
356
|
+
const { done, value } = await chunks.next();
|
|
357
|
+
if (done) ended = true;
|
|
358
|
+
else queue.push(value);
|
|
359
|
+
}
|
|
360
|
+
return queue.length >= n;
|
|
361
|
+
};
|
|
362
|
+
try {
|
|
363
|
+
if (!(await fill(PREAMBLE))) throw new Error("This is not an application backup.");
|
|
364
|
+
const preamble = queue.take(PREAMBLE);
|
|
365
|
+
if (!isAppBackup(preamble)) throw new Error("This is not an application backup.");
|
|
366
|
+
const version = preamble[APP_BACKUP_MAGIC.length];
|
|
367
|
+
if (version !== APP_BACKUP_VERSION) {
|
|
368
|
+
throw new Error(
|
|
369
|
+
`This backup was written in format ${version}, and this build reads ${APP_BACKUP_VERSION}. ` +
|
|
370
|
+
"Upgrade @le-space/orbitdb-storage-bridge to open it.",
|
|
371
|
+
);
|
|
372
|
+
}
|
|
373
|
+
const length = new DataView(preamble.buffer, preamble.byteOffset, preamble.byteLength).getUint32(
|
|
374
|
+
APP_BACKUP_MAGIC.length + 1,
|
|
375
|
+
);
|
|
376
|
+
if (length > MAX_HEADER_BYTES) throw new Error(`This backup names a header of ${length} bytes; a header has at most ${MAX_HEADER_BYTES}.`);
|
|
377
|
+
if (!(await fill(length))) throw new Error("This backup is cut short.");
|
|
378
|
+
// Its own copy: it outlives the chunk it arrived in.
|
|
379
|
+
const header = new Uint8Array(queue.take(length));
|
|
380
|
+
return {
|
|
381
|
+
version,
|
|
382
|
+
header,
|
|
383
|
+
body: (async function* () {
|
|
384
|
+
try {
|
|
385
|
+
while (queue.length) yield queue.take(queue.chunks[0].length);
|
|
386
|
+
while (!ended) {
|
|
387
|
+
const { done, value } = await chunks.next();
|
|
388
|
+
if (done) ended = true;
|
|
389
|
+
else if (value?.length) yield value;
|
|
390
|
+
}
|
|
391
|
+
} finally {
|
|
392
|
+
await chunks.return?.();
|
|
393
|
+
}
|
|
394
|
+
})(),
|
|
395
|
+
};
|
|
396
|
+
} catch (error) {
|
|
397
|
+
await chunks.return?.();
|
|
398
|
+
throw error;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** The longest header a streamed reader holds: a keyring, not a payload. */
|
|
403
|
+
export const MAX_HEADER_BYTES = 16 * 1024 * 1024;
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* @typedef {{ stage: "opened", frames: number, bytes: number }
|
|
407
|
+
* | { stage: "stored", blocks: number, bytes: number }
|
|
408
|
+
* | AppRestoreProgress} AppRestoreStreamProgress
|
|
409
|
+
*/
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* Restore a backup as it arrives (#187): {@link openAppBackup} and
|
|
413
|
+
* {@link restoreAppBackup} in one pass, for a file too large to hold. Each
|
|
414
|
+
* frame is opened as it comes, each block checked and put into the blockstore
|
|
415
|
+
* and let go, so the memory this needs is a frame's and a block's, whatever
|
|
416
|
+
* the backup's size.
|
|
417
|
+
*
|
|
418
|
+
* The same refusals as there, at the earliest moment each can be known: a
|
|
419
|
+
* wrong key before the first frame; another application's backup, or one of
|
|
420
|
+
* other books, at the manifest, which a backup made here carries first, so
|
|
421
|
+
* before any block is stored. **The databases are joined only after the last
|
|
422
|
+
* frame opened**: a file that is cut short, or damaged anywhere, throws with
|
|
423
|
+
* the books untouched. The blocks it stored until then stay in the blockstore,
|
|
424
|
+
* sealed and pointed at by nothing.
|
|
425
|
+
*
|
|
426
|
+
* A backup sealed in one piece (with `encrypt`) cannot be opened as it
|
|
427
|
+
* arrives. It is restored all the same when `decrypt` is given, held whole as
|
|
428
|
+
* {@link openAppBackup} holds it.
|
|
429
|
+
*
|
|
430
|
+
* @param {object} params
|
|
431
|
+
* @param {any} params.orbitdb
|
|
432
|
+
* @param {AsyncIterable<Uint8Array> | Iterable<Uint8Array> | Uint8Array} params.source -
|
|
433
|
+
* the file, e.g. a backend's `getBlobStream`
|
|
434
|
+
* @param {Uint8Array | CryptoKey} [params.key] - opens a backup sealed in frames
|
|
435
|
+
* @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} [params.decrypt] -
|
|
436
|
+
* opens a backup sealed in one piece
|
|
437
|
+
* @param {string} [params.app] - refuse a backup of another application
|
|
438
|
+
* @param {Record<string, string> | string[]} params.addresses - as {@link restoreAppBackup}
|
|
439
|
+
* @param {Record<string, any>} [params.open]
|
|
440
|
+
* @param {(header: Uint8Array) => Promise<{ key?: Uint8Array | CryptoKey, decrypt?: Function } | void> | { key?: Uint8Array | CryptoKey, decrypt?: Function } | void} [params.onHeader] -
|
|
441
|
+
* sees the header before anything is opened, and may answer with the key it
|
|
442
|
+
* leads to: how a keyring in the header is used
|
|
443
|
+
* @param {(progress: AppRestoreStreamProgress) => void} [params.onProgress] -
|
|
444
|
+
* frames opened, blocks stored, databases joined; bytes fetched are the
|
|
445
|
+
* source's to report (`getBlobStream`'s `onProgress`)
|
|
446
|
+
* @returns {Promise<{ header: Uint8Array, manifest: AppBackupManifest, blocks: number, streamed: boolean,
|
|
447
|
+
* databases: { address: string, collection?: string, joined: number, entries: number | null }[] }>}
|
|
448
|
+
*/
|
|
449
|
+
export async function restoreAppBackupStream({
|
|
450
|
+
orbitdb,
|
|
451
|
+
source,
|
|
452
|
+
key,
|
|
453
|
+
decrypt,
|
|
454
|
+
app,
|
|
455
|
+
addresses,
|
|
456
|
+
open,
|
|
457
|
+
onHeader,
|
|
458
|
+
onProgress,
|
|
459
|
+
}) {
|
|
460
|
+
if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
|
|
461
|
+
const byAddress = collectionsByAddress(addresses);
|
|
462
|
+
const { header, body } = await readAppBackupHeaderStream(source);
|
|
463
|
+
const chunks = body[Symbol.asyncIterator]();
|
|
464
|
+
try {
|
|
465
|
+
const found = (await onHeader?.(header)) ?? {};
|
|
466
|
+
key = found.key ?? key;
|
|
467
|
+
decrypt = found.decrypt ?? decrypt;
|
|
468
|
+
|
|
469
|
+
// The envelope's version is in its first five bytes.
|
|
470
|
+
const start = new ByteQueue();
|
|
471
|
+
let ended = false;
|
|
472
|
+
while (!ended && start.length < 5) {
|
|
473
|
+
const { done, value } = await chunks.next();
|
|
474
|
+
if (done) ended = true;
|
|
475
|
+
else start.push(value);
|
|
476
|
+
}
|
|
477
|
+
const first = start.take(start.length);
|
|
478
|
+
const rest = (async function* () {
|
|
479
|
+
if (first.length) yield first;
|
|
480
|
+
if (!ended) yield* { [Symbol.asyncIterator]: () => chunks };
|
|
481
|
+
})();
|
|
482
|
+
|
|
483
|
+
if (envelopeVersion(first) !== 2) {
|
|
484
|
+
// In one piece: there is no opening it before all of it is here.
|
|
485
|
+
const opened = await openEnvelope(await collectBytes(rest), header, { decrypt, app });
|
|
486
|
+
const restored = await restoreAppBackup({ orbitdb, opened, addresses, open, onProgress });
|
|
487
|
+
return { header, manifest: opened.manifest, blocks: opened.blocks.size, streamed: false, ...restored };
|
|
488
|
+
}
|
|
489
|
+
if (key === undefined) throw new Error("This backup is sealed in frames: restoring it needs its `key`");
|
|
490
|
+
|
|
491
|
+
let manifest;
|
|
492
|
+
const stored = await storeBlocksFromCAR(orbitdb.ipfs.blockstore, openFrames(rest, { key, context: header, onProgress }), {
|
|
493
|
+
onProgress,
|
|
494
|
+
onBlock: async ({ cid, bytes }, { roots }) => {
|
|
495
|
+
if (manifest || !roots.some((root) => root.equals(cid))) return;
|
|
496
|
+
const value = manifestIn(bytes);
|
|
497
|
+
if (!value) return;
|
|
498
|
+
await assertManifest(value, header, app);
|
|
499
|
+
assertTheseBooks(value, byAddress);
|
|
500
|
+
manifest = value;
|
|
501
|
+
},
|
|
502
|
+
});
|
|
503
|
+
if (!manifest) throw new Error("This backup holds no manifest.");
|
|
504
|
+
const restored = await putBack(byAddress, onProgress, (options) =>
|
|
505
|
+
restoreFromStoredBlocks(orbitdb, stored, manifest.metadata, { open, ...options }),
|
|
506
|
+
);
|
|
507
|
+
return { header, manifest, blocks: stored.blocks, streamed: true, ...restored };
|
|
508
|
+
} finally {
|
|
509
|
+
await chunks.return?.();
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/** A block's value, when it is an application backup's manifest. */
|
|
514
|
+
function manifestIn(bytes) {
|
|
515
|
+
let value;
|
|
516
|
+
try {
|
|
517
|
+
value = dagCbor.decode(bytes);
|
|
518
|
+
} catch {
|
|
519
|
+
return null;
|
|
520
|
+
}
|
|
521
|
+
return value?.kind === APP_BACKUP_KIND && value?.v === APP_BACKUP_VERSION ? value : null;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
/** The manifest is of this application, and names the header the file has. */
|
|
525
|
+
async function assertManifest(manifest, header, app) {
|
|
526
|
+
if (app && manifest.app !== app) throw new Error(`This is a backup of ${manifest.app}, not of ${app}.`);
|
|
527
|
+
// The header is outside the seal; the manifest inside it says which header belongs.
|
|
528
|
+
const expected = manifest.header ?? null;
|
|
529
|
+
const actual = header.length ? await sha256Hex(header) : null;
|
|
530
|
+
if (expected !== actual) throw new Error("This backup's header is not the one it was made with.");
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/** An envelope in one piece, opened: {@link openAppBackup} behind the header. */
|
|
534
|
+
async function openEnvelope(body, header, { decrypt, app }) {
|
|
535
|
+
if (typeof decrypt !== "function") {
|
|
536
|
+
throw new Error("This backup is sealed in one piece: restoring it needs a `decrypt` function");
|
|
537
|
+
}
|
|
538
|
+
const { ciphertext, iv } = readEnvelope(body);
|
|
539
|
+
const car = await decrypt(ciphertext, iv);
|
|
540
|
+
const blocks = await readBlocksFromCAR(car, { verify: true });
|
|
541
|
+
let manifest;
|
|
542
|
+
for (const [name, { bytes: block }] of blocks) {
|
|
543
|
+
if (CID.parse(name).code !== dagCbor.code) continue;
|
|
544
|
+
manifest = manifestIn(block);
|
|
545
|
+
if (manifest) break;
|
|
546
|
+
}
|
|
547
|
+
if (!manifest) throw new Error("This backup holds no manifest.");
|
|
548
|
+
await assertManifest(manifest, header, app);
|
|
549
|
+
return { header, manifest, blocks };
|
|
550
|
+
}
|
|
@@ -358,12 +358,16 @@ export async function waitForMessage(itemHash, options = {}) {
|
|
|
358
358
|
* coming back later, e.g. to an increment that was still pending when its
|
|
359
359
|
* flush ended (`createLiveBackup().reconcile`).
|
|
360
360
|
*
|
|
361
|
+
* `unknown` right after sending is normal: a node answers 404 until the
|
|
362
|
+
* message has reached it. `reconcile` therefore takes `unknown` as refused
|
|
363
|
+
* only for an increment shipped more than `UNKNOWN_GRACE_MS` ago.
|
|
364
|
+
*
|
|
361
365
|
* @param {string} itemHash
|
|
362
366
|
* @param {object} [options]
|
|
363
367
|
* @param {string} [options.apiHost]
|
|
364
368
|
* @param {typeof fetch} [options.fetch]
|
|
365
369
|
* @returns {Promise<{ itemHash: string, status: string, errorCode?: number, details?: unknown }>}
|
|
366
|
-
* @throws when Aleph does not answer (anything but 200 or 404)
|
|
370
|
+
* @throws when Aleph does not answer (anything but 200 or 404), or answers 200 without a status
|
|
367
371
|
*/
|
|
368
372
|
export async function messageStatus(itemHash, options = {}) {
|
|
369
373
|
const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
|
|
@@ -382,10 +386,17 @@ export async function messageStatus(itemHash, options = {}) {
|
|
|
382
386
|
"UNSUPPORTED",
|
|
383
387
|
`Aleph did not answer: ${response.status}`,
|
|
384
388
|
);
|
|
385
|
-
|
|
389
|
+
// A body that says nothing is no answer – not "unknown", which a caller
|
|
390
|
+
// takes as refused and ships again.
|
|
391
|
+
const body = await response.json().catch(() => null);
|
|
392
|
+
if (typeof body?.status !== "string")
|
|
393
|
+
throw new BackendError(
|
|
394
|
+
"UNSUPPORTED",
|
|
395
|
+
`Aleph answered ${response.status} without a message status`,
|
|
396
|
+
);
|
|
386
397
|
return {
|
|
387
398
|
itemHash,
|
|
388
|
-
status:
|
|
399
|
+
status: body.status,
|
|
389
400
|
...(body?.error_code != null ? { errorCode: body.error_code } : {}),
|
|
390
401
|
...(body?.details != null ? { details: body.details } : {}),
|
|
391
402
|
};
|
|
@@ -96,6 +96,101 @@ export async function unixfsCid(bytes) {
|
|
|
96
96
|
return last.cid.toString();
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
+
const IMPORT_AS_IPFS_ADD = {
|
|
100
|
+
cidVersion: 0,
|
|
101
|
+
rawLeaves: false,
|
|
102
|
+
reduceSingleLeafToSelf: true,
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* {@link matchesUnixfsCid} for a file that arrives in pieces (#187): the DAG
|
|
107
|
+
* is built again as the chunks come, each one let go once it is hashed, and
|
|
108
|
+
* `done()` says whether its root is the CID. `null` for a CID this check
|
|
109
|
+
* cannot prove, as there.
|
|
110
|
+
*
|
|
111
|
+
* `write` resolves once the importer has taken the chunk, so a fast source
|
|
112
|
+
* does not pile chunks up in front of a slow hash.
|
|
113
|
+
*
|
|
114
|
+
* @param {string} cid
|
|
115
|
+
* @returns {{ write(chunk: Uint8Array): Promise<void>, done(): Promise<boolean> } | null}
|
|
116
|
+
*/
|
|
117
|
+
export function unixfsCidVerifier(cid) {
|
|
118
|
+
let parsed;
|
|
119
|
+
try {
|
|
120
|
+
parsed = CID.parse(String(cid));
|
|
121
|
+
} catch {
|
|
122
|
+
return null;
|
|
123
|
+
}
|
|
124
|
+
if (parsed.code !== 0x70 || parsed.multihash.code !== 0x12) return null;
|
|
125
|
+
|
|
126
|
+
/** @type {Uint8Array[]} */
|
|
127
|
+
const waiting = [];
|
|
128
|
+
let ended = false;
|
|
129
|
+
let wake = () => {};
|
|
130
|
+
let taken = () => {};
|
|
131
|
+
async function* content() {
|
|
132
|
+
for (;;) {
|
|
133
|
+
if (waiting.length) {
|
|
134
|
+
const chunk = waiting.shift();
|
|
135
|
+
taken();
|
|
136
|
+
yield chunk;
|
|
137
|
+
} else if (ended) {
|
|
138
|
+
return;
|
|
139
|
+
} else {
|
|
140
|
+
await new Promise((resolve) => (wake = resolve));
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
const sink = {
|
|
145
|
+
put: async (key) => key,
|
|
146
|
+
get: async () => {
|
|
147
|
+
throw new Error("unixfsCidVerifier keeps no blocks");
|
|
148
|
+
},
|
|
149
|
+
has: async () => false,
|
|
150
|
+
};
|
|
151
|
+
let failure = null;
|
|
152
|
+
const root = (async () => {
|
|
153
|
+
let last;
|
|
154
|
+
for await (const entry of importer(
|
|
155
|
+
[{ content: content() }],
|
|
156
|
+
sink,
|
|
157
|
+
IMPORT_AS_IPFS_ADD,
|
|
158
|
+
)) {
|
|
159
|
+
last = entry;
|
|
160
|
+
}
|
|
161
|
+
return last?.cid ?? null;
|
|
162
|
+
})().catch((error) => {
|
|
163
|
+
failure = error;
|
|
164
|
+
taken();
|
|
165
|
+
return null;
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
return {
|
|
169
|
+
async write(chunk) {
|
|
170
|
+
if (failure) throw failure;
|
|
171
|
+
if (!chunk?.length) return;
|
|
172
|
+
waiting.push(chunk);
|
|
173
|
+
const handed = new Promise((resolve) => (taken = resolve));
|
|
174
|
+
wake();
|
|
175
|
+
await handed;
|
|
176
|
+
if (failure) throw failure;
|
|
177
|
+
},
|
|
178
|
+
async done() {
|
|
179
|
+
ended = true;
|
|
180
|
+
wake();
|
|
181
|
+
const made = await root;
|
|
182
|
+
if (failure) throw failure;
|
|
183
|
+
if (!made) return false;
|
|
184
|
+
const digest = made.multihash.digest;
|
|
185
|
+
const expected = parsed.multihash.digest;
|
|
186
|
+
if (digest.length !== expected.length) return false;
|
|
187
|
+
for (let i = 0; i < digest.length; i++)
|
|
188
|
+
if (digest[i] !== expected[i]) return false;
|
|
189
|
+
return true;
|
|
190
|
+
},
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
99
194
|
/**
|
|
100
195
|
* Whether `bytes` are the file a UnixFS CID names – one made with `ipfs add`'s
|
|
101
196
|
* defaults, as Aleph's hosts and `unixfsCid` make it. A gateway assembles the
|
package/lib/backends/aleph.js
CHANGED
|
@@ -64,7 +64,7 @@
|
|
|
64
64
|
*/
|
|
65
65
|
|
|
66
66
|
import { defineBackend, handleId, BackendError } from "./types.js";
|
|
67
|
-
import { fetchFromGateways } from "../gateway-fetch.js";
|
|
67
|
+
import { fetchFromGateways, streamFromGateways } from "../gateway-fetch.js";
|
|
68
68
|
import {
|
|
69
69
|
ALEPH_API_FORM_HEADROOM,
|
|
70
70
|
ALEPH_API_SIGNED_LIMIT,
|
|
@@ -75,6 +75,7 @@ import {
|
|
|
75
75
|
postFile,
|
|
76
76
|
unixfsCid,
|
|
77
77
|
matchesUnixfsCid,
|
|
78
|
+
unixfsCidVerifier,
|
|
78
79
|
uploadTimeout,
|
|
79
80
|
} from "./aleph-upload.js";
|
|
80
81
|
|
|
@@ -349,6 +350,49 @@ export function createAlephBackend(options = {}) {
|
|
|
349
350
|
);
|
|
350
351
|
}
|
|
351
352
|
},
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* The blob as it arrives, never all of it at once (#187): for a file of
|
|
356
|
+
* several hundred MB on a device that cannot hold it. Checked like
|
|
357
|
+
* `getBlob` – the size the STORE names, the UnixFS CID built again from
|
|
358
|
+
* the bytes – but **at the end**, after the chunks were handed on: see
|
|
359
|
+
* `streamFromGateways` for what a caller may rely on meanwhile. A dropped
|
|
360
|
+
* connection picks up where it was.
|
|
361
|
+
*
|
|
362
|
+
* @param {import("./types.js").BackendHandle|string} handle
|
|
363
|
+
* @param {{ size?: number, offset?: number, timeout?: number, signal?: AbortSignal,
|
|
364
|
+
* onProgress?: (received: number, total: number) => void }} [options]
|
|
365
|
+
* @returns {AsyncGenerator<Uint8Array>}
|
|
366
|
+
*/
|
|
367
|
+
async *getBlobStream(handle, options = {}) {
|
|
368
|
+
const id = handleId(handle);
|
|
369
|
+
const size = Number.isInteger(options?.size)
|
|
370
|
+
? options.size
|
|
371
|
+
: Number.isInteger(handle?.size)
|
|
372
|
+
? handle.size
|
|
373
|
+
: null;
|
|
374
|
+
try {
|
|
375
|
+
yield* streamFromGateways(id, {
|
|
376
|
+
gateways,
|
|
377
|
+
timeout: options?.timeout ?? timeout,
|
|
378
|
+
verifier: unixfsCidVerifier,
|
|
379
|
+
fetchImpl: doFetch,
|
|
380
|
+
...(options?.signal ? { signal: options.signal } : {}),
|
|
381
|
+
...(options?.offset ? { offset: options.offset } : {}),
|
|
382
|
+
...(size != null && size > 0 ? { expectedSize: size } : {}),
|
|
383
|
+
...(typeof options?.onProgress === "function"
|
|
384
|
+
? { onProgress: options.onProgress }
|
|
385
|
+
: {}),
|
|
386
|
+
});
|
|
387
|
+
} catch (error) {
|
|
388
|
+
const failed = new BackendError(
|
|
389
|
+
"NOT_FOUND",
|
|
390
|
+
`No blob for ${id}: ${error.message}`,
|
|
391
|
+
);
|
|
392
|
+
if (Number.isInteger(error?.received)) failed.received = error.received;
|
|
393
|
+
throw failed;
|
|
394
|
+
}
|
|
395
|
+
},
|
|
352
396
|
};
|
|
353
397
|
|
|
354
398
|
if (pin) {
|
|
@@ -46,7 +46,11 @@ import { defineBackend, BackendError, handleId } from "./types.js";
|
|
|
46
46
|
// buffer the engine will not seal.
|
|
47
47
|
export const ENVELOPE_MAGIC = new Uint8Array([0x4f, 0x53, 0x42, 0x45]);
|
|
48
48
|
|
|
49
|
-
/**
|
|
49
|
+
/**
|
|
50
|
+
* The envelope this module writes: one ciphertext under one IV, by the
|
|
51
|
+
* caller's `encrypt`. Version 2 is the envelope in frames, sealed with the key
|
|
52
|
+
* itself and opened as it arrives: ./frames.js.
|
|
53
|
+
*/
|
|
50
54
|
export const ENVELOPE_VERSION = 1;
|
|
51
55
|
|
|
52
56
|
const HEADER_LENGTH = ENVELOPE_MAGIC.length + 2;
|
|
@@ -101,6 +105,15 @@ export function readEnvelope(bytes) {
|
|
|
101
105
|
);
|
|
102
106
|
}
|
|
103
107
|
const version = bytes[ENVELOPE_MAGIC.length];
|
|
108
|
+
if (version === 2) {
|
|
109
|
+
// Sealed in frames (./frames.js): there is no one ciphertext and no one
|
|
110
|
+
// IV to hand to a caller's `decrypt`.
|
|
111
|
+
throw new BackendError(
|
|
112
|
+
"INVALID_BACKEND",
|
|
113
|
+
"This backup is sealed in frames (envelope version 2). It opens with the key itself: " +
|
|
114
|
+
"openFrames or openFramed from backends/frames.",
|
|
115
|
+
);
|
|
116
|
+
}
|
|
104
117
|
if (version !== ENVELOPE_VERSION) {
|
|
105
118
|
throw new BackendError(
|
|
106
119
|
"INVALID_BACKEND",
|