@le-space/orbitdb-storage-bridge 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/live-backup.js +2 -2
- package/lib/restore-cid.js +48 -4
- package/package.json +1 -1
package/lib/live-backup.js
CHANGED
|
@@ -56,7 +56,7 @@ import { CID } from "multiformats/cid";
|
|
|
56
56
|
import { sha256 } from "multiformats/hashes/sha2";
|
|
57
57
|
import { base58btc } from "multiformats/bases/base58";
|
|
58
58
|
import { createCARFromBlocks } from "./backup-car.js";
|
|
59
|
-
import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId } from "./restore-cid.js";
|
|
59
|
+
import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId, standalone } from "./restore-cid.js";
|
|
60
60
|
|
|
61
61
|
/** What the manifest at an increment's root calls itself. */
|
|
62
62
|
export const INCREMENT_KIND = "increment";
|
|
@@ -517,7 +517,7 @@ export async function joinIncrements(
|
|
|
517
517
|
// OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
|
|
518
518
|
// its own names, and identities are looked up by theirs, so each block
|
|
519
519
|
// goes in under the name its reader will ask for.
|
|
520
|
-
await database.log.storage.put(cid.toV1().toString(base58btc), bytes);
|
|
520
|
+
await database.log.storage.put(cid.toV1().toString(base58btc), standalone(bytes));
|
|
521
521
|
}
|
|
522
522
|
|
|
523
523
|
let joined = 0;
|
package/lib/restore-cid.js
CHANGED
|
@@ -116,7 +116,7 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
|
|
|
116
116
|
}
|
|
117
117
|
}
|
|
118
118
|
|
|
119
|
-
blocks.set(name, { bytes: block.bytes });
|
|
119
|
+
blocks.set(name, { bytes: standalone(block.bytes) });
|
|
120
120
|
}
|
|
121
121
|
|
|
122
122
|
return blocks;
|
|
@@ -146,6 +146,24 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
|
|
|
146
146
|
* The CAR was verified block by block on the way in, so this decodes without
|
|
147
147
|
* hashing again.
|
|
148
148
|
*/
|
|
149
|
+
/**
|
|
150
|
+
* The same bytes, owning their buffer.
|
|
151
|
+
*
|
|
152
|
+
* A CAR arrives as one `ArrayBuffer` and `CarReader` hands out each block as a
|
|
153
|
+
* view into it. IndexedDB's structured clone stores *the buffer behind a view*,
|
|
154
|
+
* not the view — so writing the 2 016 blocks of a 3.7 MB backup wrote 3.7 MB
|
|
155
|
+
* apiece, some 7.5 GB in all, and the browser refused everything past about
|
|
156
|
+
* block 500. The copy costs one pass over bytes that are going to disk anyway.
|
|
157
|
+
*
|
|
158
|
+
* @param {Uint8Array} bytes
|
|
159
|
+
* @returns {Uint8Array} `bytes` itself when it already owns its buffer
|
|
160
|
+
*/
|
|
161
|
+
export function standalone(bytes) {
|
|
162
|
+
return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength
|
|
163
|
+
? bytes
|
|
164
|
+
: new Uint8Array(bytes);
|
|
165
|
+
}
|
|
166
|
+
|
|
149
167
|
export function entryDatabaseId(bytes) {
|
|
150
168
|
let value;
|
|
151
169
|
try {
|
|
@@ -335,11 +353,17 @@ function assertRestorable(metadata) {
|
|
|
335
353
|
* the backup names (a node without pubsub needs `{ sync: false }`)
|
|
336
354
|
* @param {(progress: { stage: "database", index: number, total: number,
|
|
337
355
|
* address: string, joined: number, heads: number }) => void} [options.onProgress]
|
|
356
|
+
* @param {() => Promise<void>|void} [options.beforeJoin] called once the blocks
|
|
357
|
+
* are in the blockstore and before any database is opened — where a caller
|
|
358
|
+
* that must close its own handles does it, late enough that a refused block
|
|
359
|
+
* has already thrown and left those handles alone
|
|
360
|
+
* @throws {Error} `BlocksNotStoredError` when the blockstore refuses any block,
|
|
361
|
+
* before `beforeJoin` and before anything is opened
|
|
338
362
|
* @returns {Promise<{ blocks: number, databases: { address: string, name?: string,
|
|
339
363
|
* database: Object, entries: number|null, heads: number, joined: number }[] }>}
|
|
340
364
|
*/
|
|
341
365
|
export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {}) {
|
|
342
|
-
const { log = SILENT, verify = true, open = {}, onProgress } = options;
|
|
366
|
+
const { log = SILENT, verify = true, open = {}, onProgress, beforeJoin } = options;
|
|
343
367
|
if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
|
|
344
368
|
if (!(blocks instanceof Map)) throw new Error("blocks must be a Map");
|
|
345
369
|
if (!isValidMetadata(metadata)) throw new Error("Invalid backup metadata");
|
|
@@ -362,14 +386,34 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
|
|
|
362
386
|
|
|
363
387
|
// Into the blockstore, where Helia looks — once, whatever the databases.
|
|
364
388
|
let stored = 0;
|
|
389
|
+
/** @type {string[]} */
|
|
390
|
+
const refused = [];
|
|
365
391
|
for (const [cidString, { bytes }] of blocks) {
|
|
366
392
|
try {
|
|
367
|
-
await orbitdb.ipfs.blockstore.put(CID.parse(cidString), bytes);
|
|
393
|
+
await orbitdb.ipfs.blockstore.put(CID.parse(cidString), standalone(bytes));
|
|
368
394
|
stored++;
|
|
369
395
|
} catch (error) {
|
|
396
|
+
if (refused.length < 3) refused.push(`${cidString.slice(0, 12)}…: ${error.message}`);
|
|
370
397
|
log.warn(` ⚠️ could not store ${cidString.slice(0, 12)}…: ${error.message}`);
|
|
371
398
|
}
|
|
372
399
|
}
|
|
400
|
+
// A block that did not land is a head the log will later look for, not find,
|
|
401
|
+
// and wait out on the network — twenty seconds of silence per head, ending
|
|
402
|
+
// in a complaint about the entry rather than about the storage. A browser
|
|
403
|
+
// whose quota is spent arrives here with most of the backup refused. Say so
|
|
404
|
+
// now, while the caller's databases are all still open.
|
|
405
|
+
if (stored !== blocks.size) {
|
|
406
|
+
const error = new Error(
|
|
407
|
+
`The blockstore took ${stored} of ${blocks.size} blocks; the rest were refused: ${refused.join("; ")}`,
|
|
408
|
+
);
|
|
409
|
+
error.name = "BlocksNotStoredError";
|
|
410
|
+
Object.assign(error, { stored, total: blocks.size, refused });
|
|
411
|
+
throw error;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// Everything is in the blockstore and nothing has been opened yet: the one
|
|
415
|
+
// moment at which a caller can safely close what it holds open.
|
|
416
|
+
await beforeJoin?.();
|
|
373
417
|
|
|
374
418
|
const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
|
|
375
419
|
const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
|
|
@@ -388,7 +432,7 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
|
|
|
388
432
|
const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
389
433
|
for (const [hash, bytes] of mine) {
|
|
390
434
|
try {
|
|
391
|
-
await opened.log.storage.put(hash, bytes);
|
|
435
|
+
await opened.log.storage.put(hash, standalone(bytes));
|
|
392
436
|
} catch (error) {
|
|
393
437
|
log.warn(` ⚠️ could not copy ${hash.slice(0, 12)}… to log storage: ${error.message}`);
|
|
394
438
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@le-space/orbitdb-storage-bridge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.24.0",
|
|
4
4
|
"description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
|
|
5
5
|
"main": "lib/orbitdb-storacha-bridge.js",
|
|
6
6
|
"svelte": "dist/components/",
|