@le-space/orbitdb-storage-bridge 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -56,7 +56,7 @@ import { CID } from "multiformats/cid";
56
56
  import { sha256 } from "multiformats/hashes/sha2";
57
57
  import { base58btc } from "multiformats/bases/base58";
58
58
  import { createCARFromBlocks } from "./backup-car.js";
59
- import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId } from "./restore-cid.js";
59
+ import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId, standalone } from "./restore-cid.js";
60
60
 
61
61
  /** What the manifest at an increment's root calls itself. */
62
62
  export const INCREMENT_KIND = "increment";
@@ -517,7 +517,7 @@ export async function joinIncrements(
517
517
  // OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
518
518
  // its own names, and identities are looked up by theirs, so each block
519
519
  // goes in under the name its reader will ask for.
520
- await database.log.storage.put(cid.toV1().toString(base58btc), bytes);
520
+ await database.log.storage.put(cid.toV1().toString(base58btc), standalone(bytes));
521
521
  }
522
522
 
523
523
  let joined = 0;
@@ -116,7 +116,7 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
116
116
  }
117
117
  }
118
118
 
119
- blocks.set(name, { bytes: block.bytes });
119
+ blocks.set(name, { bytes: standalone(block.bytes) });
120
120
  }
121
121
 
122
122
  return blocks;
@@ -146,6 +146,24 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
146
146
  * The CAR was verified block by block on the way in, so this decodes without
147
147
  * hashing again.
148
148
  */
149
+ /**
150
+ * The same bytes, owning their buffer.
151
+ *
152
+ * A CAR arrives as one `ArrayBuffer` and `CarReader` hands out each block as a
153
+ * view into it. IndexedDB's structured clone stores *the buffer behind a view*,
154
+ * not the view — so writing the 2 016 blocks of a 3.7 MB backup wrote 3.7 MB
155
+ * apiece, some 7.5 GB in all, and the browser refused everything past about
156
+ * block 500. The copy costs one pass over bytes that are going to disk anyway.
157
+ *
158
+ * @param {Uint8Array} bytes
159
+ * @returns {Uint8Array} `bytes` itself when it already owns its buffer
160
+ */
161
+ export function standalone(bytes) {
162
+ return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength
163
+ ? bytes
164
+ : new Uint8Array(bytes);
165
+ }
166
+
149
167
  export function entryDatabaseId(bytes) {
150
168
  let value;
151
169
  try {
@@ -335,11 +353,17 @@ function assertRestorable(metadata) {
335
353
  * the backup names (a node without pubsub needs `{ sync: false }`)
336
354
  * @param {(progress: { stage: "database", index: number, total: number,
337
355
  * address: string, joined: number, heads: number }) => void} [options.onProgress]
356
+ * @param {() => Promise<void>|void} [options.beforeJoin] called once the blocks
357
+ * are in the blockstore and before any database is opened — where a caller
358
+ * that must close its own handles does it, late enough that a refused block
359
+ * has already thrown and left those handles alone
360
+ * @throws {Error} `BlocksNotStoredError` when the blockstore refuses any block,
361
+ * before `beforeJoin` and before anything is opened
338
362
  * @returns {Promise<{ blocks: number, databases: { address: string, name?: string,
339
363
  * database: Object, entries: number|null, heads: number, joined: number }[] }>}
340
364
  */
341
365
  export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {}) {
342
- const { log = SILENT, verify = true, open = {}, onProgress } = options;
366
+ const { log = SILENT, verify = true, open = {}, onProgress, beforeJoin } = options;
343
367
  if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
344
368
  if (!(blocks instanceof Map)) throw new Error("blocks must be a Map");
345
369
  if (!isValidMetadata(metadata)) throw new Error("Invalid backup metadata");
@@ -362,14 +386,34 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
362
386
 
363
387
  // Into the blockstore, where Helia looks — once, whatever the databases.
364
388
  let stored = 0;
389
+ /** @type {string[]} */
390
+ const refused = [];
365
391
  for (const [cidString, { bytes }] of blocks) {
366
392
  try {
367
- await orbitdb.ipfs.blockstore.put(CID.parse(cidString), bytes);
393
+ await orbitdb.ipfs.blockstore.put(CID.parse(cidString), standalone(bytes));
368
394
  stored++;
369
395
  } catch (error) {
396
+ if (refused.length < 3) refused.push(`${cidString.slice(0, 12)}…: ${error.message}`);
370
397
  log.warn(` ⚠️ could not store ${cidString.slice(0, 12)}…: ${error.message}`);
371
398
  }
372
399
  }
400
+ // A block that did not land is a head the log will later look for, not find,
401
+ // and wait out on the network — twenty seconds of silence per head, ending
402
+ // in a complaint about the entry rather than about the storage. A browser
403
+ // whose quota is spent arrives here with most of the backup refused. Say so
404
+ // now, while the caller's databases are all still open.
405
+ if (stored !== blocks.size) {
406
+ const error = new Error(
407
+ `The blockstore took ${stored} of ${blocks.size} blocks; the rest were refused: ${refused.join("; ")}`,
408
+ );
409
+ error.name = "BlocksNotStoredError";
410
+ Object.assign(error, { stored, total: blocks.size, refused });
411
+ throw error;
412
+ }
413
+
414
+ // Everything is in the blockstore and nothing has been opened yet: the one
415
+ // moment at which a caller can safely close what it holds open.
416
+ await beforeJoin?.();
373
417
 
374
418
  const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
375
419
  const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
@@ -388,7 +432,7 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
388
432
  const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
389
433
  for (const [hash, bytes] of mine) {
390
434
  try {
391
- await opened.log.storage.put(hash, bytes);
435
+ await opened.log.storage.put(hash, standalone(bytes));
392
436
  } catch (error) {
393
437
  log.warn(` ⚠️ could not copy ${hash.slice(0, 12)}… to log storage: ${error.message}`);
394
438
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@le-space/orbitdb-storage-bridge",
3
- "version": "0.23.0",
3
+ "version": "0.24.0",
4
4
  "description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
5
5
  "main": "lib/orbitdb-storacha-bridge.js",
6
6
  "svelte": "dist/components/",