@le-space/orbitdb-storage-bridge 0.22.1 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -56,7 +56,7 @@ import { CID } from "multiformats/cid";
56
56
  import { sha256 } from "multiformats/hashes/sha2";
57
57
  import { base58btc } from "multiformats/bases/base58";
58
58
  import { createCARFromBlocks } from "./backup-car.js";
59
- import { readBlocksFromCAR, restoreFromBlocks } from "./restore-cid.js";
59
+ import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId, standalone } from "./restore-cid.js";
60
60
 
61
61
  /** What the manifest at an increment's root calls itself. */
62
62
  export const INCREMENT_KIND = "increment";
@@ -492,6 +492,7 @@ export async function joinIncrements(
492
492
  throw new Error("These increments hold nothing for this database.");
493
493
 
494
494
  let skippedRaw = 0;
495
+ let skippedForeign = 0;
495
496
  for (const [name, { bytes }] of blocks) {
496
497
  const cid = CID.parse(name);
497
498
  if (cid.code !== DAG_CBOR) {
@@ -502,10 +503,21 @@ export async function joinIncrements(
502
503
  else skippedRaw++;
503
504
  continue;
504
505
  }
506
+ // An entry belongs to the database it names. A chain merged for several
507
+ // databases carries all of their entries, and writing every one into every
508
+ // database multiplies the writes by the database count — in a browser each
509
+ // is an IndexedDB transaction (#152). Blocks that are not entries — the
510
+ // manifest, the access controller, an identity — have no database of their
511
+ // own and are looked up by their own name here, so they still go in.
512
+ const belongsTo = entryDatabaseId(bytes);
513
+ if (belongsTo !== null && belongsTo !== address) {
514
+ skippedForeign++;
515
+ continue;
516
+ }
505
517
  // OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
506
518
  // its own names, and identities are looked up by theirs, so each block
507
519
  // goes in under the name its reader will ask for.
508
- await database.log.storage.put(cid.toV1().toString(base58btc), bytes);
520
+ await database.log.storage.put(cid.toV1().toString(base58btc), standalone(bytes));
509
521
  }
510
522
 
511
523
  let joined = 0;
@@ -522,6 +534,7 @@ export async function joinIncrements(
522
534
  gaps: merged.gaps,
523
535
  duplicates: merged.duplicates,
524
536
  ...(skippedRaw ? { skippedRaw } : {}),
537
+ ...(skippedForeign ? { skippedForeign } : {}),
525
538
  };
526
539
  }
527
540
 
@@ -116,7 +116,7 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
116
116
  }
117
117
  }
118
118
 
119
- blocks.set(name, { bytes: block.bytes });
119
+ blocks.set(name, { bytes: standalone(block.bytes) });
120
120
  }
121
121
 
122
122
  return blocks;
@@ -130,6 +130,75 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
130
130
  * entry only if it is dag-cbor *and* carries a signature, key and identity —
131
131
  * the manifest and the access controller are dag-cbor too.
132
132
  */
133
+ /**
134
+ * Which database each entry block belongs to.
135
+ *
136
+ * A log entry names its database in `id`, so a block only has to reach the log
137
+ * storage of that one database. Writing every block to every database — which
138
+ * is what this used to do — multiplies the writes by the number of databases,
139
+ * and in a browser each of those is an IndexedDB transaction: 2 000 entries in
140
+ * nine databases meant 18 099 writes where 2 011 blocks exist (issue #152).
141
+ *
142
+ * Blocks that are not entries (manifest, access controller, identity) and
143
+ * entries nothing here can read (an encrypted database hides `id`) are left
144
+ * out: they are already in the blockstore, which the log reads through.
145
+ *
146
+ * The CAR was verified block by block on the way in, so this decodes without
147
+ * hashing again.
148
+ */
149
+ /**
150
+ * The same bytes, owning their buffer.
151
+ *
152
+ * A CAR arrives as one `ArrayBuffer` and `CarReader` hands out each block as a
153
+ * view into it. IndexedDB's structured clone stores *the buffer behind a view*,
154
+ * not the view — so writing the 2 016 blocks of a 3.7 MB backup wrote 3.7 MB
155
+ * apiece, some 7.5 GB in all, and the browser refused everything past about
156
+ * block 500. The copy costs one pass over bytes that are going to disk anyway.
157
+ *
158
+ * @param {Uint8Array} bytes
159
+ * @returns {Uint8Array} `bytes` itself when it already owns its buffer
160
+ */
161
+ export function standalone(bytes) {
162
+ return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength
163
+ ? bytes
164
+ : new Uint8Array(bytes);
165
+ }
166
+
167
+ export function entryDatabaseId(bytes) {
168
+ let value;
169
+ try {
170
+ value = dagCbor.decode(bytes);
171
+ } catch {
172
+ return null;
173
+ }
174
+ // An entry carries a signature, the signer's key and its identity; a
175
+ // manifest, an access controller or an identity block carries none of them.
176
+ if (!value?.sig || !value?.key || !value?.identity) return null;
177
+ return typeof value.id === "string" ? value.id : null;
178
+ }
179
+
180
+ function entryBlocksByDatabase(blocks) {
181
+ const byDatabase = new Map();
182
+ for (const [cidString, { bytes }] of blocks) {
183
+ let cid;
184
+ try {
185
+ cid = CID.parse(cidString);
186
+ } catch {
187
+ continue;
188
+ }
189
+ if (cid.code !== DAG_CBOR) continue;
190
+
191
+ const id = entryDatabaseId(bytes);
192
+ if (id === null) continue;
193
+
194
+ const forDatabase = byDatabase.get(id) ?? [];
195
+ // The log addresses blocks in base58btc, the CAR names them in base32.
196
+ forDatabase.push([cid.toV1().toString(base58btc), bytes]);
197
+ byDatabase.set(id, forDatabase);
198
+ }
199
+ return byDatabase;
200
+ }
201
+
133
202
  async function headsIn(blocks) {
134
203
  const entries = [];
135
204
  const pointedAt = new Set();
@@ -284,11 +353,17 @@ function assertRestorable(metadata) {
284
353
  * the backup names (a node without pubsub needs `{ sync: false }`)
285
354
  * @param {(progress: { stage: "database", index: number, total: number,
286
355
  * address: string, joined: number, heads: number }) => void} [options.onProgress]
356
+ * @param {() => Promise<void>|void} [options.beforeJoin] called once the blocks
357
+ * are in the blockstore and before any database is opened — where a caller
358
+ * that must close its own handles does it, late enough that a refused block
359
+ * has already thrown and left those handles alone
360
+ * @throws {Error} `BlocksNotStoredError` when the blockstore refuses any block,
361
+ * before `beforeJoin` and before anything is opened
287
362
  * @returns {Promise<{ blocks: number, databases: { address: string, name?: string,
288
363
  * database: Object, entries: number|null, heads: number, joined: number }[] }>}
289
364
  */
290
365
  export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {}) {
291
- const { log = SILENT, verify = true, open = {}, onProgress } = options;
366
+ const { log = SILENT, verify = true, open = {}, onProgress, beforeJoin } = options;
292
367
  if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
293
368
  if (!(blocks instanceof Map)) throw new Error("blocks must be a Map");
294
369
  if (!isValidMetadata(metadata)) throw new Error("Invalid backup metadata");
@@ -311,38 +386,62 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
311
386
 
312
387
  // Into the blockstore, where Helia looks — once, whatever the databases.
313
388
  let stored = 0;
389
+ /** @type {string[]} */
390
+ const refused = [];
314
391
  for (const [cidString, { bytes }] of blocks) {
315
392
  try {
316
- await orbitdb.ipfs.blockstore.put(CID.parse(cidString), bytes);
393
+ await orbitdb.ipfs.blockstore.put(CID.parse(cidString), standalone(bytes));
317
394
  stored++;
318
395
  } catch (error) {
396
+ if (refused.length < 3) refused.push(`${cidString.slice(0, 12)}…: ${error.message}`);
319
397
  log.warn(` ⚠️ could not store ${cidString.slice(0, 12)}…: ${error.message}`);
320
398
  }
321
399
  }
400
+ // A block that did not land is a head the log will later look for, not find,
401
+ // and wait out on the network — twenty seconds of silence per head, ending
402
+ // in a complaint about the entry rather than about the storage. A browser
403
+ // whose quota is spent arrives here with most of the backup refused. Say so
404
+ // now, while the caller's databases are all still open.
405
+ if (stored !== blocks.size) {
406
+ const error = new Error(
407
+ `The blockstore took ${stored} of ${blocks.size} blocks; the rest were refused: ${refused.join("; ")}`,
408
+ );
409
+ error.name = "BlocksNotStoredError";
410
+ Object.assign(error, { stored, total: blocks.size, refused });
411
+ throw error;
412
+ }
413
+
414
+ // Everything is in the blockstore and nothing has been opened yet: the one
415
+ // moment at which a caller can safely close what it holds open.
416
+ await beforeJoin?.();
322
417
 
323
418
  const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
324
419
  const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
325
420
 
421
+ const entryBlocks = entryBlocksByDatabase(blocks);
422
+
326
423
  const databases = [];
327
424
  for (const [index, dbInfo] of metadata.databases.entries()) {
328
- const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
329
- // OrbitDB's log reads from its own store and addresses blocks in base58btc
330
- // rather than the CAR's base32. A restore that fills only the blockstore
331
- // opens a database that is empty in a way nothing reports.
332
- for (const [cidString, { bytes }] of blocks) {
333
- try {
334
- const cid = CID.parse(cidString);
335
- if (cid.code !== DAG_CBOR) continue;
336
- await opened.log.storage.put(cid.toV1().toString(base58btc), bytes);
337
- } catch (error) {
338
- log.warn(` ⚠️ could not copy ${cidString.slice(0, 12)}… to log storage: ${error.message}`);
425
+ // Its own entries, into its own log storage — the rest of the CAR belongs
426
+ // to the other databases, and the blockstore above already holds all of it.
427
+ const mine = entryBlocks.get(dbInfo.address) ?? [];
428
+ let database;
429
+ if (mine.length === 0) {
430
+ database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
431
+ } else {
432
+ const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
433
+ for (const [hash, bytes] of mine) {
434
+ try {
435
+ await opened.log.storage.put(hash, standalone(bytes));
436
+ } catch (error) {
437
+ log.warn(` ⚠️ could not copy ${hash.slice(0, 12)}… to log storage: ${error.message}`);
438
+ }
339
439
  }
440
+ // Reopened so the log reads what we just put underneath it.
441
+ await opened.close();
442
+ database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
340
443
  }
341
444
 
342
- // Reopened so the log reads what we just put underneath it.
343
- await opened.close();
344
- const database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
345
-
346
445
  // Tell the log where its history ends.
347
446
  const heads = Array.isArray(dbInfo.heads)
348
447
  ? (await Promise.all(dbInfo.heads.map((hash) => headFrom(byCanonical, hash)))).filter(Boolean)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@le-space/orbitdb-storage-bridge",
3
- "version": "0.22.1",
3
+ "version": "0.24.0",
4
4
  "description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
5
5
  "main": "lib/orbitdb-storacha-bridge.js",
6
6
  "svelte": "dist/components/",
@@ -43,6 +43,7 @@
43
43
  "test:timestamped-backup": "npm test -- test/timestamped-backup.test.js",
44
44
  "test:network-download": "npm test -- test/network-download.test.js",
45
45
  "test:backends": "npm test -- test/backends/",
46
+ "test:no-secrets": "npm test -- --forceExit --testPathIgnorePatterns '/node_modules/' '/examples/' '/dist/' '/coverage/' 'test/integration\\.test\\.js' 'test/backup-car\\.test\\.js' 'test/p256-ucan-security\\.test\\.js' 'test/timestamped-backup\\.test\\.js' 'test/network-download\\.test\\.js' 'test/access-control-integration\\.test\\.js'",
46
47
  "clear-space": "node examples/storacha/clear-space.js",
47
48
  "car-demo": "node examples/storacha/car-backup-demo.js",
48
49
  "ucan-demo": "node examples/storacha/ucan-demo.js",