@le-space/orbitdb-storage-bridge 0.23.0 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -35,15 +35,20 @@ import { CID } from "multiformats/cid";
35
35
  export const DEFAULT_GATEWAYS = ["https://ipfs.aleph.cloud/ipfs"];
36
36
 
37
37
  /**
38
- * Gateways that serve **verifiable** responses only: a single block, or a CAR.
38
+ * Gateways that answer a block request with the block, and say so.
39
39
  *
40
- * `trustless-gateway.link` was still healthy when the path gateways were
41
- * retired, and it answers with CORS — but it refuses an ordinary path request
42
- * with a 406, so it is only usable with `accept` set, and only for a CID whose
43
- * bytes are one block. A UnixFS file split into chunks needs its DAG walked,
44
- * which this module deliberately cannot do.
40
+ * `trustless-gateway.link` used to be the entry here. Measured 2026-10-24 it
41
+ * answers `525` to everything, path and trustless alike, so it has followed
42
+ * `ipfs.io` and `dweb.link` out. What is left and healthy is Aleph's, which
43
+ * honours `accept: application/vnd.ipld.raw` and sets the content type to
44
+ * match — 200 in 0.15 s with CORS `*`.
45
+ *
46
+ * Only for a CID whose bytes are one block: a UnixFS file split into chunks
47
+ * needs its DAG walked, which this module deliberately cannot do. And the
48
+ * list is a convenience, never a reason to trust an answer — every byte that
49
+ * comes back is checked against the CID that asked for it.
45
50
  */
46
- export const TRUSTLESS_GATEWAYS = ["https://trustless-gateway.link/ipfs"];
51
+ export const TRUSTLESS_GATEWAYS = ["https://ipfs.aleph.cloud/ipfs"];
47
52
 
48
53
  /** `accept` for a single block from a trustless gateway. */
49
54
  export const RAW_BLOCK = "application/vnd.ipld.raw";
@@ -56,7 +56,7 @@ import { CID } from "multiformats/cid";
56
56
  import { sha256 } from "multiformats/hashes/sha2";
57
57
  import { base58btc } from "multiformats/bases/base58";
58
58
  import { createCARFromBlocks } from "./backup-car.js";
59
- import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId } from "./restore-cid.js";
59
+ import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId, standalone } from "./restore-cid.js";
60
60
 
61
61
  /** What the manifest at an increment's root calls itself. */
62
62
  export const INCREMENT_KIND = "increment";
@@ -517,7 +517,7 @@ export async function joinIncrements(
517
517
  // OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
518
518
  // its own names, and identities are looked up by theirs, so each block
519
519
  // goes in under the name its reader will ask for.
520
- await database.log.storage.put(cid.toV1().toString(base58btc), bytes);
520
+ await database.log.storage.put(cid.toV1().toString(base58btc), standalone(bytes));
521
521
  }
522
522
 
523
523
  let joined = 0;
@@ -70,6 +70,28 @@ const HASHERS = new Map([
70
70
  [sha512.code, sha512],
71
71
  ]);
72
72
 
73
+ /**
74
+ * Bytes against the CID that names them, or an exception.
75
+ *
76
+ * @param {CID} cid
77
+ * @param {Uint8Array} bytes
78
+ * @param {string} blame what it means when they do not match
79
+ */
80
+ async function assertHashes(cid, bytes, blame) {
81
+ const hasher = HASHERS.get(cid.multihash.code);
82
+ // Refused rather than waved through: a hash we cannot compute is a block we
83
+ // cannot vouch for, and silently accepting it would defeat the point.
84
+ if (!hasher) {
85
+ throw new Error(
86
+ `Block ${cid} uses multihash 0x${cid.multihash.code.toString(16)}, which this reader cannot verify`,
87
+ );
88
+ }
89
+ const digest = await hasher.digest(bytes);
90
+ if (!sameBytes(digest.digest, cid.multihash.digest)) {
91
+ throw new Error(`Block ${cid} does not hash to its own CID — ${blame}`);
92
+ }
93
+ }
94
+
73
95
  /**
74
96
  * Every block in a CAR, by CID string — each one checked against its own name.
75
97
  *
@@ -102,21 +124,10 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
102
124
  const name = block.cid.toString();
103
125
 
104
126
  if (verify) {
105
- const hasher = HASHERS.get(block.cid.multihash.code);
106
- // Refused rather than waved through: a hash we cannot compute is a block
107
- // we cannot vouch for, and silently accepting it would defeat the point.
108
- if (!hasher) {
109
- throw new Error(
110
- `Block ${name} uses multihash 0x${block.cid.multihash.code.toString(16)}, which this reader cannot verify`,
111
- );
112
- }
113
- const digest = await hasher.digest(block.bytes);
114
- if (!sameBytes(digest.digest, block.cid.multihash.digest)) {
115
- throw new Error(`Block ${name} does not hash to its own CID — the CAR has been tampered with`);
116
- }
127
+ await assertHashes(block.cid, block.bytes, "the CAR has been tampered with");
117
128
  }
118
129
 
119
- blocks.set(name, { bytes: block.bytes });
130
+ blocks.set(name, { bytes: standalone(block.bytes) });
120
131
  }
121
132
 
122
133
  return blocks;
@@ -146,6 +157,24 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
146
157
  * The CAR was verified block by block on the way in, so this decodes without
147
158
  * hashing again.
148
159
  */
160
+ /**
161
+ * The same bytes, owning their buffer.
162
+ *
163
+ * A CAR arrives as one `ArrayBuffer` and `CarReader` hands out each block as a
164
+ * view into it. IndexedDB's structured clone stores *the buffer behind a view*,
165
+ * not the view — so writing the 2 016 blocks of a 3.7 MB backup wrote 3.7 MB
166
+ * apiece, some 7.5 GB in all, and the browser refused everything past about
167
+ * block 500. The copy costs one pass over bytes that are going to disk anyway.
168
+ *
169
+ * @param {Uint8Array} bytes
170
+ * @returns {Uint8Array} `bytes` itself when it already owns its buffer
171
+ */
172
+ export function standalone(bytes) {
173
+ return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength
174
+ ? bytes
175
+ : new Uint8Array(bytes);
176
+ }
177
+
149
178
  export function entryDatabaseId(bytes) {
150
179
  let value;
151
180
  try {
@@ -335,11 +364,37 @@ function assertRestorable(metadata) {
335
364
  * the backup names (a node without pubsub needs `{ sync: false }`)
336
365
  * @param {(progress: { stage: "database", index: number, total: number,
337
366
  * address: string, joined: number, heads: number }) => void} [options.onProgress]
338
- * @returns {Promise<{ blocks: number, databases: { address: string, name?: string,
367
+ * @param {((cid: string) => Promise<Uint8Array>) | null} [options.fetchBytes] a
368
+ * second source for a head block the blocks do not carry — `fetchFromGateways`
369
+ * bound to the gateways an application is willing to ask. Off by default: a
370
+ * restore that reaches the network unasked is not something an offline-first
371
+ * application can allow. Whatever comes back is checked against its CID.
372
+ * @param {boolean} [options.requireHeads=false] throw `MissingHeadsError` when a
373
+ * head the metadata names cannot be supplied — before `beforeJoin`, so a
374
+ * caller that closes its databases there still has them
375
+ * @param {() => Promise<void>|void} [options.beforeJoin] called once the blocks
376
+ * are in the blockstore and before any database is opened — where a caller
377
+ * that must close its own handles does it, late enough that a refused block
378
+ * has already thrown and left those handles alone
379
+ * @throws {Error} `BlocksNotStoredError` when the blockstore refuses any block,
380
+ * before `beforeJoin` and before anything is opened
381
+ * `missingHeads` are the heads the metadata names that neither the blocks nor
382
+ * the second source could supply: not an error here, because a caller may know
383
+ * the database is reachable another way, but never silence either.
384
+ *
385
+ * @returns {Promise<{ blocks: number, missingHeads: string[], databases: { address: string, name?: string,
339
386
  * database: Object, entries: number|null, heads: number, joined: number }[] }>}
340
387
  */
341
388
  export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {}) {
342
- const { log = SILENT, verify = true, open = {}, onProgress } = options;
389
+ const {
390
+ log = SILENT,
391
+ verify = true,
392
+ open = {},
393
+ onProgress,
394
+ beforeJoin,
395
+ fetchBytes = null,
396
+ requireHeads = false,
397
+ } = options;
343
398
  if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
344
399
  if (!(blocks instanceof Map)) throw new Error("blocks must be a Map");
345
400
  if (!isValidMetadata(metadata)) throw new Error("Invalid backup metadata");
@@ -362,20 +417,89 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
362
417
 
363
418
  // Into the blockstore, where Helia looks — once, whatever the databases.
364
419
  let stored = 0;
420
+ /** @type {string[]} */
421
+ const refused = [];
365
422
  for (const [cidString, { bytes }] of blocks) {
366
423
  try {
367
- await orbitdb.ipfs.blockstore.put(CID.parse(cidString), bytes);
424
+ await orbitdb.ipfs.blockstore.put(CID.parse(cidString), standalone(bytes));
368
425
  stored++;
369
426
  } catch (error) {
427
+ if (refused.length < 3) refused.push(`${cidString.slice(0, 12)}…: ${error.message}`);
370
428
  log.warn(` ⚠️ could not store ${cidString.slice(0, 12)}…: ${error.message}`);
371
429
  }
372
430
  }
431
+ // A block that did not land is a head the log will later look for, not find,
432
+ // and wait out on the network — twenty seconds of silence per head, ending
433
+ // in a complaint about the entry rather than about the storage. A browser
434
+ // whose quota is spent arrives here with most of the backup refused. Say so
435
+ // now, while the caller's databases are all still open.
436
+ if (stored !== blocks.size) {
437
+ const error = new Error(
438
+ `The blockstore took ${stored} of ${blocks.size} blocks; the rest were refused: ${refused.join("; ")}`,
439
+ );
440
+ error.name = "BlocksNotStoredError";
441
+ Object.assign(error, { stored, total: blocks.size, refused });
442
+ throw error;
443
+ }
373
444
 
374
445
  const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
375
446
  const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
376
447
 
377
448
  const entryBlocks = entryBlocksByDatabase(blocks);
378
449
 
450
+ // A head the blocks do not carry used to be dropped here without a word, so
451
+ // a backup missing one came back "restored" with nothing joined. Name it —
452
+ // and, when the caller offered a second source, ask that source for it. The
453
+ // bytes are checked against the CID that asked for them before they are
454
+ // believed, so a gateway is a convenience here and never a party to trust.
455
+ /** @type {string[]} */
456
+ const missingHeads = [];
457
+ for (const dbInfo of metadata.databases) {
458
+ for (const hash of Array.isArray(dbInfo.heads) ? dbInfo.heads : []) {
459
+ const name = canonical(hash);
460
+ if (name && byCanonical.has(name)) continue;
461
+ if (!name || !fetchBytes) {
462
+ missingHeads.push(String(hash));
463
+ continue;
464
+ }
465
+ try {
466
+ const cid = CID.parse(name);
467
+ const bytes = standalone(await fetchBytes(name));
468
+ await assertHashes(cid, bytes, "the second source answered with something else");
469
+ byCanonical.set(name, { bytes });
470
+ await orbitdb.ipfs.blockstore.put(cid, bytes);
471
+ log.info(` ↓ head ${name.slice(0, 12)}… came from the second source`);
472
+ } catch (error) {
473
+ missingHeads.push(String(hash));
474
+ log.warn(
475
+ ` ⚠️ head ${String(hash).slice(0, 12)}… is not in the backup and could not be fetched: ${error.message}`,
476
+ );
477
+ }
478
+ }
479
+ }
480
+
481
+ // An application whose books are replaced by this restore cannot live with a
482
+ // partial one: better to refuse now, while its databases are still open and
483
+ // still hold what they had. Off by default, because a caller that replicates
484
+ // may well reach the rest another way.
485
+ if (requireHeads && missingHeads.length) {
486
+ const error = new Error(
487
+ `The backup names ${missingHeads.length} head${missingHeads.length === 1 ? "" : "s"} ` +
488
+ `that neither it nor a second source could supply: ` +
489
+ missingHeads.map((h) => `${String(h).slice(0, 12)}…`).join(", "),
490
+ );
491
+ error.name = "MissingHeadsError";
492
+ Object.assign(error, { missingHeads });
493
+ throw error;
494
+ }
495
+
496
+ // Every block is in the blockstore, every head the metadata names is in hand,
497
+ // and nothing has been opened yet: the one moment at which a caller can
498
+ // safely close what it holds open. Later than it used to be, deliberately —
499
+ // asking a second source for a missing head can fail, and a caller's books
500
+ // should not already be shut when it does.
501
+ await beforeJoin?.();
502
+
379
503
  const databases = [];
380
504
  for (const [index, dbInfo] of metadata.databases.entries()) {
381
505
  // Its own entries, into its own log storage — the rest of the CAR belongs
@@ -388,7 +512,7 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
388
512
  const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
389
513
  for (const [hash, bytes] of mine) {
390
514
  try {
391
- await opened.log.storage.put(hash, bytes);
515
+ await opened.log.storage.put(hash, standalone(bytes));
392
516
  } catch (error) {
393
517
  log.warn(` ⚠️ could not copy ${hash.slice(0, 12)}… to log storage: ${error.message}`);
394
518
  }
@@ -438,7 +562,7 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
438
562
  });
439
563
  }
440
564
 
441
- return { blocks: stored, databases };
565
+ return { blocks: stored, databases, missingHeads };
442
566
  }
443
567
 
444
568
  /** A stated head, decoded from the blocks; null when the blocks lack it. */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@le-space/orbitdb-storage-bridge",
3
- "version": "0.23.0",
3
+ "version": "0.25.0",
4
4
  "description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
5
5
  "main": "lib/orbitdb-storacha-bridge.js",
6
6
  "svelte": "dist/components/",