@le-space/orbitdb-storage-bridge 0.14.1 → 0.16.0

package/lib/restore-cid.js

@@ -157,7 +157,8 @@ async function headsIn(blocks) {
 }
 
 /**
- * Restore a database from the CID of a CAR backup's metadata.
+ * Restore from the CID of a CAR backup's metadata: every database it names
+ * (`restoreFromBlocks` does the work once the blocks are fetched).
  *
  * Everything hangs off that one CID: the metadata names the CAR, the CAR holds
  * the blocks, and the blocks carry their own addresses. Nothing here talks to
@@ -191,7 +192,9 @@ async function headsIn(blocks) {
  *   OrbitDB's Sync subscribes on open, and on a libp2p built without a pubsub
  *   service that throws before the database is ever handed back.
  * @returns {Promise<{ address: string, database: Object, blocks: number,
- *   entries: number, heads: number, joined: number }>}
+ *   entries: number, heads: number, joined: number, databases: Object[] }>}
+ *   the first database's at the top, as a one-database backup always had it;
+ *   `databases` lists them all
  */
 export async function restoreFromCID(orbitdb, options = {}) {
   const { metadataCID, fetchBytes = fetchFromGateways, log = SILENT, verify = true, open = {}, ...rest } = options;
@@ -216,40 +219,97 @@ export async function restoreFromCID(orbitdb, options = {}) {
   // address to open it into wastes the whole transfer.
   const carCID = metadata.carCID;
   if (!carCID) throw new Error("Backup metadata names no CAR file");
-
-  const dbInfo = metadata.databases?.[0];
-  if (!dbInfo?.address) {
-    // `isValidMetadata` also accepts the older `{ root, path }` shape, which
-    // predates CAR backups and has no address to open. Say which it is.
-    throw new Error(
-      dbInfo ? "This is a pre-CAR backup; restoreFromCID needs a CAR backup" : "Backup metadata names no database",
-    );
-  }
+  assertRestorable(metadata);
 
   // 2 · the CAR holds the blocks, and each one is checked against its own CID
   const blocks = await readBlocksFromCAR(await fetchBytes(carCID, { ...rest, log }), { verify });
   log.info(`   ✅ ${blocks.size} blocks`);
 
+  // 3 · every database the metadata names
+  const restored = await restoreFromBlocks(orbitdb, blocks, metadata, { log, verify, open });
+  const [first] = restored.databases;
+
+  // The one-database answer this always gave, and the whole list beside it.
+  return {
+    address: first.address,
+    database: first.database,
+    blocks: restored.blocks,
+    entries: metadata.totalEntries ?? first.entries,
+    heads: first.heads,
+    joined: first.joined,
+    databases: restored.databases,
+  };
+}
+
+/**
+ * Refuse metadata that names nothing a restore could open, and say which kind
+ * it is. `isValidMetadata` also accepts the older `{ root, path }` shape, which
+ * predates CAR backups and has no address to open.
+ */
+function assertRestorable(metadata) {
+  const dbs = metadata.databases ?? [];
+  if (dbs.length === 0) throw new Error("Backup metadata names no database");
+  if (!dbs.every((db) => db?.address)) {
+    throw new Error("This is a pre-CAR backup; restoreFromCID needs a CAR backup");
+  }
+}
+
+/**
+ * Restore every database a backup names, from blocks already in hand.
+ *
+ * `restoreFromCID` fetches and then calls this. An application that holds the
+ * blocks some other way — a backup it sealed itself and has just opened, a file
+ * on a USB stick, several small CARs written as it went — calls it directly.
+ * The blocks are checked against the metadata, never trusted for it: each
+ * database's manifest must be among them.
+ *
+ * Per database: the blocks go into the blockstore (once, for all of them) and
+ * into that database's own log storage (its dag-cbor blocks only — a receipt
+ * file's raw chunks are no log's business); the database is reopened so the
+ * log reads what was put underneath it; and its heads are joined. The heads
+ * are the ones the metadata states, or — for metadata that states none — the
+ * entries of this log nothing else points back to.
+ *
+ * **Each returned `database` replaces any handle the caller held on that
+ * address**, for the reason `restoreFromCID` gives.
+ *
+ * @param {Object} orbitdb an OrbitDB instance to restore into
+ * @param {Map<string, { bytes: Uint8Array }>} blocks by CID string, in any base —
+ *   what `readBlocksFromCAR` returns
+ * @param {Object} metadata as `bundleDatabases` or `backupDatabaseCAR` write it
+ * @param {Object} [options]
+ * @param {boolean} [options.verify=true] each database's manifest must be among the blocks
+ * @param {{ info: Function, warn: Function, debug: Function }} [options.log]
+ * @param {Object} [options.open] options for `orbitdb.open`, merged over the type
+ *   the backup names (a node without pubsub needs `{ sync: false }`)
+ * @param {(progress: { stage: "database", index: number, total: number,
+ *   address: string, joined: number, heads: number }) => void} [options.onProgress]
+ * @returns {Promise<{ blocks: number, databases: { address: string, name?: string,
+ *   database: Object, entries: number|null, heads: number, joined: number }[] }>}
+ */
+export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {}) {
+  const { log = SILENT, verify = true, open = {}, onProgress } = options;
+  if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
+  if (!(blocks instanceof Map)) throw new Error("blocks must be a Map");
+  if (!isValidMetadata(metadata)) throw new Error("Invalid backup metadata");
+  assertRestorable(metadata);
+
   // The blocks verify individually; this is what ties them to *this* backup.
   // Without it a CAR full of perfectly valid blocks from some other database
   // would restore happily, because every block would be honest about itself.
-  const manifestCID = dbInfo.manifestCID ?? metadata.manifestCID;
-  if (verify && manifestCID) {
-    const wanted = canonical(manifestCID);
-    const present = new Set([...blocks.keys()].map(canonical));
-    if (!wanted || !present.has(wanted)) {
-      throw new Error(
-        `The CAR does not contain the manifest ${manifestCID} that this backup names`,
-      );
+  const present = new Set([...blocks.keys()].map(canonical));
+  if (verify) {
+    for (const db of metadata.databases) {
+      const manifestCID = db.manifestCID ?? metadata.manifestCID;
+      if (!manifestCID) continue;
+      const wanted = canonical(manifestCID);
+      if (!wanted || !present.has(wanted)) {
+        throw new Error(`The CAR does not contain the manifest ${manifestCID} that this backup names`);
+      }
     }
   }
 
-  // 3 · into the blockstore, and into the log's own storage
-  //
-  // Both, and not by accident: the blockstore is where Helia looks, while
-  // OrbitDB's log reads from its own store and addresses blocks in base58btc
-  // rather than the CAR's base32. A restore that fills only one of them opens
-  // a database that is empty in a way nothing reports.
+  // Into the blockstore, where Helia looks — once, whatever the databases.
   let stored = 0;
   for (const [cidString, { bytes }] of blocks) {
     try {
@@ -260,43 +320,75 @@ export async function restoreFromCID(orbitdb, options = {}) {
     }
   }
 
-  const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
-  for (const [cidString, { bytes }] of blocks) {
-    try {
-      await opened.log.storage.put(CID.parse(cidString).toV1().toString(base58btc), bytes);
-    } catch (error) {
-      log.warn(`   ⚠️ could not copy ${cidString.slice(0, 12)}… to log storage: ${error.message}`);
+  const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
+  const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
+
+  const databases = [];
+  for (const [index, dbInfo] of metadata.databases.entries()) {
+    const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
+    // OrbitDB's log reads from its own store and addresses blocks in base58btc
+    // rather than the CAR's base32. A restore that fills only the blockstore
+    // opens a database that is empty in a way nothing reports.
+    for (const [cidString, { bytes }] of blocks) {
+      try {
+        const cid = CID.parse(cidString);
+        if (cid.code !== DAG_CBOR) continue;
+        await opened.log.storage.put(cid.toV1().toString(base58btc), bytes);
+      } catch (error) {
+        log.warn(`   ⚠️ could not copy ${cidString.slice(0, 12)}… to log storage: ${error.message}`);
+      }
     }
-  }
 
-  // Reopened so the log reads what we just put underneath it.
-  await opened.close();
-  const database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
+    // Reopened so the log reads what we just put underneath it.
+    await opened.close();
+    const database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
 
-  // 4 · tell the log where its history ends
-  const heads = await headsIn(blocks);
-  let joined = 0;
-  for (const head of heads) {
-    try {
-      const { v, id, key, sig, next, refs, clock, payload, identity } = head.value;
-      if (await database.log.joinEntry({ hash: head.hash, v, id, key, sig, next, refs, clock, payload, identity })) {
-        joined++;
+    // Tell the log where its history ends.
+    const heads = Array.isArray(dbInfo.heads)
+      ? (await Promise.all(dbInfo.heads.map((hash) => headFrom(byCanonical, hash)))).filter(Boolean)
+      : found.filter((head) => head.value.id === dbInfo.address || metadata.databases.length === 1);
+    let joined = 0;
+    for (const head of heads) {
+      try {
+        const { v, id, key, sig, next, refs, clock, payload, identity } = head.value;
+        if (await database.log.joinEntry({ hash: head.hash, v, id, key, sig, next, refs, clock, payload, identity })) {
+          joined++;
+        }
+      } catch (error) {
+        log.warn(`   ⚠️ could not join head ${head.hash.slice(0, 12)}…: ${error.message}`);
       }
-    } catch (error) {
-      log.warn(`   ⚠️ could not join head ${head.hash.slice(0, 12)}…: ${error.message}`);
     }
+
+    log.info(`✅ Restored ${dbInfo.address} — ${joined}/${heads.length} heads`);
+    onProgress?.({
+      stage: "database",
+      index,
+      total: metadata.databases.length,
+      address: dbInfo.address,
+      joined,
+      heads: heads.length,
+    });
+    databases.push({
+      address: dbInfo.address,
+      name: dbInfo.name,
+      database,
+      entries: dbInfo.entryCount ?? null,
+      heads: heads.length,
+      joined,
+    });
   }
 
-  log.info(`✅ Restored ${dbInfo.address} — ${stored} blocks, ${joined}/${heads.length} heads`);
+  return { blocks: stored, databases };
+}
 
-  return {
-    address: dbInfo.address,
-    database,
-    blocks: stored,
-    entries: metadata.totalEntries ?? dbInfo.entryCount ?? null,
-    heads: heads.length,
-    joined,
-  };
+/** A stated head, decoded from the blocks; null when the blocks lack it. */
+async function headFrom(byCanonical, hash) {
+  const name = canonical(hash);
+  const block = name ? byCanonical.get(name) : undefined;
+  if (!block) return null;
+  const cid = CID.parse(name);
+  const { value } = await Block.decode({ cid, bytes: block.bytes, codec: dagCbor, hasher: sha256 });
+  return { hash: cid.toV1().toString(base58btc), value };
 }
 
-export default { restoreFromCID, fetchFromGateways, readBlocksFromCAR, DEFAULT_GATEWAYS, SILENT };
+export default { restoreFromCID, restoreFromBlocks, fetchFromGateways, readBlocksFromCAR, DEFAULT_GATEWAYS, SILENT };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@le-space/orbitdb-storage-bridge",
-  "version": "0.14.1",
+  "version": "0.16.0",
   "description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
   "main": "lib/orbitdb-storacha-bridge.js",
   "svelte": "dist/components/",
@@ -15,6 +15,7 @@
     "./pointer-ipns": "./lib/pointer-ipns.js",
     "./dehydrate": "./lib/dehydrate.js",
     "./restore-cid": "./lib/restore-cid.js",
+    "./extract-blocks": "./lib/extract-blocks.js",
     "./gateway-fetch": "./lib/gateway-fetch.js",
     "./peer-fetch": "./lib/peer-fetch.js",
     "./backends/types": "./lib/backends/types.js",
@@ -46,6 +47,7 @@
     "clear-space": "node examples/storacha/clear-space.js",
     "car-demo": "node examples/storacha/car-backup-demo.js",
     "ucan-demo": "node examples/storacha/ucan-demo.js",
+    "check:links": "node scripts/check-links.mjs",
     "lint": "eslint --config eslint.config.js '{lib,examples,test}/**/*.js' --ignore-pattern 'examples/svelte/*/build/**' --ignore-pattern 'examples/svelte/*/dist/**' --ignore-pattern 'examples/svelte/**/.svelte-kit/**' --ignore-pattern 'node_modules/**'",
     "format": "prettier --write lib/ examples/ test/ --ignore-path '.prettierignore'",
     "clean": "rm -rf ./cid-bridge-test* ./demo-test*",