@le-space/orbitdb-storage-bridge 0.10.0

package/lib/dehydrate.js

@@ -0,0 +1,146 @@
+/**
+ * Put a database somewhere a second device can find it, and get it back.
+ *
+ * Two calls, and between them nothing but a secret both devices can produce —
+ * a passkey's PRF output, in the case this was written for
+ * ([funkpost#93](https://github.com/NiKrause/funkpost/issues/93)):
+ *
+ *     // the device that still has the database, while it has internet
+ *     await dehydrate({ orbitdb, address, seed, backend })
+ *
+ *     // any device holding the same seed, later, knowing nothing else
+ *     const { db } = await hydrate({ orbitdb, seed })
+ *
+ * What makes that work is that the *name* is computed rather than remembered:
+ * `derivePointerKey` stretches the seed into a key, the IPNS name follows from
+ * the key, and the pointer under that name says where the backup is. The
+ * second device needs no CID, no address, no file — see `pointer-ipns.js`.
+ *
+ * **There is no encrypted identity archive here, and that is not an
+ * oversight.** The design this follows kept one because the signing key was
+ * generated at random and therefore had to be carried. A key *derived* from
+ * the same PRF output does not: the device recomputes it, the access
+ * controller recognises it, and there is nothing to keep secret in the open.
+ * Deriving it is the identity provider's business, not this module's — the
+ * seed arrives here as bytes for a name, and nothing about who holds it.
+ *
+ * What this inherits from the pieces underneath, and says plainly rather than
+ * hiding: a pointer lives as long as the routing endpoints keep it, and a
+ * backup as long as the storage backend keeps it. Aleph takes an upload
+ * without an account and keeps it without a promise. For a device that comes
+ * back in a week this is enough; for an archive it is not.
+ */
+
+import {
+  derivePointerKey,
+  publishPointer,
+  resolvePointer,
+  DEFAULT_ENDPOINTS,
+} from "./pointer-ipns.js";
+import logger from "./logger.js";
+
+/**
+ * Back the database up and publish a pointer to it under a name the seed
+ * derives.
+ *
+ * @param {Object} params
+ * @param {Object} params.orbitdb
+ * @param {string} params.address Database address to back up.
+ * @param {Uint8Array} params.seed Secret bytes both devices can produce.
+ * @param {string} [params.label] Distinguishes several pointers of one seed.
+ * @param {Object} [params.backend] Storage backend (Aleph, Pinata, …).
+ * @param {string[]} [params.endpoints] Routing endpoints to publish to.
+ * @param {bigint|number} [params.sequence] Overrides the default, which is
+ *   seconds since the epoch — monotonic, and readable in a log.
+ * @param {Object} [params.backup] Extra options for `backupDatabaseCAR`.
+ * @returns {Promise<{name: string, metadataCID: string, carCID: string, sequence: bigint, blocks: number}>}
+ */
+export async function dehydrate({
+  orbitdb,
+  address,
+  seed,
+  label,
+  backend,
+  endpoints = DEFAULT_ENDPOINTS,
+  sequence,
+  backup = {},
+}) {
+  if (!address) throw new Error("dehydrate needs the database address");
+  const { backupDatabaseCAR } = await import("./backup-car.js");
+
+  const result = await backupDatabaseCAR(orbitdb, address, {
+    backend,
+    ...backup,
+  });
+  if (!result?.success) {
+    throw new Error(`the backup failed: ${result?.error ?? "no reason given"}`);
+  }
+  const metadataCID = result.backupFiles?.metadataCID;
+  if (!metadataCID)
+    throw new Error("the backup named no metadata CID to point at");
+
+  const privateKey = await derivePointerKey(seed, { label });
+  const published = await publishPointer({
+    privateKey,
+    cid: metadataCID,
+    endpoints,
+    ...(sequence === undefined ? {} : { sequence }),
+  });
+
+  logger.info(`💧 dehydrated ${address} → ${published.name}`);
+  return {
+    name: published.name,
+    metadataCID,
+    carCID: result.backupFiles?.carCID,
+    sequence: published.sequence,
+    blocks: result.blocksTotal ?? 0,
+  };
+}
+
+/**
+ * Find the pointer the seed names, and restore what it points at.
+ *
+ * The database comes back open. It replaces any handle the caller had for the
+ * same address: `restoreFromCID` opens it itself, and writing through an older
+ * handle fails once this one exists.
+ *
+ * @param {Object} params
+ * @param {Object} params.orbitdb
+ * @param {Uint8Array} params.seed
+ * @param {string} [params.label]
+ * @param {string[]} [params.endpoints]
+ * @param {Object} [params.open] Options for `orbitdb.open` — a node without a
+ *   pubsub service needs `{ sync: false }`.
+ * @param {Object} [params.restore] Extra options for `restoreFromCID`.
+ * @returns {Promise<{db: Object, address: string, name: string, metadataCID: string, blocks: number, entries: number|null}>}
+ */
+export async function hydrate({
+  orbitdb,
+  seed,
+  label,
+  endpoints = DEFAULT_ENDPOINTS,
+  open = {},
+  restore = {},
+}) {
+  const privateKey = await derivePointerKey(seed, { label });
+  const pointer = await resolvePointer({ privateKey, endpoints });
+
+  const { restoreFromCID } = await import("./restore-cid.js");
+  const result = await restoreFromCID(orbitdb, {
+    metadataCID: pointer.cid,
+    open,
+    ...restore,
+  });
+
+  logger.info(
+    `💦 hydrated ${pointer.name} → ${result.address ?? "a database"}`,
+  );
+  return {
+    db: result.database,
+    address: result.address,
+    name: pointer.name,
+    metadataCID: pointer.cid,
+    blocks: result.blocks,
+    entries: result.entries,
+  };
+}

package/lib/extract-blocks.js

@@ -0,0 +1,258 @@
+/**
+ * Extracting a database's blocks — the half of a backup that reads.
+ *
+ * Its own module, and not part of the main entry, because the main entry
+ * imports `@storacha/client` at the top: a browser that backs up to Aleph
+ * would otherwise carry the whole Storacha SDK to call this one function.
+ * The same reason `restore-cid.js` exists (issue #58).
+ */
+
+import { CID } from "multiformats/cid";
+import * as Block from "multiformats/block";
+import * as dagCbor from "@ipld/dag-cbor";
+import { sha256 } from "multiformats/hashes/sha2";
+import { logger } from "./logger.js";
+
+/**
+ * Extract blocks from an OrbitDB database
+ *
+ * @param {Object} database - OrbitDB database instance
+ * @param {Object} options - Extraction options
+ * @param {boolean} [options.logEntriesOnly] - If true, only extract log entries (for fallback reconstruction)
+ * @returns {Promise<Object>} - { blocks, blockSources, manifestCID }
+ */
+export async function extractDatabaseBlocks(database, options = {}) {
+  const logEntriesOnly = options.logEntriesOnly || false;
+  const extractionMode = logEntriesOnly
+    ? "log entries only (fallback mode)"
+    : "all blocks";
+
+  logger.info(
+    `🔍 Extracting ${extractionMode} from database: ${database.name}`,
+  );
+
+  const blocks = new Map();
+  const blockSources = new Map();
+
+  // 1. Get all log entries
+  const entries = await database.log.values();
+  logger.info(`   Found ${entries.length} log entries`);
+
+  for (const entry of entries) {
+    try {
+      const entryBytes = await database.log.storage.get(entry.hash);
+      if (entryBytes) {
+        const entryCid = CID.parse(entry.hash);
+        blocks.set(entry.hash, { cid: entryCid, bytes: entryBytes });
+        blockSources.set(entry.hash, "log_entry");
+        logger.info(`   ✓ Entry block: ${entry.hash}`);
+      }
+    } catch (error) {
+      logger.warn(`   ⚠️ Failed to get entry ${entry.hash}: ${error.message}`);
+    }
+  }
+
+  // Get manifest CID for metadata (always extract this regardless of mode)
+  const addressParts = database.address.split("/");
+  const manifestCID = addressParts[addressParts.length - 1];
+
+  // Only extract metadata blocks if NOT in log-entries-only mode
+  if (!logEntriesOnly) {
+    // 2. Get database manifest
+    try {
+      const manifestBytes = await database.log.storage.get(manifestCID);
+      if (manifestBytes) {
+        const manifestParsedCid = CID.parse(manifestCID);
+        blocks.set(manifestCID, {
+          cid: manifestParsedCid,
+          bytes: manifestBytes,
+        });
+        blockSources.set(manifestCID, "manifest");
+        logger.info(`   ✓ Manifest block: ${manifestCID}`);
+
+        // Decode manifest to get access controller
+        try {
+          const manifestBlock = await Block.decode({
+            cid: manifestParsedCid,
+            bytes: manifestBytes,
+            codec: dagCbor,
+            hasher: sha256,
+          });
+
+          // Get access controller block
+          if (manifestBlock.value.accessController) {
+            const accessControllerCID =
+              manifestBlock.value.accessController.replace("/ipfs/", "");
+            try {
+              const accessBytes =
+                await database.log.storage.get(accessControllerCID);
+              if (accessBytes) {
+                const accessParsedCid = CID.parse(accessControllerCID);
+                blocks.set(accessControllerCID, {
+                  cid: accessParsedCid,
+                  bytes: accessBytes,
+                });
+                blockSources.set(accessControllerCID, "access_controller");
+                logger.info(`   ✓ Access controller: ${accessControllerCID}`);
+              }
+            } catch (error) {
+              logger.warn(
+                `   ⚠️ Could not get access controller: ${error.message}`,
+              );
+            }
+          }
+        } catch (error) {
+          logger.warn(`   ⚠️ Could not decode manifest: ${error.message}`);
+        }
+      }
+    } catch (error) {
+      logger.warn(`   ⚠️ Could not get manifest: ${error.message}`);
+    }
+
+    // 3. Get identity blocks using identities system and from log entries
+    logger.debug(
+      `Getting identity blocks from identities system and log entries...`,
+    );
+
+    // Collect all identity references from log entries
+    const referencedIdentities = new Set();
+    for (const entry of entries) {
+      if (entry.identity) {
+        referencedIdentities.add(entry.identity);
+      }
+    }
+
+    logger.info(
+      `   📝 Found ${referencedIdentities.size} unique identity references in log entries`,
+    );
+
+    // Get identity blocks - try multiple approaches for robustness
+    for (const identityHash of referencedIdentities) {
+      try {
+        // Method 1: the database's own identity, which carries its block. The
+        // log's storage may not hold it — `Identities()` without `ipfs` keeps
+        // identities in memory — and a Helia blockstore asked for it searches
+        // the network until OrbitDB's 30-second timeout, then goes without.
+        const ownIdentity = database.identity ?? database.log.identity;
+        if (ownIdentity?.hash === identityHash && ownIdentity.bytes) {
+          blocks.set(identityHash, {
+            cid: CID.parse(identityHash),
+            bytes: ownIdentity.bytes,
+          });
+          blockSources.set(identityHash, "identity_own");
+          logger.info(`   ✓ Identity block (own): ${identityHash}`);
+          continue;
+        }
+
+        // Method 2: Try to get identity from identities system (if available)
+        if (
+          database.log.identities &&
+          typeof database.log.identities.getIdentity === "function"
+        ) {
+          try {
+            const identity =
+              await database.log.identities.getIdentity(identityHash);
+            if (identity && identity.hash) {
+              // Get the identity block from storage
+              const identityBytes = await database.log.storage.get(
+                identity.hash,
+              );
+              if (identityBytes) {
+                const identityCid = CID.parse(identity.hash);
+                blocks.set(identity.hash, {
+                  cid: identityCid,
+                  bytes: identityBytes,
+                });
+                blockSources.set(identity.hash, "identity_system");
+                logger.info(`   ✓ Identity block (system): ${identity.hash}`);
+                continue; // Skip other methods if this works
+              }
+            }
+          } catch (systemError) {
+            logger.warn(
+              `   ⚠️ Identity system failed for ${identityHash}: ${systemError.message}`,
+            );
+          }
+        } else {
+          logger.info(
+            `   ℹ️ Identity system not available, using direct storage access`,
+          );
+        }
+
+        // Method 3: Try to get the identity hash directly from storage
+        try {
+          const identityBytes = await database.log.storage.get(identityHash);
+          if (identityBytes && !blocks.has(identityHash)) {
+            const identityCid = CID.parse(identityHash);
+            blocks.set(identityHash, {
+              cid: identityCid,
+              bytes: identityBytes,
+            });
+            blockSources.set(identityHash, "identity_direct");
+            logger.info(`   ✓ Identity block (direct): ${identityHash}`);
+            continue; // Skip scanning if direct access works
+          }
+        } catch (directError) {
+          logger.warn(
+            `   ⚠️ Could not get identity ${identityHash} directly: ${directError.message}`,
+          );
+        }
+      } catch (error) {
+        logger.warn(
+          `   ⚠️ Failed to get identity ${identityHash}: ${error.message}`,
+        );
+      }
+    }
+
+    // Additional scan through all storage blocks for any missed identity blocks
+    logger.debug(`Scanning remaining storage blocks for missed identities...`);
+    let discoveredIdentities = 0;
+
+    for await (const [hash, bytes] of database.log.storage.iterator()) {
+      try {
+        // Skip if we already have this block
+        if (blocks.has(hash)) {
+          continue;
+        }
+
+        // Try to decode as CBOR to check if it's an identity block
+        const cid = CID.parse(hash);
+        if (cid.code === 0x71) {
+          // dag-cbor codec
+          const block = await Block.decode({
+            cid,
+            bytes,
+            codec: dagCbor,
+            hasher: sha256,
+          });
+
+          const content = block.value;
+
+          // Check if this is an identity block (enhanced detection)
+          if (content && content.id && (content.type || content.publicKey)) {
+            blocks.set(hash, { cid, bytes });
+            blockSources.set(hash, "identity_discovered");
+            discoveredIdentities++;
+            logger.info(
+              `   ✓ Identity block discovered: ${hash}${referencedIdentities.has(hash) ? " (was referenced)" : " (unreferenced)"}`,
+            );
+          }
+        }
+      } catch {
+        // Skip blocks that can't be decoded - they might be raw data or other formats
+        continue;
+      }
+    }
+
+    logger.info(
+      `   📊 Identity blocks: ${referencedIdentities.size} referenced, ${discoveredIdentities} discovered`,
+    );
+  } else {
+    logger.info(
+      `   ⚡ Skipping manifest, access controller, and identity blocks (fallback mode)`,
+    );
+  }
+
+  logger.info(`   📊 Extracted ${blocks.size} total blocks`);
+  return { blocks, blockSources, manifestCID };
+}

package/lib/gateway-fetch.js

@@ -0,0 +1,116 @@
+/**
+ * Fetching bytes by CID from public IPFS gateways.
+ *
+ * Extracted from `restore-cid.js` when a second caller appeared. It is not
+ * about restoring and it is not about CARs — it is "resolve this name, from
+ * whoever will serve it" — and leaving it where it was would have meant a
+ * storage driver importing a CAR reader to make an HTTP request.
+ *
+ * Nothing here imports anything. That is deliberate and worth keeping: this is
+ * the module a browser reaches for when it wants one blob and nothing else.
+ *
+ * @module gateway-fetch
+ */
+
+/** Tried in order. The first is Storacha's own, the rest are public. */
+export const DEFAULT_GATEWAYS = [
+  "https://w3s.link/ipfs",
+  "https://storacha.link/ipfs",
+  "https://dweb.link/ipfs",
+  "https://ipfs.io/ipfs",
+];
+
+/**
+ * Quiet by default. A restore reports itself through its return value and its
+ * exceptions; anything more is the caller's choice, and importing a logger to
+ * offer it would cost more than the feature.
+ */
+export const SILENT = { info() {}, warn() {}, debug() {} };
+
+const DEFAULT_TIMEOUT_MS = 60_000;
+const MAX_ATTEMPTS_PER_GATEWAY = 3;
+
+/**
+ * A gateway that cannot serve a CID usually says so in HTML, with a 200.
+ *
+ * That is the trap this guards: the bytes arrive, the status is fine, and what
+ * you have is an error page that fails much later as an unreadable CAR. Two
+ * cheap checks — the declared type, and what the bytes actually start with,
+ * because the header is not always honest.
+ */
+const looksLikeAnErrorPage = (bytes, contentType = "") => {
+  if (contentType.includes("text/html") || contentType.includes("xhtml")) return true;
+  const start = new TextDecoder("utf-8", { fatal: false })
+    .decode(bytes.subarray(0, Math.min(100, bytes.length)))
+    .trim();
+  return start.startsWith("<!DOCTYPE") || start.startsWith("<html") || start.startsWith("<?xml");
+};
+
+const waitFor = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * How long a 429 wants us to wait, in the order the answer is trustworthy:
+ * what the server said, when it says the window resets, then a backoff.
+ */
+const backoffFor = (response, attempt) => {
+  const retryAfter = response.headers?.get?.("Retry-After");
+  if (retryAfter) {
+    const seconds = Number.parseInt(retryAfter, 10);
+    if (Number.isFinite(seconds)) return seconds * 1000;
+    const date = new Date(retryAfter);
+    if (!Number.isNaN(date.getTime())) return Math.max(0, date.getTime() - Date.now());
+  }
+  const reset = Number.parseInt(response.headers?.get?.("X-RateLimit-Reset") ?? "", 10);
+  if (Number.isFinite(reset)) return Math.max(0, reset * 1000 - Date.now());
+  return 2000 * (attempt + 1);
+};
+
+/**
+ * Fetch the bytes behind a CID, trying each gateway in turn.
+ *
+ * @param {string} cid
+ * @param {Object} [options]
+ * @param {string[]} [options.gateways]
+ * @param {number} [options.timeout] per request, in ms
+ * @param {AbortSignal} [options.signal]
+ * @returns {Promise<Uint8Array>}
+ */
+export async function fetchFromGateways(cid, { gateways = DEFAULT_GATEWAYS, timeout = DEFAULT_TIMEOUT_MS, signal = null, log = SILENT } = {}) {
+  let lastError = null;
+
+  for (const gateway of gateways) {
+    const url = `${gateway}/${cid}`;
+    for (let attempt = 0; attempt < MAX_ATTEMPTS_PER_GATEWAY; attempt++) {
+      const timer = AbortSignal.timeout ? AbortSignal.timeout(timeout) : null;
+      try {
+        const response = await fetch(url, { signal: signal ?? timer ?? undefined });
+
+        if (response.status === 429 && attempt < MAX_ATTEMPTS_PER_GATEWAY - 1) {
+          const wait = backoffFor(response, attempt);
+          log.warn(`   ⚠️ ${gateway} rate-limited; waiting ${Math.round(wait / 1000)}s`);
+          await waitFor(wait);
+          continue;
+        }
+        if (!response.ok) {
+          log.debug(`   ⚠️ ${gateway} answered ${response.status}`);
+          break; // a status this gateway will keep giving — move on
+        }
+
+        const bytes = new Uint8Array(await response.arrayBuffer());
+        if (looksLikeAnErrorPage(bytes, response.headers?.get?.("content-type") ?? "")) {
+          log.warn(`   ⚠️ ${gateway} returned an error page with a 200`);
+          break;
+        }
+        log.info(`   ✅ ${bytes.length} bytes from ${gateway}`);
+        return bytes;
+      } catch (error) {
+        lastError = error;
+        log.debug(`   ⚠️ ${gateway} failed: ${error.message}`);
+      }
+    }
+  }
+
+  throw new Error(
+    `Could not fetch ${cid} from any gateway${lastError ? `. Last error: ${lastError.message}` : ""}`,
+  );
+}