@le-space/orbitdb-storage-bridge 0.28.1 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -30,7 +30,7 @@
30
30
  * @module restore-cid
31
31
  */
32
32
 
33
- import { CarReader } from "@ipld/car";
33
+ import { CarReader, CarBlockIterator } from "@ipld/car";
34
34
  import { CID } from "multiformats/cid";
35
35
  import { base58btc } from "multiformats/bases/base58";
36
36
  import * as Block from "multiformats/block";
@@ -386,15 +386,7 @@ function assertRestorable(metadata) {
386
386
  * database: Object, entries: number|null, heads: number, joined: number }[] }>}
387
387
  */
388
388
  export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {}) {
389
- const {
390
- log = SILENT,
391
- verify = true,
392
- open = {},
393
- onProgress,
394
- beforeJoin,
395
- fetchBytes = null,
396
- requireHeads = false,
397
- } = options;
389
+ const { log = SILENT, verify = true } = options;
398
390
  if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
399
391
  if (!(blocks instanceof Map)) throw new Error("blocks must be a Map");
400
392
  if (!isValidMetadata(metadata)) throw new Error("Invalid backup metadata");
@@ -444,9 +436,35 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
444
436
 
445
437
  const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
446
438
  const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
447
-
448
439
  const entryBlocks = entryBlocksByDatabase(blocks);
449
440
 
441
+ return joinDatabases(orbitdb, metadata, options, {
442
+ stored,
443
+ found,
444
+ has: (name) => byCanonical.has(name),
445
+ get: (name) => byCanonical.get(name)?.bytes,
446
+ remember: (name, bytes) => byCanonical.set(name, { bytes }),
447
+ entriesOf: (address) => entryBlocks.get(address) ?? [],
448
+ });
449
+ }
450
+
451
+ /**
452
+ * The second half of a restore, whatever holds the blocks: a Map
453
+ * ({@link restoreFromBlocks}) or the blockstore they were streamed into
454
+ * ({@link restoreFromStoredBlocks}). Every block is in the blockstore by now.
455
+ *
456
+ * @param {Object} orbitdb
457
+ * @param {Object} metadata
458
+ * @param {Object} options as `restoreFromBlocks` takes them
459
+ * @param {{ stored: number, found: any[], has(name: string): boolean | Promise<boolean>,
460
+ * get(name: string): Uint8Array | undefined | Promise<Uint8Array | undefined>,
461
+ * remember(name: string, bytes: Uint8Array): void,
462
+ * entriesOf(address: string): [string, Uint8Array | null][] }} source
463
+ */
464
+ async function joinDatabases(orbitdb, metadata, options, source) {
465
+ const { log = SILENT, open = {}, onProgress, beforeJoin, fetchBytes = null, requireHeads = false } = options;
466
+ const { found } = source;
467
+
450
468
  // A head the blocks do not carry used to be dropped here without a word, so
451
469
  // a backup missing one came back "restored" with nothing joined. Name it —
452
470
  // and, when the caller offered a second source, ask that source for it. The
@@ -457,7 +475,7 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
457
475
  for (const dbInfo of metadata.databases) {
458
476
  for (const hash of Array.isArray(dbInfo.heads) ? dbInfo.heads : []) {
459
477
  const name = canonical(hash);
460
- if (name && byCanonical.has(name)) continue;
478
+ if (name && (await source.has(name))) continue;
461
479
  if (!name || !fetchBytes) {
462
480
  missingHeads.push(String(hash));
463
481
  continue;
@@ -466,8 +484,8 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
466
484
  const cid = CID.parse(name);
467
485
  const bytes = standalone(await fetchBytes(name));
468
486
  await assertHashes(cid, bytes, "the second source answered with something else");
469
- byCanonical.set(name, { bytes });
470
487
  await orbitdb.ipfs.blockstore.put(cid, bytes);
488
+ source.remember(name, bytes);
471
489
  log.info(` ↓ head ${name.slice(0, 12)}… came from the second source`);
472
490
  } catch (error) {
473
491
  missingHeads.push(String(hash));
@@ -504,14 +522,16 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
504
522
  for (const [index, dbInfo] of metadata.databases.entries()) {
505
523
  // Its own entries, into its own log storage — the rest of the CAR belongs
506
524
  // to the other databases, and the blockstore above already holds all of it.
507
- const mine = entryBlocks.get(dbInfo.address) ?? [];
525
+ const mine = source.entriesOf(dbInfo.address);
508
526
  let database;
509
527
  if (mine.length === 0) {
510
528
  database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
511
529
  } else {
512
530
  const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
513
- for (const [hash, bytes] of mine) {
531
+ for (const [hash, held] of mine) {
514
532
  try {
533
+ const bytes = held ?? (await source.get(canonical(hash)));
534
+ if (!bytes) throw new Error("the block is not in the blockstore");
515
535
  await opened.log.storage.put(hash, standalone(bytes));
516
536
  } catch (error) {
517
537
  log.warn(` ⚠️ could not copy ${hash.slice(0, 12)}… to log storage: ${error.message}`);
@@ -524,7 +544,7 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
524
544
 
525
545
  // Tell the log where its history ends.
526
546
  const heads = Array.isArray(dbInfo.heads)
527
- ? (await Promise.all(dbInfo.heads.map((hash) => headFrom(byCanonical, hash)))).filter(Boolean)
547
+ ? (await Promise.all(dbInfo.heads.map((hash) => headFrom(source, hash)))).filter(Boolean)
528
548
  : found.filter((head) => head.value.id === dbInfo.address || metadata.databases.length === 1);
529
549
  let joined = 0;
530
550
  for (const head of heads) {
@@ -562,17 +582,232 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
562
582
  });
563
583
  }
564
584
 
565
- return { blocks: stored, databases, missingHeads };
585
+ return { blocks: source.stored, databases, missingHeads };
586
+ }
587
+
588
+
589
+ /** Whatever a blockstore's `get` gives, as bytes: Helia's answers with them, or with their pieces. */
590
+ async function bytesFrom(got) {
591
+ const value = await got;
592
+ if (value instanceof Uint8Array) return value;
593
+ const parts = [];
594
+ let total = 0;
595
+ for await (const part of value) {
596
+ parts.push(part);
597
+ total += part.length;
598
+ }
599
+ if (parts.length === 1) return parts[0];
600
+ const out = new Uint8Array(total);
601
+ let at = 0;
602
+ for (const part of parts) {
603
+ out.set(part, at);
604
+ at += part.length;
605
+ }
606
+ return out;
607
+ }
608
+
609
+ /**
610
+ * A CAR as it arrives: its roots, known from the first bytes, and its blocks
611
+ * one at a time — each checked against its own CID, as {@link readBlocksFromCAR}
612
+ * checks them, and none kept (#187).
613
+ *
614
+ * @param {AsyncIterable<Uint8Array> | Iterable<Uint8Array> | Uint8Array} source
615
+ * @param {Object} [options]
616
+ * @param {boolean} [options.verify=true] see {@link readBlocksFromCAR}
617
+ * @returns {Promise<{ roots: CID[], blocks(): AsyncGenerator<{ cid: CID, bytes: Uint8Array }> }>}
618
+ */
619
+ export async function openCARStream(source, { verify = true } = {}) {
620
+ const chunks =
621
+ source instanceof Uint8Array
622
+ ? (async function* () {
623
+ yield source;
624
+ })()
625
+ : source;
626
+ const reader = await CarBlockIterator.fromIterable(chunks);
627
+ return {
628
+ roots: await reader.getRoots(),
629
+ async *blocks() {
630
+ for await (const block of reader) {
631
+ if (verify) await assertHashes(block.cid, block.bytes, "the CAR has been tampered with");
632
+ yield { cid: block.cid, bytes: block.bytes };
633
+ }
634
+ },
635
+ };
636
+ }
637
+
638
+ /**
639
+ * @typedef {object} StoredBlocks what {@link storeBlocksFromCAR} knows of a CAR
640
+ * it put into a blockstore: names, never bytes
641
+ * @property {string[]} roots the CAR's roots, CIDv1
642
+ * @property {number} blocks how many it stored
643
+ * @property {number} bytes their bytes
644
+ * @property {Set<string>} names every block's CID, CIDv1 in its own base
645
+ * @property {Map<string, string[]>} entries by database address, the hashes
646
+ * (base58btc, as a log names them) of the log entries that say which
647
+ * database they are of; an encrypted database's entries say nothing and
648
+ * are read through the blockstore
649
+ */
650
+
651
+ /**
652
+ * Put a CAR's blocks into a blockstore as they arrive, keeping only their
653
+ * names (#187).
654
+ *
655
+ * {@link readBlocksFromCAR} answers with every block in a Map, and
656
+ * {@link restoreFromBlocks} then copies them into the blockstore: the backup
657
+ * twice in memory. Here a block is checked, stored and let go, so the memory
658
+ * a restore needs is a block's and a list of CIDs, whatever the backup's size.
659
+ * Hand the result to {@link restoreFromStoredBlocks}; several CARs (a base and
660
+ * its increments) go into one result through `into`.
661
+ *
662
+ * A block the blockstore refuses ends it at once (`BlocksNotStoredError`), as
663
+ * a browser whose quota is spent refuses all that follow. What was stored
664
+ * before stays: content-addressed blocks nothing points at yet.
665
+ *
666
+ * @param {Object} blockstore a Helia blockstore, e.g. `orbitdb.ipfs.blockstore`
667
+ * @param {AsyncIterable<Uint8Array> | Iterable<Uint8Array> | Uint8Array} source the CAR
668
+ * @param {Object} [options]
669
+ * @param {boolean} [options.verify=true]
670
+ * @param {StoredBlocks} [options.into] add to what an earlier CAR left
671
+ * @param {(block: { cid: CID, bytes: Uint8Array }, about: { roots: CID[], index: number }) => void | Promise<void>} [options.onBlock]
672
+ * sees each block before it is stored, and may throw to refuse the CAR: the
673
+ * place to read a manifest (a root) and decide whether the rest is wanted
674
+ * @param {(progress: { stage: "stored", blocks: number, bytes: number }) => void} [options.onProgress]
675
+ * after each block, counted over `into` as well
676
+ * @returns {Promise<StoredBlocks>}
677
+ */
678
+ export async function storeBlocksFromCAR(blockstore, source, { verify = true, into, onBlock, onProgress } = {}) {
679
+ if (typeof blockstore?.put !== "function") throw new Error("A blockstore is required");
680
+ const car = await openCARStream(source, { verify });
681
+ return storeBlocks(blockstore, car.blocks(), { roots: car.roots, into, onBlock, onProgress });
682
+ }
683
+
684
+ /**
685
+ * {@link storeBlocksFromCAR} for blocks that are not in a CAR any more: put
686
+ * into the blockstore one at a time, their names kept. For a caller that
687
+ * opened a file whole (a backup sealed in one piece) and wants it in the same
688
+ * {@link StoredBlocks} as the files it streamed.
689
+ *
690
+ * The blocks are taken as they are: check them first (`readBlocksFromCAR`,
691
+ * `openCARStream`).
692
+ *
693
+ * @param {Object} blockstore
694
+ * @param {AsyncIterable<{ cid: CID, bytes: Uint8Array }> | Iterable<{ cid: CID, bytes: Uint8Array }>} blocks
695
+ * @param {Object} [options] as there
696
+ * @param {StoredBlocks} [options.into]
697
+ * @param {(block: { cid: CID, bytes: Uint8Array }, about: { roots: CID[], index: number }) => void | Promise<void>} [options.onBlock]
698
+ * @param {(progress: { stage: "stored", blocks: number, bytes: number }) => void} [options.onProgress]
699
+ * @param {CID[]} [options.roots] what the blocks' file names as its roots
700
+ * @returns {Promise<StoredBlocks>}
701
+ */
702
+ export async function storeBlocks(blockstore, blocks, { roots = [], into, onBlock, onProgress } = {}) {
703
+ if (typeof blockstore?.put !== "function") throw new Error("A blockstore is required");
704
+ const stored = into ?? { roots: [], blocks: 0, bytes: 0, names: new Set(), entries: new Map() };
705
+ for (const root of roots) {
706
+ const name = root.toV1().toString();
707
+ if (!stored.roots.includes(name)) stored.roots.push(name);
708
+ }
709
+ let index = 0;
710
+ for await (const block of blocks) {
711
+ await onBlock?.(block, { roots, index });
712
+ index++;
713
+ const name = block.cid.toV1().toString();
714
+ try {
715
+ await blockstore.put(block.cid, standalone(block.bytes));
716
+ } catch (cause) {
717
+ const error = new Error(
718
+ `The blockstore took ${stored.blocks} blocks and refused the next: ${name.slice(0, 12)}…: ${cause.message}`,
719
+ { cause },
720
+ );
721
+ error.name = "BlocksNotStoredError";
722
+ Object.assign(error, { stored: stored.blocks, refused: [`${name.slice(0, 12)}…: ${cause.message}`] });
723
+ throw error;
724
+ }
725
+ if (!stored.names.has(name)) {
726
+ stored.names.add(name);
727
+ if (block.cid.code === DAG_CBOR) {
728
+ const id = entryDatabaseId(block.bytes);
729
+ if (id !== null) {
730
+ const hashes = stored.entries.get(id) ?? [];
731
+ hashes.push(block.cid.toV1().toString(base58btc));
732
+ stored.entries.set(id, hashes);
733
+ }
734
+ }
735
+ }
736
+ stored.blocks++;
737
+ stored.bytes += block.bytes.length;
738
+ onProgress?.({ stage: "stored", blocks: stored.blocks, bytes: stored.bytes });
739
+ }
740
+ return stored;
741
+ }
742
+
743
+ /**
744
+ * {@link restoreFromBlocks} for blocks that are in the blockstore already,
745
+ * put there by {@link storeBlocksFromCAR}: each database reopened and joined
746
+ * at its heads, the blocks read back one at a time where one is needed.
747
+ *
748
+ * The metadata must state every database's heads, as `bundleDatabases` and
749
+ * the increments write it: deriving them means reading every entry, which is
750
+ * what this exists to avoid.
751
+ *
752
+ * @param {Object} orbitdb
753
+ * @param {StoredBlocks} stored
754
+ * @param {Object} metadata
755
+ * @param {Object} [options] as {@link restoreFromBlocks} takes them
756
+ * @returns {ReturnType<typeof restoreFromBlocks>}
757
+ */
758
+ export async function restoreFromStoredBlocks(orbitdb, stored, metadata, options = {}) {
759
+ const { verify = true } = options;
760
+ const blockstore = orbitdb?.ipfs?.blockstore;
761
+ if (!blockstore) throw new Error("An OrbitDB instance is required");
762
+ if (!(stored?.names instanceof Set)) throw new Error("stored must be what storeBlocksFromCAR returned");
763
+ if (!isValidMetadata(metadata)) throw new Error("Invalid backup metadata");
764
+ assertRestorable(metadata);
765
+ if (!metadata.databases.every((db) => Array.isArray(db.heads))) {
766
+ throw new Error("A streamed restore needs metadata that states each database's heads");
767
+ }
768
+ if (verify) {
769
+ for (const db of metadata.databases) {
770
+ const manifestCID = db.manifestCID ?? metadata.manifestCID;
771
+ if (!manifestCID) continue;
772
+ const wanted = canonical(manifestCID);
773
+ if (!wanted || !stored.names.has(wanted)) {
774
+ throw new Error(`The CAR does not contain the manifest ${manifestCID} that this backup names`);
775
+ }
776
+ }
777
+ }
778
+ /** @type {Map<string, Uint8Array>} heads a second source supplied */
779
+ const fetched = new Map();
780
+ return joinDatabases(orbitdb, metadata, options, {
781
+ stored: stored.blocks,
782
+ found: [],
783
+ has: (name) => stored.names.has(name) || fetched.has(name),
784
+ get: async (name) =>
785
+ fetched.get(name) ??
786
+ (stored.names.has(name) ? bytesFrom(blockstore.get(CID.parse(name))).catch(() => undefined) : undefined),
787
+ remember: (name, bytes) => fetched.set(name, bytes),
788
+ entriesOf: (address) => (stored.entries.get(address) ?? []).map((hash) => [hash, null]),
789
+ });
566
790
  }
567
791
 
568
792
  /** A stated head, decoded from the blocks; null when the blocks lack it. */
569
- async function headFrom(byCanonical, hash) {
793
+ async function headFrom(source, hash) {
570
794
  const name = canonical(hash);
571
- const block = name ? byCanonical.get(name) : undefined;
572
- if (!block) return null;
795
+ const bytes = name ? await source.get(name) : undefined;
796
+ if (!bytes) return null;
573
797
  const cid = CID.parse(name);
574
- const { value } = await Block.decode({ cid, bytes: block.bytes, codec: dagCbor, hasher: sha256 });
798
+ const { value } = await Block.decode({ cid, bytes, codec: dagCbor, hasher: sha256 });
575
799
  return { hash: cid.toV1().toString(base58btc), value };
576
800
  }
577
801
 
578
- export default { restoreFromCID, restoreFromBlocks, fetchFromGateways, readBlocksFromCAR, DEFAULT_GATEWAYS, SILENT };
802
+ export default {
803
+ restoreFromCID,
804
+ restoreFromBlocks,
805
+ restoreFromStoredBlocks,
806
+ storeBlocksFromCAR,
807
+ storeBlocks,
808
+ openCARStream,
809
+ fetchFromGateways,
810
+ readBlocksFromCAR,
811
+ DEFAULT_GATEWAYS,
812
+ SILENT,
813
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@le-space/orbitdb-storage-bridge",
3
- "version": "0.28.1",
3
+ "version": "0.29.0",
4
4
  "description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
5
5
  "main": "lib/orbitdb-storacha-bridge.js",
6
6
  "type": "module",
@@ -26,6 +26,7 @@
26
26
  "./backends/lighthouse": "./lib/backends/lighthouse.js",
27
27
  "./backends/mirror": "./lib/backends/mirror.js",
28
28
  "./backends/encryption": "./lib/backends/encryption.js",
29
+ "./backends/frames": "./lib/backends/frames.js",
29
30
  "./backends/choose": "./lib/backends/choose.js",
30
31
  "./backends/resolve": "./lib/backends/resolve.js",
31
32
  "./memory-courier": "./lib/memory-courier.js",