@le-space/orbitdb-storage-bridge 0.22.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,11 +17,8 @@
17
17
  * @see {@link ../../docs/STORAGE-BACKENDS.md} for what replaced it
18
18
  */
19
19
 
20
- import * as Client from "@storacha/client";
21
- import { StoreMemory } from "@storacha/client/stores/memory";
22
- import { Signer } from "@storacha/client/principal/ed25519";
23
- import * as Proof from "@storacha/client/proof";
24
20
  import { CID } from "multiformats/cid";
21
+ import { loadStorachaClient } from "../storacha-client.js";
25
22
  import { defineBackend, handleId, BackendError } from "./types.js";
26
23
 
27
24
  /**
@@ -61,6 +58,7 @@ async function resolveClient(options) {
61
58
  );
62
59
  }
63
60
 
61
+ const { Client, StoreMemory, Signer, Proof } = await loadStorachaClient();
64
62
  const clientOptions = {
65
63
  principal: Signer.parse(options.storachaKey),
66
64
  store: new StoreMemory(),
@@ -56,7 +56,7 @@ import { CID } from "multiformats/cid";
56
56
  import { sha256 } from "multiformats/hashes/sha2";
57
57
  import { base58btc } from "multiformats/bases/base58";
58
58
  import { createCARFromBlocks } from "./backup-car.js";
59
- import { readBlocksFromCAR, restoreFromBlocks } from "./restore-cid.js";
59
+ import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId } from "./restore-cid.js";
60
60
 
61
61
  /** What the manifest at an increment's root calls itself. */
62
62
  export const INCREMENT_KIND = "increment";
@@ -492,6 +492,7 @@ export async function joinIncrements(
492
492
  throw new Error("These increments hold nothing for this database.");
493
493
 
494
494
  let skippedRaw = 0;
495
+ let skippedForeign = 0;
495
496
  for (const [name, { bytes }] of blocks) {
496
497
  const cid = CID.parse(name);
497
498
  if (cid.code !== DAG_CBOR) {
@@ -502,6 +503,17 @@ export async function joinIncrements(
502
503
  else skippedRaw++;
503
504
  continue;
504
505
  }
506
+ // An entry belongs to the database it names. A chain merged for several
507
+ // databases carries all of their entries, and writing every one into every
508
+ // database multiplies the writes by the database count — in a browser each
509
+ // is an IndexedDB transaction (#152). Blocks that are not entries — the
510
+ // manifest, the access controller, an identity — have no database of their
511
+ // own and are looked up by their own name here, so they still go in.
512
+ const belongsTo = entryDatabaseId(bytes);
513
+ if (belongsTo !== null && belongsTo !== address) {
514
+ skippedForeign++;
515
+ continue;
516
+ }
505
517
  // OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
506
518
  // its own names, and identities are looked up by theirs, so each block
507
519
  // goes in under the name its reader will ask for.
@@ -522,6 +534,7 @@ export async function joinIncrements(
522
534
  gaps: merged.gaps,
523
535
  duplicates: merged.duplicates,
524
536
  ...(skippedRaw ? { skippedRaw } : {}),
537
+ ...(skippedForeign ? { skippedForeign } : {}),
525
538
  };
526
539
  }
527
540
 
@@ -34,6 +34,7 @@ import { readBlockBytes } from "./block-bytes.js";
34
34
  // every vintage look for it.
35
35
  import { extractDatabaseBlocks } from "./extract-blocks.js";
36
36
  import { resolveBackend } from "./backends/resolve.js";
37
+ import { loadStorachaClient } from "./storacha-client.js";
37
38
 
38
39
  export { extractDatabaseBlocks };
39
40
 
@@ -100,34 +101,6 @@ export function extractManifestCID(databaseAddress) {
100
101
  }
101
102
 
102
103
 
103
- /**
104
- * The Storacha client, loaded when first needed and never at import time.
105
- * `@storacha/client` is an optional peer dependency: without it the main
106
- * entry still loads, and only the Storacha paths fail, here, with a message
107
- * that names the package.
108
- *
109
- * @returns {Promise<{Client: object, StoreMemory: Function, Signer: {parse: Function}, Proof: {parse: Function}}>}
110
- */
111
- async function loadStorachaClient() {
112
- try {
113
- const [Client, { StoreMemory }, { Signer }, Proof] = await Promise.all([
114
- import("@storacha/client"),
115
- import("@storacha/client/stores/memory"),
116
- import("@storacha/client/principal/ed25519"),
117
- import("@storacha/client/proof"),
118
- ]);
119
- return { Client, StoreMemory, Signer, Proof };
120
- } catch (error) {
121
- if (error?.code === "ERR_MODULE_NOT_FOUND" || error?.code === "MODULE_NOT_FOUND") {
122
- throw new Error(
123
- "The Storacha backend needs @storacha/client, an optional peer dependency since 0.22.0: npm install @storacha/client",
124
- { cause: error },
125
- );
126
- }
127
- throw error;
128
- }
129
- }
130
-
131
104
  /**
132
105
  * Initialize Storacha client with credentials
133
106
  *
@@ -130,6 +130,57 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
130
130
  * entry only if it is dag-cbor *and* carries a signature, key and identity —
131
131
  * the manifest and the access controller are dag-cbor too.
132
132
  */
133
+ /**
134
+ * Which database each entry block belongs to.
135
+ *
136
+ * A log entry names its database in `id`, so a block only has to reach the log
137
+ * storage of that one database. Writing every block to every database — which
138
+ * is what this used to do — multiplies the writes by the number of databases,
139
+ * and in a browser each of those is an IndexedDB transaction: 2 000 entries in
140
+ * nine databases meant 18 099 writes where 2 011 blocks exist (issue #152).
141
+ *
142
+ * Blocks that are not entries (manifest, access controller, identity) and
143
+ * entries nothing here can read (an encrypted database hides `id`) are left
144
+ * out: they are already in the blockstore, which the log reads through.
145
+ *
146
+ * The CAR was verified block by block on the way in, so this decodes without
147
+ * hashing again.
148
+ */
149
+ export function entryDatabaseId(bytes) {
150
+ let value;
151
+ try {
152
+ value = dagCbor.decode(bytes);
153
+ } catch {
154
+ return null;
155
+ }
156
+ // An entry carries a signature, the signer's key and its identity; a
157
+ // manifest, an access controller or an identity block carries none of them.
158
+ if (!value?.sig || !value?.key || !value?.identity) return null;
159
+ return typeof value.id === "string" ? value.id : null;
160
+ }
161
+
162
+ function entryBlocksByDatabase(blocks) {
163
+ const byDatabase = new Map();
164
+ for (const [cidString, { bytes }] of blocks) {
165
+ let cid;
166
+ try {
167
+ cid = CID.parse(cidString);
168
+ } catch {
169
+ continue;
170
+ }
171
+ if (cid.code !== DAG_CBOR) continue;
172
+
173
+ const id = entryDatabaseId(bytes);
174
+ if (id === null) continue;
175
+
176
+ const forDatabase = byDatabase.get(id) ?? [];
177
+ // The log addresses blocks in base58btc, the CAR names them in base32.
178
+ forDatabase.push([cid.toV1().toString(base58btc), bytes]);
179
+ byDatabase.set(id, forDatabase);
180
+ }
181
+ return byDatabase;
182
+ }
183
+
133
184
  async function headsIn(blocks) {
134
185
  const entries = [];
135
186
  const pointedAt = new Set();
@@ -323,26 +374,30 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
323
374
  const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
324
375
  const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
325
376
 
377
+ const entryBlocks = entryBlocksByDatabase(blocks);
378
+
326
379
  const databases = [];
327
380
  for (const [index, dbInfo] of metadata.databases.entries()) {
328
- const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
329
- // OrbitDB's log reads from its own store and addresses blocks in base58btc
330
- // rather than the CAR's base32. A restore that fills only the blockstore
331
- // opens a database that is empty in a way nothing reports.
332
- for (const [cidString, { bytes }] of blocks) {
333
- try {
334
- const cid = CID.parse(cidString);
335
- if (cid.code !== DAG_CBOR) continue;
336
- await opened.log.storage.put(cid.toV1().toString(base58btc), bytes);
337
- } catch (error) {
338
- log.warn(` ⚠️ could not copy ${cidString.slice(0, 12)}… to log storage: ${error.message}`);
381
+ // Its own entries, into its own log storage — the rest of the CAR belongs
382
+ // to the other databases, and the blockstore above already holds all of it.
383
+ const mine = entryBlocks.get(dbInfo.address) ?? [];
384
+ let database;
385
+ if (mine.length === 0) {
386
+ database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
387
+ } else {
388
+ const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
389
+ for (const [hash, bytes] of mine) {
390
+ try {
391
+ await opened.log.storage.put(hash, bytes);
392
+ } catch (error) {
393
+ log.warn(` ⚠️ could not copy ${hash.slice(0, 12)}… to log storage: ${error.message}`);
394
+ }
339
395
  }
396
+ // Reopened so the log reads what we just put underneath it.
397
+ await opened.close();
398
+ database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
340
399
  }
341
400
 
342
- // Reopened so the log reads what we just put underneath it.
343
- await opened.close();
344
- const database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
345
-
346
401
  // Tell the log where its history ends.
347
402
  const heads = Array.isArray(dbInfo.heads)
348
403
  ? (await Promise.all(dbInfo.heads.map((hash) => headFrom(byCanonical, hash)))).filter(Boolean)
@@ -0,0 +1,36 @@
1
+ /**
2
+ * The Storacha client, loaded when first needed and never at import time.
3
+ *
4
+ * `@storacha/client` is an optional peer dependency since 0.22.0. A static
5
+ * `import` of it anywhere under lib/ would break a consumer's bundle: Vite and
6
+ * Rollup follow dynamic imports into chunks and resolve their named imports at
7
+ * build time, so even a driver that is only ever `import()`ed — `resolve.js`
8
+ * reaches `backends/storacha.js` that way — fails the build of an application
9
+ * that never chose Storacha. So the client is imported here, dynamically, and
10
+ * nowhere else; without the package the Storacha paths fail at the call, with
11
+ * a message that names it. test/no-static-storacha-import.test.js keeps it so.
12
+ *
13
+ * @returns {Promise<{ Client: typeof import("@storacha/client"), StoreMemory: Function, Signer: { parse: Function }, Proof: { parse: Function } }>}
14
+ */
15
+ export async function loadStorachaClient() {
16
+ try {
17
+ const [Client, { StoreMemory }, { Signer }, Proof] = await Promise.all([
18
+ import("@storacha/client"),
19
+ import("@storacha/client/stores/memory"),
20
+ import("@storacha/client/principal/ed25519"),
21
+ import("@storacha/client/proof"),
22
+ ]);
23
+ return { Client, StoreMemory, Signer, Proof };
24
+ } catch (error) {
25
+ if (
26
+ error?.code === "ERR_MODULE_NOT_FOUND" ||
27
+ error?.code === "MODULE_NOT_FOUND"
28
+ ) {
29
+ throw new Error(
30
+ "The Storacha backend needs @storacha/client, an optional peer dependency since 0.22.0: npm install @storacha/client",
31
+ { cause: error },
32
+ );
33
+ }
34
+ throw error;
35
+ }
36
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@le-space/orbitdb-storage-bridge",
3
- "version": "0.22.0",
3
+ "version": "0.23.0",
4
4
  "description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
5
5
  "main": "lib/orbitdb-storacha-bridge.js",
6
6
  "svelte": "dist/components/",
@@ -43,6 +43,7 @@
43
43
  "test:timestamped-backup": "npm test -- test/timestamped-backup.test.js",
44
44
  "test:network-download": "npm test -- test/network-download.test.js",
45
45
  "test:backends": "npm test -- test/backends/",
46
+ "test:no-secrets": "npm test -- --forceExit --testPathIgnorePatterns '/node_modules/' '/examples/' '/dist/' '/coverage/' 'test/integration\\.test\\.js' 'test/backup-car\\.test\\.js' 'test/p256-ucan-security\\.test\\.js' 'test/timestamped-backup\\.test\\.js' 'test/network-download\\.test\\.js' 'test/access-control-integration\\.test\\.js'",
46
47
  "clear-space": "node examples/storacha/clear-space.js",
47
48
  "car-demo": "node examples/storacha/car-backup-demo.js",
48
49
  "ucan-demo": "node examples/storacha/ucan-demo.js",