@le-space/orbitdb-storage-bridge 0.22.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/live-backup.js +14 -1
- package/lib/restore-cid.js +70 -15
- package/package.json +2 -1
package/lib/live-backup.js
CHANGED
|
@@ -56,7 +56,7 @@ import { CID } from "multiformats/cid";
|
|
|
56
56
|
import { sha256 } from "multiformats/hashes/sha2";
|
|
57
57
|
import { base58btc } from "multiformats/bases/base58";
|
|
58
58
|
import { createCARFromBlocks } from "./backup-car.js";
|
|
59
|
-
import { readBlocksFromCAR, restoreFromBlocks } from "./restore-cid.js";
|
|
59
|
+
import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId } from "./restore-cid.js";
|
|
60
60
|
|
|
61
61
|
/** What the manifest at an increment's root calls itself. */
|
|
62
62
|
export const INCREMENT_KIND = "increment";
|
|
@@ -492,6 +492,7 @@ export async function joinIncrements(
|
|
|
492
492
|
throw new Error("These increments hold nothing for this database.");
|
|
493
493
|
|
|
494
494
|
let skippedRaw = 0;
|
|
495
|
+
let skippedForeign = 0;
|
|
495
496
|
for (const [name, { bytes }] of blocks) {
|
|
496
497
|
const cid = CID.parse(name);
|
|
497
498
|
if (cid.code !== DAG_CBOR) {
|
|
@@ -502,6 +503,17 @@ export async function joinIncrements(
|
|
|
502
503
|
else skippedRaw++;
|
|
503
504
|
continue;
|
|
504
505
|
}
|
|
506
|
+
// An entry belongs to the database it names. A chain merged for several
|
|
507
|
+
// databases carries all of their entries, and writing every one into every
|
|
508
|
+
// database multiplies the writes by the database count — in a browser each
|
|
509
|
+
// is an IndexedDB transaction (#152). Blocks that are not entries — the
|
|
510
|
+
// manifest, the access controller, an identity — have no database of their
|
|
511
|
+
// own and are looked up by their own name here, so they still go in.
|
|
512
|
+
const belongsTo = entryDatabaseId(bytes);
|
|
513
|
+
if (belongsTo !== null && belongsTo !== address) {
|
|
514
|
+
skippedForeign++;
|
|
515
|
+
continue;
|
|
516
|
+
}
|
|
505
517
|
// OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
|
|
506
518
|
// its own names, and identities are looked up by theirs, so each block
|
|
507
519
|
// goes in under the name its reader will ask for.
|
|
@@ -522,6 +534,7 @@ export async function joinIncrements(
|
|
|
522
534
|
gaps: merged.gaps,
|
|
523
535
|
duplicates: merged.duplicates,
|
|
524
536
|
...(skippedRaw ? { skippedRaw } : {}),
|
|
537
|
+
...(skippedForeign ? { skippedForeign } : {}),
|
|
525
538
|
};
|
|
526
539
|
}
|
|
527
540
|
|
package/lib/restore-cid.js
CHANGED
|
@@ -130,6 +130,57 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
|
|
|
130
130
|
* entry only if it is dag-cbor *and* carries a signature, key and identity —
|
|
131
131
|
* the manifest and the access controller are dag-cbor too.
|
|
132
132
|
*/
|
|
133
|
+
/**
|
|
134
|
+
* Which database each entry block belongs to.
|
|
135
|
+
*
|
|
136
|
+
* A log entry names its database in `id`, so a block only has to reach the log
|
|
137
|
+
* storage of that one database. Writing every block to every database — which
|
|
138
|
+
* is what this used to do — multiplies the writes by the number of databases,
|
|
139
|
+
* and in a browser each of those is an IndexedDB transaction: 2 000 entries in
|
|
140
|
+
* nine databases meant 18 099 writes where 2 011 blocks exist (issue #152).
|
|
141
|
+
*
|
|
142
|
+
* Blocks that are not entries (manifest, access controller, identity) and
|
|
143
|
+
* entries nothing here can read (an encrypted database hides `id`) are left
|
|
144
|
+
* out: they are already in the blockstore, which the log reads through.
|
|
145
|
+
*
|
|
146
|
+
* The CAR was verified block by block on the way in, so this decodes without
|
|
147
|
+
* hashing again.
|
|
148
|
+
*/
|
|
149
|
+
export function entryDatabaseId(bytes) {
|
|
150
|
+
let value;
|
|
151
|
+
try {
|
|
152
|
+
value = dagCbor.decode(bytes);
|
|
153
|
+
} catch {
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
// An entry carries a signature, the signer's key and its identity; a
|
|
157
|
+
// manifest, an access controller or an identity block carries none of them.
|
|
158
|
+
if (!value?.sig || !value?.key || !value?.identity) return null;
|
|
159
|
+
return typeof value.id === "string" ? value.id : null;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function entryBlocksByDatabase(blocks) {
|
|
163
|
+
const byDatabase = new Map();
|
|
164
|
+
for (const [cidString, { bytes }] of blocks) {
|
|
165
|
+
let cid;
|
|
166
|
+
try {
|
|
167
|
+
cid = CID.parse(cidString);
|
|
168
|
+
} catch {
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
if (cid.code !== DAG_CBOR) continue;
|
|
172
|
+
|
|
173
|
+
const id = entryDatabaseId(bytes);
|
|
174
|
+
if (id === null) continue;
|
|
175
|
+
|
|
176
|
+
const forDatabase = byDatabase.get(id) ?? [];
|
|
177
|
+
// The log addresses blocks in base58btc, the CAR names them in base32.
|
|
178
|
+
forDatabase.push([cid.toV1().toString(base58btc), bytes]);
|
|
179
|
+
byDatabase.set(id, forDatabase);
|
|
180
|
+
}
|
|
181
|
+
return byDatabase;
|
|
182
|
+
}
|
|
183
|
+
|
|
133
184
|
async function headsIn(blocks) {
|
|
134
185
|
const entries = [];
|
|
135
186
|
const pointedAt = new Set();
|
|
@@ -323,26 +374,30 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
|
|
|
323
374
|
const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
|
|
324
375
|
const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
|
|
325
376
|
|
|
377
|
+
const entryBlocks = entryBlocksByDatabase(blocks);
|
|
378
|
+
|
|
326
379
|
const databases = [];
|
|
327
380
|
for (const [index, dbInfo] of metadata.databases.entries()) {
|
|
328
|
-
|
|
329
|
-
//
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
381
|
+
// Its own entries, into its own log storage — the rest of the CAR belongs
|
|
382
|
+
// to the other databases, and the blockstore above already holds all of it.
|
|
383
|
+
const mine = entryBlocks.get(dbInfo.address) ?? [];
|
|
384
|
+
let database;
|
|
385
|
+
if (mine.length === 0) {
|
|
386
|
+
database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
387
|
+
} else {
|
|
388
|
+
const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
389
|
+
for (const [hash, bytes] of mine) {
|
|
390
|
+
try {
|
|
391
|
+
await opened.log.storage.put(hash, bytes);
|
|
392
|
+
} catch (error) {
|
|
393
|
+
log.warn(` ⚠️ could not copy ${hash.slice(0, 12)}… to log storage: ${error.message}`);
|
|
394
|
+
}
|
|
339
395
|
}
|
|
396
|
+
// Reopened so the log reads what we just put underneath it.
|
|
397
|
+
await opened.close();
|
|
398
|
+
database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
340
399
|
}
|
|
341
400
|
|
|
342
|
-
// Reopened so the log reads what we just put underneath it.
|
|
343
|
-
await opened.close();
|
|
344
|
-
const database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
345
|
-
|
|
346
401
|
// Tell the log where its history ends.
|
|
347
402
|
const heads = Array.isArray(dbInfo.heads)
|
|
348
403
|
? (await Promise.all(dbInfo.heads.map((hash) => headFrom(byCanonical, hash)))).filter(Boolean)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@le-space/orbitdb-storage-bridge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.23.0",
|
|
4
4
|
"description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
|
|
5
5
|
"main": "lib/orbitdb-storacha-bridge.js",
|
|
6
6
|
"svelte": "dist/components/",
|
|
@@ -43,6 +43,7 @@
|
|
|
43
43
|
"test:timestamped-backup": "npm test -- test/timestamped-backup.test.js",
|
|
44
44
|
"test:network-download": "npm test -- test/network-download.test.js",
|
|
45
45
|
"test:backends": "npm test -- test/backends/",
|
|
46
|
+
"test:no-secrets": "npm test -- --forceExit --testPathIgnorePatterns '/node_modules/' '/examples/' '/dist/' '/coverage/' 'test/integration\\.test\\.js' 'test/backup-car\\.test\\.js' 'test/p256-ucan-security\\.test\\.js' 'test/timestamped-backup\\.test\\.js' 'test/network-download\\.test\\.js' 'test/access-control-integration\\.test\\.js'",
|
|
46
47
|
"clear-space": "node examples/storacha/clear-space.js",
|
|
47
48
|
"car-demo": "node examples/storacha/car-backup-demo.js",
|
|
48
49
|
"ucan-demo": "node examples/storacha/ucan-demo.js",
|