@le-space/orbitdb-storage-bridge 0.22.1 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/live-backup.js +15 -2
- package/lib/restore-cid.js +117 -18
- package/package.json +2 -1
package/lib/live-backup.js
CHANGED
|
@@ -56,7 +56,7 @@ import { CID } from "multiformats/cid";
|
|
|
56
56
|
import { sha256 } from "multiformats/hashes/sha2";
|
|
57
57
|
import { base58btc } from "multiformats/bases/base58";
|
|
58
58
|
import { createCARFromBlocks } from "./backup-car.js";
|
|
59
|
-
import { readBlocksFromCAR, restoreFromBlocks } from "./restore-cid.js";
|
|
59
|
+
import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId, standalone } from "./restore-cid.js";
|
|
60
60
|
|
|
61
61
|
/** What the manifest at an increment's root calls itself. */
|
|
62
62
|
export const INCREMENT_KIND = "increment";
|
|
@@ -492,6 +492,7 @@ export async function joinIncrements(
|
|
|
492
492
|
throw new Error("These increments hold nothing for this database.");
|
|
493
493
|
|
|
494
494
|
let skippedRaw = 0;
|
|
495
|
+
let skippedForeign = 0;
|
|
495
496
|
for (const [name, { bytes }] of blocks) {
|
|
496
497
|
const cid = CID.parse(name);
|
|
497
498
|
if (cid.code !== DAG_CBOR) {
|
|
@@ -502,10 +503,21 @@ export async function joinIncrements(
|
|
|
502
503
|
else skippedRaw++;
|
|
503
504
|
continue;
|
|
504
505
|
}
|
|
506
|
+
// An entry belongs to the database it names. A chain merged for several
|
|
507
|
+
// databases carries all of their entries, and writing every one into every
|
|
508
|
+
// database multiplies the writes by the database count — in a browser each
|
|
509
|
+
// is an IndexedDB transaction (#152). Blocks that are not entries — the
|
|
510
|
+
// manifest, the access controller, an identity — have no database of their
|
|
511
|
+
// own and are looked up by their own name here, so they still go in.
|
|
512
|
+
const belongsTo = entryDatabaseId(bytes);
|
|
513
|
+
if (belongsTo !== null && belongsTo !== address) {
|
|
514
|
+
skippedForeign++;
|
|
515
|
+
continue;
|
|
516
|
+
}
|
|
505
517
|
// OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
|
|
506
518
|
// its own names, and identities are looked up by theirs, so each block
|
|
507
519
|
// goes in under the name its reader will ask for.
|
|
508
|
-
await database.log.storage.put(cid.toV1().toString(base58btc), bytes);
|
|
520
|
+
await database.log.storage.put(cid.toV1().toString(base58btc), standalone(bytes));
|
|
509
521
|
}
|
|
510
522
|
|
|
511
523
|
let joined = 0;
|
|
@@ -522,6 +534,7 @@ export async function joinIncrements(
|
|
|
522
534
|
gaps: merged.gaps,
|
|
523
535
|
duplicates: merged.duplicates,
|
|
524
536
|
...(skippedRaw ? { skippedRaw } : {}),
|
|
537
|
+
...(skippedForeign ? { skippedForeign } : {}),
|
|
525
538
|
};
|
|
526
539
|
}
|
|
527
540
|
|
package/lib/restore-cid.js
CHANGED
|
@@ -116,7 +116,7 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
|
|
|
116
116
|
}
|
|
117
117
|
}
|
|
118
118
|
|
|
119
|
-
blocks.set(name, { bytes: block.bytes });
|
|
119
|
+
blocks.set(name, { bytes: standalone(block.bytes) });
|
|
120
120
|
}
|
|
121
121
|
|
|
122
122
|
return blocks;
|
|
@@ -130,6 +130,75 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
|
|
|
130
130
|
* entry only if it is dag-cbor *and* carries a signature, key and identity —
|
|
131
131
|
* the manifest and the access controller are dag-cbor too.
|
|
132
132
|
*/
|
|
133
|
+
/**
|
|
134
|
+
* Which database each entry block belongs to.
|
|
135
|
+
*
|
|
136
|
+
* A log entry names its database in `id`, so a block only has to reach the log
|
|
137
|
+
* storage of that one database. Writing every block to every database — which
|
|
138
|
+
* is what this used to do — multiplies the writes by the number of databases,
|
|
139
|
+
* and in a browser each of those is an IndexedDB transaction: 2 000 entries in
|
|
140
|
+
* nine databases meant 18 099 writes where 2 011 blocks exist (issue #152).
|
|
141
|
+
*
|
|
142
|
+
* Blocks that are not entries (manifest, access controller, identity) and
|
|
143
|
+
* entries nothing here can read (an encrypted database hides `id`) are left
|
|
144
|
+
* out: they are already in the blockstore, which the log reads through.
|
|
145
|
+
*
|
|
146
|
+
* The CAR was verified block by block on the way in, so this decodes without
|
|
147
|
+
* hashing again.
|
|
148
|
+
*/
|
|
149
|
+
/**
|
|
150
|
+
* The same bytes, owning their buffer.
|
|
151
|
+
*
|
|
152
|
+
* A CAR arrives as one `ArrayBuffer` and `CarReader` hands out each block as a
|
|
153
|
+
* view into it. IndexedDB's structured clone stores *the buffer behind a view*,
|
|
154
|
+
* not the view — so writing the 2 016 blocks of a 3.7 MB backup wrote 3.7 MB
|
|
155
|
+
* apiece, some 7.5 GB in all, and the browser refused everything past about
|
|
156
|
+
* block 500. The copy costs one pass over bytes that are going to disk anyway.
|
|
157
|
+
*
|
|
158
|
+
* @param {Uint8Array} bytes
|
|
159
|
+
* @returns {Uint8Array} `bytes` itself when it already owns its buffer
|
|
160
|
+
*/
|
|
161
|
+
export function standalone(bytes) {
|
|
162
|
+
return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength
|
|
163
|
+
? bytes
|
|
164
|
+
: new Uint8Array(bytes);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export function entryDatabaseId(bytes) {
|
|
168
|
+
let value;
|
|
169
|
+
try {
|
|
170
|
+
value = dagCbor.decode(bytes);
|
|
171
|
+
} catch {
|
|
172
|
+
return null;
|
|
173
|
+
}
|
|
174
|
+
// An entry carries a signature, the signer's key and its identity; a
|
|
175
|
+
// manifest, an access controller or an identity block carries none of them.
|
|
176
|
+
if (!value?.sig || !value?.key || !value?.identity) return null;
|
|
177
|
+
return typeof value.id === "string" ? value.id : null;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function entryBlocksByDatabase(blocks) {
|
|
181
|
+
const byDatabase = new Map();
|
|
182
|
+
for (const [cidString, { bytes }] of blocks) {
|
|
183
|
+
let cid;
|
|
184
|
+
try {
|
|
185
|
+
cid = CID.parse(cidString);
|
|
186
|
+
} catch {
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
if (cid.code !== DAG_CBOR) continue;
|
|
190
|
+
|
|
191
|
+
const id = entryDatabaseId(bytes);
|
|
192
|
+
if (id === null) continue;
|
|
193
|
+
|
|
194
|
+
const forDatabase = byDatabase.get(id) ?? [];
|
|
195
|
+
// The log addresses blocks in base58btc, the CAR names them in base32.
|
|
196
|
+
forDatabase.push([cid.toV1().toString(base58btc), bytes]);
|
|
197
|
+
byDatabase.set(id, forDatabase);
|
|
198
|
+
}
|
|
199
|
+
return byDatabase;
|
|
200
|
+
}
|
|
201
|
+
|
|
133
202
|
async function headsIn(blocks) {
|
|
134
203
|
const entries = [];
|
|
135
204
|
const pointedAt = new Set();
|
|
@@ -284,11 +353,17 @@ function assertRestorable(metadata) {
|
|
|
284
353
|
* the backup names (a node without pubsub needs `{ sync: false }`)
|
|
285
354
|
* @param {(progress: { stage: "database", index: number, total: number,
|
|
286
355
|
* address: string, joined: number, heads: number }) => void} [options.onProgress]
|
|
356
|
+
* @param {() => Promise<void>|void} [options.beforeJoin] called once the blocks
|
|
357
|
+
* are in the blockstore and before any database is opened — where a caller
|
|
358
|
+
* that must close its own handles does it, late enough that a refused block
|
|
359
|
+
* has already thrown and left those handles alone
|
|
360
|
+
* @throws {Error} `BlocksNotStoredError` when the blockstore refuses any block,
|
|
361
|
+
* before `beforeJoin` and before anything is opened
|
|
287
362
|
* @returns {Promise<{ blocks: number, databases: { address: string, name?: string,
|
|
288
363
|
* database: Object, entries: number|null, heads: number, joined: number }[] }>}
|
|
289
364
|
*/
|
|
290
365
|
export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {}) {
|
|
291
|
-
const { log = SILENT, verify = true, open = {}, onProgress } = options;
|
|
366
|
+
const { log = SILENT, verify = true, open = {}, onProgress, beforeJoin } = options;
|
|
292
367
|
if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
|
|
293
368
|
if (!(blocks instanceof Map)) throw new Error("blocks must be a Map");
|
|
294
369
|
if (!isValidMetadata(metadata)) throw new Error("Invalid backup metadata");
|
|
@@ -311,38 +386,62 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
|
|
|
311
386
|
|
|
312
387
|
// Into the blockstore, where Helia looks — once, whatever the databases.
|
|
313
388
|
let stored = 0;
|
|
389
|
+
/** @type {string[]} */
|
|
390
|
+
const refused = [];
|
|
314
391
|
for (const [cidString, { bytes }] of blocks) {
|
|
315
392
|
try {
|
|
316
|
-
await orbitdb.ipfs.blockstore.put(CID.parse(cidString), bytes);
|
|
393
|
+
await orbitdb.ipfs.blockstore.put(CID.parse(cidString), standalone(bytes));
|
|
317
394
|
stored++;
|
|
318
395
|
} catch (error) {
|
|
396
|
+
if (refused.length < 3) refused.push(`${cidString.slice(0, 12)}…: ${error.message}`);
|
|
319
397
|
log.warn(` ⚠️ could not store ${cidString.slice(0, 12)}…: ${error.message}`);
|
|
320
398
|
}
|
|
321
399
|
}
|
|
400
|
+
// A block that did not land is a head the log will later look for, not find,
|
|
401
|
+
// and wait out on the network — twenty seconds of silence per head, ending
|
|
402
|
+
// in a complaint about the entry rather than about the storage. A browser
|
|
403
|
+
// whose quota is spent arrives here with most of the backup refused. Say so
|
|
404
|
+
// now, while the caller's databases are all still open.
|
|
405
|
+
if (stored !== blocks.size) {
|
|
406
|
+
const error = new Error(
|
|
407
|
+
`The blockstore took ${stored} of ${blocks.size} blocks; the rest were refused: ${refused.join("; ")}`,
|
|
408
|
+
);
|
|
409
|
+
error.name = "BlocksNotStoredError";
|
|
410
|
+
Object.assign(error, { stored, total: blocks.size, refused });
|
|
411
|
+
throw error;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// Everything is in the blockstore and nothing has been opened yet: the one
|
|
415
|
+
// moment at which a caller can safely close what it holds open.
|
|
416
|
+
await beforeJoin?.();
|
|
322
417
|
|
|
323
418
|
const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
|
|
324
419
|
const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
|
|
325
420
|
|
|
421
|
+
const entryBlocks = entryBlocksByDatabase(blocks);
|
|
422
|
+
|
|
326
423
|
const databases = [];
|
|
327
424
|
for (const [index, dbInfo] of metadata.databases.entries()) {
|
|
328
|
-
|
|
329
|
-
//
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
425
|
+
// Its own entries, into its own log storage — the rest of the CAR belongs
|
|
426
|
+
// to the other databases, and the blockstore above already holds all of it.
|
|
427
|
+
const mine = entryBlocks.get(dbInfo.address) ?? [];
|
|
428
|
+
let database;
|
|
429
|
+
if (mine.length === 0) {
|
|
430
|
+
database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
431
|
+
} else {
|
|
432
|
+
const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
433
|
+
for (const [hash, bytes] of mine) {
|
|
434
|
+
try {
|
|
435
|
+
await opened.log.storage.put(hash, standalone(bytes));
|
|
436
|
+
} catch (error) {
|
|
437
|
+
log.warn(` ⚠️ could not copy ${hash.slice(0, 12)}… to log storage: ${error.message}`);
|
|
438
|
+
}
|
|
339
439
|
}
|
|
440
|
+
// Reopened so the log reads what we just put underneath it.
|
|
441
|
+
await opened.close();
|
|
442
|
+
database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
340
443
|
}
|
|
341
444
|
|
|
342
|
-
// Reopened so the log reads what we just put underneath it.
|
|
343
|
-
await opened.close();
|
|
344
|
-
const database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
345
|
-
|
|
346
445
|
// Tell the log where its history ends.
|
|
347
446
|
const heads = Array.isArray(dbInfo.heads)
|
|
348
447
|
? (await Promise.all(dbInfo.heads.map((hash) => headFrom(byCanonical, hash)))).filter(Boolean)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@le-space/orbitdb-storage-bridge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.24.0",
|
|
4
4
|
"description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
|
|
5
5
|
"main": "lib/orbitdb-storacha-bridge.js",
|
|
6
6
|
"svelte": "dist/components/",
|
|
@@ -43,6 +43,7 @@
|
|
|
43
43
|
"test:timestamped-backup": "npm test -- test/timestamped-backup.test.js",
|
|
44
44
|
"test:network-download": "npm test -- test/network-download.test.js",
|
|
45
45
|
"test:backends": "npm test -- test/backends/",
|
|
46
|
+
"test:no-secrets": "npm test -- --forceExit --testPathIgnorePatterns '/node_modules/' '/examples/' '/dist/' '/coverage/' 'test/integration\\.test\\.js' 'test/backup-car\\.test\\.js' 'test/p256-ucan-security\\.test\\.js' 'test/timestamped-backup\\.test\\.js' 'test/network-download\\.test\\.js' 'test/access-control-integration\\.test\\.js'",
|
|
46
47
|
"clear-space": "node examples/storacha/clear-space.js",
|
|
47
48
|
"car-demo": "node examples/storacha/car-backup-demo.js",
|
|
48
49
|
"ucan-demo": "node examples/storacha/ucan-demo.js",
|