@le-space/orbitdb-storage-bridge 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/gateway-fetch.js +12 -7
- package/lib/live-backup.js +2 -2
- package/lib/restore-cid.js +142 -18
- package/package.json +1 -1
package/lib/gateway-fetch.js
CHANGED
|
@@ -35,15 +35,20 @@ import { CID } from "multiformats/cid";
|
|
|
35
35
|
export const DEFAULT_GATEWAYS = ["https://ipfs.aleph.cloud/ipfs"];
|
|
36
36
|
|
|
37
37
|
/**
|
|
38
|
-
* Gateways that
|
|
38
|
+
* Gateways that answer a block request with the block, and say so.
|
|
39
39
|
*
|
|
40
|
-
* `trustless-gateway.link`
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
40
|
+
* `trustless-gateway.link` used to be the entry here. Measured 2026-10-24 it
|
|
41
|
+
* answers `525` to everything, path and trustless alike, so it has followed
|
|
42
|
+
* `ipfs.io` and `dweb.link` out. What is left and healthy is Aleph's, which
|
|
43
|
+
* honours `accept: application/vnd.ipld.raw` and sets the content type to
|
|
44
|
+
* match — 200 in 0.15 s with CORS `*`.
|
|
45
|
+
*
|
|
46
|
+
* Only for a CID whose bytes are one block: a UnixFS file split into chunks
|
|
47
|
+
* needs its DAG walked, which this module deliberately cannot do. And the
|
|
48
|
+
* list is a convenience, never a reason to trust an answer — every byte that
|
|
49
|
+
* comes back is checked against the CID that asked for it.
|
|
45
50
|
*/
|
|
46
|
-
export const TRUSTLESS_GATEWAYS = ["https://
|
|
51
|
+
export const TRUSTLESS_GATEWAYS = ["https://ipfs.aleph.cloud/ipfs"];
|
|
47
52
|
|
|
48
53
|
/** `accept` for a single block from a trustless gateway. */
|
|
49
54
|
export const RAW_BLOCK = "application/vnd.ipld.raw";
|
package/lib/live-backup.js
CHANGED
|
@@ -56,7 +56,7 @@ import { CID } from "multiformats/cid";
|
|
|
56
56
|
import { sha256 } from "multiformats/hashes/sha2";
|
|
57
57
|
import { base58btc } from "multiformats/bases/base58";
|
|
58
58
|
import { createCARFromBlocks } from "./backup-car.js";
|
|
59
|
-
import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId } from "./restore-cid.js";
|
|
59
|
+
import { readBlocksFromCAR, restoreFromBlocks, entryDatabaseId, standalone } from "./restore-cid.js";
|
|
60
60
|
|
|
61
61
|
/** What the manifest at an increment's root calls itself. */
|
|
62
62
|
export const INCREMENT_KIND = "increment";
|
|
@@ -517,7 +517,7 @@ export async function joinIncrements(
|
|
|
517
517
|
// OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
|
|
518
518
|
// its own names, and identities are looked up by theirs, so each block
|
|
519
519
|
// goes in under the name its reader will ask for.
|
|
520
|
-
await database.log.storage.put(cid.toV1().toString(base58btc), bytes);
|
|
520
|
+
await database.log.storage.put(cid.toV1().toString(base58btc), standalone(bytes));
|
|
521
521
|
}
|
|
522
522
|
|
|
523
523
|
let joined = 0;
|
package/lib/restore-cid.js
CHANGED
|
@@ -70,6 +70,28 @@ const HASHERS = new Map([
|
|
|
70
70
|
[sha512.code, sha512],
|
|
71
71
|
]);
|
|
72
72
|
|
|
73
|
+
/**
|
|
74
|
+
* Bytes against the CID that names them, or an exception.
|
|
75
|
+
*
|
|
76
|
+
* @param {CID} cid
|
|
77
|
+
* @param {Uint8Array} bytes
|
|
78
|
+
* @param {string} blame what it means when they do not match
|
|
79
|
+
*/
|
|
80
|
+
async function assertHashes(cid, bytes, blame) {
|
|
81
|
+
const hasher = HASHERS.get(cid.multihash.code);
|
|
82
|
+
// Refused rather than waved through: a hash we cannot compute is a block we
|
|
83
|
+
// cannot vouch for, and silently accepting it would defeat the point.
|
|
84
|
+
if (!hasher) {
|
|
85
|
+
throw new Error(
|
|
86
|
+
`Block ${cid} uses multihash 0x${cid.multihash.code.toString(16)}, which this reader cannot verify`,
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
const digest = await hasher.digest(bytes);
|
|
90
|
+
if (!sameBytes(digest.digest, cid.multihash.digest)) {
|
|
91
|
+
throw new Error(`Block ${cid} does not hash to its own CID — ${blame}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
73
95
|
/**
|
|
74
96
|
* Every block in a CAR, by CID string — each one checked against its own name.
|
|
75
97
|
*
|
|
@@ -102,21 +124,10 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
|
|
|
102
124
|
const name = block.cid.toString();
|
|
103
125
|
|
|
104
126
|
if (verify) {
|
|
105
|
-
|
|
106
|
-
// Refused rather than waved through: a hash we cannot compute is a block
|
|
107
|
-
// we cannot vouch for, and silently accepting it would defeat the point.
|
|
108
|
-
if (!hasher) {
|
|
109
|
-
throw new Error(
|
|
110
|
-
`Block ${name} uses multihash 0x${block.cid.multihash.code.toString(16)}, which this reader cannot verify`,
|
|
111
|
-
);
|
|
112
|
-
}
|
|
113
|
-
const digest = await hasher.digest(block.bytes);
|
|
114
|
-
if (!sameBytes(digest.digest, block.cid.multihash.digest)) {
|
|
115
|
-
throw new Error(`Block ${name} does not hash to its own CID — the CAR has been tampered with`);
|
|
116
|
-
}
|
|
127
|
+
await assertHashes(block.cid, block.bytes, "the CAR has been tampered with");
|
|
117
128
|
}
|
|
118
129
|
|
|
119
|
-
blocks.set(name, { bytes: block.bytes });
|
|
130
|
+
blocks.set(name, { bytes: standalone(block.bytes) });
|
|
120
131
|
}
|
|
121
132
|
|
|
122
133
|
return blocks;
|
|
@@ -146,6 +157,24 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
|
|
|
146
157
|
* The CAR was verified block by block on the way in, so this decodes without
|
|
147
158
|
* hashing again.
|
|
148
159
|
*/
|
|
160
|
+
/**
|
|
161
|
+
* The same bytes, owning their buffer.
|
|
162
|
+
*
|
|
163
|
+
* A CAR arrives as one `ArrayBuffer` and `CarReader` hands out each block as a
|
|
164
|
+
* view into it. IndexedDB's structured clone stores *the buffer behind a view*,
|
|
165
|
+
* not the view — so writing the 2 016 blocks of a 3.7 MB backup wrote 3.7 MB
|
|
166
|
+
* apiece, some 7.5 GB in all, and the browser refused everything past about
|
|
167
|
+
* block 500. The copy costs one pass over bytes that are going to disk anyway.
|
|
168
|
+
*
|
|
169
|
+
* @param {Uint8Array} bytes
|
|
170
|
+
* @returns {Uint8Array} `bytes` itself when it already owns its buffer
|
|
171
|
+
*/
|
|
172
|
+
export function standalone(bytes) {
|
|
173
|
+
return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength
|
|
174
|
+
? bytes
|
|
175
|
+
: new Uint8Array(bytes);
|
|
176
|
+
}
|
|
177
|
+
|
|
149
178
|
export function entryDatabaseId(bytes) {
|
|
150
179
|
let value;
|
|
151
180
|
try {
|
|
@@ -335,11 +364,37 @@ function assertRestorable(metadata) {
|
|
|
335
364
|
* the backup names (a node without pubsub needs `{ sync: false }`)
|
|
336
365
|
* @param {(progress: { stage: "database", index: number, total: number,
|
|
337
366
|
* address: string, joined: number, heads: number }) => void} [options.onProgress]
|
|
338
|
-
* @
|
|
367
|
+
* @param {((cid: string) => Promise<Uint8Array>) | null} [options.fetchBytes] a
|
|
368
|
+
* second source for a head block the blocks do not carry — `fetchFromGateways`
|
|
369
|
+
* bound to the gateways an application is willing to ask. Off by default: a
|
|
370
|
+
* restore that reaches the network unasked is not something an offline-first
|
|
371
|
+
* application can allow. Whatever comes back is checked against its CID.
|
|
372
|
+
* @param {boolean} [options.requireHeads=false] throw `MissingHeadsError` when a
|
|
373
|
+
* head the metadata names cannot be supplied — before `beforeJoin`, so a
|
|
374
|
+
* caller that closes its databases there still has them
|
|
375
|
+
* @param {() => Promise<void>|void} [options.beforeJoin] called once the blocks
|
|
376
|
+
* are in the blockstore and before any database is opened — where a caller
|
|
377
|
+
* that must close its own handles does it, late enough that a refused block
|
|
378
|
+
* has already thrown and left those handles alone
|
|
379
|
+
* @throws {Error} `BlocksNotStoredError` when the blockstore refuses any block,
|
|
380
|
+
* before `beforeJoin` and before anything is opened
|
|
381
|
+
* `missingHeads` are the heads the metadata names that neither the blocks nor
|
|
382
|
+
* the second source could supply: not an error here, because a caller may know
|
|
383
|
+
* the database is reachable another way, but never silence either.
|
|
384
|
+
*
|
|
385
|
+
* @returns {Promise<{ blocks: number, missingHeads: string[], databases: { address: string, name?: string,
|
|
339
386
|
* database: Object, entries: number|null, heads: number, joined: number }[] }>}
|
|
340
387
|
*/
|
|
341
388
|
export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {}) {
|
|
342
|
-
const {
|
|
389
|
+
const {
|
|
390
|
+
log = SILENT,
|
|
391
|
+
verify = true,
|
|
392
|
+
open = {},
|
|
393
|
+
onProgress,
|
|
394
|
+
beforeJoin,
|
|
395
|
+
fetchBytes = null,
|
|
396
|
+
requireHeads = false,
|
|
397
|
+
} = options;
|
|
343
398
|
if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
|
|
344
399
|
if (!(blocks instanceof Map)) throw new Error("blocks must be a Map");
|
|
345
400
|
if (!isValidMetadata(metadata)) throw new Error("Invalid backup metadata");
|
|
@@ -362,20 +417,89 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
|
|
|
362
417
|
|
|
363
418
|
// Into the blockstore, where Helia looks — once, whatever the databases.
|
|
364
419
|
let stored = 0;
|
|
420
|
+
/** @type {string[]} */
|
|
421
|
+
const refused = [];
|
|
365
422
|
for (const [cidString, { bytes }] of blocks) {
|
|
366
423
|
try {
|
|
367
|
-
await orbitdb.ipfs.blockstore.put(CID.parse(cidString), bytes);
|
|
424
|
+
await orbitdb.ipfs.blockstore.put(CID.parse(cidString), standalone(bytes));
|
|
368
425
|
stored++;
|
|
369
426
|
} catch (error) {
|
|
427
|
+
if (refused.length < 3) refused.push(`${cidString.slice(0, 12)}…: ${error.message}`);
|
|
370
428
|
log.warn(` ⚠️ could not store ${cidString.slice(0, 12)}…: ${error.message}`);
|
|
371
429
|
}
|
|
372
430
|
}
|
|
431
|
+
// A block that did not land is a head the log will later look for, not find,
|
|
432
|
+
// and wait out on the network — twenty seconds of silence per head, ending
|
|
433
|
+
// in a complaint about the entry rather than about the storage. A browser
|
|
434
|
+
// whose quota is spent arrives here with most of the backup refused. Say so
|
|
435
|
+
// now, while the caller's databases are all still open.
|
|
436
|
+
if (stored !== blocks.size) {
|
|
437
|
+
const error = new Error(
|
|
438
|
+
`The blockstore took ${stored} of ${blocks.size} blocks; the rest were refused: ${refused.join("; ")}`,
|
|
439
|
+
);
|
|
440
|
+
error.name = "BlocksNotStoredError";
|
|
441
|
+
Object.assign(error, { stored, total: blocks.size, refused });
|
|
442
|
+
throw error;
|
|
443
|
+
}
|
|
373
444
|
|
|
374
445
|
const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
|
|
375
446
|
const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
|
|
376
447
|
|
|
377
448
|
const entryBlocks = entryBlocksByDatabase(blocks);
|
|
378
449
|
|
|
450
|
+
// A head the blocks do not carry used to be dropped here without a word, so
|
|
451
|
+
// a backup missing one came back "restored" with nothing joined. Name it —
|
|
452
|
+
// and, when the caller offered a second source, ask that source for it. The
|
|
453
|
+
// bytes are checked against the CID that asked for them before they are
|
|
454
|
+
// believed, so a gateway is a convenience here and never a party to trust.
|
|
455
|
+
/** @type {string[]} */
|
|
456
|
+
const missingHeads = [];
|
|
457
|
+
for (const dbInfo of metadata.databases) {
|
|
458
|
+
for (const hash of Array.isArray(dbInfo.heads) ? dbInfo.heads : []) {
|
|
459
|
+
const name = canonical(hash);
|
|
460
|
+
if (name && byCanonical.has(name)) continue;
|
|
461
|
+
if (!name || !fetchBytes) {
|
|
462
|
+
missingHeads.push(String(hash));
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
try {
|
|
466
|
+
const cid = CID.parse(name);
|
|
467
|
+
const bytes = standalone(await fetchBytes(name));
|
|
468
|
+
await assertHashes(cid, bytes, "the second source answered with something else");
|
|
469
|
+
byCanonical.set(name, { bytes });
|
|
470
|
+
await orbitdb.ipfs.blockstore.put(cid, bytes);
|
|
471
|
+
log.info(` ↓ head ${name.slice(0, 12)}… came from the second source`);
|
|
472
|
+
} catch (error) {
|
|
473
|
+
missingHeads.push(String(hash));
|
|
474
|
+
log.warn(
|
|
475
|
+
` ⚠️ head ${String(hash).slice(0, 12)}… is not in the backup and could not be fetched: ${error.message}`,
|
|
476
|
+
);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// An application whose books are replaced by this restore cannot live with a
|
|
482
|
+
// partial one: better to refuse now, while its databases are still open and
|
|
483
|
+
// still hold what they had. Off by default, because a caller that replicates
|
|
484
|
+
// may well reach the rest another way.
|
|
485
|
+
if (requireHeads && missingHeads.length) {
|
|
486
|
+
const error = new Error(
|
|
487
|
+
`The backup names ${missingHeads.length} head${missingHeads.length === 1 ? "" : "s"} ` +
|
|
488
|
+
`that neither it nor a second source could supply: ` +
|
|
489
|
+
missingHeads.map((h) => `${String(h).slice(0, 12)}…`).join(", "),
|
|
490
|
+
);
|
|
491
|
+
error.name = "MissingHeadsError";
|
|
492
|
+
Object.assign(error, { missingHeads });
|
|
493
|
+
throw error;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
// Every block is in the blockstore, every head the metadata names is in hand,
|
|
497
|
+
// and nothing has been opened yet: the one moment at which a caller can
|
|
498
|
+
// safely close what it holds open. Later than it used to be, deliberately —
|
|
499
|
+
// asking a second source for a missing head can fail, and a caller's books
|
|
500
|
+
// should not already be shut when it does.
|
|
501
|
+
await beforeJoin?.();
|
|
502
|
+
|
|
379
503
|
const databases = [];
|
|
380
504
|
for (const [index, dbInfo] of metadata.databases.entries()) {
|
|
381
505
|
// Its own entries, into its own log storage — the rest of the CAR belongs
|
|
@@ -388,7 +512,7 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
|
|
|
388
512
|
const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
389
513
|
for (const [hash, bytes] of mine) {
|
|
390
514
|
try {
|
|
391
|
-
await opened.log.storage.put(hash, bytes);
|
|
515
|
+
await opened.log.storage.put(hash, standalone(bytes));
|
|
392
516
|
} catch (error) {
|
|
393
517
|
log.warn(` ⚠️ could not copy ${hash.slice(0, 12)}… to log storage: ${error.message}`);
|
|
394
518
|
}
|
|
@@ -438,7 +562,7 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
|
|
|
438
562
|
});
|
|
439
563
|
}
|
|
440
564
|
|
|
441
|
-
return { blocks: stored, databases };
|
|
565
|
+
return { blocks: stored, databases, missingHeads };
|
|
442
566
|
}
|
|
443
567
|
|
|
444
568
|
/** A stated head, decoded from the blocks; null when the blocks lack it. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@le-space/orbitdb-storage-bridge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.25.0",
|
|
4
4
|
"description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
|
|
5
5
|
"main": "lib/orbitdb-storacha-bridge.js",
|
|
6
6
|
"svelte": "dist/components/",
|