@le-space/orbitdb-storage-bridge 0.24.0 → 0.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/gateway-fetch.js +27 -22
- package/lib/restore-cid.js +99 -19
- package/package.json +1 -1
package/lib/gateway-fetch.js
CHANGED
|
@@ -35,15 +35,20 @@ import { CID } from "multiformats/cid";
|
|
|
35
35
|
export const DEFAULT_GATEWAYS = ["https://ipfs.aleph.cloud/ipfs"];
|
|
36
36
|
|
|
37
37
|
/**
|
|
38
|
-
* Gateways that
|
|
38
|
+
* Gateways that answer a block request with the block, and say so.
|
|
39
39
|
*
|
|
40
|
-
* `trustless-gateway.link`
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
40
|
+
* `trustless-gateway.link` used to be the entry here. Measured 2026-10-24 it
|
|
41
|
+
* answers `525` to everything, path and trustless alike, so it has followed
|
|
42
|
+
* `ipfs.io` and `dweb.link` out. What is left and healthy is Aleph's, which
|
|
43
|
+
* honours `accept: application/vnd.ipld.raw` and sets the content type to
|
|
44
|
+
* match — 200 in 0.15 s with CORS `*`.
|
|
45
|
+
*
|
|
46
|
+
* Only for a CID whose bytes are one block: a UnixFS file split into chunks
|
|
47
|
+
* needs its DAG walked, which this module deliberately cannot do. And the
|
|
48
|
+
* list is a convenience, never a reason to trust an answer — every byte that
|
|
49
|
+
* comes back is checked against the CID that asked for it.
|
|
45
50
|
*/
|
|
46
|
-
export const TRUSTLESS_GATEWAYS = ["https://
|
|
51
|
+
export const TRUSTLESS_GATEWAYS = ["https://ipfs.aleph.cloud/ipfs"];
|
|
47
52
|
|
|
48
53
|
/** `accept` for a single block from a trustless gateway. */
|
|
49
54
|
export const RAW_BLOCK = "application/vnd.ipld.raw";
|
|
@@ -111,21 +116,6 @@ const backoffFor = (response, attempt) => {
|
|
|
111
116
|
return 2000 * (attempt + 1);
|
|
112
117
|
};
|
|
113
118
|
|
|
114
|
-
/**
|
|
115
|
-
* Fetch the bytes behind a CID, trying each gateway in turn.
|
|
116
|
-
*
|
|
117
|
-
* @param {string} cid
|
|
118
|
-
* @param {Object} [options]
|
|
119
|
-
* @param {string[]} [options.gateways]
|
|
120
|
-
* @param {number} [options.timeout] per request, in ms
|
|
121
|
-
* @param {AbortSignal} [options.signal]
|
|
122
|
-
* @param {string} [options.accept] sent as `Accept`; `RAW_BLOCK` for a single
|
|
123
|
-
* block from a trustless gateway
|
|
124
|
-
* @param {number} [options.maxBackoff] longest 429 wait to honour, in ms
|
|
125
|
-
* @param {(ms: number) => Promise<void>} [options.sleep] injectable for tests
|
|
126
|
-
* @param {typeof fetch} [options.fetchImpl] injectable for tests
|
|
127
|
-
* @returns {Promise<Uint8Array>}
|
|
128
|
-
*/
|
|
129
119
|
/**
|
|
130
120
|
* True unless the CID is a raw sha-256 block whose digest the bytes miss.
|
|
131
121
|
* Anything this cannot check (another codec, another hash, an unparsable
|
|
@@ -152,6 +142,21 @@ export async function matchesRawCid(cid, bytes) {
|
|
|
152
142
|
return true;
|
|
153
143
|
}
|
|
154
144
|
|
|
145
|
+
/**
|
|
146
|
+
* Fetch the bytes behind a CID, trying each gateway in turn.
|
|
147
|
+
*
|
|
148
|
+
* @param {string} cid
|
|
149
|
+
* @param {Object} [options]
|
|
150
|
+
* @param {string[]} [options.gateways]
|
|
151
|
+
* @param {number} [options.timeout] per request, in ms
|
|
152
|
+
* @param {AbortSignal} [options.signal]
|
|
153
|
+
* @param {string} [options.accept] sent as `Accept`; `RAW_BLOCK` for a single
|
|
154
|
+
* block from a trustless gateway
|
|
155
|
+
* @param {number} [options.maxBackoff] longest 429 wait to honour, in ms
|
|
156
|
+
* @param {(ms: number) => Promise<void>} [options.sleep] injectable for tests
|
|
157
|
+
* @param {typeof fetch} [options.fetchImpl] injectable for tests
|
|
158
|
+
* @returns {Promise<Uint8Array>}
|
|
159
|
+
*/
|
|
155
160
|
export async function fetchFromGateways(
|
|
156
161
|
cid,
|
|
157
162
|
{
|
package/lib/restore-cid.js
CHANGED
|
@@ -70,6 +70,28 @@ const HASHERS = new Map([
|
|
|
70
70
|
[sha512.code, sha512],
|
|
71
71
|
]);
|
|
72
72
|
|
|
73
|
+
/**
|
|
74
|
+
* Bytes against the CID that names them, or an exception.
|
|
75
|
+
*
|
|
76
|
+
* @param {CID} cid
|
|
77
|
+
* @param {Uint8Array} bytes
|
|
78
|
+
* @param {string} blame what it means when they do not match
|
|
79
|
+
*/
|
|
80
|
+
async function assertHashes(cid, bytes, blame) {
|
|
81
|
+
const hasher = HASHERS.get(cid.multihash.code);
|
|
82
|
+
// Refused rather than waved through: a hash we cannot compute is a block we
|
|
83
|
+
// cannot vouch for, and silently accepting it would defeat the point.
|
|
84
|
+
if (!hasher) {
|
|
85
|
+
throw new Error(
|
|
86
|
+
`Block ${cid} uses multihash 0x${cid.multihash.code.toString(16)}, which this reader cannot verify`,
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
const digest = await hasher.digest(bytes);
|
|
90
|
+
if (!sameBytes(digest.digest, cid.multihash.digest)) {
|
|
91
|
+
throw new Error(`Block ${cid} does not hash to its own CID — ${blame}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
73
95
|
/**
|
|
74
96
|
* Every block in a CAR, by CID string — each one checked against its own name.
|
|
75
97
|
*
|
|
@@ -102,18 +124,7 @@ export async function readBlocksFromCAR(carBytes, { verify = true } = {}) {
|
|
|
102
124
|
const name = block.cid.toString();
|
|
103
125
|
|
|
104
126
|
if (verify) {
|
|
105
|
-
|
|
106
|
-
// Refused rather than waved through: a hash we cannot compute is a block
|
|
107
|
-
// we cannot vouch for, and silently accepting it would defeat the point.
|
|
108
|
-
if (!hasher) {
|
|
109
|
-
throw new Error(
|
|
110
|
-
`Block ${name} uses multihash 0x${block.cid.multihash.code.toString(16)}, which this reader cannot verify`,
|
|
111
|
-
);
|
|
112
|
-
}
|
|
113
|
-
const digest = await hasher.digest(block.bytes);
|
|
114
|
-
if (!sameBytes(digest.digest, block.cid.multihash.digest)) {
|
|
115
|
-
throw new Error(`Block ${name} does not hash to its own CID — the CAR has been tampered with`);
|
|
116
|
-
}
|
|
127
|
+
await assertHashes(block.cid, block.bytes, "the CAR has been tampered with");
|
|
117
128
|
}
|
|
118
129
|
|
|
119
130
|
blocks.set(name, { bytes: standalone(block.bytes) });
|
|
@@ -353,17 +364,37 @@ function assertRestorable(metadata) {
|
|
|
353
364
|
* the backup names (a node without pubsub needs `{ sync: false }`)
|
|
354
365
|
* @param {(progress: { stage: "database", index: number, total: number,
|
|
355
366
|
* address: string, joined: number, heads: number }) => void} [options.onProgress]
|
|
367
|
+
* @param {((cid: string) => Promise<Uint8Array>) | null} [options.fetchBytes] a
|
|
368
|
+
* second source for a head block the blocks do not carry — `fetchFromGateways`
|
|
369
|
+
* bound to the gateways an application is willing to ask. Off by default: a
|
|
370
|
+
* restore that reaches the network unasked is not something an offline-first
|
|
371
|
+
* application can allow. Whatever comes back is checked against its CID.
|
|
372
|
+
* @param {boolean} [options.requireHeads=false] throw `MissingHeadsError` when a
|
|
373
|
+
* head the metadata names cannot be supplied — before `beforeJoin`, so a
|
|
374
|
+
* caller that closes its databases there still has them
|
|
356
375
|
* @param {() => Promise<void>|void} [options.beforeJoin] called once the blocks
|
|
357
376
|
* are in the blockstore and before any database is opened — where a caller
|
|
358
377
|
* that must close its own handles does it, late enough that a refused block
|
|
359
378
|
* has already thrown and left those handles alone
|
|
360
379
|
* @throws {Error} `BlocksNotStoredError` when the blockstore refuses any block,
|
|
361
380
|
* before `beforeJoin` and before anything is opened
|
|
362
|
-
*
|
|
381
|
+
* `missingHeads` are the heads the metadata names that neither the blocks nor
|
|
382
|
+
* the second source could supply: not an error here, because a caller may know
|
|
383
|
+
* the database is reachable another way, but never silence either.
|
|
384
|
+
*
|
|
385
|
+
* @returns {Promise<{ blocks: number, missingHeads: string[], databases: { address: string, name?: string,
|
|
363
386
|
* database: Object, entries: number|null, heads: number, joined: number }[] }>}
|
|
364
387
|
*/
|
|
365
388
|
export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {}) {
|
|
366
|
-
const {
|
|
389
|
+
const {
|
|
390
|
+
log = SILENT,
|
|
391
|
+
verify = true,
|
|
392
|
+
open = {},
|
|
393
|
+
onProgress,
|
|
394
|
+
beforeJoin,
|
|
395
|
+
fetchBytes = null,
|
|
396
|
+
requireHeads = false,
|
|
397
|
+
} = options;
|
|
367
398
|
if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
|
|
368
399
|
if (!(blocks instanceof Map)) throw new Error("blocks must be a Map");
|
|
369
400
|
if (!isValidMetadata(metadata)) throw new Error("Invalid backup metadata");
|
|
@@ -411,15 +442,64 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
|
|
|
411
442
|
throw error;
|
|
412
443
|
}
|
|
413
444
|
|
|
414
|
-
// Everything is in the blockstore and nothing has been opened yet: the one
|
|
415
|
-
// moment at which a caller can safely close what it holds open.
|
|
416
|
-
await beforeJoin?.();
|
|
417
|
-
|
|
418
445
|
const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
|
|
419
446
|
const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
|
|
420
447
|
|
|
421
448
|
const entryBlocks = entryBlocksByDatabase(blocks);
|
|
422
449
|
|
|
450
|
+
// A head the blocks do not carry used to be dropped here without a word, so
|
|
451
|
+
// a backup missing one came back "restored" with nothing joined. Name it —
|
|
452
|
+
// and, when the caller offered a second source, ask that source for it. The
|
|
453
|
+
// bytes are checked against the CID that asked for them before they are
|
|
454
|
+
// believed, so a gateway is a convenience here and never a party to trust.
|
|
455
|
+
/** @type {string[]} */
|
|
456
|
+
const missingHeads = [];
|
|
457
|
+
for (const dbInfo of metadata.databases) {
|
|
458
|
+
for (const hash of Array.isArray(dbInfo.heads) ? dbInfo.heads : []) {
|
|
459
|
+
const name = canonical(hash);
|
|
460
|
+
if (name && byCanonical.has(name)) continue;
|
|
461
|
+
if (!name || !fetchBytes) {
|
|
462
|
+
missingHeads.push(String(hash));
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
try {
|
|
466
|
+
const cid = CID.parse(name);
|
|
467
|
+
const bytes = standalone(await fetchBytes(name));
|
|
468
|
+
await assertHashes(cid, bytes, "the second source answered with something else");
|
|
469
|
+
byCanonical.set(name, { bytes });
|
|
470
|
+
await orbitdb.ipfs.blockstore.put(cid, bytes);
|
|
471
|
+
log.info(` ↓ head ${name.slice(0, 12)}… came from the second source`);
|
|
472
|
+
} catch (error) {
|
|
473
|
+
missingHeads.push(String(hash));
|
|
474
|
+
log.warn(
|
|
475
|
+
` ⚠️ head ${String(hash).slice(0, 12)}… is not in the backup and could not be fetched: ${error.message}`,
|
|
476
|
+
);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// An application whose books are replaced by this restore cannot live with a
|
|
482
|
+
// partial one: better to refuse now, while its databases are still open and
|
|
483
|
+
// still hold what they had. Off by default, because a caller that replicates
|
|
484
|
+
// may well reach the rest another way.
|
|
485
|
+
if (requireHeads && missingHeads.length) {
|
|
486
|
+
const error = new Error(
|
|
487
|
+
`The backup names ${missingHeads.length} head${missingHeads.length === 1 ? "" : "s"} ` +
|
|
488
|
+
`that neither it nor a second source could supply: ` +
|
|
489
|
+
missingHeads.map((h) => `${String(h).slice(0, 12)}…`).join(", "),
|
|
490
|
+
);
|
|
491
|
+
error.name = "MissingHeadsError";
|
|
492
|
+
Object.assign(error, { missingHeads });
|
|
493
|
+
throw error;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
// Every block is in the blockstore, every head the metadata names is in hand,
|
|
497
|
+
// and nothing has been opened yet: the one moment at which a caller can
|
|
498
|
+
// safely close what it holds open. Later than it used to be, deliberately —
|
|
499
|
+
// asking a second source for a missing head can fail, and a caller's books
|
|
500
|
+
// should not already be shut when it does.
|
|
501
|
+
await beforeJoin?.();
|
|
502
|
+
|
|
423
503
|
const databases = [];
|
|
424
504
|
for (const [index, dbInfo] of metadata.databases.entries()) {
|
|
425
505
|
// Its own entries, into its own log storage — the rest of the CAR belongs
|
|
@@ -482,7 +562,7 @@ export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {})
|
|
|
482
562
|
});
|
|
483
563
|
}
|
|
484
564
|
|
|
485
|
-
return { blocks: stored, databases };
|
|
565
|
+
return { blocks: stored, databases, missingHeads };
|
|
486
566
|
}
|
|
487
567
|
|
|
488
568
|
/** A stated head, decoded from the blocks; null when the blocks lack it. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@le-space/orbitdb-storage-bridge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.25.1",
|
|
4
4
|
"description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
|
|
5
5
|
"main": "lib/orbitdb-storacha-bridge.js",
|
|
6
6
|
"svelte": "dist/components/",
|