@le-space/orbitdb-storage-bridge 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/lib/backends/aleph.js +10 -6
- package/lib/backends/choose.js +44 -3
- package/lib/backends/encryption.js +246 -0
- package/lib/backends/storacha.js +13 -8
- package/lib/backup-car.js +13 -6
- package/lib/dehydrate.js +32 -1
- package/lib/gateway-fetch.js +65 -10
- package/lib/orbitdb-storacha-bridge.js +8 -9
- package/lib/peer-fetch.js +391 -0
- package/lib/ucan-bridge.js +1 -1
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -60,6 +60,18 @@
|
|
|
60
60
|
- [License](#license)
|
|
61
61
|
|
|
62
62
|
|
|
63
|
+
## Does this work from a browser? Measure it
|
|
64
|
+
|
|
65
|
+
`examples/browser/storage-probe/` is a static page that asks the services this package talks to,
|
|
66
|
+
from a real browser, with no server: do Aleph, Pinata and Lighthouse answer a page at all, does
|
|
67
|
+
their CORS survive a refusal as well as a success, and does anybody on IPFS hold a given CID at an
|
|
68
|
+
address a browser can dial. With your own key it does a real upload and reads it back. The key is
|
|
69
|
+
kept in that browser and sent only to the service it belongs to.
|
|
70
|
+
|
|
71
|
+
It is published from this repository's Pages, and it opens from a checkout just as well — there is
|
|
72
|
+
no build step, which is the point: what it measures is a browser talking to a service, with
|
|
73
|
+
nothing in between.
|
|
74
|
+
|
|
63
75
|
## Status: Storacha sunset (May 2026)
|
|
64
76
|
|
|
65
77
|
Storacha switched off user writes in **May 2026** and has since decommissioned the service.
|
|
@@ -73,6 +85,17 @@ Verified on 2026-09-05:
|
|
|
73
85
|
| the widget demo CID linked in the roadmap below | `504` on `w3s.link`, `dweb.link`, `ipfs.io` and `trustless-gateway.link` |
|
|
74
86
|
| `@storacha/client` on npm | last release `2.1.4`, 2026-05-15, not marked deprecated |
|
|
75
87
|
|
|
88
|
+
The gateway those redirects pointed at is gone too. On **2026-09-21** Protocol Labs retired
|
|
89
|
+
`ipfs.io` and `dweb.link`: both answer `429` with an RFC 8594 `Sunset` header and a link to
|
|
90
|
+
[gatewaychanges.ipfs.io](https://gatewaychanges.ipfs.io/), so `storacha.link` and `w3s.link`
|
|
91
|
+
now redirect to a closed door. Retrieval defaults here are `ipfs.aleph.cloud` — the one free
|
|
92
|
+
path gateway measured still serving arbitrary CIDs on 2026-09-23 — with
|
|
93
|
+
`trustless-gateway.link` available for verifiable single-block requests
|
|
94
|
+
(`Accept: application/vnd.ipld.raw`). One host is not a fallback chain, which is why `peer-fetch.js` fetches from the providers that
|
|
95
|
+
hold the blocks instead — bitswap over libp2p, measured from a real page at 0.73 s to dial and
|
|
96
|
+
0.26 s for the block, with no credential in the path. See
|
|
97
|
+
[docs/RECOVERY-ON-A-SECOND-DEVICE.md](docs/RECOVERY-ON-A-SECOND-DEVICE.md).
|
|
98
|
+
|
|
76
99
|
The shutdown is traceable in the open:
|
|
77
100
|
[`upload-service#708`](https://github.com/storacha/upload-service/pull/708) added a `writesDisabled`
|
|
78
101
|
kill switch that makes the eight user-initiated write capabilities
|
package/lib/backends/aleph.js
CHANGED
|
@@ -47,12 +47,16 @@
|
|
|
47
47
|
import { defineBackend, handleId, BackendError } from "./types.js";
|
|
48
48
|
import { fetchFromGateways } from "../gateway-fetch.js";
|
|
49
49
|
|
|
50
|
-
/**
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
50
|
+
/**
|
|
51
|
+
* Aleph's own, and only Aleph's own.
|
|
52
|
+
*
|
|
53
|
+
* `dweb.link` and `ipfs.io` were the two fallbacks behind it until Protocol
|
|
54
|
+
* Labs retired them on 2026-09-21; they answer 429 with a `Sunset` header now.
|
|
55
|
+
* Nothing free was found to replace them that will serve an arbitrary CID, so
|
|
56
|
+
* this driver's retrieval depends on one host until the peer path in #112
|
|
57
|
+
* lands. A caller with its own gateway passes `gateways`.
|
|
58
|
+
*/
|
|
59
|
+
export const ALEPH_GATEWAYS = Object.freeze(["https://ipfs.aleph.cloud/ipfs"]);
|
|
56
60
|
|
|
57
61
|
const ALEPH_INGEST = "https://ipfs.aleph.cloud/api/v0/add";
|
|
58
62
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
package/lib/backends/choose.js
CHANGED
|
@@ -21,6 +21,16 @@
|
|
|
21
21
|
* {@link ../backends/mirror.js createMirrorBackend} over them, so the caller's
|
|
22
22
|
* code is the same whether one service was ticked or three.
|
|
23
23
|
*
|
|
24
|
+
* ## A gateway belongs to the service it came from
|
|
25
|
+
*
|
|
26
|
+
* Since the public path gateways were retired (#111), the gateway a reader can
|
|
27
|
+
* actually use is usually their own account's — `<name>.mypinata.cloud`,
|
|
28
|
+
* `<name>.lighthouseweb3.xyz` — and those are not interchangeable: Pinata's
|
|
29
|
+
* answers 401 for a Lighthouse CID, Lighthouse's answers 402. So `gateway` is
|
|
30
|
+
* a string only while one service is chosen, and an object keyed by service
|
|
31
|
+
* once several are. Handing one string to three drivers is refused rather than
|
|
32
|
+
* half-working.
|
|
33
|
+
*
|
|
24
34
|
* ## What it refuses
|
|
25
35
|
*
|
|
26
36
|
* A key that is missing is refused here, with the name of the service in the
|
|
@@ -36,6 +46,16 @@
|
|
|
36
46
|
|
|
37
47
|
import { BackendError } from "./types.js";
|
|
38
48
|
|
|
49
|
+
/**
|
|
50
|
+
* Aleph's driver tries a list of `…/ipfs` prefixes, while a reader types a
|
|
51
|
+
* host. Accept either, and a bare domain as https, like the other drivers do.
|
|
52
|
+
*/
|
|
53
|
+
const alephGateway = (value) => {
|
|
54
|
+
const withScheme = /^https?:\/\//i.test(value) ? value : `https://${value}`;
|
|
55
|
+
const trimmed = withScheme.replace(/\/+$/, "");
|
|
56
|
+
return trimmed.endsWith("/ipfs") ? trimmed : `${trimmed}/ipfs`;
|
|
57
|
+
};
|
|
58
|
+
|
|
39
59
|
/** The names a caller may ask for. */
|
|
40
60
|
export const BACKEND_KINDS = Object.freeze([
|
|
41
61
|
"aleph",
|
|
@@ -55,7 +75,11 @@ export const BACKEND_KINDS = Object.freeze([
|
|
|
55
75
|
* @param {string} [choice.apiKey] - Lighthouse
|
|
56
76
|
* @param {"shared"|"user"} [choice.keyOwnership] - Lighthouse: whose key it is,
|
|
57
77
|
* which is what decides `browserSafeAuth`
|
|
58
|
-
* @param {string
|
|
78
|
+
* @param {string|Record<string, string>} [choice.gateway] - retrieval gateway:
|
|
79
|
+
* a string for a single kind, or `{ pinata: "…", lighthouse: "…" }` for several.
|
|
80
|
+
* A bare domain is read as https, as each driver already accepts
|
|
81
|
+
* @param {string[]} [choice.gateways] - retrieval gateways, tried in order; wins
|
|
82
|
+
* over `gateway` where a driver takes a list
|
|
59
83
|
* @param {object} [choice.options] - passed through to the driver, for anything
|
|
60
84
|
* this signature does not name
|
|
61
85
|
* @param {"one"|"all"} [choice.require] - for several kinds: how many must accept a write
|
|
@@ -80,6 +104,14 @@ export async function createBackendFromChoice(choice = {}) {
|
|
|
80
104
|
}
|
|
81
105
|
}
|
|
82
106
|
|
|
107
|
+
if (wanted.length > 1 && typeof choice.gateway === "string") {
|
|
108
|
+
throw new BackendError(
|
|
109
|
+
"INVALID_BACKEND",
|
|
110
|
+
`A gateway belongs to one service — with several kinds pass an object: ` +
|
|
111
|
+
`{ gateway: { ${wanted.map((k) => `${k}: "…"`).join(", ")} } }`,
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
|
|
83
115
|
if (wanted.length > 1) {
|
|
84
116
|
const backends = [];
|
|
85
117
|
for (const kind of wanted) {
|
|
@@ -90,12 +122,19 @@ export async function createBackendFromChoice(choice = {}) {
|
|
|
90
122
|
}
|
|
91
123
|
|
|
92
124
|
const [kind] = wanted;
|
|
93
|
-
const gateways = choice.gateways ? { gateways: choice.gateways } : {};
|
|
94
125
|
const extra = choice.options ?? {};
|
|
126
|
+
const gateway =
|
|
127
|
+
typeof choice.gateway === "string" ? choice.gateway : choice.gateway?.[kind];
|
|
128
|
+
const gateways = choice.gateways ? { gateways: choice.gateways } : {};
|
|
129
|
+
// Pinata and Lighthouse take one gateway; Aleph takes the list it tries in
|
|
130
|
+
// order, so a single choice becomes a list of one.
|
|
131
|
+
const oneGateway = gateway ? { gateway } : {};
|
|
95
132
|
|
|
96
133
|
if (kind === "aleph") {
|
|
97
134
|
const { createAlephBackend } = await import("./aleph.js");
|
|
98
|
-
|
|
135
|
+
const fromOne =
|
|
136
|
+
gateway && !choice.gateways ? { gateways: [alephGateway(gateway)] } : {};
|
|
137
|
+
return createAlephBackend({ ...gateways, ...fromOne, ...extra });
|
|
99
138
|
}
|
|
100
139
|
|
|
101
140
|
if (kind === "pinata") {
|
|
@@ -114,6 +153,7 @@ export async function createBackendFromChoice(choice = {}) {
|
|
|
114
153
|
return createPinataBackend({
|
|
115
154
|
...(choice.jwt ? { jwt: choice.jwt } : {}),
|
|
116
155
|
...(choice.getUploadUrl ? { getUploadUrl: choice.getUploadUrl } : {}),
|
|
156
|
+
...oneGateway,
|
|
117
157
|
...gateways,
|
|
118
158
|
...extra,
|
|
119
159
|
});
|
|
@@ -128,6 +168,7 @@ export async function createBackendFromChoice(choice = {}) {
|
|
|
128
168
|
...(choice.apiKey ? { apiKey: choice.apiKey } : {}),
|
|
129
169
|
// Whose key it is decides browserSafeAuth, and a page should say which it got.
|
|
130
170
|
keyOwnership: choice.keyOwnership ?? "shared",
|
|
171
|
+
...oneGateway,
|
|
131
172
|
...gateways,
|
|
132
173
|
...extra,
|
|
133
174
|
});
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Encrypting what goes to a storage service, and decrypting what
|
|
3
|
+
* comes back.
|
|
4
|
+
*
|
|
5
|
+
* A backup leaves the browser as two opaque blobs — a CAR and a small metadata
|
|
6
|
+
* JSON — and the services that hold them neither need nor should have their
|
|
7
|
+
* contents. This wraps a backend so `putBlob` encrypts and `getBlob` decrypts,
|
|
8
|
+
* and **nothing above it changes**: `dehydrate`, `restoreFromCID`, the mirror,
|
|
9
|
+
* the gateway path and the peer path all move opaque bytes either way.
|
|
10
|
+
*
|
|
11
|
+
* ## The keys are the caller's
|
|
12
|
+
*
|
|
13
|
+
* This package knows nothing about passkeys, and should not: it takes two
|
|
14
|
+
* functions, exactly as `createBackendFromChoice` takes `normaliseAddress`
|
|
15
|
+
* rather than importing viem. A browser consumer derives them from the
|
|
16
|
+
* security key's PRF output — `@le-space/orbitdb-identity-provider-webauthn-did`
|
|
17
|
+
* ships `getPrfOutput`, `encryptWithAESGCM` and `decryptWithAESGCM` — and
|
|
18
|
+
* should derive a **separate** key for this, from the same secret with a
|
|
19
|
+
* different info string, so that a compromised backup key is not a signing key.
|
|
20
|
+
*
|
|
21
|
+
* ## The envelope, and why it is not just ciphertext
|
|
22
|
+
*
|
|
23
|
+
* Bytes on the way back can be one of three things: this envelope, a plaintext
|
|
24
|
+
* CAR from before backups were encrypted, or a plaintext CAR written with
|
|
25
|
+
* `dontEncrypt`. A restore has to tell them apart without being told, because
|
|
26
|
+
* a pointer does not carry that knowledge. So every ciphertext starts with a
|
|
27
|
+
* magic number and a version:
|
|
28
|
+
*
|
|
29
|
+
* ```
|
|
30
|
+
* "OSBE" | version | ivLength | iv | ciphertext
|
|
31
|
+
* 4 1 1 ~12 …
|
|
32
|
+
* ```
|
|
33
|
+
*
|
|
34
|
+
* A CAR begins with a varint length and a dag-cbor header, and JSON with `{`,
|
|
35
|
+
* so neither collides with the magic. The version is there for the day the
|
|
36
|
+
* algorithm changes: a backup from before it can then be refused with a
|
|
37
|
+
* reason rather than decrypted into noise.
|
|
38
|
+
*
|
|
39
|
+
* @module backends/encryption
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
import { defineBackend, BackendError } from "./types.js";
|
|
43
|
+
|
|
44
|
+
/** `OSBE` — orbitdb-storage-bridge, encrypted. */
|
|
45
|
+
// Not frozen: freezing a typed array throws, because its elements live in a
|
|
46
|
+
// buffer the engine will not seal.
|
|
47
|
+
export const ENVELOPE_MAGIC = new Uint8Array([0x4f, 0x53, 0x42, 0x45]);
|
|
48
|
+
|
|
49
|
+
/** Bumped when the envelope or the algorithm changes in a way readers must notice. */
|
|
50
|
+
export const ENVELOPE_VERSION = 1;
|
|
51
|
+
|
|
52
|
+
const HEADER_LENGTH = ENVELOPE_MAGIC.length + 2;
|
|
53
|
+
|
|
54
|
+
/** Do these bytes start with an envelope this module wrote? */
|
|
55
|
+
export function isEncrypted(bytes) {
|
|
56
|
+
if (!(bytes instanceof Uint8Array) || bytes.length < HEADER_LENGTH) return false;
|
|
57
|
+
return ENVELOPE_MAGIC.every((byte, index) => bytes[index] === byte);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Wrap ciphertext and its IV so a reader can recognise both without being told.
|
|
62
|
+
*
|
|
63
|
+
* @param {Uint8Array} ciphertext
|
|
64
|
+
* @param {Uint8Array} iv
|
|
65
|
+
* @returns {Uint8Array}
|
|
66
|
+
*/
|
|
67
|
+
export function wrapEnvelope(ciphertext, iv) {
|
|
68
|
+
if (!(ciphertext instanceof Uint8Array) || !(iv instanceof Uint8Array)) {
|
|
69
|
+
throw new BackendError("INVALID_BACKEND", "encrypt must return Uint8Array ciphertext and iv");
|
|
70
|
+
}
|
|
71
|
+
if (iv.length > 255) {
|
|
72
|
+
throw new BackendError("INVALID_BACKEND", `An IV of ${iv.length} bytes does not fit the envelope`);
|
|
73
|
+
}
|
|
74
|
+
const out = new Uint8Array(HEADER_LENGTH + iv.length + ciphertext.length);
|
|
75
|
+
out.set(ENVELOPE_MAGIC, 0);
|
|
76
|
+
out[ENVELOPE_MAGIC.length] = ENVELOPE_VERSION;
|
|
77
|
+
out[ENVELOPE_MAGIC.length + 1] = iv.length;
|
|
78
|
+
out.set(iv, HEADER_LENGTH);
|
|
79
|
+
out.set(ciphertext, HEADER_LENGTH + iv.length);
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Read an envelope back, refusing a version this build does not know.
|
|
85
|
+
*
|
|
86
|
+
* @param {Uint8Array} bytes
|
|
87
|
+
* @returns {{ ciphertext: Uint8Array, iv: Uint8Array, version: number }}
|
|
88
|
+
*/
|
|
89
|
+
export function readEnvelope(bytes) {
|
|
90
|
+
if (!isEncrypted(bytes)) {
|
|
91
|
+
throw new BackendError("INVALID_BACKEND", "These bytes are not an encrypted backup");
|
|
92
|
+
}
|
|
93
|
+
const version = bytes[ENVELOPE_MAGIC.length];
|
|
94
|
+
if (version !== ENVELOPE_VERSION) {
|
|
95
|
+
throw new BackendError(
|
|
96
|
+
"INVALID_BACKEND",
|
|
97
|
+
`This backup was written with envelope version ${version}, and this build reads ${ENVELOPE_VERSION}. ` +
|
|
98
|
+
"Upgrade @le-space/orbitdb-storage-bridge to restore it.",
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
const ivLength = bytes[ENVELOPE_MAGIC.length + 1];
|
|
102
|
+
const iv = bytes.subarray(HEADER_LENGTH, HEADER_LENGTH + ivLength);
|
|
103
|
+
const ciphertext = bytes.subarray(HEADER_LENGTH + ivLength);
|
|
104
|
+
return { ciphertext, iv, version };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Encrypt on the way out, decrypt on the way back.
|
|
109
|
+
*
|
|
110
|
+
* @param {import("./types.js").StorageBackend} backend - the one that stores bytes
|
|
111
|
+
* @param {object} options
|
|
112
|
+
* @param {(plaintext: Uint8Array) => Promise<{ciphertext: Uint8Array, iv: Uint8Array}>} options.encrypt
|
|
113
|
+
* @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} options.decrypt
|
|
114
|
+
* @param {boolean} [options.allowPlaintextReads=true] - restore a backup written
|
|
115
|
+
* before backups were encrypted, or written with `dontEncrypt`. On by
|
|
116
|
+
* default: a pointer does not say which kind it names, and refusing would
|
|
117
|
+
* make yesterday's backups unreadable for no gain in secrecy.
|
|
118
|
+
* @returns {import("./types.js").StorageBackend}
|
|
119
|
+
*/
|
|
120
|
+
export function withEncryption(backend, { encrypt, decrypt, allowPlaintextReads = true } = {}) {
|
|
121
|
+
if (!backend || typeof backend.putBlob !== "function") {
|
|
122
|
+
throw new BackendError("INVALID_BACKEND", "withEncryption needs a backend to wrap");
|
|
123
|
+
}
|
|
124
|
+
if (typeof encrypt !== "function" || typeof decrypt !== "function") {
|
|
125
|
+
throw new BackendError(
|
|
126
|
+
"INVALID_BACKEND",
|
|
127
|
+
"withEncryption needs encrypt and decrypt functions — this package holds no keys of its own",
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const inner = backend;
|
|
132
|
+
|
|
133
|
+
return defineBackend({
|
|
134
|
+
...inner,
|
|
135
|
+
name: `encrypted(${inner.name})`,
|
|
136
|
+
capabilities: {
|
|
137
|
+
...inner.capabilities,
|
|
138
|
+
// The stored bytes are an envelope, not a CAR: nothing downstream may
|
|
139
|
+
// treat them as one, and there are no inner CIDs to preserve.
|
|
140
|
+
carImport: false,
|
|
141
|
+
preservesInnerCids: false,
|
|
142
|
+
},
|
|
143
|
+
|
|
144
|
+
async putBlob(bytes, meta) {
|
|
145
|
+
const { ciphertext, iv } = (await encrypt(bytes)) ?? {};
|
|
146
|
+
return inner.putBlob(wrapEnvelope(ciphertext, iv), meta);
|
|
147
|
+
},
|
|
148
|
+
|
|
149
|
+
async getBlob(handle) {
|
|
150
|
+
const bytes = await inner.getBlob(handle);
|
|
151
|
+
if (!isEncrypted(bytes)) {
|
|
152
|
+
if (allowPlaintextReads) return bytes;
|
|
153
|
+
throw new BackendError(
|
|
154
|
+
"INVALID_BACKEND",
|
|
155
|
+
"This backup is not encrypted, and allowPlaintextReads is off",
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
const { ciphertext, iv } = readEnvelope(bytes);
|
|
159
|
+
try {
|
|
160
|
+
return await decrypt(ciphertext, iv);
|
|
161
|
+
} catch (error) {
|
|
162
|
+
// The common cause by far is the wrong key — a different security key,
|
|
163
|
+
// or a key derived with a different info string. Say that, rather than
|
|
164
|
+
// letting an unreadable CAR surface three layers down.
|
|
165
|
+
throw new BackendError(
|
|
166
|
+
"INVALID_BACKEND",
|
|
167
|
+
`Could not decrypt this backup: ${error.message}. ` +
|
|
168
|
+
"It was written with a different key, or by a different derivation.",
|
|
169
|
+
{ cause: error },
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
},
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Decrypt on the way back, for the path a restore actually takes.
|
|
178
|
+
*
|
|
179
|
+
* `restoreFromCID` does not read through a backend: it takes `fetchBytes(cid)`
|
|
180
|
+
* and gets the bytes from a gateway or from peers, so `withEncryption`'s
|
|
181
|
+
* `getBlob` never runs during a restore. This is the other half — wrap the
|
|
182
|
+
* fetcher, and a backup written through an encrypting backend comes back
|
|
183
|
+
* readable however it was fetched.
|
|
184
|
+
*
|
|
185
|
+
* @param {(cid: string, options?: object) => Promise<Uint8Array>} fetchBytes
|
|
186
|
+
* @param {object} options
|
|
187
|
+
* @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} options.decrypt
|
|
188
|
+
* @param {boolean} [options.allowPlaintextReads=true]
|
|
189
|
+
* @returns {(cid: string, options?: object) => Promise<Uint8Array>}
|
|
190
|
+
*/
|
|
191
|
+
export function decryptingFetch(fetchBytes, { decrypt, allowPlaintextReads = true } = {}) {
|
|
192
|
+
if (typeof fetchBytes !== "function") {
|
|
193
|
+
throw new BackendError("INVALID_BACKEND", "decryptingFetch needs a fetcher to wrap");
|
|
194
|
+
}
|
|
195
|
+
if (typeof decrypt !== "function") {
|
|
196
|
+
throw new BackendError("INVALID_BACKEND", "decryptingFetch needs a decrypt function");
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
return async function fetchAndDecrypt(cid, options) {
|
|
200
|
+
const bytes = await fetchBytes(cid, options);
|
|
201
|
+
if (!isEncrypted(bytes)) {
|
|
202
|
+
if (allowPlaintextReads) return bytes;
|
|
203
|
+
throw new BackendError(
|
|
204
|
+
"INVALID_BACKEND",
|
|
205
|
+
`The backup at ${cid} is not encrypted, and allowPlaintextReads is off`,
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
const { ciphertext, iv } = readEnvelope(bytes);
|
|
209
|
+
try {
|
|
210
|
+
return await decrypt(ciphertext, iv);
|
|
211
|
+
} catch (error) {
|
|
212
|
+
throw new BackendError(
|
|
213
|
+
"INVALID_BACKEND",
|
|
214
|
+
`Could not decrypt the backup at ${cid}: ${error.message}. ` +
|
|
215
|
+
"It was written with a different key, or by a different derivation.",
|
|
216
|
+
{ cause: error },
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Recognise an encrypted backup when there is no key to open it.
|
|
224
|
+
*
|
|
225
|
+
* Without this, a restore of an encrypted backup without `decrypt` fails deep
|
|
226
|
+
* inside the CAR reader — "unexpected end of data", or a block that will not
|
|
227
|
+
* verify — and the reason has nothing to do with what actually happened.
|
|
228
|
+
*
|
|
229
|
+
* @param {(cid: string, options?: object) => Promise<Uint8Array>} fetchBytes
|
|
230
|
+
* @returns {(cid: string, options?: object) => Promise<Uint8Array>}
|
|
231
|
+
*/
|
|
232
|
+
export function explainIfEncrypted(fetchBytes) {
|
|
233
|
+
return async function fetchAndCheck(cid, options) {
|
|
234
|
+
const bytes = await fetchBytes(cid, options);
|
|
235
|
+
if (isEncrypted(bytes)) {
|
|
236
|
+
throw new BackendError(
|
|
237
|
+
"INVALID_BACKEND",
|
|
238
|
+
`The backup at ${cid} is encrypted, and no way to decrypt it was given. ` +
|
|
239
|
+
"Pass `decrypt` to hydrate() — a browser derives it from the security key.",
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
return bytes;
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export default withEncryption;
|
package/lib/backends/storacha.js
CHANGED
|
@@ -7,9 +7,10 @@
|
|
|
7
7
|
* credentials against a self-hosted w3up deployment can still use it, and it is the
|
|
8
8
|
* reference for what a UCAN-delegated backend looked like when one existed.
|
|
9
9
|
*
|
|
10
|
-
* Retrieval
|
|
11
|
-
* answer 301 to `dweb.link
|
|
12
|
-
*
|
|
10
|
+
* Retrieval defaults to a gateway belonging to nobody in this story: `storacha.link`
|
|
11
|
+
* and `w3s.link` answer 301 to `dweb.link`, which Protocol Labs retired on 2026-09-21
|
|
12
|
+
* along with `ipfs.io`. A caller passes its own list — which is what the in-memory
|
|
13
|
+
* service does.
|
|
13
14
|
*
|
|
14
15
|
* @author @NiKrause
|
|
15
16
|
* @requires ./types.js - the backend contract
|
|
@@ -23,11 +24,15 @@ import * as Proof from "@storacha/client/proof";
|
|
|
23
24
|
import { CID } from "multiformats/cid";
|
|
24
25
|
import { defineBackend, handleId, BackendError } from "./types.js";
|
|
25
26
|
|
|
26
|
-
/**
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
27
|
+
/**
|
|
28
|
+
* Gateways tried in order by `getBlob()`, unless the caller supplies its own.
|
|
29
|
+
*
|
|
30
|
+
* Storacha's own hosts redirected here, and here was retired on 2026-09-21
|
|
31
|
+
* together with `ipfs.io`. What is left is a gateway belonging to a different
|
|
32
|
+
* service, which will serve a Storacha CID only while somebody still provides
|
|
33
|
+
* those blocks to the network.
|
|
34
|
+
*/
|
|
35
|
+
export const DEFAULT_GATEWAYS = Object.freeze(["https://ipfs.aleph.cloud"]);
|
|
31
36
|
|
|
32
37
|
/**
|
|
33
38
|
* Build a Storacha client from key and proof, or adopt one that already exists.
|
package/lib/backup-car.js
CHANGED
|
@@ -33,13 +33,14 @@ const spaceHelpers = () => import("./orbitdb-storacha-bridge.js");
|
|
|
33
33
|
import { unixfs } from "@helia/unixfs";
|
|
34
34
|
import logger from "./logger.js";
|
|
35
35
|
import { readBlockBytes } from "./block-bytes.js";
|
|
36
|
+
import { DEFAULT_GATEWAYS, MAX_BACKOFF_MS } from "./gateway-fetch.js";
|
|
36
37
|
|
|
37
38
|
/**
|
|
38
39
|
* Default configuration options
|
|
39
40
|
*/
|
|
40
41
|
const DEFAULT_OPTIONS = {
|
|
41
42
|
timeout: 30000,
|
|
42
|
-
gateway: "https://w3s.link
|
|
43
|
+
gateway: "https://ipfs.aleph.cloud", // w3s.link redirects to a gateway retired on 2026-09-21
|
|
43
44
|
verbose: false,
|
|
44
45
|
// Network download options
|
|
45
46
|
useIPFSNetwork: true, // Enable network downloads by default
|
|
@@ -812,12 +813,11 @@ export async function restoreFromSpaceCAR(orbitdb, options = {}) {
|
|
|
812
813
|
|
|
813
814
|
// Fallback to gateway if network failed or disabled
|
|
814
815
|
if (!carBytes) {
|
|
815
|
-
//
|
|
816
|
+
// The configured gateway, then whatever the package still trusts.
|
|
817
|
+
// `storacha.link`, `dweb.link` and `ipfs.io` were all retired on
|
|
818
|
+
// 2026-09-21 and stood here until then.
|
|
816
819
|
const gateways = [
|
|
817
|
-
`${config.gateway}/ipfs`,
|
|
818
|
-
"https://storacha.link/ipfs",
|
|
819
|
-
"https://dweb.link/ipfs",
|
|
820
|
-
"https://ipfs.io/ipfs",
|
|
820
|
+
...new Set([`${config.gateway}/ipfs`, ...DEFAULT_GATEWAYS]),
|
|
821
821
|
];
|
|
822
822
|
|
|
823
823
|
let carResponse;
|
|
@@ -943,6 +943,13 @@ export async function restoreFromSpaceCAR(orbitdb, options = {}) {
|
|
|
943
943
|
}
|
|
944
944
|
}
|
|
945
945
|
|
|
946
|
+
if (waitTime > MAX_BACKOFF_MS) {
|
|
947
|
+
logger.warn(
|
|
948
|
+
` ⚠️ ${gateway} wants ${Math.round(waitTime / 1000)}s — treating it as closed`,
|
|
949
|
+
);
|
|
950
|
+
break; // try the next gateway rather than wait out a retirement
|
|
951
|
+
}
|
|
952
|
+
|
|
946
953
|
logger.warn(
|
|
947
954
|
` Waiting ${Math.round(waitTime / 1000)}s before retry (attempt ${attempts + 1}/${maxAttempts})...`,
|
|
948
955
|
);
|
package/lib/dehydrate.js
CHANGED
|
@@ -61,15 +61,33 @@ export async function dehydrate({
|
|
|
61
61
|
seed,
|
|
62
62
|
label,
|
|
63
63
|
backend,
|
|
64
|
+
encrypt,
|
|
65
|
+
decrypt,
|
|
66
|
+
dontEncrypt = false,
|
|
64
67
|
endpoints = DEFAULT_ENDPOINTS,
|
|
65
68
|
sequence,
|
|
66
69
|
backup = {},
|
|
67
70
|
}) {
|
|
68
71
|
if (!address) throw new Error("dehydrate needs the database address");
|
|
72
|
+
|
|
73
|
+
// Encrypted by default. A caller who wants the backup readable by anyone
|
|
74
|
+
// holding the CID has to write that down, because it is a decision rather
|
|
75
|
+
// than an oversight — and an oversight is what this refusal exists to catch.
|
|
76
|
+
if (!encrypt && !dontEncrypt) {
|
|
77
|
+
throw new Error(
|
|
78
|
+
"Backups are encrypted by default. Pass `encrypt` and `decrypt` — a browser " +
|
|
79
|
+
"derives them from the security key — or `dontEncrypt: true` if this backup " +
|
|
80
|
+
"is meant to be readable by anyone holding the CID.",
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
|
|
69
84
|
const { backupDatabaseCAR } = await import("./backup-car.js");
|
|
85
|
+
const storeIn = encrypt
|
|
86
|
+
? (await import("./backends/encryption.js")).withEncryption(backend, { encrypt, decrypt })
|
|
87
|
+
: backend;
|
|
70
88
|
|
|
71
89
|
const result = await backupDatabaseCAR(orbitdb, address, {
|
|
72
|
-
backend,
|
|
90
|
+
backend: storeIn,
|
|
73
91
|
...backup,
|
|
74
92
|
});
|
|
75
93
|
if (!result?.success) {
|
|
@@ -118,6 +136,7 @@ export async function hydrate({
|
|
|
118
136
|
orbitdb,
|
|
119
137
|
seed,
|
|
120
138
|
label,
|
|
139
|
+
decrypt,
|
|
121
140
|
endpoints = DEFAULT_ENDPOINTS,
|
|
122
141
|
open = {},
|
|
123
142
|
restore = {},
|
|
@@ -126,10 +145,22 @@ export async function hydrate({
|
|
|
126
145
|
const pointer = await resolvePointer({ privateKey, endpoints });
|
|
127
146
|
|
|
128
147
|
const { restoreFromCID } = await import("./restore-cid.js");
|
|
148
|
+
const { decryptingFetch, explainIfEncrypted } = await import("./backends/encryption.js");
|
|
149
|
+
const { fetchFromGateways } = await import("./gateway-fetch.js");
|
|
150
|
+
|
|
151
|
+
// A restore does not read through a backend — it fetches from a gateway or
|
|
152
|
+
// from peers — so decryption belongs here, around the fetcher. Without a
|
|
153
|
+
// key, an encrypted backup is still recognised, and says so.
|
|
154
|
+
const fetchBytes = restore.fetchBytes ?? fetchFromGateways;
|
|
155
|
+
const reader = decrypt
|
|
156
|
+
? decryptingFetch(fetchBytes, { decrypt })
|
|
157
|
+
: explainIfEncrypted(fetchBytes);
|
|
158
|
+
|
|
129
159
|
const result = await restoreFromCID(orbitdb, {
|
|
130
160
|
metadataCID: pointer.cid,
|
|
131
161
|
open,
|
|
132
162
|
...restore,
|
|
163
|
+
fetchBytes: reader,
|
|
133
164
|
});
|
|
134
165
|
|
|
135
166
|
logger.info(
|
package/lib/gateway-fetch.js
CHANGED
|
@@ -12,13 +12,39 @@
|
|
|
12
12
|
* @module gateway-fetch
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
/**
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
15
|
+
/**
|
|
16
|
+
* Tried in order.
|
|
17
|
+
*
|
|
18
|
+
* This used to be four entries and is now one, because on 2026-09-21 Protocol
|
|
19
|
+
* Labs retired `ipfs.io` and `dweb.link` — they answer 429 with an RFC 8594
|
|
20
|
+
* `Sunset` header — and `w3s.link` and `storacha.link` redirect to `dweb.link`.
|
|
21
|
+
* All four were dead at once, which is what a list of gateways run by one
|
|
22
|
+
* organisation is worth.
|
|
23
|
+
*
|
|
24
|
+
* Measured 2026-09-23: `ipfs.aleph.cloud` answers in 0.2 s with CORS, and no
|
|
25
|
+
* other free path gateway found would serve an arbitrary CID —
|
|
26
|
+
* `gateway.pinata.cloud` answers 403 for content it does not hold,
|
|
27
|
+
* `4everland.io` redirects after 15 s, the rest do not answer at all.
|
|
28
|
+
*
|
|
29
|
+
* One entry is not a fallback chain, and pretending otherwise is how this
|
|
30
|
+
* rotted unnoticed. The replacement is not a longer list: it is fetching from
|
|
31
|
+
* the providers that hold the blocks, over libp2p — see issue #112.
|
|
32
|
+
*/
|
|
33
|
+
export const DEFAULT_GATEWAYS = ["https://ipfs.aleph.cloud/ipfs"];
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Gateways that serve **verifiable** responses only: a single block, or a CAR.
|
|
37
|
+
*
|
|
38
|
+
* `trustless-gateway.link` was still healthy when the path gateways were
|
|
39
|
+
* retired, and it answers with CORS — but it refuses an ordinary path request
|
|
40
|
+
* with a 406, so it is only usable with `accept` set, and only for a CID whose
|
|
41
|
+
* bytes are one block. A UnixFS file split into chunks needs its DAG walked,
|
|
42
|
+
* which this module deliberately cannot do.
|
|
43
|
+
*/
|
|
44
|
+
export const TRUSTLESS_GATEWAYS = ["https://trustless-gateway.link/ipfs"];
|
|
45
|
+
|
|
46
|
+
/** `accept` for a single block from a trustless gateway. */
|
|
47
|
+
export const RAW_BLOCK = "application/vnd.ipld.raw";
|
|
22
48
|
|
|
23
49
|
/**
|
|
24
50
|
* Quiet by default. A restore reports itself through its return value and its
|
|
@@ -30,6 +56,15 @@ export const SILENT = { info() {}, warn() {}, debug() {} };
|
|
|
30
56
|
const DEFAULT_TIMEOUT_MS = 60_000;
|
|
31
57
|
const MAX_ATTEMPTS_PER_GATEWAY = 3;
|
|
32
58
|
|
|
59
|
+
/**
|
|
60
|
+
* Longest 429 wait worth honouring.
|
|
61
|
+
*
|
|
62
|
+
* A retired gateway asks for 900 s, and three attempts against four of them is
|
|
63
|
+
* most of an afternoon spent waiting for an answer that will not change. Past
|
|
64
|
+
* this, the gateway is not rate-limiting us, it is closed.
|
|
65
|
+
*/
|
|
66
|
+
export const MAX_BACKOFF_MS = 30_000;
|
|
67
|
+
|
|
33
68
|
/**
|
|
34
69
|
* A gateway that cannot serve a CID usually says so in HTML, with a 200.
|
|
35
70
|
*
|
|
@@ -73,9 +108,14 @@ const backoffFor = (response, attempt) => {
|
|
|
73
108
|
* @param {string[]} [options.gateways]
|
|
74
109
|
* @param {number} [options.timeout] per request, in ms
|
|
75
110
|
* @param {AbortSignal} [options.signal]
|
|
111
|
+
* @param {string} [options.accept] sent as `Accept`; `RAW_BLOCK` for a single
|
|
112
|
+
* block from a trustless gateway
|
|
113
|
+
* @param {number} [options.maxBackoff] longest 429 wait to honour, in ms
|
|
114
|
+
* @param {(ms: number) => Promise<void>} [options.sleep] injectable for tests
|
|
115
|
+
* @param {typeof fetch} [options.fetchImpl] injectable for tests
|
|
76
116
|
* @returns {Promise<Uint8Array>}
|
|
77
117
|
*/
|
|
78
|
-
export async function fetchFromGateways(cid, { gateways = DEFAULT_GATEWAYS, timeout = DEFAULT_TIMEOUT_MS, signal = null, log = SILENT } = {}) {
|
|
118
|
+
export async function fetchFromGateways(cid, { gateways = DEFAULT_GATEWAYS, timeout = DEFAULT_TIMEOUT_MS, signal = null, log = SILENT, accept = null, maxBackoff = MAX_BACKOFF_MS, sleep = waitFor, fetchImpl = fetch } = {}) {
|
|
79
119
|
let lastError = null;
|
|
80
120
|
|
|
81
121
|
for (const gateway of gateways) {
|
|
@@ -83,12 +123,27 @@ export async function fetchFromGateways(cid, { gateways = DEFAULT_GATEWAYS, time
|
|
|
83
123
|
for (let attempt = 0; attempt < MAX_ATTEMPTS_PER_GATEWAY; attempt++) {
|
|
84
124
|
const timer = AbortSignal.timeout ? AbortSignal.timeout(timeout) : null;
|
|
85
125
|
try {
|
|
86
|
-
const response = await
|
|
126
|
+
const response = await fetchImpl(url, {
|
|
127
|
+
signal: signal ?? timer ?? undefined,
|
|
128
|
+
...(accept ? { headers: { Accept: accept } } : {}),
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
// An RFC 8594 Sunset header means the gateway is going away or already
|
|
132
|
+
// has. Retrying is pointless, and so is coming back next time.
|
|
133
|
+
const sunset = response.headers?.get?.("Sunset");
|
|
134
|
+
if (sunset && !response.ok) {
|
|
135
|
+
log.warn(` ⚠️ ${gateway} is retired (Sunset: ${sunset})`);
|
|
136
|
+
break;
|
|
137
|
+
}
|
|
87
138
|
|
|
88
139
|
if (response.status === 429 && attempt < MAX_ATTEMPTS_PER_GATEWAY - 1) {
|
|
89
140
|
const wait = backoffFor(response, attempt);
|
|
141
|
+
if (wait > maxBackoff) {
|
|
142
|
+
log.warn(` ⚠️ ${gateway} wants ${Math.round(wait / 1000)}s — treating it as closed`);
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
90
145
|
log.warn(` ⚠️ ${gateway} rate-limited; waiting ${Math.round(wait / 1000)}s`);
|
|
91
|
-
await
|
|
146
|
+
await sleep(wait);
|
|
92
147
|
continue;
|
|
93
148
|
}
|
|
94
149
|
if (!response.ok) {
|
|
@@ -17,6 +17,7 @@ import { sha256 } from "multiformats/hashes/sha2";
|
|
|
17
17
|
import { bases } from "multiformats/basics";
|
|
18
18
|
import { EventEmitter } from "events";
|
|
19
19
|
import { createHeliaOrbitDB, cleanupOrbitDBDirectories } from "./utils.js";
|
|
20
|
+
import { DEFAULT_GATEWAYS } from "./gateway-fetch.js";
|
|
20
21
|
import {
|
|
21
22
|
generateBackupPrefix,
|
|
22
23
|
getBackupFilenames,
|
|
@@ -47,7 +48,7 @@ function logHeliaActivity(message) {
|
|
|
47
48
|
const DEFAULT_OPTIONS = {
|
|
48
49
|
// Core configuration
|
|
49
50
|
timeout: 30000, // Timeout in milliseconds
|
|
50
|
-
gateway: "https://
|
|
51
|
+
gateway: "https://ipfs.aleph.cloud", // dweb.link was retired on 2026-09-21
|
|
51
52
|
verbose: false, // Enable verbose debug logging
|
|
52
53
|
|
|
53
54
|
// Network download options
|
|
@@ -823,14 +824,12 @@ export async function downloadBlock(cid, options = {}) {
|
|
|
823
824
|
}
|
|
824
825
|
}
|
|
825
826
|
|
|
826
|
-
// Fallback to gateway download
|
|
827
|
-
//
|
|
828
|
-
//
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
"https://ipfs.io/ipfs",
|
|
833
|
-
];
|
|
827
|
+
// Fallback to gateway download.
|
|
828
|
+
// `dweb.link` and `ipfs.io` stood here until Protocol Labs retired them on
|
|
829
|
+
// 2026-09-21; `storacha.link` and `w3s.link` had already become redirects to
|
|
830
|
+
// the first of those. What is left is one host, which is not a fallback
|
|
831
|
+
// chain — see issue #112 for the path that does not depend on one.
|
|
832
|
+
const gateways = [...new Set([`${config.gateway}/ipfs`, ...DEFAULT_GATEWAYS])];
|
|
834
833
|
|
|
835
834
|
for (const gateway of gateways) {
|
|
836
835
|
try {
|
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Fetching bytes by CID from peers, when a gateway will not do.
|
|
3
|
+
*
|
|
4
|
+
* `gateway-fetch.js` is the HTTP half of this: ask a host, hope it answers.
|
|
5
|
+
* That stopped being a plan on 2026-09-21, when the public path gateways were
|
|
6
|
+
* retired (#111) and every fallback list in this package turned out to be one
|
|
7
|
+
* live entry and a dead tail. This is the other half — ask the peers that
|
|
8
|
+
* actually hold the blocks.
|
|
9
|
+
*
|
|
10
|
+
* ## Measured before it was written
|
|
11
|
+
*
|
|
12
|
+
* From a real page (Chrome, no build step, Helia loaded as ES modules), on
|
|
13
|
+
* 2026-09-23:
|
|
14
|
+
*
|
|
15
|
+
* ```
|
|
16
|
+
* Pinata dial 729 ms over wss bitswap 263 ms
|
|
17
|
+
* Lighthouse dial 581 ms over webrtc-direct bitswap 323 ms
|
|
18
|
+
* Aleph dial 450 ms over webrtc-direct bitswap 345 ms
|
|
19
|
+
* ```
|
|
20
|
+
*
|
|
21
|
+
* No credential in any of it, while both paid providers' HTTP gateways want
|
|
22
|
+
* that account's key and Lighthouse's shared gateway answers 402 for content
|
|
23
|
+
* it does not hold. Three services, two transports, about a second each.
|
|
24
|
+
*
|
|
25
|
+
* ## The peer list follows the backends, and nothing else
|
|
26
|
+
*
|
|
27
|
+
* There is no default provider here, and that is deliberate. Whoever stores
|
|
28
|
+
* with a service already shares their CIDs with it, so reading from it adds no
|
|
29
|
+
* new party — but a page that dials Pinata having never uploaded there would
|
|
30
|
+
* be telling Pinata what its reader is looking for, in exchange for nothing.
|
|
31
|
+
* The caller names the providers. {@link PINATA_BITSWAP} is a constant because
|
|
32
|
+
* Pinata publishes it in DNS; everyone else has to be looked up.
|
|
33
|
+
*
|
|
34
|
+
* ## What the caller has to bring
|
|
35
|
+
*
|
|
36
|
+
* A Helia with bitswap **and identify**. This module has no libp2p of its own —
|
|
37
|
+
* the page that restores already has a node, and a second one would be a second
|
|
38
|
+
* identity on the network.
|
|
39
|
+
*
|
|
40
|
+
* Two things about that node are easy to get wrong, and both were:
|
|
41
|
+
*
|
|
42
|
+
* - **`identify` is not optional.** `withLibp2pLight` does not include it, and
|
|
43
|
+
* without it bitswap never learns that the peer it just dialled speaks
|
|
44
|
+
* bitswap: no want is sent, and the fetch fails with "Failed to load block"
|
|
45
|
+
* after the full timeout. Measured against Aleph: 60 s of nothing without
|
|
46
|
+
* identify, 1.6 s for 400 kB with it, over the same dialled connection.
|
|
47
|
+
* - **Helia's browser defaults try to listen** on `/webrtc` and `/p2p-circuit`
|
|
48
|
+
* and **throw on start** when no transport serves them, so a fetch-only node
|
|
49
|
+
* wants `addresses: { listen: [] }`.
|
|
50
|
+
*
|
|
51
|
+
* ```js
|
|
52
|
+
* const helia = await withBitswap(withLibp2pLight(createHeliaLight({ … }), {
|
|
53
|
+
* addresses: { listen: [] },
|
|
54
|
+
* transports: [webSockets(), webRTCDirect()],
|
|
55
|
+
* connectionEncrypters: [noise()],
|
|
56
|
+
* streamMuxers: [yamux()],
|
|
57
|
+
* services: { identify: identify() }, // ← without this, nothing arrives
|
|
58
|
+
* })).start()
|
|
59
|
+
* ```
|
|
60
|
+
*
|
|
61
|
+
* @module peer-fetch
|
|
62
|
+
*/
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Pinata's bitswap endpoint, published in DNS:
|
|
66
|
+
*
|
|
67
|
+
* ```
|
|
68
|
+
* $ dig +short TXT _dnsaddr.bitswap-v3.pinata.cloud
|
|
69
|
+
* "dnsaddr=/dns4/bitswap-v3.pinata.cloud/tcp/443/wss/p2p/Qmdv6yNikmUWUWXufLJLRNkv6Y9sY5cmgeX5RVWA4WNMz4"
|
|
70
|
+
* ```
|
|
71
|
+
*
|
|
72
|
+
* A DNS name on 443 with a CA certificate is dialable from a page, which is
|
|
73
|
+
* what makes it worth naming here rather than looking up per CID.
|
|
74
|
+
*/
|
|
75
|
+
export const PINATA_BITSWAP =
|
|
76
|
+
"/dns4/bitswap-v3.pinata.cloud/tcp/443/wss/p2p/Qmdv6yNikmUWUWXufLJLRNkv6Y9sY5cmgeX5RVWA4WNMz4";
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Aleph's own node, which took some finding.
|
|
80
|
+
*
|
|
81
|
+
* It publishes no `_dnsaddr` and answers `404` to `/api/v0/id`, so looking for
|
|
82
|
+
* it the way Pinata is found says it has no peer — which is what an earlier
|
|
83
|
+
* version of this file claimed. It does: ask a router for the providers of a
|
|
84
|
+
* CID Aleph holds, and the record is `46.255.204.211`, the address
|
|
85
|
+
* `ipfs.aleph.cloud` resolves to, with `webrtc-direct` and `webtransport`
|
|
86
|
+
* among its addresses. A page dialled it in 450 ms and had the block 345 ms
|
|
87
|
+
* later.
|
|
88
|
+
*
|
|
89
|
+
* Two caveats make this a starting point rather than a guarantee:
|
|
90
|
+
*
|
|
91
|
+
* - **The certhash rotates.** A `webrtc-direct` address carries the
|
|
92
|
+
* certificate's hash, and the node generates a new certificate when it
|
|
93
|
+
* restarts. When this address stops working, look the CID up again — the
|
|
94
|
+
* peer id is the stable part.
|
|
95
|
+
* - **Only one router will tell you.** Aleph announces over the DHT rather
|
|
96
|
+
* than IPNI, so `cid.contact` — the one router that answers a browser with
|
|
97
|
+
* CORS — does not know its CIDs, while `delegated-ipfs.dev` does and sends
|
|
98
|
+
* no CORS header for provider lookups. From a page, this constant is the way
|
|
99
|
+
* in; from Node, {@link providersFor} with `delegated-ipfs.dev` is better.
|
|
100
|
+
*/
|
|
101
|
+
export const ALEPH_BITSWAP = Object.freeze([
|
|
102
|
+
"/ip4/46.255.204.211/udp/4001/webrtc-direct/certhash/uEiCbM9yMfnP02vviIL26n8bI0-vU0DGUHx20POBLKGjEmg/p2p/12D3KooWACE5dRw5V9WXuDTcngjE3ZaDSZ4qYJGfuhXZbENnL54y",
|
|
103
|
+
"/ip4/46.255.204.211/udp/4001/quic-v1/webtransport/certhash/uEiDMxOK9kZFH5SW6zcmNpXU4EGgBgvZYqqlJhw1cci50KA/certhash/uEiCeRpuyQWojWx2cP8guNiY3EDfIF25D2k8CMTZd1qtSfw/p2p/12D3KooWACE5dRw5V9WXuDTcngjE3ZaDSZ4qYJGfuhXZbENnL54y",
|
|
104
|
+
]);
|
|
105
|
+
|
|
106
|
+
/** Aleph's peer id, which outlives the certificate hashes above. */
|
|
107
|
+
export const ALEPH_PEER_ID = "12D3KooWACE5dRw5V9WXuDTcngjE3ZaDSZ4qYJGfuhXZbENnL54y";
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Routers that answer a page.
|
|
111
|
+
*
|
|
112
|
+
* Both send `access-control-allow-origin: *` for provider lookups. An earlier
|
|
113
|
+
* version of this file said `delegated-ipfs.dev` did not, from one request that
|
|
114
|
+
* came back without the header; repeated from a deployed page it answers with
|
|
115
|
+
* it, so the restriction was wrong and this list is both.
|
|
116
|
+
*
|
|
117
|
+
* What is true, and matters more, is that they know **different things**:
|
|
118
|
+
* `cid.contact` is an IPNI index and knows what is announced to it — Pinata's
|
|
119
|
+
* CIDs and Lighthouse's were there — while a CID that Aleph holds appeared only
|
|
120
|
+
* at `delegated-ipfs.dev`, because Aleph announces over the DHT. Asking one
|
|
121
|
+
* router is asking half the network.
|
|
122
|
+
*/
|
|
123
|
+
export const DEFAULT_ROUTERS = Object.freeze([
|
|
124
|
+
"https://cid.contact",
|
|
125
|
+
"https://delegated-ipfs.dev",
|
|
126
|
+
]);
|
|
127
|
+
|
|
128
|
+
/** @deprecated Same as {@link DEFAULT_ROUTERS}; kept so an import still works. */
|
|
129
|
+
export const ALL_ROUTERS = DEFAULT_ROUTERS;
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Transports a browser can dial.
|
|
133
|
+
*
|
|
134
|
+
* `tcp` and plain `quic-v1` are not here: a page has no raw sockets. `tls/ws`
|
|
135
|
+
* needs a name rather than an IP, since the certificate has to match — an
|
|
136
|
+
* AutoTLS `libp2p.direct` address qualifies only while that name resolves,
|
|
137
|
+
* which for one provider it did not on the day this was written.
|
|
138
|
+
*/
|
|
139
|
+
export const BROWSER_TRANSPORTS = Object.freeze([
|
|
140
|
+
"/wss",
|
|
141
|
+
"/tls/ws",
|
|
142
|
+
"/webtransport",
|
|
143
|
+
"/webrtc-direct",
|
|
144
|
+
]);
|
|
145
|
+
|
|
146
|
+
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
147
|
+
|
|
148
|
+
/** Quiet by default, like `gateway-fetch.js`. */
|
|
149
|
+
export const SILENT = { info() {}, warn() {}, debug() {} };
|
|
150
|
+
|
|
151
|
+
/** Does this multiaddr use a transport a page can open? */
|
|
152
|
+
export function isBrowserDialable(addr) {
|
|
153
|
+
if (typeof addr !== "string") return false;
|
|
154
|
+
if (!BROWSER_TRANSPORTS.some((transport) => addr.includes(transport))) return false;
|
|
155
|
+
// A certificate cannot be issued for a bare IP, so ws/wss on one is not
|
|
156
|
+
// dialable however well-formed it looks; webtransport and webrtc-direct
|
|
157
|
+
// carry a certhash instead and are fine.
|
|
158
|
+
const needsName = addr.includes("/wss") || addr.includes("/tls/ws");
|
|
159
|
+
return !needsName || addr.includes("/dns");
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Who says they hold this CID, and at what address.
|
|
164
|
+
*
|
|
165
|
+
* @param {string} cid
|
|
166
|
+
* @param {Object} [options]
|
|
167
|
+
* @param {string[]} [options.routers] - delegated routing endpoints
|
|
168
|
+
* @param {boolean} [options.dialableOnly=true] - keep only what a page can open
|
|
169
|
+
* @param {AbortSignal} [options.signal]
|
|
170
|
+
* @param {typeof fetch} [options.fetchImpl]
|
|
171
|
+
* @returns {Promise<Array<{ id: string, addrs: string[] }>>}
|
|
172
|
+
*/
|
|
173
|
+
export async function providersFor(cid, {
|
|
174
|
+
routers = DEFAULT_ROUTERS,
|
|
175
|
+
dialableOnly = true,
|
|
176
|
+
signal = null,
|
|
177
|
+
fetchImpl = fetch,
|
|
178
|
+
log = SILENT,
|
|
179
|
+
} = {}) {
|
|
180
|
+
const found = new Map();
|
|
181
|
+
|
|
182
|
+
for (const router of routers) {
|
|
183
|
+
try {
|
|
184
|
+
const response = await fetchImpl(`${router}/routing/v1/providers/${cid}`, {
|
|
185
|
+
headers: { Accept: "application/json" },
|
|
186
|
+
signal,
|
|
187
|
+
});
|
|
188
|
+
if (!response.ok) {
|
|
189
|
+
log.debug(` ⚠️ ${router} answered ${response.status}`);
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
const body = await response.text();
|
|
193
|
+
for (const record of parseProviderRecords(body)) {
|
|
194
|
+
const id = record.ID ?? record.id;
|
|
195
|
+
if (!id) continue;
|
|
196
|
+
const addrs = (record.Addrs ?? record.addrs ?? []).filter(
|
|
197
|
+
(addr) => !dialableOnly || isBrowserDialable(addr),
|
|
198
|
+
);
|
|
199
|
+
if (addrs.length === 0) continue;
|
|
200
|
+
const already = found.get(id);
|
|
201
|
+
if (already) {
|
|
202
|
+
for (const addr of addrs) if (!already.addrs.includes(addr)) already.addrs.push(addr);
|
|
203
|
+
} else {
|
|
204
|
+
found.set(id, { id, addrs: [...addrs] });
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
} catch (error) {
|
|
208
|
+
log.debug(` ⚠️ ${router} failed: ${error.message}`);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
return [...found.values()];
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* A routing answer is JSON, or one JSON object per line when the router
|
|
217
|
+
* streams — both shapes are in the spec, and both are in the wild.
|
|
218
|
+
*/
|
|
219
|
+
function parseProviderRecords(body) {
|
|
220
|
+
const records = [];
|
|
221
|
+
const push = (value) => {
|
|
222
|
+
if (!value) return;
|
|
223
|
+
if (Array.isArray(value.Providers)) records.push(...value.Providers);
|
|
224
|
+
else if (Array.isArray(value.providers)) records.push(...value.providers);
|
|
225
|
+
else records.push(value);
|
|
226
|
+
};
|
|
227
|
+
try {
|
|
228
|
+
push(JSON.parse(body));
|
|
229
|
+
return records;
|
|
230
|
+
} catch {
|
|
231
|
+
// not one document — try it as a stream of them
|
|
232
|
+
}
|
|
233
|
+
for (const line of body.split("\n")) {
|
|
234
|
+
const trimmed = line.trim();
|
|
235
|
+
if (!trimmed) continue;
|
|
236
|
+
try {
|
|
237
|
+
push(JSON.parse(trimmed));
|
|
238
|
+
} catch {
|
|
239
|
+
// a partial line at the end of a stream; nothing to do with it
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return records;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Build a `fetchBytes(cid)` that goes over libp2p.
|
|
247
|
+
*
|
|
248
|
+
* Fits where `restoreFromCID`'s `fetchBytes` goes, so nothing downstream
|
|
249
|
+
* changes: the blocks are verified against their CIDs exactly as before, which
|
|
250
|
+
* is what makes a stranger's bytes as safe as a gateway's.
|
|
251
|
+
*
|
|
252
|
+
* @param {Object} options
|
|
253
|
+
* @param {Object} options.helia - a started Helia **with bitswap**
|
|
254
|
+
* @param {string[] | ((cid: string) => Promise<string[]>)} options.providers -
|
|
255
|
+
* multiaddrs to dial, or a function that finds them for a CID
|
|
256
|
+
* @param {number} [options.timeout]
|
|
257
|
+
* @param {(addr: string) => Promise<unknown>} [options.dial] - defaults to the node's
|
|
258
|
+
* @param {(cid: any, options?: Object) => AsyncIterable<Uint8Array>} [options.cat] -
|
|
259
|
+
* defaults to `@helia/unixfs`, which reads a chunked file as well as a single block
|
|
260
|
+
* @returns {(cid: string, options?: Object) => Promise<Uint8Array>}
|
|
261
|
+
*/
|
|
262
|
+
export function createPeerFetch({
|
|
263
|
+
helia,
|
|
264
|
+
providers = [],
|
|
265
|
+
timeout = DEFAULT_TIMEOUT_MS,
|
|
266
|
+
dial = null,
|
|
267
|
+
cat = null,
|
|
268
|
+
log = SILENT,
|
|
269
|
+
} = {}) {
|
|
270
|
+
if (!helia && (!dial || !cat)) {
|
|
271
|
+
throw new Error("createPeerFetch needs a Helia node, or both dial and cat");
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
return async function fetchOverPeers(cid, options = {}) {
|
|
275
|
+
const signal = options.signal ?? AbortSignal.timeout(options.timeout ?? timeout);
|
|
276
|
+
const addrs =
|
|
277
|
+
typeof providers === "function" ? await providers(cid) : [...providers];
|
|
278
|
+
|
|
279
|
+
if (addrs.length === 0) {
|
|
280
|
+
throw new Error(`No provider to dial for ${cid}`);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const dialOne = dial ?? (async (addr) => {
|
|
284
|
+
const { multiaddr } = await import("@multiformats/multiaddr");
|
|
285
|
+
return helia.libp2p.dial(multiaddr(addr), { signal });
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
// One connection per peer, not one per address. A provider usually
|
|
289
|
+
// advertises the same peer several times — Aleph offers webrtc-direct and
|
|
290
|
+
// webtransport — and dialling both gets two connections to one node, which
|
|
291
|
+
// buys nothing and makes a peer count read double.
|
|
292
|
+
const reached = new Set();
|
|
293
|
+
for (const addr of addrs) {
|
|
294
|
+
const peer = addr.match(/\/p2p\/([^/]+)/)?.[1] ?? addr;
|
|
295
|
+
if (reached.has(peer)) continue;
|
|
296
|
+
try {
|
|
297
|
+
await dialOne(addr);
|
|
298
|
+
reached.add(peer);
|
|
299
|
+
log.debug(` ✅ dialled ${addr}`);
|
|
300
|
+
} catch (error) {
|
|
301
|
+
log.debug(` ⚠️ could not dial ${addr}: ${error.message}`);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
const dialed = reached.size;
|
|
305
|
+
if (dialed === 0) {
|
|
306
|
+
throw new Error(`Could not dial any provider for ${cid} (tried ${addrs.length})`);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const read = cat ?? (await unixfsCat(helia));
|
|
310
|
+
const parts = [];
|
|
311
|
+
let total = 0;
|
|
312
|
+
for await (const chunk of read(cid, { signal })) {
|
|
313
|
+
parts.push(chunk);
|
|
314
|
+
total += chunk.length;
|
|
315
|
+
}
|
|
316
|
+
const bytes = new Uint8Array(total);
|
|
317
|
+
let at = 0;
|
|
318
|
+
for (const part of parts) {
|
|
319
|
+
bytes.set(part, at);
|
|
320
|
+
at += part.length;
|
|
321
|
+
}
|
|
322
|
+
log.info(` ✅ ${bytes.length} bytes over libp2p from ${dialed} peer(s)`);
|
|
323
|
+
return bytes;
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* `@helia/unixfs` rather than the blockstore: a backup large enough to be
|
|
329
|
+
* chunked is a DAG, and `blockstore.get` would return its root block and call
|
|
330
|
+
* that the file. Imported here so a caller that supplies `cat` never loads it.
|
|
331
|
+
*/
|
|
332
|
+
async function unixfsCat(helia) {
|
|
333
|
+
const [{ unixfs }, { CID }] = await Promise.all([
|
|
334
|
+
import("@helia/unixfs"),
|
|
335
|
+
import("multiformats/cid"),
|
|
336
|
+
]);
|
|
337
|
+
const fs = unixfs(helia);
|
|
338
|
+
return (cid, options) => fs.cat(typeof cid === "string" ? CID.parse(cid) : cid, options);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Try HTTP first, then peers — and say which one delivered.
|
|
343
|
+
*
|
|
344
|
+
* The order is not a preference for HTTP: a warm gateway answered in 0.23 s
|
|
345
|
+
* against 0.86–1.9 s over libp2p, and a phone on a rationed connection should
|
|
346
|
+
* not open a swarm for a file one request would have fetched. But the timeout
|
|
347
|
+
* is short on purpose, because the other measurement from the same day is a
|
|
348
|
+
* gateway taking 29 s for a block a peer served in under one.
|
|
349
|
+
*
|
|
350
|
+
* @param {Object} options
|
|
351
|
+
* @param {(cid: string, options?: Object) => Promise<Uint8Array>} options.viaGateway
|
|
352
|
+
* @param {(cid: string, options?: Object) => Promise<Uint8Array>} options.viaPeers
|
|
353
|
+
* @param {number} [options.gatewayTimeout=3000] - before the peer path starts
|
|
354
|
+
* @param {(path: "gateway"|"peers", info: Object) => void} [options.onPath] -
|
|
355
|
+
* told which path was taken and how long it took, so a page can show it
|
|
356
|
+
* @returns {(cid: string, options?: Object) => Promise<Uint8Array>}
|
|
357
|
+
*/
|
|
358
|
+
export function createGatewayFirstFetch({
|
|
359
|
+
viaGateway,
|
|
360
|
+
viaPeers,
|
|
361
|
+
gatewayTimeout = 3000,
|
|
362
|
+
onPath = () => {},
|
|
363
|
+
now = () => Date.now(),
|
|
364
|
+
} = {}) {
|
|
365
|
+
return async function fetchBytes(cid, options = {}) {
|
|
366
|
+
const started = now();
|
|
367
|
+
try {
|
|
368
|
+
const bytes = await viaGateway(cid, { ...options, timeout: gatewayTimeout });
|
|
369
|
+
onPath("gateway", { cid, ms: now() - started, bytes: bytes.length });
|
|
370
|
+
return bytes;
|
|
371
|
+
} catch (error) {
|
|
372
|
+
const gatewayMs = now() - started;
|
|
373
|
+
const peersStarted = now();
|
|
374
|
+
try {
|
|
375
|
+
const bytes = await viaPeers(cid, options);
|
|
376
|
+
onPath("peers", {
|
|
377
|
+
cid,
|
|
378
|
+
ms: now() - peersStarted,
|
|
379
|
+
bytes: bytes.length,
|
|
380
|
+
after: { path: "gateway", ms: gatewayMs, error: error.message },
|
|
381
|
+
});
|
|
382
|
+
return bytes;
|
|
383
|
+
} catch (peerError) {
|
|
384
|
+
throw new Error(
|
|
385
|
+
`Could not fetch ${cid}: the gateway said "${error.message}" and the peers said "${peerError.message}"`,
|
|
386
|
+
{ cause: peerError },
|
|
387
|
+
);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
}
|
package/lib/ucan-bridge.js
CHANGED
|
@@ -102,7 +102,7 @@ function findCorrectManifest(analysis) {
|
|
|
102
102
|
*/
|
|
103
103
|
const DEFAULT_UCAN_OPTIONS = {
|
|
104
104
|
timeout: 30000,
|
|
105
|
-
gateway: "https://w3s.link
|
|
105
|
+
gateway: "https://ipfs.aleph.cloud", // w3s.link redirects to a gateway retired on 2026-09-21
|
|
106
106
|
batchSize: 10,
|
|
107
107
|
maxConcurrency: 3,
|
|
108
108
|
// UCAN-specific options
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@le-space/orbitdb-storage-bridge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.13.0",
|
|
4
4
|
"description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
|
|
5
5
|
"main": "lib/orbitdb-storacha-bridge.js",
|
|
6
6
|
"svelte": "dist/components/",
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
"./dehydrate": "./lib/dehydrate.js",
|
|
17
17
|
"./restore-cid": "./lib/restore-cid.js",
|
|
18
18
|
"./gateway-fetch": "./lib/gateway-fetch.js",
|
|
19
|
+
"./peer-fetch": "./lib/peer-fetch.js",
|
|
19
20
|
"./backends/types": "./lib/backends/types.js",
|
|
20
21
|
"./backends/memory": "./lib/backends/memory.js",
|
|
21
22
|
"./backends/storacha": "./lib/backends/storacha.js",
|
|
@@ -24,6 +25,7 @@
|
|
|
24
25
|
"./backends/pinata": "./lib/backends/pinata.js",
|
|
25
26
|
"./backends/lighthouse": "./lib/backends/lighthouse.js",
|
|
26
27
|
"./backends/mirror": "./lib/backends/mirror.js",
|
|
28
|
+
"./backends/encryption": "./lib/backends/encryption.js",
|
|
27
29
|
"./backends/choose": "./lib/backends/choose.js",
|
|
28
30
|
"./backends/resolve": "./lib/backends/resolve.js",
|
|
29
31
|
"./memory-courier": "./lib/memory-courier.js",
|
|
@@ -86,7 +88,6 @@
|
|
|
86
88
|
"dependencies": {
|
|
87
89
|
"@chainsafe/libp2p-noise": "^17.0.0",
|
|
88
90
|
"@chainsafe/libp2p-yamux": "^8.0.1",
|
|
89
|
-
"@helia/block-brokers": "^5.2.4",
|
|
90
91
|
"@helia/routers": "^5.1.1",
|
|
91
92
|
"@helia/unixfs": "^8.0.7",
|
|
92
93
|
"@ipld/car": "^5.4.1",
|
|
@@ -108,7 +109,7 @@
|
|
|
108
109
|
"cbor-web": "^10.0.11",
|
|
109
110
|
"datastore-level": "^13.0.1",
|
|
110
111
|
"dotenv": "^17.2.1",
|
|
111
|
-
"helia": "^7.1.
|
|
112
|
+
"helia": "^7.1.15",
|
|
112
113
|
"libp2p": "^3.3.6",
|
|
113
114
|
"multiformats": "^14.0.5"
|
|
114
115
|
},
|