@le-space/orbitdb-storage-bridge 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/backends/encryption.js +246 -0
- package/lib/courier-sync.js +143 -15
- package/lib/dehydrate.js +32 -1
- package/lib/peer-fetch.js +45 -19
- package/package.json +2 -1
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Encrypting what goes to a storage service, and decrypting what
|
|
3
|
+
* comes back.
|
|
4
|
+
*
|
|
5
|
+
* A backup leaves the browser as two opaque blobs — a CAR and a small metadata
|
|
6
|
+
* JSON — and the services that hold them neither need nor should have their
|
|
7
|
+
* contents. This wraps a backend so `putBlob` encrypts and `getBlob` decrypts,
|
|
8
|
+
* and **nothing above it changes**: `dehydrate`, `restoreFromCID`, the mirror,
|
|
9
|
+
* the gateway path and the peer path all move opaque bytes either way.
|
|
10
|
+
*
|
|
11
|
+
* ## The keys are the caller's
|
|
12
|
+
*
|
|
13
|
+
* This package knows nothing about passkeys, and should not: it takes two
|
|
14
|
+
* functions, exactly as `createBackendFromChoice` takes `normaliseAddress`
|
|
15
|
+
* rather than importing viem. A browser consumer derives them from the
|
|
16
|
+
* security key's PRF output — `@le-space/orbitdb-identity-provider-webauthn-did`
|
|
17
|
+
* ships `getPrfOutput`, `encryptWithAESGCM` and `decryptWithAESGCM` — and
|
|
18
|
+
* should derive a **separate** key for this, from the same secret with a
|
|
19
|
+
* different info string, so that a compromised backup key is not a signing key.
|
|
20
|
+
*
|
|
21
|
+
* ## The envelope, and why it is not just ciphertext
|
|
22
|
+
*
|
|
23
|
+
* Bytes on the way back can be one of three things: this envelope, a plaintext
|
|
24
|
+
* CAR from before backups were encrypted, or a plaintext CAR written with
|
|
25
|
+
* `dontEncrypt`. A restore has to tell them apart without being told, because
|
|
26
|
+
* a pointer does not carry that knowledge. So every ciphertext starts with a
|
|
27
|
+
* magic number and a version:
|
|
28
|
+
*
|
|
29
|
+
* ```
|
|
30
|
+
* "OSBE" | version | ivLength | iv | ciphertext
|
|
31
|
+
* 4 1 1 ~12 …
|
|
32
|
+
* ```
|
|
33
|
+
*
|
|
34
|
+
* A CAR begins with a varint length and a dag-cbor header, and JSON with `{`,
|
|
35
|
+
* so neither collides with the magic. The version is there for the day the
|
|
36
|
+
* algorithm changes: a backup from before it can then be refused with a
|
|
37
|
+
* reason rather than decrypted into noise.
|
|
38
|
+
*
|
|
39
|
+
* @module backends/encryption
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
import { defineBackend, BackendError } from "./types.js";
|
|
43
|
+
|
|
44
|
+
/** `OSBE` — orbitdb-storage-bridge, encrypted. */
|
|
45
|
+
// Not frozen: freezing a typed array throws, because its elements live in a
|
|
46
|
+
// buffer the engine will not seal.
|
|
47
|
+
export const ENVELOPE_MAGIC = new Uint8Array([0x4f, 0x53, 0x42, 0x45]);
|
|
48
|
+
|
|
49
|
+
/** Bumped when the envelope or the algorithm changes in a way readers must notice. */
|
|
50
|
+
export const ENVELOPE_VERSION = 1;
|
|
51
|
+
|
|
52
|
+
const HEADER_LENGTH = ENVELOPE_MAGIC.length + 2;
|
|
53
|
+
|
|
54
|
+
/** Do these bytes start with an envelope this module wrote? */
|
|
55
|
+
export function isEncrypted(bytes) {
|
|
56
|
+
if (!(bytes instanceof Uint8Array) || bytes.length < HEADER_LENGTH) return false;
|
|
57
|
+
return ENVELOPE_MAGIC.every((byte, index) => bytes[index] === byte);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Wrap ciphertext and its IV so a reader can recognise both without being told.
|
|
62
|
+
*
|
|
63
|
+
* @param {Uint8Array} ciphertext
|
|
64
|
+
* @param {Uint8Array} iv
|
|
65
|
+
* @returns {Uint8Array}
|
|
66
|
+
*/
|
|
67
|
+
export function wrapEnvelope(ciphertext, iv) {
|
|
68
|
+
if (!(ciphertext instanceof Uint8Array) || !(iv instanceof Uint8Array)) {
|
|
69
|
+
throw new BackendError("INVALID_BACKEND", "encrypt must return Uint8Array ciphertext and iv");
|
|
70
|
+
}
|
|
71
|
+
if (iv.length > 255) {
|
|
72
|
+
throw new BackendError("INVALID_BACKEND", `An IV of ${iv.length} bytes does not fit the envelope`);
|
|
73
|
+
}
|
|
74
|
+
const out = new Uint8Array(HEADER_LENGTH + iv.length + ciphertext.length);
|
|
75
|
+
out.set(ENVELOPE_MAGIC, 0);
|
|
76
|
+
out[ENVELOPE_MAGIC.length] = ENVELOPE_VERSION;
|
|
77
|
+
out[ENVELOPE_MAGIC.length + 1] = iv.length;
|
|
78
|
+
out.set(iv, HEADER_LENGTH);
|
|
79
|
+
out.set(ciphertext, HEADER_LENGTH + iv.length);
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Read an envelope back, refusing a version this build does not know.
|
|
85
|
+
*
|
|
86
|
+
* @param {Uint8Array} bytes
|
|
87
|
+
* @returns {{ ciphertext: Uint8Array, iv: Uint8Array, version: number }}
|
|
88
|
+
*/
|
|
89
|
+
export function readEnvelope(bytes) {
|
|
90
|
+
if (!isEncrypted(bytes)) {
|
|
91
|
+
throw new BackendError("INVALID_BACKEND", "These bytes are not an encrypted backup");
|
|
92
|
+
}
|
|
93
|
+
const version = bytes[ENVELOPE_MAGIC.length];
|
|
94
|
+
if (version !== ENVELOPE_VERSION) {
|
|
95
|
+
throw new BackendError(
|
|
96
|
+
"INVALID_BACKEND",
|
|
97
|
+
`This backup was written with envelope version ${version}, and this build reads ${ENVELOPE_VERSION}. ` +
|
|
98
|
+
"Upgrade @le-space/orbitdb-storage-bridge to restore it.",
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
const ivLength = bytes[ENVELOPE_MAGIC.length + 1];
|
|
102
|
+
const iv = bytes.subarray(HEADER_LENGTH, HEADER_LENGTH + ivLength);
|
|
103
|
+
const ciphertext = bytes.subarray(HEADER_LENGTH + ivLength);
|
|
104
|
+
return { ciphertext, iv, version };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Encrypt on the way out, decrypt on the way back.
|
|
109
|
+
*
|
|
110
|
+
* @param {import("./types.js").StorageBackend} backend - the one that stores bytes
|
|
111
|
+
* @param {object} options
|
|
112
|
+
* @param {(plaintext: Uint8Array) => Promise<{ciphertext: Uint8Array, iv: Uint8Array}>} options.encrypt
|
|
113
|
+
* @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} options.decrypt
|
|
114
|
+
* @param {boolean} [options.allowPlaintextReads=true] - restore a backup written
|
|
115
|
+
* before backups were encrypted, or written with `dontEncrypt`. On by
|
|
116
|
+
* default: a pointer does not say which kind it names, and refusing would
|
|
117
|
+
* make yesterday's backups unreadable for no gain in secrecy.
|
|
118
|
+
* @returns {import("./types.js").StorageBackend}
|
|
119
|
+
*/
|
|
120
|
+
export function withEncryption(backend, { encrypt, decrypt, allowPlaintextReads = true } = {}) {
|
|
121
|
+
if (!backend || typeof backend.putBlob !== "function") {
|
|
122
|
+
throw new BackendError("INVALID_BACKEND", "withEncryption needs a backend to wrap");
|
|
123
|
+
}
|
|
124
|
+
if (typeof encrypt !== "function" || typeof decrypt !== "function") {
|
|
125
|
+
throw new BackendError(
|
|
126
|
+
"INVALID_BACKEND",
|
|
127
|
+
"withEncryption needs encrypt and decrypt functions — this package holds no keys of its own",
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const inner = backend;
|
|
132
|
+
|
|
133
|
+
return defineBackend({
|
|
134
|
+
...inner,
|
|
135
|
+
name: `encrypted(${inner.name})`,
|
|
136
|
+
capabilities: {
|
|
137
|
+
...inner.capabilities,
|
|
138
|
+
// The stored bytes are an envelope, not a CAR: nothing downstream may
|
|
139
|
+
// treat them as one, and there are no inner CIDs to preserve.
|
|
140
|
+
carImport: false,
|
|
141
|
+
preservesInnerCids: false,
|
|
142
|
+
},
|
|
143
|
+
|
|
144
|
+
async putBlob(bytes, meta) {
|
|
145
|
+
const { ciphertext, iv } = (await encrypt(bytes)) ?? {};
|
|
146
|
+
return inner.putBlob(wrapEnvelope(ciphertext, iv), meta);
|
|
147
|
+
},
|
|
148
|
+
|
|
149
|
+
async getBlob(handle) {
|
|
150
|
+
const bytes = await inner.getBlob(handle);
|
|
151
|
+
if (!isEncrypted(bytes)) {
|
|
152
|
+
if (allowPlaintextReads) return bytes;
|
|
153
|
+
throw new BackendError(
|
|
154
|
+
"INVALID_BACKEND",
|
|
155
|
+
"This backup is not encrypted, and allowPlaintextReads is off",
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
const { ciphertext, iv } = readEnvelope(bytes);
|
|
159
|
+
try {
|
|
160
|
+
return await decrypt(ciphertext, iv);
|
|
161
|
+
} catch (error) {
|
|
162
|
+
// The common cause by far is the wrong key — a different security key,
|
|
163
|
+
// or a key derived with a different info string. Say that, rather than
|
|
164
|
+
// letting an unreadable CAR surface three layers down.
|
|
165
|
+
throw new BackendError(
|
|
166
|
+
"INVALID_BACKEND",
|
|
167
|
+
`Could not decrypt this backup: ${error.message}. ` +
|
|
168
|
+
"It was written with a different key, or by a different derivation.",
|
|
169
|
+
{ cause: error },
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
},
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Decrypt on the way back, for the path a restore actually takes.
|
|
178
|
+
*
|
|
179
|
+
* `restoreFromCID` does not read through a backend: it takes `fetchBytes(cid)`
|
|
180
|
+
* and gets the bytes from a gateway or from peers, so `withEncryption`'s
|
|
181
|
+
* `getBlob` never runs during a restore. This is the other half — wrap the
|
|
182
|
+
* fetcher, and a backup written through an encrypting backend comes back
|
|
183
|
+
* readable however it was fetched.
|
|
184
|
+
*
|
|
185
|
+
* @param {(cid: string, options?: object) => Promise<Uint8Array>} fetchBytes
|
|
186
|
+
* @param {object} options
|
|
187
|
+
* @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} options.decrypt
|
|
188
|
+
* @param {boolean} [options.allowPlaintextReads=true]
|
|
189
|
+
* @returns {(cid: string, options?: object) => Promise<Uint8Array>}
|
|
190
|
+
*/
|
|
191
|
+
export function decryptingFetch(fetchBytes, { decrypt, allowPlaintextReads = true } = {}) {
|
|
192
|
+
if (typeof fetchBytes !== "function") {
|
|
193
|
+
throw new BackendError("INVALID_BACKEND", "decryptingFetch needs a fetcher to wrap");
|
|
194
|
+
}
|
|
195
|
+
if (typeof decrypt !== "function") {
|
|
196
|
+
throw new BackendError("INVALID_BACKEND", "decryptingFetch needs a decrypt function");
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
return async function fetchAndDecrypt(cid, options) {
|
|
200
|
+
const bytes = await fetchBytes(cid, options);
|
|
201
|
+
if (!isEncrypted(bytes)) {
|
|
202
|
+
if (allowPlaintextReads) return bytes;
|
|
203
|
+
throw new BackendError(
|
|
204
|
+
"INVALID_BACKEND",
|
|
205
|
+
`The backup at ${cid} is not encrypted, and allowPlaintextReads is off`,
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
const { ciphertext, iv } = readEnvelope(bytes);
|
|
209
|
+
try {
|
|
210
|
+
return await decrypt(ciphertext, iv);
|
|
211
|
+
} catch (error) {
|
|
212
|
+
throw new BackendError(
|
|
213
|
+
"INVALID_BACKEND",
|
|
214
|
+
`Could not decrypt the backup at ${cid}: ${error.message}. ` +
|
|
215
|
+
"It was written with a different key, or by a different derivation.",
|
|
216
|
+
{ cause: error },
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Recognise an encrypted backup when there is no key to open it.
|
|
224
|
+
*
|
|
225
|
+
* Without this, a restore of an encrypted backup without `decrypt` fails deep
|
|
226
|
+
* inside the CAR reader — "unexpected end of data", or a block that will not
|
|
227
|
+
* verify — and the reason has nothing to do with what actually happened.
|
|
228
|
+
*
|
|
229
|
+
* @param {(cid: string, options?: object) => Promise<Uint8Array>} fetchBytes
|
|
230
|
+
* @returns {(cid: string, options?: object) => Promise<Uint8Array>}
|
|
231
|
+
*/
|
|
232
|
+
export function explainIfEncrypted(fetchBytes) {
|
|
233
|
+
return async function fetchAndCheck(cid, options) {
|
|
234
|
+
const bytes = await fetchBytes(cid, options);
|
|
235
|
+
if (isEncrypted(bytes)) {
|
|
236
|
+
throw new BackendError(
|
|
237
|
+
"INVALID_BACKEND",
|
|
238
|
+
`The backup at ${cid} is encrypted, and no way to decrypt it was given. ` +
|
|
239
|
+
"Pass `decrypt` to hydrate() — a browser derives it from the security key.",
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
return bytes;
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export default withEncryption;
|
package/lib/courier-sync.js
CHANGED
|
@@ -165,6 +165,49 @@ function isOplogEntry(value) {
|
|
|
165
165
|
);
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
+
/**
|
|
169
|
+
* What the peer can be assumed to hold, given the heads it named.
|
|
170
|
+
*
|
|
171
|
+
* The heads alone are not a stop set. An OrbitDB entry's `refs` are skip-list
|
|
172
|
+
* back-references that point *past* its parent, so a walk that stops only at
|
|
173
|
+
* the named hashes follows a ref around them and carries on to the root: a peer
|
|
174
|
+
* missing one entry was sent the whole log, measured at 12 blocks and 8742 B
|
|
175
|
+
* where 2 blocks and 1533 B were owed (funkpost's two phones over LoRa, #127).
|
|
176
|
+
* At half a kilobyte a minute that is the difference between a list that syncs
|
|
177
|
+
* and one that cannot.
|
|
178
|
+
*
|
|
179
|
+
* A head is a claim about everything below it, so the closure below those heads
|
|
180
|
+
* is what the peer holds. It is walked over blocks *we* hold; where we cannot
|
|
181
|
+
* follow, that ancestry stays unknown and so stays out of the stop set, which
|
|
182
|
+
* errs towards sending — the direction that costs bytes rather than
|
|
183
|
+
* correctness.
|
|
184
|
+
*
|
|
185
|
+
* Reading the whole ancestry locally to avoid transmitting it is a good trade
|
|
186
|
+
* on any carrier: the reads are a blockstore away, the bytes are airtime.
|
|
187
|
+
*/
|
|
188
|
+
async function reachableFrom(db, roots) {
|
|
189
|
+
const held = new Set();
|
|
190
|
+
const queue = [...roots];
|
|
191
|
+
while (queue.length > 0) {
|
|
192
|
+
const hash = queue.shift();
|
|
193
|
+
if (held.has(hash)) continue;
|
|
194
|
+
held.add(hash);
|
|
195
|
+
const bytes = await db.log.storage.get(hash).catch(() => null);
|
|
196
|
+
if (!bytes) continue; // not ours to follow; their ancestry ends here for us
|
|
197
|
+
let value;
|
|
198
|
+
try {
|
|
199
|
+
value = dagCbor.decode(bytes);
|
|
200
|
+
} catch {
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
if (!isOplogEntry(value)) continue;
|
|
204
|
+
for (const parent of [...value.next, ...(value.refs || [])]) {
|
|
205
|
+
if (!held.has(parent)) queue.push(parent);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return held;
|
|
209
|
+
}
|
|
210
|
+
|
|
168
211
|
/**
|
|
169
212
|
* Compute the delta a peer with `theirHeads` is missing: entry blocks from our
|
|
170
213
|
* heads down to their heads, the identity blocks those entries reference, and
|
|
@@ -184,10 +227,19 @@ function isOplogEntry(value) {
|
|
|
184
227
|
* @returns {Promise<{heads: Array<string>, blocks: Array<{hash: string, bytes: Uint8Array}>}>}
|
|
185
228
|
*/
|
|
186
229
|
export async function createDelta({ db, theirHeads = [] }) {
|
|
187
|
-
const stop = new Set(theirHeads);
|
|
188
230
|
const heads = await db.log.heads();
|
|
189
231
|
const headHashes = heads.map((entry) => entry.hash);
|
|
190
232
|
|
|
233
|
+
// The peer stands exactly where we do: nothing is owed, and the ancestry
|
|
234
|
+
// need not be read to find that out. This is the steady state between two
|
|
235
|
+
// quiet peers, so it is worth answering before the walk below.
|
|
236
|
+
const named = new Set(theirHeads);
|
|
237
|
+
if (headHashes.length > 0 && headHashes.every((hash) => named.has(hash))) {
|
|
238
|
+
return { heads: headHashes, blocks: [] };
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const stop = await reachableFrom(db, theirHeads);
|
|
242
|
+
|
|
191
243
|
const seen = new Set();
|
|
192
244
|
const identityHashes = new Set();
|
|
193
245
|
const entryBlocks = [];
|
|
@@ -267,7 +319,9 @@ export async function createDelta({ db, theirHeads = [] }) {
|
|
|
267
319
|
* @param {Object} params
|
|
268
320
|
* @param {Object} params.db An open OrbitDB database
|
|
269
321
|
* @param {{heads: Array<string>, blocks: Array<{hash: string, bytes: Uint8Array}>}} params.delta
|
|
270
|
-
* @returns {Promise<{complete: boolean, joined: number, missing: Array<string>,
|
|
322
|
+
* @returns {Promise<{complete: boolean, joined: number, missing: Array<string>,
|
|
323
|
+
* entries: Array<Object>, heads: number, outcome: Object}>} `heads` is how
|
|
324
|
+
* many were offered and `outcome` says what became of each — see `noJoins`.
|
|
271
325
|
*/
|
|
272
326
|
export async function applyDelta({ db, delta }) {
|
|
273
327
|
return applyDeltaToStores({
|
|
@@ -289,7 +343,39 @@ function dbBlockstore(db) {
|
|
|
289
343
|
};
|
|
290
344
|
}
|
|
291
345
|
|
|
346
|
+
/**
|
|
347
|
+
* Why a head did not join.
|
|
348
|
+
*
|
|
349
|
+
* The join loop below has exactly five exits, and from outside four of them
|
|
350
|
+
* look the same: no join, and — since "synced" only fires when something
|
|
351
|
+
* joined — no event at all. That silence is what left a day of field logs
|
|
352
|
+
* unreadable. Two phones over LoRa received five complete deltas and joined
|
|
353
|
+
* nothing all day, and the log could not say whether the courier was working
|
|
354
|
+
* or broken, because the two findings that matter produce identical silence:
|
|
355
|
+
*
|
|
356
|
+
* held the entry was already in the log. Another route brought it first;
|
|
357
|
+
* the courier delivered something nobody needed, which is wasteful
|
|
358
|
+
* but correct.
|
|
359
|
+
* absent the sender named a head and did not send it. A defect in the
|
|
360
|
+
* delta it built, and the database does not move.
|
|
361
|
+
*
|
|
362
|
+
* Opposite repairs, one symptom. `malformed` and `refused` should not happen
|
|
363
|
+
* at all; they are counted separately rather than folded into `absent` so
|
|
364
|
+
* that "should not happen" stays falsifiable in a field log.
|
|
365
|
+
*
|
|
366
|
+
* Reported for every delivery through the "applied" event, including the
|
|
367
|
+
* deliveries that came to nothing — those are the interesting ones.
|
|
368
|
+
*/
|
|
369
|
+
const noJoins = () => ({
|
|
370
|
+
joined: 0,
|
|
371
|
+
held: 0,
|
|
372
|
+
absent: 0,
|
|
373
|
+
malformed: 0,
|
|
374
|
+
refused: 0,
|
|
375
|
+
});
|
|
376
|
+
|
|
292
377
|
async function applyDeltaToStores({ blockstore, log, events, delta }) {
|
|
378
|
+
const heads = delta.heads || [];
|
|
293
379
|
const inDelta = new Map();
|
|
294
380
|
for (const block of delta.blocks || []) {
|
|
295
381
|
inDelta.set(block.hash, block.bytes);
|
|
@@ -312,32 +398,56 @@ async function applyDeltaToStores({ blockstore, log, events, delta }) {
|
|
|
312
398
|
}
|
|
313
399
|
}
|
|
314
400
|
if (missing.length > 0) {
|
|
315
|
-
return {
|
|
401
|
+
return {
|
|
402
|
+
complete: false,
|
|
403
|
+
joined: 0,
|
|
404
|
+
missing,
|
|
405
|
+
entries: [],
|
|
406
|
+
heads: heads.length,
|
|
407
|
+
outcome: noJoins(),
|
|
408
|
+
};
|
|
316
409
|
}
|
|
317
410
|
|
|
318
|
-
|
|
411
|
+
const outcome = noJoins();
|
|
319
412
|
const entries = [];
|
|
320
|
-
for (const hash of
|
|
321
|
-
if (await log.has(hash))
|
|
413
|
+
for (const hash of heads) {
|
|
414
|
+
if (await log.has(hash)) {
|
|
415
|
+
outcome.held++;
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
322
418
|
const bytes = inDelta.get(hash);
|
|
323
|
-
if (!bytes)
|
|
419
|
+
if (!bytes) {
|
|
420
|
+
outcome.absent++;
|
|
421
|
+
continue;
|
|
422
|
+
}
|
|
324
423
|
const value = dagCbor.decode(bytes);
|
|
325
|
-
if (!isOplogEntry(value))
|
|
424
|
+
if (!isOplogEntry(value)) {
|
|
425
|
+
outcome.malformed++;
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
326
428
|
const entry = { ...value, hash };
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
entries.push(entry);
|
|
429
|
+
if (!(await log.joinEntry(entry))) {
|
|
430
|
+
outcome.refused++;
|
|
431
|
+
continue;
|
|
331
432
|
}
|
|
433
|
+
outcome.joined++;
|
|
434
|
+
entries.push(entry);
|
|
332
435
|
}
|
|
333
436
|
|
|
334
437
|
// Database.applyOperation emits 'update' when the pubsub Sync delivers an
|
|
335
438
|
// entry; a courier delivery is the same event from the application's side.
|
|
336
|
-
if (events && joined > 0) {
|
|
439
|
+
if (events && outcome.joined > 0) {
|
|
337
440
|
for (const entry of entries) events.emit("update", entry);
|
|
338
441
|
}
|
|
339
442
|
|
|
340
|
-
return {
|
|
443
|
+
return {
|
|
444
|
+
complete: true,
|
|
445
|
+
joined: outcome.joined,
|
|
446
|
+
missing: [],
|
|
447
|
+
entries,
|
|
448
|
+
heads: heads.length,
|
|
449
|
+
outcome,
|
|
450
|
+
};
|
|
341
451
|
}
|
|
342
452
|
|
|
343
453
|
/**
|
|
@@ -373,6 +483,14 @@ async function applyDeltaToStores({ blockstore, log, events, delta }) {
|
|
|
373
483
|
* carrier that cannot keep up before the oldest is dropped.
|
|
374
484
|
* @returns {Promise<Object>} sync handle: { start, stop, announce, hello,
|
|
375
485
|
* presence, forgetPeers, db(), events }
|
|
486
|
+
*
|
|
487
|
+
* Events, via `sync.on(name, cb)`:
|
|
488
|
+
* "message" { direction, type, bytes } one message on or off the carrier
|
|
489
|
+
* "synced" { joined, entries } the database moved
|
|
490
|
+
* "applied" { complete, heads, missing, joined, held, absent, malformed,
|
|
491
|
+
* refused } what a `blocks` delivery came to,
|
|
492
|
+
* fired even when it came to nothing
|
|
493
|
+
* "error" Error a delivery that threw
|
|
376
494
|
*/
|
|
377
495
|
export async function createCourierSync({
|
|
378
496
|
db = null,
|
|
@@ -411,7 +529,7 @@ export async function createCourierSync({
|
|
|
411
529
|
const pendingBlocks = new Map(); // hash -> bytes, parked until the database can open
|
|
412
530
|
const peers = new Map(); // sender id (hex) -> when we last heard it
|
|
413
531
|
let lastHeardAt = null; // any traffic for this database, identified or not
|
|
414
|
-
const listeners = { synced: [], message: [], error: [] };
|
|
532
|
+
const listeners = { synced: [], applied: [], message: [], error: [] };
|
|
415
533
|
let database = db;
|
|
416
534
|
// Opened on first contact but not handed out: the bootstrap is not in it yet.
|
|
417
535
|
// The protocol works on it all the same, so repair stays incremental.
|
|
@@ -723,6 +841,16 @@ export async function createCourierSync({
|
|
|
723
841
|
} finally {
|
|
724
842
|
applying = false;
|
|
725
843
|
}
|
|
844
|
+
// What the delivery came to, always — including when it came to nothing.
|
|
845
|
+
// A delta that moves the database is a "synced"; a delta that moves
|
|
846
|
+
// nothing is either a courier doing no harm or a courier doing no good,
|
|
847
|
+
// and this is the only line that tells them apart.
|
|
848
|
+
emit("applied", {
|
|
849
|
+
complete: result.complete,
|
|
850
|
+
heads: result.heads,
|
|
851
|
+
missing: result.missing.length,
|
|
852
|
+
...result.outcome,
|
|
853
|
+
});
|
|
726
854
|
if (!result.complete) {
|
|
727
855
|
post(
|
|
728
856
|
{
|
package/lib/dehydrate.js
CHANGED
|
@@ -61,15 +61,33 @@ export async function dehydrate({
|
|
|
61
61
|
seed,
|
|
62
62
|
label,
|
|
63
63
|
backend,
|
|
64
|
+
encrypt,
|
|
65
|
+
decrypt,
|
|
66
|
+
dontEncrypt = false,
|
|
64
67
|
endpoints = DEFAULT_ENDPOINTS,
|
|
65
68
|
sequence,
|
|
66
69
|
backup = {},
|
|
67
70
|
}) {
|
|
68
71
|
if (!address) throw new Error("dehydrate needs the database address");
|
|
72
|
+
|
|
73
|
+
// Encrypted by default. A caller who wants the backup readable by anyone
|
|
74
|
+
// holding the CID has to write that down, because it is a decision rather
|
|
75
|
+
// than an oversight — and an oversight is what this refusal exists to catch.
|
|
76
|
+
if (!encrypt && !dontEncrypt) {
|
|
77
|
+
throw new Error(
|
|
78
|
+
"Backups are encrypted by default. Pass `encrypt` and `decrypt` — a browser " +
|
|
79
|
+
"derives them from the security key — or `dontEncrypt: true` if this backup " +
|
|
80
|
+
"is meant to be readable by anyone holding the CID.",
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
|
|
69
84
|
const { backupDatabaseCAR } = await import("./backup-car.js");
|
|
85
|
+
const storeIn = encrypt
|
|
86
|
+
? (await import("./backends/encryption.js")).withEncryption(backend, { encrypt, decrypt })
|
|
87
|
+
: backend;
|
|
70
88
|
|
|
71
89
|
const result = await backupDatabaseCAR(orbitdb, address, {
|
|
72
|
-
backend,
|
|
90
|
+
backend: storeIn,
|
|
73
91
|
...backup,
|
|
74
92
|
});
|
|
75
93
|
if (!result?.success) {
|
|
@@ -118,6 +136,7 @@ export async function hydrate({
|
|
|
118
136
|
orbitdb,
|
|
119
137
|
seed,
|
|
120
138
|
label,
|
|
139
|
+
decrypt,
|
|
121
140
|
endpoints = DEFAULT_ENDPOINTS,
|
|
122
141
|
open = {},
|
|
123
142
|
restore = {},
|
|
@@ -126,10 +145,22 @@ export async function hydrate({
|
|
|
126
145
|
const pointer = await resolvePointer({ privateKey, endpoints });
|
|
127
146
|
|
|
128
147
|
const { restoreFromCID } = await import("./restore-cid.js");
|
|
148
|
+
const { decryptingFetch, explainIfEncrypted } = await import("./backends/encryption.js");
|
|
149
|
+
const { fetchFromGateways } = await import("./gateway-fetch.js");
|
|
150
|
+
|
|
151
|
+
// A restore does not read through a backend — it fetches from a gateway or
|
|
152
|
+
// from peers — so decryption belongs here, around the fetcher. Without a
|
|
153
|
+
// key, an encrypted backup is still recognised, and says so.
|
|
154
|
+
const fetchBytes = restore.fetchBytes ?? fetchFromGateways;
|
|
155
|
+
const reader = decrypt
|
|
156
|
+
? decryptingFetch(fetchBytes, { decrypt })
|
|
157
|
+
: explainIfEncrypted(fetchBytes);
|
|
158
|
+
|
|
129
159
|
const result = await restoreFromCID(orbitdb, {
|
|
130
160
|
metadataCID: pointer.cid,
|
|
131
161
|
open,
|
|
132
162
|
...restore,
|
|
163
|
+
fetchBytes: reader,
|
|
133
164
|
});
|
|
134
165
|
|
|
135
166
|
logger.info(
|
package/lib/peer-fetch.js
CHANGED
|
@@ -33,11 +33,30 @@
|
|
|
33
33
|
*
|
|
34
34
|
* ## What the caller has to bring
|
|
35
35
|
*
|
|
36
|
-
* A Helia with bitswap
|
|
37
|
-
* restores already has a node, and a second one would be a second
|
|
38
|
-
* the network.
|
|
39
|
-
*
|
|
40
|
-
*
|
|
36
|
+
* A Helia with bitswap **and identify**. This module has no libp2p of its own —
|
|
37
|
+
* the page that restores already has a node, and a second one would be a second
|
|
38
|
+
* identity on the network.
|
|
39
|
+
*
|
|
40
|
+
* Two things about that node are easy to get wrong, and both were:
|
|
41
|
+
*
|
|
42
|
+
* - **`identify` is not optional.** `withLibp2pLight` does not include it, and
|
|
43
|
+
* without it bitswap never learns that the peer it just dialled speaks
|
|
44
|
+
* bitswap: no want is sent, and the fetch fails with "Failed to load block"
|
|
45
|
+
* after the full timeout. Measured against Aleph: 60 s of nothing without
|
|
46
|
+
* identify, 1.6 s for 400 kB with it, over the same dialled connection.
|
|
47
|
+
* - **Helia's browser defaults try to listen** on `/webrtc` and `/p2p-circuit`
|
|
48
|
+
* and **throw on start** when no transport serves them, so a fetch-only node
|
|
49
|
+
* wants `addresses: { listen: [] }`.
|
|
50
|
+
*
|
|
51
|
+
* ```js
|
|
52
|
+
* const helia = await withBitswap(withLibp2pLight(createHeliaLight({ … }), {
|
|
53
|
+
* addresses: { listen: [] },
|
|
54
|
+
* transports: [webSockets(), webRTCDirect()],
|
|
55
|
+
* connectionEncrypters: [noise()],
|
|
56
|
+
* streamMuxers: [yamux()],
|
|
57
|
+
* services: { identify: identify() }, // ← without this, nothing arrives
|
|
58
|
+
* })).start()
|
|
59
|
+
* ```
|
|
41
60
|
*
|
|
42
61
|
* @module peer-fetch
|
|
43
62
|
*/
|
|
@@ -90,25 +109,25 @@ export const ALEPH_PEER_ID = "12D3KooWACE5dRw5V9WXuDTcngjE3ZaDSZ4qYJGfuhXZbENnL5
|
|
|
90
109
|
/**
|
|
91
110
|
* Routers that answer a page.
|
|
92
111
|
*
|
|
93
|
-
*
|
|
94
|
-
* `delegated-ipfs.dev` did not,
|
|
95
|
-
*
|
|
112
|
+
* Both send `access-control-allow-origin: *` for provider lookups. An earlier
|
|
113
|
+
* version of this file said `delegated-ipfs.dev` did not, from one request that
|
|
114
|
+
* came back without the header; repeated from a deployed page it answers with
|
|
115
|
+
* it, so the restriction was wrong and this list is both.
|
|
96
116
|
*
|
|
97
|
-
*
|
|
117
|
+
* What is true, and matters more, is that they know **different things**:
|
|
98
118
|
* `cid.contact` is an IPNI index and knows what is announced to it — Pinata's
|
|
99
|
-
* CIDs and Lighthouse's were there — while a CID that Aleph holds appeared
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
* {@link ALEPH_BITSWAP} is for.
|
|
119
|
+
* CIDs and Lighthouse's were there — while a CID that Aleph holds appeared only
|
|
120
|
+
* at `delegated-ipfs.dev`, because Aleph announces over the DHT. Asking one
|
|
121
|
+
* router is asking half the network.
|
|
103
122
|
*/
|
|
104
|
-
export const DEFAULT_ROUTERS = Object.freeze([
|
|
105
|
-
|
|
106
|
-
/** Everything a Node caller can ask, where CORS does not apply. */
|
|
107
|
-
export const ALL_ROUTERS = Object.freeze([
|
|
123
|
+
export const DEFAULT_ROUTERS = Object.freeze([
|
|
108
124
|
"https://cid.contact",
|
|
109
125
|
"https://delegated-ipfs.dev",
|
|
110
126
|
]);
|
|
111
127
|
|
|
128
|
+
/** @deprecated Same as {@link DEFAULT_ROUTERS}; kept so an import still works. */
|
|
129
|
+
export const ALL_ROUTERS = DEFAULT_ROUTERS;
|
|
130
|
+
|
|
112
131
|
/**
|
|
113
132
|
* Transports a browser can dial.
|
|
114
133
|
*
|
|
@@ -266,16 +285,23 @@ export function createPeerFetch({
|
|
|
266
285
|
return helia.libp2p.dial(multiaddr(addr), { signal });
|
|
267
286
|
});
|
|
268
287
|
|
|
269
|
-
|
|
288
|
+
// One connection per peer, not one per address. A provider usually
|
|
289
|
+
// advertises the same peer several times — Aleph offers webrtc-direct and
|
|
290
|
+
// webtransport — and dialling both gets two connections to one node, which
|
|
291
|
+
// buys nothing and makes a peer count read double.
|
|
292
|
+
const reached = new Set();
|
|
270
293
|
for (const addr of addrs) {
|
|
294
|
+
const peer = addr.match(/\/p2p\/([^/]+)/)?.[1] ?? addr;
|
|
295
|
+
if (reached.has(peer)) continue;
|
|
271
296
|
try {
|
|
272
297
|
await dialOne(addr);
|
|
273
|
-
|
|
298
|
+
reached.add(peer);
|
|
274
299
|
log.debug(` ✅ dialled ${addr}`);
|
|
275
300
|
} catch (error) {
|
|
276
301
|
log.debug(` ⚠️ could not dial ${addr}: ${error.message}`);
|
|
277
302
|
}
|
|
278
303
|
}
|
|
304
|
+
const dialed = reached.size;
|
|
279
305
|
if (dialed === 0) {
|
|
280
306
|
throw new Error(`Could not dial any provider for ${cid} (tried ${addrs.length})`);
|
|
281
307
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@le-space/orbitdb-storage-bridge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
|
|
5
5
|
"main": "lib/orbitdb-storacha-bridge.js",
|
|
6
6
|
"svelte": "dist/components/",
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
"./backends/pinata": "./lib/backends/pinata.js",
|
|
26
26
|
"./backends/lighthouse": "./lib/backends/lighthouse.js",
|
|
27
27
|
"./backends/mirror": "./lib/backends/mirror.js",
|
|
28
|
+
"./backends/encryption": "./lib/backends/encryption.js",
|
|
28
29
|
"./backends/choose": "./lib/backends/choose.js",
|
|
29
30
|
"./backends/resolve": "./lib/backends/resolve.js",
|
|
30
31
|
"./memory-courier": "./lib/memory-courier.js",
|