@le-space/orbitdb-storage-bridge 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/backends/aleph-pin.js +147 -6
- package/lib/courier-sync.js +61 -5
- package/lib/extract-blocks.js +73 -0
- package/lib/restore-cid.js +147 -55
- package/package.json +2 -1
|
@@ -65,16 +65,21 @@ async function sha256Hex(payload) {
|
|
|
65
65
|
* Exported so it can be tested without a wallet, and read without running one.
|
|
66
66
|
*
|
|
67
67
|
* @param {object} args
|
|
68
|
-
* @param {string} args.sender - the wallet address
|
|
68
|
+
* @param {string} args.sender - the wallet address that signs
|
|
69
|
+
* @param {string} [args.owner] - the account the STORE is for, when the sender
|
|
70
|
+
* signs on its behalf (see {@link createAlephAuthorizer}); the sender itself
|
|
71
|
+
* by default
|
|
69
72
|
* @param {string} args.cid - what to keep
|
|
70
73
|
* @param {string} [args.channel]
|
|
71
74
|
* @param {number} [args.now] - seconds; injected so a test is not a clock
|
|
72
75
|
* @param {(payload: string) => Promise<string>} [args.hasher]
|
|
73
76
|
*/
|
|
74
|
-
export async function buildStoreMessage({ sender, cid, channel = DEFAULT_ALEPH_CHANNEL, now, hasher = sha256Hex }) {
|
|
77
|
+
export async function buildStoreMessage({ sender, owner, cid, channel = DEFAULT_ALEPH_CHANNEL, now, hasher = sha256Hex }) {
|
|
75
78
|
const time = now ?? Date.now() / 1000;
|
|
76
79
|
const content = {
|
|
77
|
-
|
|
80
|
+
// Whose STORE this is. Equal to the sender unless the owner's `security`
|
|
81
|
+
// aggregate authorizes the sender to send it on the owner's behalf.
|
|
82
|
+
address: owner || sender,
|
|
78
83
|
// "the thing to keep is an IPFS CID" — not the envelope's item_type
|
|
79
84
|
item_type: "ipfs",
|
|
80
85
|
item_hash: cid,
|
|
@@ -102,7 +107,9 @@ export const signaturePayload = (message) =>
|
|
|
102
107
|
* A `pin` function for {@link createAlephBackend}.
|
|
103
108
|
*
|
|
104
109
|
* @param {object} options
|
|
105
|
-
* @param {string} options.sender - the wallet address
|
|
110
|
+
* @param {string} options.sender - the wallet address that signs
|
|
111
|
+
* @param {string} [options.owner] - the account the STORE is for, when it has
|
|
112
|
+
* authorized `sender` (a delegate) to send STORE messages on its behalf
|
|
106
113
|
* @param {(address: string, message: string) => Promise<string>} options.sign -
|
|
107
114
|
* `personal_sign`, or anything shaped like it. The library never sees a key.
|
|
108
115
|
* @param {string} [options.apiHost]
|
|
@@ -113,7 +120,7 @@ export const signaturePayload = (message) =>
|
|
|
113
120
|
* @returns {(cid: string, meta?: object) => Promise<{ itemHash: string, status: string }>}
|
|
114
121
|
*/
|
|
115
122
|
export function createAlephPin(options = {}) {
|
|
116
|
-
const { sender, sign } = options;
|
|
123
|
+
const { sender, sign, owner } = options;
|
|
117
124
|
const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
|
|
118
125
|
const channel = options.channel || DEFAULT_ALEPH_CHANNEL;
|
|
119
126
|
const hasher = options.hasher || sha256Hex;
|
|
@@ -125,7 +132,7 @@ export function createAlephPin(options = {}) {
|
|
|
125
132
|
if (typeof doFetch !== "function") throw new BackendError("INVALID_BACKEND", "createAlephPin needs fetch");
|
|
126
133
|
|
|
127
134
|
return async function pin(cid) {
|
|
128
|
-
const unsigned = await buildStoreMessage({ sender, cid, channel, hasher, now: now?.() });
|
|
135
|
+
const unsigned = await buildStoreMessage({ sender, owner, cid, channel, hasher, now: now?.() });
|
|
129
136
|
const signature = await sign(sender, signaturePayload(unsigned));
|
|
130
137
|
const message = {
|
|
131
138
|
...unsigned,
|
|
@@ -158,4 +165,138 @@ export function createAlephPin(options = {}) {
|
|
|
158
165
|
};
|
|
159
166
|
}
|
|
160
167
|
|
|
168
|
+
/** The reserved channel and aggregate key Aleph keeps permissions in. */
|
|
169
|
+
export const SECURITY = "security";
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* @typedef {object} AlephAuthorization one entry of an owner's `security.authorizations`
|
|
173
|
+
* @property {string} address - the delegate, which may then send on the owner's behalf
|
|
174
|
+
* @property {string[]} [types] - e.g. `["STORE"]`; every type when absent
|
|
175
|
+
* @property {string[]} [channels] - e.g. `["BELEGE-BACKUP"]`; every channel when absent
|
|
176
|
+
* @property {string} [chain] - only the delegate's address on this chain, e.g. `"ETH"`
|
|
177
|
+
* @property {string[]} [post_types]
|
|
178
|
+
* @property {string[]} [aggregate_keys]
|
|
179
|
+
*/
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Build the unsigned AGGREGATE that sets an owner's authorizations.
|
|
183
|
+
*
|
|
184
|
+
* Aleph keeps permissions in the owner's `security` aggregate, written only by
|
|
185
|
+
* the owner itself (`sender == content.address`) on the `security` channel. An
|
|
186
|
+
* aggregate key is replaced as a whole, so `authorizations` is the **complete**
|
|
187
|
+
* list after the change, not an addition: {@link createAlephAuthorizer} reads
|
|
188
|
+
* the current list first.
|
|
189
|
+
*
|
|
190
|
+
* @param {object} args
|
|
191
|
+
* @param {string} args.owner - the account granting, and signing
|
|
192
|
+
* @param {AlephAuthorization[]} args.authorizations
|
|
193
|
+
* @param {number} [args.now] - seconds
|
|
194
|
+
* @param {(payload: string) => Promise<string>} [args.hasher]
|
|
195
|
+
*/
|
|
196
|
+
export async function buildAuthorizationMessage({ owner, authorizations, now, hasher = sha256Hex }) {
|
|
197
|
+
if (!owner) throw new BackendError("INVALID_BACKEND", "an authorization needs the owner's address");
|
|
198
|
+
if (!Array.isArray(authorizations) || authorizations.some((a) => typeof a?.address !== "string" || !a.address)) {
|
|
199
|
+
throw new BackendError("INVALID_BACKEND", "every authorization needs the delegate's address");
|
|
200
|
+
}
|
|
201
|
+
const time = now ?? Date.now() / 1000;
|
|
202
|
+
const item_content = JSON.stringify({
|
|
203
|
+
address: owner,
|
|
204
|
+
key: SECURITY,
|
|
205
|
+
content: { authorizations },
|
|
206
|
+
time,
|
|
207
|
+
});
|
|
208
|
+
return {
|
|
209
|
+
sender: owner,
|
|
210
|
+
chain: "ETH",
|
|
211
|
+
type: "AGGREGATE",
|
|
212
|
+
item_hash: await hasher(item_content),
|
|
213
|
+
item_type: "inline",
|
|
214
|
+
item_content,
|
|
215
|
+
time,
|
|
216
|
+
channel: SECURITY,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Let other keys send STORE messages for an account: read, grant and revoke its
|
|
222
|
+
* authorizations.
|
|
223
|
+
*
|
|
224
|
+
* The use it was built for: one funded account pays for keeping backups, and
|
|
225
|
+
* the keys that actually sign them are others — a key derived from a passkey in
|
|
226
|
+
* the browser, one per person or device. Each is granted only what it needs
|
|
227
|
+
* (`types: ["STORE"]`, one channel), and revoked on its own. The owner signs
|
|
228
|
+
* these grants; the delegates then pass `owner` to {@link createAlephPin}.
|
|
229
|
+
*
|
|
230
|
+
* Which balance Aleph charges for a STORE sent by a delegate is not stated in
|
|
231
|
+
* its documentation; measure it before relying on it.
|
|
232
|
+
*
|
|
233
|
+
* Aleph keys accounts by their EIP-55 checksummed address; pass `owner` in that
|
|
234
|
+
* form, or `read()` finds nothing.
|
|
235
|
+
*
|
|
236
|
+
* @param {object} options
|
|
237
|
+
* @param {string} options.owner - the account granting
|
|
238
|
+
* @param {(address: string, message: string) => Promise<string>} options.sign - the owner's `personal_sign`
|
|
239
|
+
* @param {string} [options.apiHost]
|
|
240
|
+
* @param {typeof fetch} [options.fetch]
|
|
241
|
+
* @param {() => number} [options.now] - seconds
|
|
242
|
+
* @param {(payload: string) => Promise<string>} [options.hasher]
|
|
243
|
+
*/
|
|
244
|
+
export function createAlephAuthorizer(options = {}) {
|
|
245
|
+
const { owner, sign } = options;
|
|
246
|
+
const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
|
|
247
|
+
const hasher = options.hasher || sha256Hex;
|
|
248
|
+
const doFetch = options.fetch || globalThis.fetch;
|
|
249
|
+
if (!owner) throw new BackendError("INVALID_BACKEND", "createAlephAuthorizer needs the owner's address");
|
|
250
|
+
if (typeof sign !== "function") throw new BackendError("INVALID_BACKEND", "createAlephAuthorizer needs a `sign` function");
|
|
251
|
+
if (typeof doFetch !== "function") throw new BackendError("INVALID_BACKEND", "createAlephAuthorizer needs fetch");
|
|
252
|
+
|
|
253
|
+
const same = (a, b) => String(a).toLowerCase() === String(b).toLowerCase();
|
|
254
|
+
|
|
255
|
+
/** @returns {Promise<AlephAuthorization[]>} */
|
|
256
|
+
async function read() {
|
|
257
|
+
const response = await doFetch(`${apiHost}/api/v0/aggregates/${owner}.json?keys=${SECURITY}`);
|
|
258
|
+
if (response.status === 404) return [];
|
|
259
|
+
if (!response.ok) {
|
|
260
|
+
throw new BackendError("UNSUPPORTED", `Aleph did not answer the owner's permissions: ${response.status}`);
|
|
261
|
+
}
|
|
262
|
+
const body = await response.json().catch(() => ({}));
|
|
263
|
+
const list = body?.data?.[SECURITY]?.authorizations;
|
|
264
|
+
return Array.isArray(list) ? list : [];
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** @param {AlephAuthorization[]} authorizations */
|
|
268
|
+
async function write(authorizations) {
|
|
269
|
+
const unsigned = await buildAuthorizationMessage({ owner, authorizations, hasher, now: options.now?.() });
|
|
270
|
+
const signature = await sign(owner, signaturePayload(unsigned));
|
|
271
|
+
const response = await doFetch(`${apiHost}/api/v0/messages`, {
|
|
272
|
+
method: "POST",
|
|
273
|
+
headers: { "content-type": "application/json" },
|
|
274
|
+
body: JSON.stringify({
|
|
275
|
+
message: { ...unsigned, signature: signature.startsWith("0x") ? signature : `0x${signature}` },
|
|
276
|
+
sync: true,
|
|
277
|
+
}),
|
|
278
|
+
});
|
|
279
|
+
if (!response.ok && response.status !== 202) {
|
|
280
|
+
const detail = await response.text().catch(() => "");
|
|
281
|
+
throw new BackendError("UNSUPPORTED", `Aleph refused the authorization: ${response.status} ${detail.slice(0, 200)}`);
|
|
282
|
+
}
|
|
283
|
+
const body = await response.json().catch(() => ({}));
|
|
284
|
+
return { itemHash: unsigned.item_hash, status: body?.message_status ?? "pending", authorizations };
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
return {
|
|
288
|
+
read,
|
|
289
|
+
write,
|
|
290
|
+
/** Grant, or replace the grant of the same address. @param {AlephAuthorization} authorization */
|
|
291
|
+
async authorize(authorization) {
|
|
292
|
+
const others = (await read()).filter((a) => !same(a.address, authorization?.address));
|
|
293
|
+
return write([...others, authorization]);
|
|
294
|
+
},
|
|
295
|
+
/** Take one address's grant away; the others stay. @param {string} address */
|
|
296
|
+
async revoke(address) {
|
|
297
|
+
return write((await read()).filter((a) => !same(a.address, address)));
|
|
298
|
+
},
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
|
|
161
302
|
export default createAlephPin;
|
package/lib/courier-sync.js
CHANGED
|
@@ -127,6 +127,13 @@ const CHANGE_ID_LENGTH = 8;
|
|
|
127
127
|
// How many change ids to remember. A duplicate arriving after this many others
|
|
128
128
|
// is applied twice: one extra entry in the log, not a wrong value.
|
|
129
129
|
const SEEN_OPS = 512;
|
|
130
|
+
// How many operations may wait for a deliberate send before the batch goes as
|
|
131
|
+
// a delta instead. Not a memory limit: the outbox holds `maxOutbox` (32) and
|
|
132
|
+
// sheds the oldest when it overflows, and an operation is the one message here
|
|
133
|
+
// that is *not* re-derivable — so a long batch of them would lose writes
|
|
134
|
+
// quietly. A delta is one message and complete by construction, which is the
|
|
135
|
+
// right shape once a batch is large enough to threaten that.
|
|
136
|
+
const MAX_PENDING_OPS = 16;
|
|
130
137
|
|
|
131
138
|
// Four bytes of sender id: enough that two peers in one conversation collide
|
|
132
139
|
// with probability ~1 in 4 billion, small enough to ride on every message.
|
|
@@ -640,6 +647,10 @@ export async function createCourierSync({
|
|
|
640
647
|
const peers = new Map(); // sender id (hex) -> when we last heard it
|
|
641
648
|
let lastHeardAt = null; // any traffic for this database, identified or not
|
|
642
649
|
const listeners = { synced: [], applied: [], message: [], error: [] };
|
|
650
|
+
// Written locally and not yet sent, oldest first. Only fills when the
|
|
651
|
+
// operation plane is on and the application has opted out of sending on
|
|
652
|
+
// every write — the batching case, where a button decides.
|
|
653
|
+
const pendingOps = [];
|
|
643
654
|
// Change ids already applied, oldest first. Insertion-ordered, so the oldest
|
|
644
655
|
// key is the first one Map iteration yields.
|
|
645
656
|
const seenOps = new Map();
|
|
@@ -847,6 +858,31 @@ export async function createCourierSync({
|
|
|
847
858
|
await send(message);
|
|
848
859
|
};
|
|
849
860
|
|
|
861
|
+
/**
|
|
862
|
+
* Everything written since the last deliberate send, as operations.
|
|
863
|
+
*
|
|
864
|
+
* Answers how many went, so the caller can tell "I sent your changes" from
|
|
865
|
+
* "there was nothing to send, so I asked the peer to reconcile" — which is
|
|
866
|
+
* what an announce means when the queue is empty.
|
|
867
|
+
*
|
|
868
|
+
* Past `MAX_PENDING_OPS` the batch goes as a delta instead. Not arbitrary:
|
|
869
|
+
* the outbox holds `maxOutbox` messages and sheds the oldest, and an
|
|
870
|
+
* operation is the one message here that cannot be re-derived, so a long
|
|
871
|
+
* batch of them would lose writes without saying so. One delta is one
|
|
872
|
+
* message and carries all of it.
|
|
873
|
+
*/
|
|
874
|
+
const flushOperations = async () => {
|
|
875
|
+
if (pendingOps.length === 0) return 0;
|
|
876
|
+
const going = pendingOps.splice(0, pendingOps.length);
|
|
877
|
+
if (going.length > MAX_PENDING_OPS) {
|
|
878
|
+
emit("message", { direction: "out", type: "batch-too-long", bytes: 0 });
|
|
879
|
+
await announce(true);
|
|
880
|
+
return going.length;
|
|
881
|
+
}
|
|
882
|
+
for (const entry of going) await shipOperation(entry);
|
|
883
|
+
return going.length;
|
|
884
|
+
};
|
|
885
|
+
|
|
850
886
|
/**
|
|
851
887
|
* Announce without waiting for delivery — for callers that sit on the
|
|
852
888
|
* receive queue, where waiting for the radio is the head-of-line block this
|
|
@@ -878,13 +914,22 @@ export async function createCourierSync({
|
|
|
878
914
|
};
|
|
879
915
|
|
|
880
916
|
const watchLocalUpdates = () => {
|
|
881
|
-
//
|
|
882
|
-
//
|
|
883
|
-
//
|
|
884
|
-
|
|
917
|
+
// On the delta plane, opting out means not watching at all: the application
|
|
918
|
+
// announces when it decides to. On the operation plane it means something
|
|
919
|
+
// else — the write still has to be *remembered*, or the deliberate send has
|
|
920
|
+
// nothing to send. So the hook goes on either way there, and only what it
|
|
921
|
+
// does with the entry changes.
|
|
922
|
+
//
|
|
923
|
+
// Incoming announces are still answered in both cases: going quiet must not
|
|
924
|
+
// mean going deaf.
|
|
925
|
+
if (!announceOnLocalUpdate && liveUpdates !== "operations") return;
|
|
885
926
|
if (!database || offUpdate) return;
|
|
886
927
|
const onUpdate = (entry) => {
|
|
887
928
|
if (applying) return; // courier-applied entries already end in an announce
|
|
929
|
+
if (liveUpdates === "operations" && !announceOnLocalUpdate) {
|
|
930
|
+
pendingOps.push(entry);
|
|
931
|
+
return; // waits for the application to ask
|
|
932
|
+
}
|
|
888
933
|
queue = queue
|
|
889
934
|
.then(() =>
|
|
890
935
|
liveUpdates === "operations" ? shipOperation(entry) : announceSoon(),
|
|
@@ -1183,7 +1228,18 @@ export async function createCourierSync({
|
|
|
1183
1228
|
}
|
|
1184
1229
|
},
|
|
1185
1230
|
/** Re-announce — recovery poke after suspected loss. */
|
|
1186
|
-
|
|
1231
|
+
/**
|
|
1232
|
+
* What an application's "send" button means.
|
|
1233
|
+
*
|
|
1234
|
+
* With changes waiting, it sends those — that is what the press was about.
|
|
1235
|
+
* With none, it is a request to reconcile, which is what it always was.
|
|
1236
|
+
* Internal announces never flush: the acknowledgement that ends a blocks
|
|
1237
|
+
* delivery must not spend somebody's queued writes on its own.
|
|
1238
|
+
*/
|
|
1239
|
+
announce: async () => {
|
|
1240
|
+
if ((await flushOperations()) > 0) return;
|
|
1241
|
+
await announce(true);
|
|
1242
|
+
},
|
|
1187
1243
|
/** This instance's sender id, as it appears on the wire. */
|
|
1188
1244
|
peerId: hex(peerId),
|
|
1189
1245
|
/**
|
package/lib/extract-blocks.js
CHANGED
|
@@ -256,3 +256,76 @@ export async function extractDatabaseBlocks(database, options = {}) {
|
|
|
256
256
|
logger.info(` 📊 Extracted ${blocks.size} total blocks`);
|
|
257
257
|
return { blocks, blockSources, manifestCID };
|
|
258
258
|
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Several databases in one backup: every block of each, in one Map, and the
|
|
262
|
+
* metadata that names them all.
|
|
263
|
+
*
|
|
264
|
+
* The metadata is the shape `backupDatabaseCAR` writes and `isValidMetadata`
|
|
265
|
+
* accepts — `databases` was always a list, and here it finally holds more than
|
|
266
|
+
* one. Each entry also names the database's **heads**: `restoreFromBlocks` can
|
|
267
|
+
* find them from the blocks alone, but a stated head says which entries the
|
|
268
|
+
* writer considered current when the backup ran, and costs a few bytes.
|
|
269
|
+
*
|
|
270
|
+
* Blocks two databases share — a writer's identity — are kept once: the Map is
|
|
271
|
+
* keyed by the CID string OrbitDB uses, and the same block has the same name.
|
|
272
|
+
*
|
|
273
|
+
* @param {Object[]|Object<string, Object>} databases opened OrbitDB databases,
|
|
274
|
+
* as a list or by name
|
|
275
|
+
* @param {Object} [options]
|
|
276
|
+
* @param {(progress: { stage: "database", index: number, total: number,
|
|
277
|
+
* name: string, address: string, entries: number, blocks: number }) => void}
|
|
278
|
+
* [options.onProgress] called after each database, with what it added
|
|
279
|
+
* @param {number} [options.timestamp] ms since the epoch; injected for tests
|
|
280
|
+
* @returns {Promise<{ blocks: Map<string, { cid: CID, bytes: Uint8Array }>,
|
|
281
|
+
* metadata: Object }>}
|
|
282
|
+
*/
|
|
283
|
+
export async function bundleDatabases(databases, options = {}) {
|
|
284
|
+
const list = Array.isArray(databases) ? databases : Object.values(databases ?? {});
|
|
285
|
+
if (list.length === 0) throw new Error("bundleDatabases needs at least one database");
|
|
286
|
+
|
|
287
|
+
const blocks = new Map();
|
|
288
|
+
const described = [];
|
|
289
|
+
let totalEntries = 0;
|
|
290
|
+
|
|
291
|
+
for (const [index, database] of list.entries()) {
|
|
292
|
+
const before = blocks.size;
|
|
293
|
+
const { blocks: own, manifestCID } = await extractDatabaseBlocks(database);
|
|
294
|
+
for (const [name, block] of own) if (!blocks.has(name)) blocks.set(name, block);
|
|
295
|
+
|
|
296
|
+
const entries = (await database.log.values()).length;
|
|
297
|
+
const heads = (await database.log.heads()).map((head) => head.hash);
|
|
298
|
+
totalEntries += entries;
|
|
299
|
+
described.push({
|
|
300
|
+
address: database.address,
|
|
301
|
+
name: database.name,
|
|
302
|
+
type: database.type,
|
|
303
|
+
manifestCID,
|
|
304
|
+
entryCount: entries,
|
|
305
|
+
heads,
|
|
306
|
+
});
|
|
307
|
+
options.onProgress?.({
|
|
308
|
+
stage: "database",
|
|
309
|
+
index,
|
|
310
|
+
total: list.length,
|
|
311
|
+
name: database.name,
|
|
312
|
+
address: database.address,
|
|
313
|
+
entries,
|
|
314
|
+
blocks: blocks.size - before,
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
return {
|
|
319
|
+
blocks,
|
|
320
|
+
metadata: {
|
|
321
|
+
version: "1.0",
|
|
322
|
+
timestamp: options.timestamp ?? Date.now(),
|
|
323
|
+
databaseCount: described.length,
|
|
324
|
+
totalBlocks: blocks.size,
|
|
325
|
+
totalEntries,
|
|
326
|
+
// The first database's, so a reader of the one-database shape still finds one.
|
|
327
|
+
manifestCID: described[0].manifestCID,
|
|
328
|
+
databases: described,
|
|
329
|
+
},
|
|
330
|
+
};
|
|
331
|
+
}
|
package/lib/restore-cid.js
CHANGED
|
@@ -157,7 +157,8 @@ async function headsIn(blocks) {
|
|
|
157
157
|
}
|
|
158
158
|
|
|
159
159
|
/**
|
|
160
|
-
* Restore
|
|
160
|
+
* Restore from the CID of a CAR backup's metadata: every database it names
|
|
161
|
+
* (`restoreFromBlocks` does the work once the blocks are fetched).
|
|
161
162
|
*
|
|
162
163
|
* Everything hangs off that one CID: the metadata names the CAR, the CAR holds
|
|
163
164
|
* the blocks, and the blocks carry their own addresses. Nothing here talks to
|
|
@@ -191,7 +192,9 @@ async function headsIn(blocks) {
|
|
|
191
192
|
* OrbitDB's Sync subscribes on open, and on a libp2p built without a pubsub
|
|
192
193
|
* service that throws before the database is ever handed back.
|
|
193
194
|
* @returns {Promise<{ address: string, database: Object, blocks: number,
|
|
194
|
-
* entries: number, heads: number, joined: number }>}
|
|
195
|
+
* entries: number, heads: number, joined: number, databases: Object[] }>}
|
|
196
|
+
* the first database's at the top, as a one-database backup always had it;
|
|
197
|
+
* `databases` lists them all
|
|
195
198
|
*/
|
|
196
199
|
export async function restoreFromCID(orbitdb, options = {}) {
|
|
197
200
|
const { metadataCID, fetchBytes = fetchFromGateways, log = SILENT, verify = true, open = {}, ...rest } = options;
|
|
@@ -216,40 +219,97 @@ export async function restoreFromCID(orbitdb, options = {}) {
|
|
|
216
219
|
// address to open it into wastes the whole transfer.
|
|
217
220
|
const carCID = metadata.carCID;
|
|
218
221
|
if (!carCID) throw new Error("Backup metadata names no CAR file");
|
|
219
|
-
|
|
220
|
-
const dbInfo = metadata.databases?.[0];
|
|
221
|
-
if (!dbInfo?.address) {
|
|
222
|
-
// `isValidMetadata` also accepts the older `{ root, path }` shape, which
|
|
223
|
-
// predates CAR backups and has no address to open. Say which it is.
|
|
224
|
-
throw new Error(
|
|
225
|
-
dbInfo ? "This is a pre-CAR backup; restoreFromCID needs a CAR backup" : "Backup metadata names no database",
|
|
226
|
-
);
|
|
227
|
-
}
|
|
222
|
+
assertRestorable(metadata);
|
|
228
223
|
|
|
229
224
|
// 2 · the CAR holds the blocks, and each one is checked against its own CID
|
|
230
225
|
const blocks = await readBlocksFromCAR(await fetchBytes(carCID, { ...rest, log }), { verify });
|
|
231
226
|
log.info(` ✅ ${blocks.size} blocks`);
|
|
232
227
|
|
|
228
|
+
// 3 · every database the metadata names
|
|
229
|
+
const restored = await restoreFromBlocks(orbitdb, blocks, metadata, { log, verify, open });
|
|
230
|
+
const [first] = restored.databases;
|
|
231
|
+
|
|
232
|
+
// The one-database answer this always gave, and the whole list beside it.
|
|
233
|
+
return {
|
|
234
|
+
address: first.address,
|
|
235
|
+
database: first.database,
|
|
236
|
+
blocks: restored.blocks,
|
|
237
|
+
entries: metadata.totalEntries ?? first.entries,
|
|
238
|
+
heads: first.heads,
|
|
239
|
+
joined: first.joined,
|
|
240
|
+
databases: restored.databases,
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Refuse metadata that names nothing a restore could open, and say which kind
|
|
246
|
+
* it is. `isValidMetadata` also accepts the older `{ root, path }` shape, which
|
|
247
|
+
* predates CAR backups and has no address to open.
|
|
248
|
+
*/
|
|
249
|
+
function assertRestorable(metadata) {
|
|
250
|
+
const dbs = metadata.databases ?? [];
|
|
251
|
+
if (dbs.length === 0) throw new Error("Backup metadata names no database");
|
|
252
|
+
if (!dbs.every((db) => db?.address)) {
|
|
253
|
+
throw new Error("This is a pre-CAR backup; restoreFromCID needs a CAR backup");
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Restore every database a backup names, from blocks already in hand.
|
|
259
|
+
*
|
|
260
|
+
* `restoreFromCID` fetches and then calls this. An application that holds the
|
|
261
|
+
* blocks some other way — a backup it sealed itself and has just opened, a file
|
|
262
|
+
* on a USB stick, several small CARs written as it went — calls it directly.
|
|
263
|
+
* The blocks are checked against the metadata, never trusted for it: each
|
|
264
|
+
* database's manifest must be among them.
|
|
265
|
+
*
|
|
266
|
+
* Per database: the blocks go into the blockstore (once, for all of them) and
|
|
267
|
+
* into that database's own log storage (its dag-cbor blocks only — a receipt
|
|
268
|
+
* file's raw chunks are no log's business); the database is reopened so the
|
|
269
|
+
* log reads what was put underneath it; and its heads are joined. The heads
|
|
270
|
+
* are the ones the metadata states, or — for metadata that states none — the
|
|
271
|
+
* entries of this log nothing else points back to.
|
|
272
|
+
*
|
|
273
|
+
* **Each returned `database` replaces any handle the caller held on that
|
|
274
|
+
* address**, for the reason `restoreFromCID` gives.
|
|
275
|
+
*
|
|
276
|
+
* @param {Object} orbitdb an OrbitDB instance to restore into
|
|
277
|
+
* @param {Map<string, { bytes: Uint8Array }>} blocks by CID string, in any base —
|
|
278
|
+
* what `readBlocksFromCAR` returns
|
|
279
|
+
* @param {Object} metadata as `bundleDatabases` or `backupDatabaseCAR` write it
|
|
280
|
+
* @param {Object} [options]
|
|
281
|
+
* @param {boolean} [options.verify=true] each database's manifest must be among the blocks
|
|
282
|
+
* @param {{ info: Function, warn: Function, debug: Function }} [options.log]
|
|
283
|
+
* @param {Object} [options.open] options for `orbitdb.open`, merged over the type
|
|
284
|
+
* the backup names (a node without pubsub needs `{ sync: false }`)
|
|
285
|
+
* @param {(progress: { stage: "database", index: number, total: number,
|
|
286
|
+
* address: string, joined: number, heads: number }) => void} [options.onProgress]
|
|
287
|
+
* @returns {Promise<{ blocks: number, databases: { address: string, name?: string,
|
|
288
|
+
* database: Object, entries: number|null, heads: number, joined: number }[] }>}
|
|
289
|
+
*/
|
|
290
|
+
export async function restoreFromBlocks(orbitdb, blocks, metadata, options = {}) {
|
|
291
|
+
const { log = SILENT, verify = true, open = {}, onProgress } = options;
|
|
292
|
+
if (!orbitdb?.ipfs?.blockstore) throw new Error("An OrbitDB instance is required");
|
|
293
|
+
if (!(blocks instanceof Map)) throw new Error("blocks must be a Map");
|
|
294
|
+
if (!isValidMetadata(metadata)) throw new Error("Invalid backup metadata");
|
|
295
|
+
assertRestorable(metadata);
|
|
296
|
+
|
|
233
297
|
// The blocks verify individually; this is what ties them to *this* backup.
|
|
234
298
|
// Without it a CAR full of perfectly valid blocks from some other database
|
|
235
299
|
// would restore happily, because every block would be honest about itself.
|
|
236
|
-
const
|
|
237
|
-
if (verify
|
|
238
|
-
const
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
300
|
+
const present = new Set([...blocks.keys()].map(canonical));
|
|
301
|
+
if (verify) {
|
|
302
|
+
for (const db of metadata.databases) {
|
|
303
|
+
const manifestCID = db.manifestCID ?? metadata.manifestCID;
|
|
304
|
+
if (!manifestCID) continue;
|
|
305
|
+
const wanted = canonical(manifestCID);
|
|
306
|
+
if (!wanted || !present.has(wanted)) {
|
|
307
|
+
throw new Error(`The CAR does not contain the manifest ${manifestCID} that this backup names`);
|
|
308
|
+
}
|
|
244
309
|
}
|
|
245
310
|
}
|
|
246
311
|
|
|
247
|
-
//
|
|
248
|
-
//
|
|
249
|
-
// Both, and not by accident: the blockstore is where Helia looks, while
|
|
250
|
-
// OrbitDB's log reads from its own store and addresses blocks in base58btc
|
|
251
|
-
// rather than the CAR's base32. A restore that fills only one of them opens
|
|
252
|
-
// a database that is empty in a way nothing reports.
|
|
312
|
+
// Into the blockstore, where Helia looks — once, whatever the databases.
|
|
253
313
|
let stored = 0;
|
|
254
314
|
for (const [cidString, { bytes }] of blocks) {
|
|
255
315
|
try {
|
|
@@ -260,43 +320,75 @@ export async function restoreFromCID(orbitdb, options = {}) {
|
|
|
260
320
|
}
|
|
261
321
|
}
|
|
262
322
|
|
|
263
|
-
const
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
323
|
+
const found = metadata.databases.some((db) => !Array.isArray(db.heads)) ? await headsIn(blocks) : [];
|
|
324
|
+
const byCanonical = new Map([...blocks].map(([name, block]) => [canonical(name), block]));
|
|
325
|
+
|
|
326
|
+
const databases = [];
|
|
327
|
+
for (const [index, dbInfo] of metadata.databases.entries()) {
|
|
328
|
+
const opened = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
329
|
+
// OrbitDB's log reads from its own store and addresses blocks in base58btc
|
|
330
|
+
// rather than the CAR's base32. A restore that fills only the blockstore
|
|
331
|
+
// opens a database that is empty in a way nothing reports.
|
|
332
|
+
for (const [cidString, { bytes }] of blocks) {
|
|
333
|
+
try {
|
|
334
|
+
const cid = CID.parse(cidString);
|
|
335
|
+
if (cid.code !== DAG_CBOR) continue;
|
|
336
|
+
await opened.log.storage.put(cid.toV1().toString(base58btc), bytes);
|
|
337
|
+
} catch (error) {
|
|
338
|
+
log.warn(` ⚠️ could not copy ${cidString.slice(0, 12)}… to log storage: ${error.message}`);
|
|
339
|
+
}
|
|
269
340
|
}
|
|
270
|
-
}
|
|
271
341
|
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
342
|
+
// Reopened so the log reads what we just put underneath it.
|
|
343
|
+
await opened.close();
|
|
344
|
+
const database = await orbitdb.open(dbInfo.address, { type: dbInfo.type, ...open });
|
|
275
345
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
346
|
+
// Tell the log where its history ends.
|
|
347
|
+
const heads = Array.isArray(dbInfo.heads)
|
|
348
|
+
? (await Promise.all(dbInfo.heads.map((hash) => headFrom(byCanonical, hash)))).filter(Boolean)
|
|
349
|
+
: found.filter((head) => head.value.id === dbInfo.address || metadata.databases.length === 1);
|
|
350
|
+
let joined = 0;
|
|
351
|
+
for (const head of heads) {
|
|
352
|
+
try {
|
|
353
|
+
const { v, id, key, sig, next, refs, clock, payload, identity } = head.value;
|
|
354
|
+
if (await database.log.joinEntry({ hash: head.hash, v, id, key, sig, next, refs, clock, payload, identity })) {
|
|
355
|
+
joined++;
|
|
356
|
+
}
|
|
357
|
+
} catch (error) {
|
|
358
|
+
log.warn(` ⚠️ could not join head ${head.hash.slice(0, 12)}…: ${error.message}`);
|
|
284
359
|
}
|
|
285
|
-
} catch (error) {
|
|
286
|
-
log.warn(` ⚠️ could not join head ${head.hash.slice(0, 12)}…: ${error.message}`);
|
|
287
360
|
}
|
|
361
|
+
|
|
362
|
+
log.info(`✅ Restored ${dbInfo.address} — ${joined}/${heads.length} heads`);
|
|
363
|
+
onProgress?.({
|
|
364
|
+
stage: "database",
|
|
365
|
+
index,
|
|
366
|
+
total: metadata.databases.length,
|
|
367
|
+
address: dbInfo.address,
|
|
368
|
+
joined,
|
|
369
|
+
heads: heads.length,
|
|
370
|
+
});
|
|
371
|
+
databases.push({
|
|
372
|
+
address: dbInfo.address,
|
|
373
|
+
name: dbInfo.name,
|
|
374
|
+
database,
|
|
375
|
+
entries: dbInfo.entryCount ?? null,
|
|
376
|
+
heads: heads.length,
|
|
377
|
+
joined,
|
|
378
|
+
});
|
|
288
379
|
}
|
|
289
380
|
|
|
290
|
-
|
|
381
|
+
return { blocks: stored, databases };
|
|
382
|
+
}
|
|
291
383
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
};
|
|
384
|
+
/** A stated head, decoded from the blocks; null when the blocks lack it. */
|
|
385
|
+
async function headFrom(byCanonical, hash) {
|
|
386
|
+
const name = canonical(hash);
|
|
387
|
+
const block = name ? byCanonical.get(name) : undefined;
|
|
388
|
+
if (!block) return null;
|
|
389
|
+
const cid = CID.parse(name);
|
|
390
|
+
const { value } = await Block.decode({ cid, bytes: block.bytes, codec: dagCbor, hasher: sha256 });
|
|
391
|
+
return { hash: cid.toV1().toString(base58btc), value };
|
|
300
392
|
}
|
|
301
393
|
|
|
302
|
-
export default { restoreFromCID, fetchFromGateways, readBlocksFromCAR, DEFAULT_GATEWAYS, SILENT };
|
|
394
|
+
export default { restoreFromCID, restoreFromBlocks, fetchFromGateways, readBlocksFromCAR, DEFAULT_GATEWAYS, SILENT };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@le-space/orbitdb-storage-bridge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0",
|
|
4
4
|
"description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
|
|
5
5
|
"main": "lib/orbitdb-storacha-bridge.js",
|
|
6
6
|
"svelte": "dist/components/",
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
"./pointer-ipns": "./lib/pointer-ipns.js",
|
|
16
16
|
"./dehydrate": "./lib/dehydrate.js",
|
|
17
17
|
"./restore-cid": "./lib/restore-cid.js",
|
|
18
|
+
"./extract-blocks": "./lib/extract-blocks.js",
|
|
18
19
|
"./gateway-fetch": "./lib/gateway-fetch.js",
|
|
19
20
|
"./peer-fetch": "./lib/peer-fetch.js",
|
|
20
21
|
"./backends/types": "./lib/backends/types.js",
|