@le-space/orbitdb-storage-bridge 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/lib/backends/aleph-pin.js +355 -27
- package/lib/live-backup.js +734 -0
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -84,6 +84,7 @@ Galaxy A57 with the same key brought the database back and wrote to it. Source:
|
|
|
84
84
|
| [docs/RECOVERY-ON-A-SECOND-DEVICE.md](docs/RECOVERY-ON-A-SECOND-DEVICE.md) | the passkey recovery procedure, step by step |
|
|
85
85
|
| [docs/CAR-BACKUP.md](docs/CAR-BACKUP.md) | CAR-based timestamped backups |
|
|
86
86
|
| [docs/APP-BACKUP.md](docs/APP-BACKUP.md) | an application's backup in one sealed file: several databases, a keyring header, kept on Aleph from the browser, restored by merging |
|
|
87
|
+
| [docs/LIVE-BACKUP.md](docs/LIVE-BACKUP.md) | increments between two full backups: what was written since the last one, shipped as one small CAR, with nobody clicking and no peer online |
|
|
87
88
|
| [SVELTE-COMPONENTS.md](SVELTE-COMPONENTS.md) | the browser components — older than this repository's copy of them; [docs/STORACHA-UI-HISTORY.md](docs/STORACHA-UI-HISTORY.md) records what each piece could do and where it went |
|
|
88
89
|
| [docs/LOGGING.md](docs/LOGGING.md) | debug namespaces, in Node and in a browser |
|
|
89
90
|
| [ROADMAP.md](ROADMAP.md) | what is planned |
|
|
@@ -77,7 +77,9 @@ async function sha256Hex(payload) {
|
|
|
77
77
|
"SHA-256",
|
|
78
78
|
new TextEncoder().encode(payload),
|
|
79
79
|
);
|
|
80
|
-
return [...new Uint8Array(digest)]
|
|
80
|
+
return [...new Uint8Array(digest)]
|
|
81
|
+
.map((b) => b.toString(16).padStart(2, "0"))
|
|
82
|
+
.join("");
|
|
81
83
|
}
|
|
82
84
|
|
|
83
85
|
/**
|
|
@@ -173,12 +175,29 @@ export function createAlephPin(options = {}) {
|
|
|
173
175
|
const doFetch = options.fetch || globalThis.fetch;
|
|
174
176
|
const now = options.now;
|
|
175
177
|
|
|
176
|
-
if (!sender)
|
|
177
|
-
|
|
178
|
-
|
|
178
|
+
if (!sender)
|
|
179
|
+
throw new BackendError(
|
|
180
|
+
"INVALID_BACKEND",
|
|
181
|
+
"createAlephPin needs the wallet address as `sender`",
|
|
182
|
+
);
|
|
183
|
+
if (typeof sign !== "function")
|
|
184
|
+
throw new BackendError(
|
|
185
|
+
"INVALID_BACKEND",
|
|
186
|
+
"createAlephPin needs a `sign` function",
|
|
187
|
+
);
|
|
188
|
+
if (typeof doFetch !== "function")
|
|
189
|
+
throw new BackendError("INVALID_BACKEND", "createAlephPin needs fetch");
|
|
179
190
|
|
|
180
191
|
return async function pin(cid) {
|
|
181
|
-
const unsigned = await buildStoreMessage({
|
|
192
|
+
const unsigned = await buildStoreMessage({
|
|
193
|
+
sender,
|
|
194
|
+
owner,
|
|
195
|
+
cid,
|
|
196
|
+
channel,
|
|
197
|
+
payment,
|
|
198
|
+
hasher,
|
|
199
|
+
now: now?.(),
|
|
200
|
+
});
|
|
182
201
|
const signature = await sign(sender, signaturePayload(unsigned));
|
|
183
202
|
const message = {
|
|
184
203
|
...unsigned,
|
|
@@ -212,7 +231,13 @@ export function createAlephPin(options = {}) {
|
|
|
212
231
|
}
|
|
213
232
|
|
|
214
233
|
/** Statuses after which a message no longer changes by itself. */
|
|
215
|
-
const SETTLED = new Set([
|
|
234
|
+
const SETTLED = new Set([
|
|
235
|
+
"processed",
|
|
236
|
+
"rejected",
|
|
237
|
+
"removing",
|
|
238
|
+
"removed",
|
|
239
|
+
"forgotten",
|
|
240
|
+
]);
|
|
216
241
|
|
|
217
242
|
/**
|
|
218
243
|
* Follow a message until Aleph has decided about it.
|
|
@@ -241,9 +266,16 @@ export async function waitForMessage(itemHash, options = {}) {
|
|
|
241
266
|
const doFetch = options.fetch || globalThis.fetch;
|
|
242
267
|
const timeout = options.timeout ?? 60_000;
|
|
243
268
|
const interval = options.interval ?? 2_000;
|
|
244
|
-
const sleep =
|
|
245
|
-
|
|
246
|
-
|
|
269
|
+
const sleep =
|
|
270
|
+
options.sleep ||
|
|
271
|
+
((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
272
|
+
if (!itemHash)
|
|
273
|
+
throw new BackendError(
|
|
274
|
+
"INVALID_BACKEND",
|
|
275
|
+
"waitForMessage needs the message's item hash",
|
|
276
|
+
);
|
|
277
|
+
if (typeof doFetch !== "function")
|
|
278
|
+
throw new BackendError("INVALID_BACKEND", "waitForMessage needs fetch");
|
|
247
279
|
|
|
248
280
|
// Counted in asks rather than read off a clock, so an injected `sleep` is enough for a test.
|
|
249
281
|
const asks = Math.max(1, Math.floor(timeout / interval) + 1);
|
|
@@ -297,8 +329,13 @@ export async function listAlephStores(options = {}) {
|
|
|
297
329
|
const { owner, channel } = options;
|
|
298
330
|
const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
|
|
299
331
|
const doFetch = options.fetch || globalThis.fetch;
|
|
300
|
-
if (!owner)
|
|
301
|
-
|
|
332
|
+
if (!owner)
|
|
333
|
+
throw new BackendError(
|
|
334
|
+
"INVALID_BACKEND",
|
|
335
|
+
"listAlephStores needs the paying account's address as `owner`",
|
|
336
|
+
);
|
|
337
|
+
if (typeof doFetch !== "function")
|
|
338
|
+
throw new BackendError("INVALID_BACKEND", "listAlephStores needs fetch");
|
|
302
339
|
|
|
303
340
|
const url = new URL(`${apiHost}/api/v0/messages.json`);
|
|
304
341
|
url.searchParams.set("owners", owner);
|
|
@@ -309,7 +346,10 @@ export async function listAlephStores(options = {}) {
|
|
|
309
346
|
|
|
310
347
|
const response = await doFetch(url.toString());
|
|
311
348
|
if (!response.ok) {
|
|
312
|
-
throw new BackendError(
|
|
349
|
+
throw new BackendError(
|
|
350
|
+
"UNSUPPORTED",
|
|
351
|
+
`Aleph did not list the STORE messages: ${response.status}`,
|
|
352
|
+
);
|
|
313
353
|
}
|
|
314
354
|
const body = await response.json().catch(() => ({}));
|
|
315
355
|
const messages = Array.isArray(body?.messages) ? body.messages : [];
|
|
@@ -328,7 +368,246 @@ export async function listAlephStores(options = {}) {
|
|
|
328
368
|
// and keeps a misread filter from handing back someone else's files.
|
|
329
369
|
.filter((store) => store.cid && store.itemHash && same(store.owner, owner))
|
|
330
370
|
.sort((a, b) => b.time - a.time);
|
|
331
|
-
return {
|
|
371
|
+
return {
|
|
372
|
+
stores,
|
|
373
|
+
total: Number.isFinite(body?.pagination_total)
|
|
374
|
+
? body.pagination_total
|
|
375
|
+
: null,
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Build the unsigned FORGET that releases what STORE messages keep.
|
|
381
|
+
*
|
|
382
|
+
* Aleph keeps a file for as long as a STORE names it; a FORGET from the
|
|
383
|
+
* STORE's owner (or a delegate it authorized for `FORGET`) names those STOREs
|
|
384
|
+
* by their item hashes, and the files they kept are unpinned and stop
|
|
385
|
+
* costing credits. Only messages of the same account can be forgotten, and a
|
|
386
|
+
* forgotten message cannot be brought back — the file has to be stored again.
|
|
387
|
+
*
|
|
388
|
+
* @param {object} args
|
|
389
|
+
* @param {string} args.sender - the address that signs
|
|
390
|
+
* @param {string} [args.owner] - the account whose messages these are, when `sender` is its delegate
|
|
391
|
+
* @param {string[]} args.hashes - item hashes of the STORE messages to forget
|
|
392
|
+
* @param {string} [args.reason] - kept with the FORGET, for whoever reads the account's history
|
|
393
|
+
* @param {string} [args.channel]
|
|
394
|
+
* @param {number} [args.now] - seconds since the epoch; injected for tests
|
|
395
|
+
* @param {(payload: string) => Promise<string>} [args.hasher]
|
|
396
|
+
*/
|
|
397
|
+
export async function buildForgetMessage({
|
|
398
|
+
sender,
|
|
399
|
+
owner,
|
|
400
|
+
hashes,
|
|
401
|
+
reason,
|
|
402
|
+
channel = DEFAULT_ALEPH_CHANNEL,
|
|
403
|
+
now,
|
|
404
|
+
hasher = sha256Hex,
|
|
405
|
+
}) {
|
|
406
|
+
if (
|
|
407
|
+
!Array.isArray(hashes) ||
|
|
408
|
+
hashes.length === 0 ||
|
|
409
|
+
!hashes.every((h) => typeof h === "string" && h)
|
|
410
|
+
) {
|
|
411
|
+
throw new BackendError(
|
|
412
|
+
"INVALID_BACKEND",
|
|
413
|
+
"a FORGET names at least one item hash",
|
|
414
|
+
);
|
|
415
|
+
}
|
|
416
|
+
const time = now ?? Date.now() / 1000;
|
|
417
|
+
const content = {
|
|
418
|
+
address: owner || sender,
|
|
419
|
+
time,
|
|
420
|
+
hashes: [...hashes],
|
|
421
|
+
...(reason ? { reason } : {}),
|
|
422
|
+
};
|
|
423
|
+
const item_content = JSON.stringify(content);
|
|
424
|
+
return {
|
|
425
|
+
sender,
|
|
426
|
+
chain: "ETH",
|
|
427
|
+
type: "FORGET",
|
|
428
|
+
item_hash: await hasher(item_content),
|
|
429
|
+
item_type: "inline",
|
|
430
|
+
item_content,
|
|
431
|
+
time,
|
|
432
|
+
channel,
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
/**
|
|
437
|
+
* Forget STORE messages: a signed FORGET, posted, and Aleph's first answer.
|
|
438
|
+
*
|
|
439
|
+
* The same shape as {@link createAlephPin}: the library never sees a key, and
|
|
440
|
+
* `pending` is what Aleph says before it has checked the signature and the
|
|
441
|
+
* ownership — follow it with {@link waitForMessage}. A FORGET that names a
|
|
442
|
+
* message of another account is `rejected` there.
|
|
443
|
+
*
|
|
444
|
+
* @param {object} options
|
|
445
|
+
* @param {string} options.sender
|
|
446
|
+
* @param {string} [options.owner]
|
|
447
|
+
* @param {(address: string, message: string) => Promise<string>} options.sign - `personal_sign`
|
|
448
|
+
* @param {string} [options.apiHost]
|
|
449
|
+
* @param {string} [options.channel]
|
|
450
|
+
* @param {(payload: string) => Promise<string>} [options.hasher]
|
|
451
|
+
* @param {typeof fetch} [options.fetch]
|
|
452
|
+
* @param {() => number} [options.now]
|
|
453
|
+
* @returns {(hashes: string[], reason?: string) => Promise<{ itemHash: string, status: string }>}
|
|
454
|
+
*/
|
|
455
|
+
export function createAlephForget(options = {}) {
|
|
456
|
+
const { sender, sign, owner } = options;
|
|
457
|
+
const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
|
|
458
|
+
const channel = options.channel || DEFAULT_ALEPH_CHANNEL;
|
|
459
|
+
const hasher = options.hasher || sha256Hex;
|
|
460
|
+
const doFetch = options.fetch || globalThis.fetch;
|
|
461
|
+
const now = options.now;
|
|
462
|
+
|
|
463
|
+
if (!sender)
|
|
464
|
+
throw new BackendError(
|
|
465
|
+
"INVALID_BACKEND",
|
|
466
|
+
"createAlephForget needs the wallet address as `sender`",
|
|
467
|
+
);
|
|
468
|
+
if (typeof sign !== "function")
|
|
469
|
+
throw new BackendError(
|
|
470
|
+
"INVALID_BACKEND",
|
|
471
|
+
"createAlephForget needs a `sign` function",
|
|
472
|
+
);
|
|
473
|
+
if (typeof doFetch !== "function")
|
|
474
|
+
throw new BackendError("INVALID_BACKEND", "createAlephForget needs fetch");
|
|
475
|
+
|
|
476
|
+
return async function forget(hashes, reason) {
|
|
477
|
+
const unsigned = await buildForgetMessage({
|
|
478
|
+
sender,
|
|
479
|
+
owner,
|
|
480
|
+
hashes,
|
|
481
|
+
reason,
|
|
482
|
+
channel,
|
|
483
|
+
hasher,
|
|
484
|
+
now: now?.(),
|
|
485
|
+
});
|
|
486
|
+
const signature = await sign(sender, signaturePayload(unsigned));
|
|
487
|
+
const message = {
|
|
488
|
+
...unsigned,
|
|
489
|
+
signature: signature.startsWith("0x") ? signature : `0x${signature}`,
|
|
490
|
+
};
|
|
491
|
+
const response = await doFetch(`${apiHost}/api/v0/messages`, {
|
|
492
|
+
method: "POST",
|
|
493
|
+
headers: { "content-type": "application/json" },
|
|
494
|
+
body: JSON.stringify({ message, sync: true }),
|
|
495
|
+
});
|
|
496
|
+
if (!response.ok && response.status !== 202) {
|
|
497
|
+
const detail = await response.text().catch(() => "");
|
|
498
|
+
throw new BackendError(
|
|
499
|
+
"UNSUPPORTED",
|
|
500
|
+
`Aleph refused the FORGET message: ${response.status} ${detail.slice(0, 200)}`,
|
|
501
|
+
);
|
|
502
|
+
}
|
|
503
|
+
const body = await response.json().catch(() => ({}));
|
|
504
|
+
return {
|
|
505
|
+
itemHash: unsigned.item_hash,
|
|
506
|
+
status: body?.message_status ?? "pending",
|
|
507
|
+
};
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Forget every STORE an account keeps on a channel — an application's whole
|
|
513
|
+
* archive — in batches, each followed until Aleph has decided.
|
|
514
|
+
*
|
|
515
|
+
* `keep` spares some: a predicate over the listed stores, e.g. the newest full
|
|
516
|
+
* backup. What was forgotten, what was kept and what Aleph rejected come back
|
|
517
|
+
* by item hash, so the caller can say so.
|
|
518
|
+
*
|
|
519
|
+
* @param {object} options
|
|
520
|
+
* @param {string} options.owner - the account the STOREs are for
|
|
521
|
+
* @param {string} options.channel - one channel; a FORGET across channels is not what anyone means
|
|
522
|
+
* @param {ReturnType<typeof createAlephForget>} options.forget - from {@link createAlephForget}, signing for that account
|
|
523
|
+
* @param {(store: { cid: string, itemHash: string, time: number }) => boolean} [options.keep]
|
|
524
|
+
* @param {string} [options.reason]
|
|
525
|
+
* @param {number} [options.batch=50] - item hashes per FORGET message
|
|
526
|
+
* @param {string} [options.apiHost]
|
|
527
|
+
* @param {typeof fetch} [options.fetch]
|
|
528
|
+
* @param {{ timeout?: number, interval?: number }} [options.settle] - for {@link waitForMessage}
|
|
529
|
+
* @param {(progress: { done: number, total: number }) => void} [options.onProgress]
|
|
530
|
+
* @returns {Promise<{ forgotten: string[], kept: string[], rejected: { itemHash: string, errorCode?: string }[] }>}
|
|
531
|
+
*/
|
|
532
|
+
export async function forgetAlephStores(options = {}) {
|
|
533
|
+
const {
|
|
534
|
+
owner,
|
|
535
|
+
channel,
|
|
536
|
+
forget,
|
|
537
|
+
keep,
|
|
538
|
+
reason,
|
|
539
|
+
batch = 50,
|
|
540
|
+
apiHost,
|
|
541
|
+
settle = {},
|
|
542
|
+
onProgress,
|
|
543
|
+
} = options;
|
|
544
|
+
if (!owner)
|
|
545
|
+
throw new BackendError(
|
|
546
|
+
"INVALID_BACKEND",
|
|
547
|
+
"forgetAlephStores needs the account's address as `owner`",
|
|
548
|
+
);
|
|
549
|
+
if (!channel)
|
|
550
|
+
throw new BackendError(
|
|
551
|
+
"INVALID_BACKEND",
|
|
552
|
+
"forgetAlephStores needs a `channel`",
|
|
553
|
+
);
|
|
554
|
+
if (typeof forget !== "function")
|
|
555
|
+
throw new BackendError(
|
|
556
|
+
"INVALID_BACKEND",
|
|
557
|
+
"forgetAlephStores needs `forget` from createAlephForget",
|
|
558
|
+
);
|
|
559
|
+
const doFetch = options.fetch;
|
|
560
|
+
|
|
561
|
+
/** @type {{ cid: string, itemHash: string, time: number }[]} */
|
|
562
|
+
const all = [];
|
|
563
|
+
for (let page = 1; ; page++) {
|
|
564
|
+
const { stores } = await listAlephStores({
|
|
565
|
+
owner,
|
|
566
|
+
channel,
|
|
567
|
+
apiHost,
|
|
568
|
+
fetch: doFetch,
|
|
569
|
+
pagination: 100,
|
|
570
|
+
page,
|
|
571
|
+
});
|
|
572
|
+
all.push(...stores);
|
|
573
|
+
if (stores.length < 100) break;
|
|
574
|
+
}
|
|
575
|
+
const kept = all
|
|
576
|
+
.filter((store) => keep?.(store))
|
|
577
|
+
.map((store) => store.itemHash);
|
|
578
|
+
const targets = all
|
|
579
|
+
.filter((store) => !kept.includes(store.itemHash))
|
|
580
|
+
.map((store) => store.itemHash);
|
|
581
|
+
|
|
582
|
+
const forgotten = [];
|
|
583
|
+
const rejected = [];
|
|
584
|
+
for (let i = 0; i < targets.length; i += batch) {
|
|
585
|
+
const hashes = targets.slice(i, i + batch);
|
|
586
|
+
const sent = await forget(hashes, reason);
|
|
587
|
+
const final =
|
|
588
|
+
sent.status === "pending"
|
|
589
|
+
? await waitForMessage(sent.itemHash, {
|
|
590
|
+
apiHost,
|
|
591
|
+
fetch: doFetch,
|
|
592
|
+
...settle,
|
|
593
|
+
})
|
|
594
|
+
: sent;
|
|
595
|
+
if (final.status === "rejected") {
|
|
596
|
+
rejected.push(
|
|
597
|
+
...hashes.map((itemHash) => ({
|
|
598
|
+
itemHash,
|
|
599
|
+
...(final.errorCode ? { errorCode: final.errorCode } : {}),
|
|
600
|
+
})),
|
|
601
|
+
);
|
|
602
|
+
} else {
|
|
603
|
+
forgotten.push(...hashes);
|
|
604
|
+
}
|
|
605
|
+
onProgress?.({
|
|
606
|
+
done: Math.min(i + batch, targets.length),
|
|
607
|
+
total: targets.length,
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
return { forgotten, kept, rejected };
|
|
332
611
|
}
|
|
333
612
|
|
|
334
613
|
/** The reserved channel and aggregate key Aleph keeps permissions in. */
|
|
@@ -359,10 +638,25 @@ export const SECURITY = "security";
|
|
|
359
638
|
* @param {number} [args.now] - seconds
|
|
360
639
|
* @param {(payload: string) => Promise<string>} [args.hasher]
|
|
361
640
|
*/
|
|
362
|
-
export async function buildAuthorizationMessage({
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
641
|
+
export async function buildAuthorizationMessage({
|
|
642
|
+
owner,
|
|
643
|
+
authorizations,
|
|
644
|
+
now,
|
|
645
|
+
hasher = sha256Hex,
|
|
646
|
+
}) {
|
|
647
|
+
if (!owner)
|
|
648
|
+
throw new BackendError(
|
|
649
|
+
"INVALID_BACKEND",
|
|
650
|
+
"an authorization needs the owner's address",
|
|
651
|
+
);
|
|
652
|
+
if (
|
|
653
|
+
!Array.isArray(authorizations) ||
|
|
654
|
+
authorizations.some((a) => typeof a?.address !== "string" || !a.address)
|
|
655
|
+
) {
|
|
656
|
+
throw new BackendError(
|
|
657
|
+
"INVALID_BACKEND",
|
|
658
|
+
"every authorization needs the delegate's address",
|
|
659
|
+
);
|
|
366
660
|
}
|
|
367
661
|
const time = now ?? Date.now() / 1000;
|
|
368
662
|
const item_content = JSON.stringify({
|
|
@@ -413,18 +707,35 @@ export function createAlephAuthorizer(options = {}) {
|
|
|
413
707
|
const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
|
|
414
708
|
const hasher = options.hasher || sha256Hex;
|
|
415
709
|
const doFetch = options.fetch || globalThis.fetch;
|
|
416
|
-
if (!owner)
|
|
417
|
-
|
|
418
|
-
|
|
710
|
+
if (!owner)
|
|
711
|
+
throw new BackendError(
|
|
712
|
+
"INVALID_BACKEND",
|
|
713
|
+
"createAlephAuthorizer needs the owner's address",
|
|
714
|
+
);
|
|
715
|
+
if (typeof sign !== "function")
|
|
716
|
+
throw new BackendError(
|
|
717
|
+
"INVALID_BACKEND",
|
|
718
|
+
"createAlephAuthorizer needs a `sign` function",
|
|
719
|
+
);
|
|
720
|
+
if (typeof doFetch !== "function")
|
|
721
|
+
throw new BackendError(
|
|
722
|
+
"INVALID_BACKEND",
|
|
723
|
+
"createAlephAuthorizer needs fetch",
|
|
724
|
+
);
|
|
419
725
|
|
|
420
726
|
const same = (a, b) => String(a).toLowerCase() === String(b).toLowerCase();
|
|
421
727
|
|
|
422
728
|
/** @returns {Promise<AlephAuthorization[]>} */
|
|
423
729
|
async function read() {
|
|
424
|
-
const response = await doFetch(
|
|
730
|
+
const response = await doFetch(
|
|
731
|
+
`${apiHost}/api/v0/aggregates/${owner}.json?keys=${SECURITY}`,
|
|
732
|
+
);
|
|
425
733
|
if (response.status === 404) return [];
|
|
426
734
|
if (!response.ok) {
|
|
427
|
-
throw new BackendError(
|
|
735
|
+
throw new BackendError(
|
|
736
|
+
"UNSUPPORTED",
|
|
737
|
+
`Aleph did not answer the owner's permissions: ${response.status}`,
|
|
738
|
+
);
|
|
428
739
|
}
|
|
429
740
|
const body = await response.json().catch(() => ({}));
|
|
430
741
|
const list = body?.data?.[SECURITY]?.authorizations;
|
|
@@ -433,22 +744,37 @@ export function createAlephAuthorizer(options = {}) {
|
|
|
433
744
|
|
|
434
745
|
/** @param {AlephAuthorization[]} authorizations */
|
|
435
746
|
async function write(authorizations) {
|
|
436
|
-
const unsigned = await buildAuthorizationMessage({
|
|
747
|
+
const unsigned = await buildAuthorizationMessage({
|
|
748
|
+
owner,
|
|
749
|
+
authorizations,
|
|
750
|
+
hasher,
|
|
751
|
+
now: options.now?.(),
|
|
752
|
+
});
|
|
437
753
|
const signature = await sign(owner, signaturePayload(unsigned));
|
|
438
754
|
const response = await doFetch(`${apiHost}/api/v0/messages`, {
|
|
439
755
|
method: "POST",
|
|
440
756
|
headers: { "content-type": "application/json" },
|
|
441
757
|
body: JSON.stringify({
|
|
442
|
-
message: {
|
|
758
|
+
message: {
|
|
759
|
+
...unsigned,
|
|
760
|
+
signature: signature.startsWith("0x") ? signature : `0x${signature}`,
|
|
761
|
+
},
|
|
443
762
|
sync: true,
|
|
444
763
|
}),
|
|
445
764
|
});
|
|
446
765
|
if (!response.ok && response.status !== 202) {
|
|
447
766
|
const detail = await response.text().catch(() => "");
|
|
448
|
-
throw new BackendError(
|
|
767
|
+
throw new BackendError(
|
|
768
|
+
"UNSUPPORTED",
|
|
769
|
+
`Aleph refused the authorization: ${response.status} ${detail.slice(0, 200)}`,
|
|
770
|
+
);
|
|
449
771
|
}
|
|
450
772
|
const body = await response.json().catch(() => ({}));
|
|
451
|
-
return {
|
|
773
|
+
return {
|
|
774
|
+
itemHash: unsigned.item_hash,
|
|
775
|
+
status: body?.message_status ?? "pending",
|
|
776
|
+
authorizations,
|
|
777
|
+
};
|
|
452
778
|
}
|
|
453
779
|
|
|
454
780
|
return {
|
|
@@ -456,7 +782,9 @@ export function createAlephAuthorizer(options = {}) {
|
|
|
456
782
|
write,
|
|
457
783
|
/** Grant, or replace the grant of the same address. @param {AlephAuthorization} authorization */
|
|
458
784
|
async authorize(authorization) {
|
|
459
|
-
const others = (await read()).filter(
|
|
785
|
+
const others = (await read()).filter(
|
|
786
|
+
(a) => !same(a.address, authorization?.address),
|
|
787
|
+
);
|
|
460
788
|
return write([...others, authorization]);
|
|
461
789
|
},
|
|
462
790
|
/** Take one address's grant away; the others stay. @param {string} address */
|
|
@@ -0,0 +1,734 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Incremental live backup: what was written since the last
|
|
3
|
+
* increment, shipped as one small CAR (#144).
|
|
4
|
+
*
|
|
5
|
+
* A full backup (`bundleDatabases`, `buildAppBackup`) is the baseline, and it
|
|
6
|
+
* stays one. This module adds what happens *between* two of them: a block
|
|
7
|
+
* should be durable somewhere else soon after it is written, without anyone
|
|
8
|
+
* clicking "back up" and without another peer being online.
|
|
9
|
+
*
|
|
10
|
+
* ## Write-behind, never write-through
|
|
11
|
+
*
|
|
12
|
+
* A local write never waits for the network. The watcher keeps, per database,
|
|
13
|
+
* the heads it last shipped; a flush walks from the current heads back over
|
|
14
|
+
* `next` until it meets an entry it has already shipped, and everything it
|
|
15
|
+
* passed on the way is the increment. Entries that arrived by replication from
|
|
16
|
+
* another device are reachable from the heads like any other, so they ship
|
|
17
|
+
* too — otherwise each device would back up only its own writes.
|
|
18
|
+
*
|
|
19
|
+
* The walk needs to know what was shipped, and the previous heads alone do not
|
|
20
|
+
* say it: a branch another writer forked from an old entry leads back past
|
|
21
|
+
* them. So the set of shipped hashes is kept in memory and, after a restart,
|
|
22
|
+
* rebuilt once from the persisted heads by walking the log. What has to be
|
|
23
|
+
* persisted is therefore tiny — a sequence number and a few hashes per
|
|
24
|
+
* database — and a crash between "written" and "shipped" loses nothing: the
|
|
25
|
+
* next flush finds the entry where it left it.
|
|
26
|
+
*
|
|
27
|
+
* ## One upload, one STORE
|
|
28
|
+
*
|
|
29
|
+
* An increment is one CAR whose root is a dag-cbor manifest
|
|
30
|
+
* (`{ kind: "increment", v: 1, sequence, base, previous, metadata }`), so a
|
|
31
|
+
* single blob is self-describing. Every increment carries each database's
|
|
32
|
+
* manifest and access controller (a few hundred bytes) and the identity of
|
|
33
|
+
* every writer it has entries from, so a chain of increments restores on its
|
|
34
|
+
* own, with no base, if it is complete from the beginning; and so
|
|
35
|
+
* `restoreFromBlocks` can verify each one the way it verifies a backup.
|
|
36
|
+
*
|
|
37
|
+
* The CAR goes through any `StorageBackend`'s `putBlob`. A backend that says
|
|
38
|
+
* `retained: false` — Aleph, where an upload is ingest and keeping is a
|
|
39
|
+
* STORE — is then asked to keep it with `pinCid`. Sealing is the caller's,
|
|
40
|
+
* through `withEncryption` around the backend: nothing here changes.
|
|
41
|
+
*
|
|
42
|
+
* ## When the backend is unreachable
|
|
43
|
+
*
|
|
44
|
+
* The flush fails, the state stays as it was, and the next flush ships
|
|
45
|
+
* everything that is still pending — the increment just grows. `status()`
|
|
46
|
+
* says how many updates are waiting, when the last increment went out and
|
|
47
|
+
* what the last failure was, so an application can show "last backed up …".
|
|
48
|
+
*
|
|
49
|
+
* @requires ./backup-car.js - createCARFromBlocks
|
|
50
|
+
* @requires ./restore-cid.js - readBlocksFromCAR, restoreFromBlocks
|
|
51
|
+
*/
|
|
52
|
+
|
|
53
|
+
import * as dagCbor from "@ipld/dag-cbor";
|
|
54
|
+
import * as Block from "multiformats/block";
|
|
55
|
+
import { CID } from "multiformats/cid";
|
|
56
|
+
import { sha256 } from "multiformats/hashes/sha2";
|
|
57
|
+
import { base58btc } from "multiformats/bases/base58";
|
|
58
|
+
import { createCARFromBlocks } from "./backup-car.js";
|
|
59
|
+
import { readBlocksFromCAR, restoreFromBlocks } from "./restore-cid.js";
|
|
60
|
+
|
|
61
|
+
/** What the manifest at an increment's root calls itself. */
|
|
62
|
+
export const INCREMENT_KIND = "increment";
|
|
63
|
+
|
|
64
|
+
/** Bumped when the manifest changes in a way readers must notice. */
|
|
65
|
+
export const INCREMENT_VERSION = 1;
|
|
66
|
+
|
|
67
|
+
/** The persisted state's format. */
|
|
68
|
+
export const LIVE_BACKUP_STATE_VERSION = 1;
|
|
69
|
+
|
|
70
|
+
const DAG_CBOR = dagCbor.code;
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* @typedef {object} LiveBackupState
|
|
74
|
+
* @property {number} v - {@link LIVE_BACKUP_STATE_VERSION}
|
|
75
|
+
* @property {number} sequence - of the last increment shipped; 0 before the first
|
|
76
|
+
* @property {string|null} base - the full backup the increments build on, as the caller named it
|
|
77
|
+
* @property {string|null} previous - the backend's id of the last increment
|
|
78
|
+
* @property {Object<string, { heads: string[] }>} databases - by address: the heads last shipped
|
|
79
|
+
*/
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* @typedef {object} IncrementManifest
|
|
83
|
+
* @property {"increment"} kind
|
|
84
|
+
* @property {number} v
|
|
85
|
+
* @property {number} sequence - 1 for the first increment after the base
|
|
86
|
+
* @property {string|null} base - the full backup this builds on, or null
|
|
87
|
+
* @property {string|null} previous - the id of the increment before this one
|
|
88
|
+
* @property {string} createdAt - ISO
|
|
89
|
+
* @property {object} metadata - the shape `restoreFromBlocks` reads: `databases`
|
|
90
|
+
* with each one's `address`, `name`, `type`, `manifestCID`, `heads`, the
|
|
91
|
+
* `since` heads it continues from and `entries`, how many it holds
|
|
92
|
+
*/
|
|
93
|
+
|
|
94
|
+
const manifestCIDOf = (database) => database.address.split("/").pop();
|
|
95
|
+
|
|
96
|
+
/** Blocks keyed the way OrbitDB names them, deduplicated by that name. */
|
|
97
|
+
const addBlock = (blocks, name, bytes) => {
|
|
98
|
+
if (!bytes || blocks.has(name)) return;
|
|
99
|
+
blocks.set(name, { cid: CID.parse(name), bytes });
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* A database's manifest and access controller: what makes a CAR of entries a
|
|
104
|
+
* backup of *this* database rather than a bag of valid blocks.
|
|
105
|
+
*/
|
|
106
|
+
async function skeletonBlocks(database, blocks) {
|
|
107
|
+
const manifestCID = manifestCIDOf(database);
|
|
108
|
+
const manifestBytes = await database.log.storage
|
|
109
|
+
.get(manifestCID)
|
|
110
|
+
.catch(() => null);
|
|
111
|
+
addBlock(blocks, manifestCID, manifestBytes);
|
|
112
|
+
if (!manifestBytes) return;
|
|
113
|
+
try {
|
|
114
|
+
const { value } = await Block.decode({
|
|
115
|
+
cid: CID.parse(manifestCID),
|
|
116
|
+
bytes: manifestBytes,
|
|
117
|
+
codec: dagCbor,
|
|
118
|
+
hasher: sha256,
|
|
119
|
+
});
|
|
120
|
+
const accessController = value?.accessController?.replace("/ipfs/", "");
|
|
121
|
+
if (accessController) {
|
|
122
|
+
addBlock(
|
|
123
|
+
blocks,
|
|
124
|
+
accessController,
|
|
125
|
+
await database.log.storage.get(accessController).catch(() => null),
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
} catch {
|
|
129
|
+
// A manifest that does not decode has no access controller to find.
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** The identity block an entry's signature refers to, from wherever this node has it. */
|
|
134
|
+
async function identityBytes(database, hash) {
|
|
135
|
+
const own = database.identity ?? database.log.identity;
|
|
136
|
+
if (own?.hash === hash && own.bytes) return own.bytes;
|
|
137
|
+
const identities = database.log.identities;
|
|
138
|
+
if (identities && typeof identities.getIdentity === "function") {
|
|
139
|
+
const identity = await identities.getIdentity(hash).catch(() => null);
|
|
140
|
+
if (identity?.bytes) return identity.bytes;
|
|
141
|
+
}
|
|
142
|
+
return database.log.storage.get(hash).catch(() => null);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Every hash reachable from these heads: what a restarted watcher counts as
|
|
147
|
+
* already shipped.
|
|
148
|
+
*/
|
|
149
|
+
async function reachable(database, heads) {
|
|
150
|
+
const seen = new Set();
|
|
151
|
+
const stack = [...heads];
|
|
152
|
+
while (stack.length) {
|
|
153
|
+
const hash = stack.pop();
|
|
154
|
+
if (seen.has(hash)) continue;
|
|
155
|
+
seen.add(hash);
|
|
156
|
+
const entry = await database.log.get(hash).catch(() => null);
|
|
157
|
+
if (entry?.next) stack.push(...entry.next);
|
|
158
|
+
}
|
|
159
|
+
return seen;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Walk from the current heads down to what was shipped, and collect what is new.
|
|
164
|
+
*
|
|
165
|
+
* @returns {Promise<{ entries: object[], heads: string[] }>} the new entries,
|
|
166
|
+
* newest first, and the heads the walk started from
|
|
167
|
+
*/
|
|
168
|
+
async function newEntries(database, shipped) {
|
|
169
|
+
const heads = await database.log.heads();
|
|
170
|
+
const entries = [];
|
|
171
|
+
const seen = new Set();
|
|
172
|
+
const stack = [...heads];
|
|
173
|
+
while (stack.length) {
|
|
174
|
+
const entry = stack.pop();
|
|
175
|
+
if (!entry || seen.has(entry.hash) || shipped.has(entry.hash)) continue;
|
|
176
|
+
seen.add(entry.hash);
|
|
177
|
+
entries.push(entry);
|
|
178
|
+
for (const hash of entry.next ?? []) {
|
|
179
|
+
if (!seen.has(hash) && !shipped.has(hash))
|
|
180
|
+
stack.push(await database.log.get(hash).catch(() => null));
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return { entries, heads: heads.map((head) => head.hash) };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Collect one increment across several databases, without shipping it.
|
|
188
|
+
*
|
|
189
|
+
* Exposed for callers that ship by other means; {@link createLiveBackup} is
|
|
190
|
+
* the usual way in.
|
|
191
|
+
*
|
|
192
|
+
* @param {Map<string, { database: object, shipped: Set<string>, collection?: string }>} watched - by address
|
|
193
|
+
* @param {object} options
|
|
194
|
+
* @param {number} options.sequence - the increment's number
|
|
195
|
+
* @param {string|null} [options.base]
|
|
196
|
+
* @param {string|null} [options.previous]
|
|
197
|
+
* @param {Map<string, { bytes: Uint8Array }>} [options.blocks] - blocks of the application's own to carry along
|
|
198
|
+
* @param {() => Date} [options.now]
|
|
199
|
+
* @returns {Promise<null | { blocks: Map<string, { cid: CID, bytes: Uint8Array }>, manifest: IncrementManifest,
|
|
200
|
+
* root: string, shipped: Map<string, { heads: string[], hashes: string[] }>, entries: number }>}
|
|
201
|
+
* null when nothing is new
|
|
202
|
+
*/
|
|
203
|
+
export async function collectIncrement(
|
|
204
|
+
watched,
|
|
205
|
+
{
|
|
206
|
+
sequence,
|
|
207
|
+
base = null,
|
|
208
|
+
previous = null,
|
|
209
|
+
blocks: own,
|
|
210
|
+
now = () => new Date(),
|
|
211
|
+
},
|
|
212
|
+
) {
|
|
213
|
+
const blocks = new Map();
|
|
214
|
+
const described = [];
|
|
215
|
+
const shippedNow = new Map();
|
|
216
|
+
let total = 0;
|
|
217
|
+
|
|
218
|
+
for (const [address, { database, shipped, collection }] of watched) {
|
|
219
|
+
const { entries, heads } = await newEntries(database, shipped);
|
|
220
|
+
if (entries.length === 0) continue;
|
|
221
|
+
await skeletonBlocks(database, blocks);
|
|
222
|
+
const identities = new Set();
|
|
223
|
+
for (const entry of entries) {
|
|
224
|
+
addBlock(
|
|
225
|
+
blocks,
|
|
226
|
+
entry.hash,
|
|
227
|
+
await database.log.storage.get(entry.hash).catch(() => null),
|
|
228
|
+
);
|
|
229
|
+
if (entry.identity) identities.add(entry.identity);
|
|
230
|
+
}
|
|
231
|
+
for (const hash of identities)
|
|
232
|
+
addBlock(blocks, hash, await identityBytes(database, hash));
|
|
233
|
+
total += entries.length;
|
|
234
|
+
described.push({
|
|
235
|
+
address,
|
|
236
|
+
name: database.name,
|
|
237
|
+
type: database.type,
|
|
238
|
+
manifestCID: manifestCIDOf(database),
|
|
239
|
+
heads,
|
|
240
|
+
since: [...(shipped.heads ?? [])],
|
|
241
|
+
entries: entries.length,
|
|
242
|
+
...(collection ? { collection } : {}),
|
|
243
|
+
});
|
|
244
|
+
shippedNow.set(address, {
|
|
245
|
+
heads,
|
|
246
|
+
hashes: entries.map((entry) => entry.hash),
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
for (const [name, block] of own ?? [])
|
|
251
|
+
if (!blocks.has(name)) blocks.set(name, block);
|
|
252
|
+
if (total === 0 && (own?.size ?? 0) === 0) return null;
|
|
253
|
+
|
|
254
|
+
const at = now();
|
|
255
|
+
/** @type {IncrementManifest} */
|
|
256
|
+
const manifest = {
|
|
257
|
+
kind: INCREMENT_KIND,
|
|
258
|
+
v: INCREMENT_VERSION,
|
|
259
|
+
sequence,
|
|
260
|
+
base,
|
|
261
|
+
previous,
|
|
262
|
+
createdAt: at.toISOString(),
|
|
263
|
+
metadata: {
|
|
264
|
+
version: "1.0",
|
|
265
|
+
timestamp: at.getTime(),
|
|
266
|
+
databaseCount: described.length,
|
|
267
|
+
totalBlocks: blocks.size,
|
|
268
|
+
totalEntries: total,
|
|
269
|
+
...(described[0] ? { manifestCID: described[0].manifestCID } : {}),
|
|
270
|
+
databases: described,
|
|
271
|
+
},
|
|
272
|
+
};
|
|
273
|
+
const rootBytes = dagCbor.encode(manifest);
|
|
274
|
+
const rootCid = CID.createV1(DAG_CBOR, await sha256.digest(rootBytes));
|
|
275
|
+
blocks.set(rootCid.toString(), { cid: rootCid, bytes: rootBytes });
|
|
276
|
+
return {
|
|
277
|
+
blocks,
|
|
278
|
+
manifest,
|
|
279
|
+
root: rootCid.toString(),
|
|
280
|
+
shipped: shippedNow,
|
|
281
|
+
entries: total,
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Read an increment back: every block checked against its CID, the manifest
|
|
287
|
+
* found among them.
|
|
288
|
+
*
|
|
289
|
+
* @param {Uint8Array} carBytes
|
|
290
|
+
* @returns {Promise<{ manifest: IncrementManifest, blocks: Map<string, { bytes: Uint8Array }> }>}
|
|
291
|
+
*/
|
|
292
|
+
export async function readIncrement(carBytes) {
|
|
293
|
+
const blocks = await readBlocksFromCAR(carBytes, { verify: true });
|
|
294
|
+
for (const [name, { bytes }] of blocks) {
|
|
295
|
+
if (CID.parse(name).code !== DAG_CBOR) continue;
|
|
296
|
+
let value;
|
|
297
|
+
try {
|
|
298
|
+
value = dagCbor.decode(bytes);
|
|
299
|
+
} catch {
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
if (value?.kind === INCREMENT_KIND) {
|
|
303
|
+
if (value.v !== INCREMENT_VERSION) {
|
|
304
|
+
throw new Error(
|
|
305
|
+
`This increment was written in format ${value.v}, and this build reads ${INCREMENT_VERSION}. ` +
|
|
306
|
+
"Upgrade @le-space/orbitdb-storage-bridge to restore it.",
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
return { manifest: value, blocks };
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
throw new Error("This CAR holds no increment manifest.");
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
const metadataOf = (source) => source?.metadata ?? source?.manifest?.metadata;
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Fold a base backup and its increments into one block Map and one metadata:
|
|
319
|
+
* every block of every source, and for each database the heads of the newest
|
|
320
|
+
* source that names it.
|
|
321
|
+
*
|
|
322
|
+
* Sources are `{ metadata | manifest, blocks }`: what `bundleDatabases`,
|
|
323
|
+
* `openAppBackup` and {@link readIncrement} return. Increments are ordered by
|
|
324
|
+
* their `sequence`; the base, if given, comes first.
|
|
325
|
+
*
|
|
326
|
+
* @param {object} params
|
|
327
|
+
* @param {{ metadata?: object, manifest?: object, blocks: Map }} [params.base]
|
|
328
|
+
* @param {{ manifest: IncrementManifest, blocks: Map }[]} params.increments
|
|
329
|
+
* @returns {{ blocks: Map, metadata: object, sequences: number[] }}
|
|
330
|
+
*/
|
|
331
|
+
export function mergeIncrements({ base, increments = [] }) {
|
|
332
|
+
const ordered = [...increments].sort(
|
|
333
|
+
(a, b) => (a.manifest?.sequence ?? 0) - (b.manifest?.sequence ?? 0),
|
|
334
|
+
);
|
|
335
|
+
const sources = base ? [base, ...ordered] : ordered;
|
|
336
|
+
if (sources.length === 0)
|
|
337
|
+
throw new Error("mergeIncrements needs a base or at least one increment");
|
|
338
|
+
|
|
339
|
+
const blocks = new Map();
|
|
340
|
+
const databases = new Map();
|
|
341
|
+
let timestamp = 0;
|
|
342
|
+
for (const source of sources) {
|
|
343
|
+
for (const [name, block] of source.blocks ?? [])
|
|
344
|
+
if (!blocks.has(name)) blocks.set(name, block);
|
|
345
|
+
const metadata = metadataOf(source);
|
|
346
|
+
if (!metadata?.databases) continue;
|
|
347
|
+
timestamp = Math.max(timestamp, Number(metadata.timestamp) || 0);
|
|
348
|
+
for (const d of metadata.databases) {
|
|
349
|
+
const known = databases.get(String(d.address)) ?? {};
|
|
350
|
+
databases.set(String(d.address), {
|
|
351
|
+
...known,
|
|
352
|
+
address: String(d.address),
|
|
353
|
+
...(d.name ? { name: d.name } : {}),
|
|
354
|
+
...(d.type ? { type: d.type } : {}),
|
|
355
|
+
...(d.manifestCID ? { manifestCID: d.manifestCID } : {}),
|
|
356
|
+
...(d.collection ? { collection: d.collection } : {}),
|
|
357
|
+
// Later sources are newer: their heads supersede.
|
|
358
|
+
...(Array.isArray(d.heads) && d.heads.length ? { heads: d.heads } : {}),
|
|
359
|
+
...(typeof d.entryCount === "number"
|
|
360
|
+
? { entryCount: d.entryCount }
|
|
361
|
+
: {}),
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
const described = [...databases.values()];
|
|
366
|
+
return {
|
|
367
|
+
blocks,
|
|
368
|
+
metadata: {
|
|
369
|
+
version: "1.0",
|
|
370
|
+
timestamp,
|
|
371
|
+
databaseCount: described.length,
|
|
372
|
+
totalBlocks: blocks.size,
|
|
373
|
+
...(described[0]?.manifestCID
|
|
374
|
+
? { manifestCID: described[0].manifestCID }
|
|
375
|
+
: {}),
|
|
376
|
+
databases: described,
|
|
377
|
+
},
|
|
378
|
+
sequences: ordered.map((i) => i.manifest?.sequence ?? 0),
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* Put a base backup and its increments back into a node, through
|
|
384
|
+
* `restoreFromBlocks`: the union of their blocks, and each database joined at
|
|
385
|
+
* the heads of the newest increment that names it.
|
|
386
|
+
*
|
|
387
|
+
* @param {object} orbitdb
|
|
388
|
+
* @param {object} params
|
|
389
|
+
* @param {{ metadata?: object, manifest?: object, blocks: Map }} [params.base]
|
|
390
|
+
* @param {{ manifest: IncrementManifest, blocks: Map }[]} params.increments
|
|
391
|
+
* @param {object} [params.open] - options for `orbitdb.open` (`encryption`, `AccessController`, …)
|
|
392
|
+
* @param {object} [params.log]
|
|
393
|
+
* @param {Function} [params.onProgress]
|
|
394
|
+
* @returns {Promise<{ blocks: number, databases: object[], sequences: number[] }>}
|
|
395
|
+
*/
|
|
396
|
+
export async function restoreFromIncrements(
|
|
397
|
+
orbitdb,
|
|
398
|
+
{ base, increments = [], open, log, onProgress } = {},
|
|
399
|
+
) {
|
|
400
|
+
const { blocks, metadata, sequences } = mergeIncrements({ base, increments });
|
|
401
|
+
const restored = await restoreFromBlocks(orbitdb, blocks, metadata, {
|
|
402
|
+
open,
|
|
403
|
+
log,
|
|
404
|
+
onProgress,
|
|
405
|
+
});
|
|
406
|
+
return { ...restored, sequences };
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* Join a base and its increments into a database that is open *now*, without
|
|
411
|
+
* closing it: the blocks go into the log's storage, and each head the newest
|
|
412
|
+
* source names is joined on the open instance.
|
|
413
|
+
*
|
|
414
|
+
* `restoreFromIncrements` reopens the database, as `restoreFromBlocks` does,
|
|
415
|
+
* which leaves an application holding a closed instance. An application whose
|
|
416
|
+
* books are open — a todo list on screen, belege's books — wants the entries
|
|
417
|
+
* to appear in what it already holds. Joining merges: what is here stays.
|
|
418
|
+
*
|
|
419
|
+
* @param {object} database - an open OrbitDB database
|
|
420
|
+
* @param {object} params
|
|
421
|
+
* @param {{ metadata?: object, manifest?: object, blocks: Map }} [params.base]
|
|
422
|
+
* @param {{ manifest: IncrementManifest, blocks: Map }[]} params.increments
|
|
423
|
+
* @returns {Promise<{ joined: number, heads: number, blocks: number }>}
|
|
424
|
+
*/
|
|
425
|
+
export async function joinIncrements(database, { base, increments = [] } = {}) {
|
|
426
|
+
const { blocks, metadata } = mergeIncrements({ base, increments });
|
|
427
|
+
const address = String(database.address);
|
|
428
|
+
const described = metadata.databases.find(
|
|
429
|
+
(d) => String(d.address) === address,
|
|
430
|
+
);
|
|
431
|
+
if (!described)
|
|
432
|
+
throw new Error("These increments hold nothing for this database.");
|
|
433
|
+
|
|
434
|
+
for (const [name, { bytes }] of blocks) {
|
|
435
|
+
const cid = CID.parse(name);
|
|
436
|
+
// OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
|
|
437
|
+
// its own names, and identities are looked up by theirs, so each block
|
|
438
|
+
// goes in under the name its reader will ask for.
|
|
439
|
+
const stored =
|
|
440
|
+
cid.code === DAG_CBOR ? cid.toV1().toString(base58btc) : name;
|
|
441
|
+
await database.log.storage.put(stored, bytes);
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
let joined = 0;
|
|
445
|
+
for (const hash of described.heads ?? []) {
|
|
446
|
+
const name = CID.parse(hash).toV1().toString(base58btc);
|
|
447
|
+
const entry = await database.log.get(name).catch(() => null);
|
|
448
|
+
if (entry && (await database.log.joinEntry(entry))) joined++;
|
|
449
|
+
}
|
|
450
|
+
return { joined, heads: (described.heads ?? []).length, blocks: blocks.size };
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
/** A state with nothing shipped yet. */
|
|
454
|
+
export const emptyLiveBackupState = () => ({
|
|
455
|
+
v: LIVE_BACKUP_STATE_VERSION,
|
|
456
|
+
sequence: 0,
|
|
457
|
+
base: null,
|
|
458
|
+
previous: null,
|
|
459
|
+
databases: {},
|
|
460
|
+
});
|
|
461
|
+
|
|
462
|
+
/**
|
|
463
|
+
* Keep the databases backed up, increment by increment.
|
|
464
|
+
*
|
|
465
|
+
* @param {object} params
|
|
466
|
+
* @param {object[]|Object<string, object>} [params.databases] - open OrbitDB
|
|
467
|
+
* databases, as a list or by name; more can be added with `watch()`
|
|
468
|
+
* @param {import("./backends/types.js").StorageBackend} params.backend - where
|
|
469
|
+
* increments go; wrap it with `withEncryption` to seal them
|
|
470
|
+
* @param {number} [params.every] - ms between flushes while something is
|
|
471
|
+
* pending; 0 or undefined for none (flush by `after`, or by hand)
|
|
472
|
+
* @param {number} [params.after] - flush once this many updates are pending
|
|
473
|
+
* @param {number} [params.settle=250] - ms to wait after an update before an
|
|
474
|
+
* `after`-triggered flush, so a burst of writes becomes one increment
|
|
475
|
+
* @param {{ load: () => Promise<LiveBackupState|null|undefined>, save: (state: LiveBackupState) => Promise<void> }} [params.state] -
|
|
476
|
+
* where the few bytes that survive a restart live; in memory when omitted
|
|
477
|
+
* @param {() => Promise<Map<string, { bytes: Uint8Array }>>|Map} [params.blocks] - blocks of the
|
|
478
|
+
* application's own to carry in the next increment (receipt files, …)
|
|
479
|
+
* @param {boolean} [params.keep=true] - ask a backend whose handle says
|
|
480
|
+
* `retained: false` to keep the increment (`pinCid`)
|
|
481
|
+
* @param {(sequence: number, at: Date) => string} [params.name] - the blob's name
|
|
482
|
+
* @param {(shipped: { sequence: number, handle: object, kept?: object, entries: number, blocks: number, bytes: number, at: Date }) => void} [params.onShipped]
|
|
483
|
+
* @param {(error: Error) => void} [params.onError]
|
|
484
|
+
* @param {() => Date} [params.now]
|
|
485
|
+
* @returns {{ watch: Function, unwatch: Function, start: Function, stop: Function,
|
|
486
|
+
* flush: () => Promise<object|null>, rebase: Function, status: () => object }}
|
|
487
|
+
*/
|
|
488
|
+
export function createLiveBackup({
|
|
489
|
+
databases,
|
|
490
|
+
backend,
|
|
491
|
+
every = 0,
|
|
492
|
+
after = 0,
|
|
493
|
+
settle = 250,
|
|
494
|
+
state: store,
|
|
495
|
+
blocks: ownBlocks,
|
|
496
|
+
keep = true,
|
|
497
|
+
name = (sequence, at) =>
|
|
498
|
+
`increment-${String(sequence).padStart(6, "0")}-${at.toISOString().replace(/[:.]/g, "-")}.car`,
|
|
499
|
+
onShipped,
|
|
500
|
+
onError,
|
|
501
|
+
now = () => new Date(),
|
|
502
|
+
} = {}) {
|
|
503
|
+
if (!backend || typeof backend.putBlob !== "function") {
|
|
504
|
+
throw new Error("createLiveBackup needs a storage backend with putBlob()");
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/** @type {Map<string, { database: object, shipped: Set<string>, collection?: string, off: Function, loaded: boolean }>} */
|
|
508
|
+
const watched = new Map();
|
|
509
|
+
/** @type {LiveBackupState} */
|
|
510
|
+
let state = emptyLiveBackupState();
|
|
511
|
+
let loaded = !store;
|
|
512
|
+
let running = false;
|
|
513
|
+
let pending = 0;
|
|
514
|
+
let inflight = null;
|
|
515
|
+
let again = false;
|
|
516
|
+
let timer = null;
|
|
517
|
+
let settling = null;
|
|
518
|
+
let lastShipped = null;
|
|
519
|
+
let lastError = null;
|
|
520
|
+
let failures = 0;
|
|
521
|
+
|
|
522
|
+
const schedule = () => {
|
|
523
|
+
if (!running || !every || timer) return;
|
|
524
|
+
timer = setTimeout(() => {
|
|
525
|
+
timer = null;
|
|
526
|
+
if (pending > 0) flush().catch(() => {});
|
|
527
|
+
schedule();
|
|
528
|
+
}, every);
|
|
529
|
+
timer.unref?.();
|
|
530
|
+
};
|
|
531
|
+
|
|
532
|
+
/** A flush soon, once the writes of this moment have settled. */
|
|
533
|
+
const scheduleFlush = () => {
|
|
534
|
+
if (!running || settling) return;
|
|
535
|
+
settling = setTimeout(() => {
|
|
536
|
+
settling = null;
|
|
537
|
+
flush().catch(() => {});
|
|
538
|
+
}, settle);
|
|
539
|
+
settling.unref?.();
|
|
540
|
+
};
|
|
541
|
+
|
|
542
|
+
const onUpdate = () => {
|
|
543
|
+
pending++;
|
|
544
|
+
if (after && pending >= after) scheduleFlush();
|
|
545
|
+
};
|
|
546
|
+
|
|
547
|
+
const watch = (database, collection) => {
|
|
548
|
+
const address = String(database.address);
|
|
549
|
+
if (watched.has(address)) return;
|
|
550
|
+
const handler = () => onUpdate();
|
|
551
|
+
database.events?.on?.("update", handler);
|
|
552
|
+
watched.set(address, {
|
|
553
|
+
database,
|
|
554
|
+
collection,
|
|
555
|
+
shipped: new Set(),
|
|
556
|
+
loaded: false,
|
|
557
|
+
off: () => database.events?.off?.("update", handler),
|
|
558
|
+
});
|
|
559
|
+
};
|
|
560
|
+
|
|
561
|
+
const unwatch = (database) => {
|
|
562
|
+
const address = String(database?.address ?? database);
|
|
563
|
+
watched.get(address)?.off();
|
|
564
|
+
watched.delete(address);
|
|
565
|
+
};
|
|
566
|
+
|
|
567
|
+
if (Array.isArray(databases)) databases.forEach((d) => watch(d));
|
|
568
|
+
else
|
|
569
|
+
for (const [collection, d] of Object.entries(databases ?? {}))
|
|
570
|
+
watch(d, collection);
|
|
571
|
+
|
|
572
|
+
const load = async () => {
|
|
573
|
+
if (!loaded) {
|
|
574
|
+
const saved = await store.load();
|
|
575
|
+
if (saved && saved.v === LIVE_BACKUP_STATE_VERSION)
|
|
576
|
+
state = { ...emptyLiveBackupState(), ...saved };
|
|
577
|
+
loaded = true;
|
|
578
|
+
}
|
|
579
|
+
for (const [address, w] of watched) {
|
|
580
|
+
if (w.loaded) continue;
|
|
581
|
+
const heads = state.databases[address]?.heads ?? [];
|
|
582
|
+
w.shipped = await reachable(w.database, heads);
|
|
583
|
+
w.shipped.heads = heads;
|
|
584
|
+
w.loaded = true;
|
|
585
|
+
}
|
|
586
|
+
};
|
|
587
|
+
|
|
588
|
+
const save = async () => {
|
|
589
|
+
// Plain JSON, so a copy is a copy and the caller may keep or serialise it.
|
|
590
|
+
if (store) await store.save(JSON.parse(JSON.stringify(state)));
|
|
591
|
+
};
|
|
592
|
+
|
|
593
|
+
const ship = async () => {
|
|
594
|
+
await load();
|
|
595
|
+
const own = typeof ownBlocks === "function" ? await ownBlocks() : ownBlocks;
|
|
596
|
+
const collected = await collectIncrement(watched, {
|
|
597
|
+
sequence: state.sequence + 1,
|
|
598
|
+
base: state.base,
|
|
599
|
+
previous: state.previous,
|
|
600
|
+
blocks: own,
|
|
601
|
+
now,
|
|
602
|
+
});
|
|
603
|
+
const countedPending = pending;
|
|
604
|
+
if (!collected) {
|
|
605
|
+
pending = 0;
|
|
606
|
+
return null;
|
|
607
|
+
}
|
|
608
|
+
const at = now();
|
|
609
|
+
const car = await createCARFromBlocks(collected.blocks, collected.root);
|
|
610
|
+
const handle = await backend.putBlob(car, {
|
|
611
|
+
name: name(collected.manifest.sequence, at),
|
|
612
|
+
contentType: "application/vnd.ipld.car",
|
|
613
|
+
});
|
|
614
|
+
let kept;
|
|
615
|
+
if (
|
|
616
|
+
keep &&
|
|
617
|
+
handle?.retained === false &&
|
|
618
|
+
typeof backend.pinCid === "function"
|
|
619
|
+
) {
|
|
620
|
+
kept = await backend.pinCid(handle.id, { name: handle.name });
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// Only now, with the bytes out of the house, does the state move.
|
|
624
|
+
state.sequence = collected.manifest.sequence;
|
|
625
|
+
state.previous = String(handle.id ?? handle.cid ?? "");
|
|
626
|
+
for (const [address, { heads, hashes }] of collected.shipped) {
|
|
627
|
+
const w = watched.get(address);
|
|
628
|
+
hashes.forEach((hash) => w.shipped.add(hash));
|
|
629
|
+
w.shipped.heads = heads;
|
|
630
|
+
state.databases[address] = { heads };
|
|
631
|
+
}
|
|
632
|
+
pending = Math.max(0, pending - countedPending);
|
|
633
|
+
await save();
|
|
634
|
+
|
|
635
|
+
const shipped = {
|
|
636
|
+
sequence: state.sequence,
|
|
637
|
+
handle,
|
|
638
|
+
...(kept ? { kept } : {}),
|
|
639
|
+
entries: collected.entries,
|
|
640
|
+
blocks: collected.blocks.size,
|
|
641
|
+
bytes: car.length,
|
|
642
|
+
at,
|
|
643
|
+
};
|
|
644
|
+
lastShipped = shipped;
|
|
645
|
+
lastError = null;
|
|
646
|
+
onShipped?.(shipped);
|
|
647
|
+
return shipped;
|
|
648
|
+
};
|
|
649
|
+
|
|
650
|
+
/** Ship what is pending; one flush at a time, and a request during one runs after it. */
|
|
651
|
+
const flush = async () => {
|
|
652
|
+
if (inflight) {
|
|
653
|
+
again = true;
|
|
654
|
+
return inflight;
|
|
655
|
+
}
|
|
656
|
+
inflight = (async () => {
|
|
657
|
+
try {
|
|
658
|
+
return await ship();
|
|
659
|
+
} catch (error) {
|
|
660
|
+
failures++;
|
|
661
|
+
lastError = error;
|
|
662
|
+
onError?.(error);
|
|
663
|
+
throw error;
|
|
664
|
+
} finally {
|
|
665
|
+
inflight = null;
|
|
666
|
+
if (again) {
|
|
667
|
+
again = false;
|
|
668
|
+
flush().catch(() => {});
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
})();
|
|
672
|
+
return inflight;
|
|
673
|
+
};
|
|
674
|
+
|
|
675
|
+
/**
|
|
676
|
+
* Everything now in the databases counts as shipped.
|
|
677
|
+
*
|
|
678
|
+
* With `base`, a full backup was just made: the increments start again at 1
|
|
679
|
+
* from it. Without, the content arrived by other means — a restore joined
|
|
680
|
+
* into the open databases — and must not be sent out again, while the
|
|
681
|
+
* numbering carries on from where it was.
|
|
682
|
+
*
|
|
683
|
+
* @param {{ base?: string|null }} [params] - what the caller calls the full backup (its id, CID, …)
|
|
684
|
+
*/
|
|
685
|
+
const rebase = async ({ base } = {}) => {
|
|
686
|
+
await load();
|
|
687
|
+
if (base !== undefined)
|
|
688
|
+
state = { ...emptyLiveBackupState(), base: base ?? null };
|
|
689
|
+
for (const [address, w] of watched) {
|
|
690
|
+
const heads = (await w.database.log.heads()).map((head) => head.hash);
|
|
691
|
+
w.shipped = await reachable(w.database, heads);
|
|
692
|
+
w.shipped.heads = heads;
|
|
693
|
+
state.databases[address] = { heads };
|
|
694
|
+
}
|
|
695
|
+
pending = 0;
|
|
696
|
+
await save();
|
|
697
|
+
};
|
|
698
|
+
|
|
699
|
+
const start = async () => {
|
|
700
|
+
if (running) return;
|
|
701
|
+
running = true;
|
|
702
|
+
await load();
|
|
703
|
+
schedule();
|
|
704
|
+
// What was written before the watcher came — after a restart, or before
|
|
705
|
+
// the archive was switched on — is as unshipped as anything, and nothing
|
|
706
|
+
// would announce it. So one flush soon, which ships nothing if there is
|
|
707
|
+
// nothing.
|
|
708
|
+
if (after || every) scheduleFlush();
|
|
709
|
+
};
|
|
710
|
+
|
|
711
|
+
const stop = async ({ flush: last = false } = {}) => {
|
|
712
|
+
running = false;
|
|
713
|
+
if (timer) clearTimeout(timer);
|
|
714
|
+
if (settling) clearTimeout(settling);
|
|
715
|
+
timer = settling = null;
|
|
716
|
+
if (last) await flush();
|
|
717
|
+
};
|
|
718
|
+
|
|
719
|
+
const status = () => ({
|
|
720
|
+
running,
|
|
721
|
+
sequence: state.sequence,
|
|
722
|
+
base: state.base,
|
|
723
|
+
previous: state.previous,
|
|
724
|
+
pending,
|
|
725
|
+
watched: watched.size,
|
|
726
|
+
lastShipped,
|
|
727
|
+
lastError,
|
|
728
|
+
failures,
|
|
729
|
+
});
|
|
730
|
+
|
|
731
|
+
return { watch, unwatch, start, stop, flush, rebase, status };
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
export default createLiveBackup;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@le-space/orbitdb-storage-bridge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
|
|
5
5
|
"main": "lib/orbitdb-storacha-bridge.js",
|
|
6
6
|
"svelte": "dist/components/",
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
"./restore-cid": "./lib/restore-cid.js",
|
|
18
18
|
"./extract-blocks": "./lib/extract-blocks.js",
|
|
19
19
|
"./app-backup": "./lib/app-backup.js",
|
|
20
|
+
"./live-backup": "./lib/live-backup.js",
|
|
20
21
|
"./gateway-fetch": "./lib/gateway-fetch.js",
|
|
21
22
|
"./peer-fetch": "./lib/peer-fetch.js",
|
|
22
23
|
"./backends/types": "./lib/backends/types.js",
|