@le-space/orbitdb-storage-bridge 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/backends/aleph-pin.js +397 -27
- package/lib/backends/encryption.js +57 -19
- package/lib/backends/types.js +4 -0
- package/lib/dehydrate.js +16 -5
- package/lib/gateway-fetch.js +75 -8
- package/lib/live-backup.js +127 -22
- package/package.json +1 -1
|
@@ -77,7 +77,9 @@ async function sha256Hex(payload) {
|
|
|
77
77
|
"SHA-256",
|
|
78
78
|
new TextEncoder().encode(payload),
|
|
79
79
|
);
|
|
80
|
-
return [...new Uint8Array(digest)]
|
|
80
|
+
return [...new Uint8Array(digest)]
|
|
81
|
+
.map((b) => b.toString(16).padStart(2, "0"))
|
|
82
|
+
.join("");
|
|
81
83
|
}
|
|
82
84
|
|
|
83
85
|
/**
|
|
@@ -173,12 +175,29 @@ export function createAlephPin(options = {}) {
|
|
|
173
175
|
const doFetch = options.fetch || globalThis.fetch;
|
|
174
176
|
const now = options.now;
|
|
175
177
|
|
|
176
|
-
if (!sender)
|
|
177
|
-
|
|
178
|
-
|
|
178
|
+
if (!sender)
|
|
179
|
+
throw new BackendError(
|
|
180
|
+
"INVALID_BACKEND",
|
|
181
|
+
"createAlephPin needs the wallet address as `sender`",
|
|
182
|
+
);
|
|
183
|
+
if (typeof sign !== "function")
|
|
184
|
+
throw new BackendError(
|
|
185
|
+
"INVALID_BACKEND",
|
|
186
|
+
"createAlephPin needs a `sign` function",
|
|
187
|
+
);
|
|
188
|
+
if (typeof doFetch !== "function")
|
|
189
|
+
throw new BackendError("INVALID_BACKEND", "createAlephPin needs fetch");
|
|
179
190
|
|
|
180
191
|
return async function pin(cid) {
|
|
181
|
-
const unsigned = await buildStoreMessage({
|
|
192
|
+
const unsigned = await buildStoreMessage({
|
|
193
|
+
sender,
|
|
194
|
+
owner,
|
|
195
|
+
cid,
|
|
196
|
+
channel,
|
|
197
|
+
payment,
|
|
198
|
+
hasher,
|
|
199
|
+
now: now?.(),
|
|
200
|
+
});
|
|
182
201
|
const signature = await sign(sender, signaturePayload(unsigned));
|
|
183
202
|
const message = {
|
|
184
203
|
...unsigned,
|
|
@@ -212,7 +231,13 @@ export function createAlephPin(options = {}) {
|
|
|
212
231
|
}
|
|
213
232
|
|
|
214
233
|
/** Statuses after which a message no longer changes by itself. */
|
|
215
|
-
const SETTLED = new Set([
|
|
234
|
+
const SETTLED = new Set([
|
|
235
|
+
"processed",
|
|
236
|
+
"rejected",
|
|
237
|
+
"removing",
|
|
238
|
+
"removed",
|
|
239
|
+
"forgotten",
|
|
240
|
+
]);
|
|
216
241
|
|
|
217
242
|
/**
|
|
218
243
|
* Follow a message until Aleph has decided about it.
|
|
@@ -241,9 +266,16 @@ export async function waitForMessage(itemHash, options = {}) {
|
|
|
241
266
|
const doFetch = options.fetch || globalThis.fetch;
|
|
242
267
|
const timeout = options.timeout ?? 60_000;
|
|
243
268
|
const interval = options.interval ?? 2_000;
|
|
244
|
-
const sleep =
|
|
245
|
-
|
|
246
|
-
|
|
269
|
+
const sleep =
|
|
270
|
+
options.sleep ||
|
|
271
|
+
((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
272
|
+
if (!itemHash)
|
|
273
|
+
throw new BackendError(
|
|
274
|
+
"INVALID_BACKEND",
|
|
275
|
+
"waitForMessage needs the message's item hash",
|
|
276
|
+
);
|
|
277
|
+
if (typeof doFetch !== "function")
|
|
278
|
+
throw new BackendError("INVALID_BACKEND", "waitForMessage needs fetch");
|
|
247
279
|
|
|
248
280
|
// Counted in asks rather than read off a clock, so an injected `sleep` is enough for a test.
|
|
249
281
|
const asks = Math.max(1, Math.floor(timeout / interval) + 1);
|
|
@@ -297,8 +329,13 @@ export async function listAlephStores(options = {}) {
|
|
|
297
329
|
const { owner, channel } = options;
|
|
298
330
|
const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
|
|
299
331
|
const doFetch = options.fetch || globalThis.fetch;
|
|
300
|
-
if (!owner)
|
|
301
|
-
|
|
332
|
+
if (!owner)
|
|
333
|
+
throw new BackendError(
|
|
334
|
+
"INVALID_BACKEND",
|
|
335
|
+
"listAlephStores needs the paying account's address as `owner`",
|
|
336
|
+
);
|
|
337
|
+
if (typeof doFetch !== "function")
|
|
338
|
+
throw new BackendError("INVALID_BACKEND", "listAlephStores needs fetch");
|
|
302
339
|
|
|
303
340
|
const url = new URL(`${apiHost}/api/v0/messages.json`);
|
|
304
341
|
url.searchParams.set("owners", owner);
|
|
@@ -309,7 +346,10 @@ export async function listAlephStores(options = {}) {
|
|
|
309
346
|
|
|
310
347
|
const response = await doFetch(url.toString());
|
|
311
348
|
if (!response.ok) {
|
|
312
|
-
throw new BackendError(
|
|
349
|
+
throw new BackendError(
|
|
350
|
+
"UNSUPPORTED",
|
|
351
|
+
`Aleph did not list the STORE messages: ${response.status}`,
|
|
352
|
+
);
|
|
313
353
|
}
|
|
314
354
|
const body = await response.json().catch(() => ({}));
|
|
315
355
|
const messages = Array.isArray(body?.messages) ? body.messages : [];
|
|
@@ -328,7 +368,288 @@ export async function listAlephStores(options = {}) {
|
|
|
328
368
|
// and keeps a misread filter from handing back someone else's files.
|
|
329
369
|
.filter((store) => store.cid && store.itemHash && same(store.owner, owner))
|
|
330
370
|
.sort((a, b) => b.time - a.time);
|
|
331
|
-
return {
|
|
371
|
+
return {
|
|
372
|
+
stores,
|
|
373
|
+
total: Number.isFinite(body?.pagination_total)
|
|
374
|
+
? body.pagination_total
|
|
375
|
+
: null,
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Build the unsigned FORGET that releases what STORE messages keep.
|
|
381
|
+
*
|
|
382
|
+
* Aleph keeps a file for as long as a STORE names it; a FORGET from the
|
|
383
|
+
* STORE's owner (or a delegate it authorized for `FORGET`) names those STOREs
|
|
384
|
+
* by their item hashes, and the files they kept are unpinned and stop
|
|
385
|
+
* costing credits. Only messages of the same account can be forgotten, and a
|
|
386
|
+
* forgotten message cannot be brought back — the file has to be stored again.
|
|
387
|
+
*
|
|
388
|
+
* @param {object} args
|
|
389
|
+
* @param {string} args.sender - the address that signs
|
|
390
|
+
* @param {string} [args.owner] - the account whose messages these are, when `sender` is its delegate
|
|
391
|
+
* @param {string[]} args.hashes - item hashes of the STORE messages to forget
|
|
392
|
+
* @param {string} [args.reason] - kept with the FORGET, for whoever reads the account's history
|
|
393
|
+
* @param {string} [args.channel]
|
|
394
|
+
* @param {number} [args.now] - seconds since the epoch; injected for tests
|
|
395
|
+
* @param {(payload: string) => Promise<string>} [args.hasher]
|
|
396
|
+
*/
|
|
397
|
+
export async function buildForgetMessage({
|
|
398
|
+
sender,
|
|
399
|
+
owner,
|
|
400
|
+
hashes,
|
|
401
|
+
reason,
|
|
402
|
+
channel = DEFAULT_ALEPH_CHANNEL,
|
|
403
|
+
now,
|
|
404
|
+
hasher = sha256Hex,
|
|
405
|
+
}) {
|
|
406
|
+
if (
|
|
407
|
+
!Array.isArray(hashes) ||
|
|
408
|
+
hashes.length === 0 ||
|
|
409
|
+
!hashes.every((h) => typeof h === "string" && h)
|
|
410
|
+
) {
|
|
411
|
+
throw new BackendError(
|
|
412
|
+
"INVALID_BACKEND",
|
|
413
|
+
"a FORGET names at least one item hash",
|
|
414
|
+
);
|
|
415
|
+
}
|
|
416
|
+
// Aleph's limits, checked before anything is signed.
|
|
417
|
+
if (hashes.length > 1000) {
|
|
418
|
+
throw new BackendError(
|
|
419
|
+
"INVALID_BACKEND",
|
|
420
|
+
`a FORGET names at most 1000 item hashes, not ${hashes.length}`,
|
|
421
|
+
);
|
|
422
|
+
}
|
|
423
|
+
if (reason != null && String(reason).length > 1000) {
|
|
424
|
+
throw new BackendError(
|
|
425
|
+
"INVALID_BACKEND",
|
|
426
|
+
"a FORGET's reason is at most 1000 characters",
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
const time = now ?? Date.now() / 1000;
|
|
430
|
+
const content = {
|
|
431
|
+
address: owner || sender,
|
|
432
|
+
time,
|
|
433
|
+
hashes: [...hashes],
|
|
434
|
+
...(reason ? { reason } : {}),
|
|
435
|
+
};
|
|
436
|
+
const item_content = JSON.stringify(content);
|
|
437
|
+
return {
|
|
438
|
+
sender,
|
|
439
|
+
chain: "ETH",
|
|
440
|
+
type: "FORGET",
|
|
441
|
+
item_hash: await hasher(item_content),
|
|
442
|
+
item_type: "inline",
|
|
443
|
+
item_content,
|
|
444
|
+
time,
|
|
445
|
+
channel,
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/**
|
|
450
|
+
* Forget STORE messages: a signed FORGET, posted, and Aleph's first answer.
|
|
451
|
+
*
|
|
452
|
+
* The same shape as {@link createAlephPin}: the library never sees a key, and
|
|
453
|
+
* `pending` is what Aleph says before it has checked the signature and the
|
|
454
|
+
* ownership — follow it with {@link waitForMessage}. A FORGET that names a
|
|
455
|
+
* message of another account is `rejected` there.
|
|
456
|
+
*
|
|
457
|
+
* @param {object} options
|
|
458
|
+
* @param {string} options.sender
|
|
459
|
+
* @param {string} [options.owner]
|
|
460
|
+
* @param {(address: string, message: string) => Promise<string>} options.sign - `personal_sign`
|
|
461
|
+
* @param {string} [options.apiHost]
|
|
462
|
+
* @param {string} [options.channel]
|
|
463
|
+
* @param {(payload: string) => Promise<string>} [options.hasher]
|
|
464
|
+
* @param {typeof fetch} [options.fetch]
|
|
465
|
+
* @param {() => number} [options.now]
|
|
466
|
+
* @returns {(hashes: string[], reason?: string) => Promise<{ itemHash: string, status: string }>}
|
|
467
|
+
*/
|
|
468
|
+
export function createAlephForget(options = {}) {
|
|
469
|
+
const { sender, sign, owner } = options;
|
|
470
|
+
const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
|
|
471
|
+
const channel = options.channel || DEFAULT_ALEPH_CHANNEL;
|
|
472
|
+
const hasher = options.hasher || sha256Hex;
|
|
473
|
+
const doFetch = options.fetch || globalThis.fetch;
|
|
474
|
+
const now = options.now;
|
|
475
|
+
|
|
476
|
+
if (!sender)
|
|
477
|
+
throw new BackendError(
|
|
478
|
+
"INVALID_BACKEND",
|
|
479
|
+
"createAlephForget needs the wallet address as `sender`",
|
|
480
|
+
);
|
|
481
|
+
if (typeof sign !== "function")
|
|
482
|
+
throw new BackendError(
|
|
483
|
+
"INVALID_BACKEND",
|
|
484
|
+
"createAlephForget needs a `sign` function",
|
|
485
|
+
);
|
|
486
|
+
if (typeof doFetch !== "function")
|
|
487
|
+
throw new BackendError("INVALID_BACKEND", "createAlephForget needs fetch");
|
|
488
|
+
|
|
489
|
+
return async function forget(hashes, reason) {
|
|
490
|
+
const unsigned = await buildForgetMessage({
|
|
491
|
+
sender,
|
|
492
|
+
owner,
|
|
493
|
+
hashes,
|
|
494
|
+
reason,
|
|
495
|
+
channel,
|
|
496
|
+
hasher,
|
|
497
|
+
now: now?.(),
|
|
498
|
+
});
|
|
499
|
+
const signature = await sign(sender, signaturePayload(unsigned));
|
|
500
|
+
const message = {
|
|
501
|
+
...unsigned,
|
|
502
|
+
signature: signature.startsWith("0x") ? signature : `0x${signature}`,
|
|
503
|
+
};
|
|
504
|
+
const response = await doFetch(`${apiHost}/api/v0/messages`, {
|
|
505
|
+
method: "POST",
|
|
506
|
+
headers: { "content-type": "application/json" },
|
|
507
|
+
body: JSON.stringify({ message, sync: true }),
|
|
508
|
+
});
|
|
509
|
+
if (!response.ok && response.status !== 202) {
|
|
510
|
+
const detail = await response.text().catch(() => "");
|
|
511
|
+
throw new BackendError(
|
|
512
|
+
"UNSUPPORTED",
|
|
513
|
+
`Aleph refused the FORGET message: ${response.status} ${detail.slice(0, 200)}`,
|
|
514
|
+
);
|
|
515
|
+
}
|
|
516
|
+
const body = await response.json().catch(() => ({}));
|
|
517
|
+
return {
|
|
518
|
+
itemHash: unsigned.item_hash,
|
|
519
|
+
status: body?.message_status ?? "pending",
|
|
520
|
+
};
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
/**
|
|
525
|
+
* Forget every STORE an account keeps on a channel — an application's whole
|
|
526
|
+
* archive — in batches, each followed until Aleph has decided.
|
|
527
|
+
*
|
|
528
|
+
* `keep` spares some: a predicate over the listed stores, e.g. the newest full
|
|
529
|
+
* backup. What was forgotten, what was kept and what Aleph rejected come back
|
|
530
|
+
* by item hash, so the caller can say so.
|
|
531
|
+
*
|
|
532
|
+
* @param {object} options
|
|
533
|
+
* @param {string} options.owner - the account the STOREs are for
|
|
534
|
+
* @param {string} options.channel - one channel; a FORGET across channels is not what anyone means
|
|
535
|
+
* @param {ReturnType<typeof createAlephForget>} options.forget - from {@link createAlephForget}, signing for that account
|
|
536
|
+
* @param {(store: { cid: string, itemHash: string, time: number }) => boolean} [options.keep]
|
|
537
|
+
* @param {string} [options.reason]
|
|
538
|
+
* @param {number} [options.batch=50] - item hashes per FORGET message
|
|
539
|
+
* @param {string} [options.apiHost]
|
|
540
|
+
* @param {typeof fetch} [options.fetch]
|
|
541
|
+
* @param {{ timeout?: number, interval?: number }} [options.settle] - for {@link waitForMessage}
|
|
542
|
+
* @param {(progress: { done: number, total: number }) => void} [options.onProgress]
|
|
543
|
+
* @returns {Promise<{ forgotten: string[], kept: string[], rejected: { itemHash: string, errorCode?: string | number }[],
|
|
544
|
+
* pending: { itemHash: string, forget: string, status: string }[], failed: { itemHash: string, error: string }[] }>}
|
|
545
|
+
* `forgotten` only once Aleph said `processed`; `pending` when it had not decided in time (follow
|
|
546
|
+
* `forget`, the FORGET's item hash, with {@link waitForMessage}); `failed` when a batch could not be sent
|
|
547
|
+
*/
|
|
548
|
+
export async function forgetAlephStores(options = {}) {
|
|
549
|
+
const {
|
|
550
|
+
owner,
|
|
551
|
+
channel,
|
|
552
|
+
forget,
|
|
553
|
+
keep,
|
|
554
|
+
reason,
|
|
555
|
+
batch = 50,
|
|
556
|
+
apiHost,
|
|
557
|
+
settle = {},
|
|
558
|
+
onProgress,
|
|
559
|
+
} = options;
|
|
560
|
+
if (!owner)
|
|
561
|
+
throw new BackendError(
|
|
562
|
+
"INVALID_BACKEND",
|
|
563
|
+
"forgetAlephStores needs the account's address as `owner`",
|
|
564
|
+
);
|
|
565
|
+
if (!channel)
|
|
566
|
+
throw new BackendError(
|
|
567
|
+
"INVALID_BACKEND",
|
|
568
|
+
"forgetAlephStores needs a `channel`",
|
|
569
|
+
);
|
|
570
|
+
if (typeof forget !== "function")
|
|
571
|
+
throw new BackendError(
|
|
572
|
+
"INVALID_BACKEND",
|
|
573
|
+
"forgetAlephStores needs `forget` from createAlephForget",
|
|
574
|
+
);
|
|
575
|
+
const doFetch = options.fetch;
|
|
576
|
+
|
|
577
|
+
/** @type {{ cid: string, itemHash: string, time: number }[]} */
|
|
578
|
+
const all = [];
|
|
579
|
+
for (let page = 1; ; page++) {
|
|
580
|
+
const { stores, total } = await listAlephStores({
|
|
581
|
+
owner,
|
|
582
|
+
channel,
|
|
583
|
+
apiHost,
|
|
584
|
+
fetch: doFetch,
|
|
585
|
+
pagination: 100,
|
|
586
|
+
page,
|
|
587
|
+
});
|
|
588
|
+
all.push(...stores);
|
|
589
|
+
// The listing filters after the page was cut, so a short page is not the
|
|
590
|
+
// end: the API's own total is.
|
|
591
|
+
if (stores.length === 0 || (total != null && all.length >= total)) break;
|
|
592
|
+
if (total == null && stores.length < 100) break;
|
|
593
|
+
}
|
|
594
|
+
const kept = new Set(
|
|
595
|
+
all.filter((store) => keep?.(store)).map((store) => store.itemHash),
|
|
596
|
+
);
|
|
597
|
+
const targets = all
|
|
598
|
+
.filter((store) => !kept.has(store.itemHash))
|
|
599
|
+
.map((store) => store.itemHash);
|
|
600
|
+
|
|
601
|
+
const forgotten = [];
|
|
602
|
+
const rejected = [];
|
|
603
|
+
const pending = [];
|
|
604
|
+
const failed = [];
|
|
605
|
+
for (let i = 0; i < targets.length; i += batch) {
|
|
606
|
+
const hashes = targets.slice(i, i + batch);
|
|
607
|
+
try {
|
|
608
|
+
const sent = await forget(hashes, reason);
|
|
609
|
+
const final =
|
|
610
|
+
sent.status === "pending"
|
|
611
|
+
? await waitForMessage(sent.itemHash, {
|
|
612
|
+
apiHost,
|
|
613
|
+
fetch: doFetch,
|
|
614
|
+
...settle,
|
|
615
|
+
})
|
|
616
|
+
: sent;
|
|
617
|
+
if (final.status === "rejected") {
|
|
618
|
+
rejected.push(
|
|
619
|
+
...hashes.map((itemHash) => ({
|
|
620
|
+
itemHash,
|
|
621
|
+
...(final.errorCode != null ? { errorCode: final.errorCode } : {}),
|
|
622
|
+
})),
|
|
623
|
+
);
|
|
624
|
+
} else if (final.status === "processed") {
|
|
625
|
+
forgotten.push(...hashes);
|
|
626
|
+
} else {
|
|
627
|
+
// Still pending when the time ran out, or a status in between: not
|
|
628
|
+
// forgotten, not refused — the caller follows the FORGET itself.
|
|
629
|
+
pending.push(
|
|
630
|
+
...hashes.map((itemHash) => ({
|
|
631
|
+
itemHash,
|
|
632
|
+
forget: sent.itemHash,
|
|
633
|
+
status: final.status,
|
|
634
|
+
})),
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
} catch (error) {
|
|
638
|
+
// One batch that could not be sent does not undo the ones before it,
|
|
639
|
+
// and the caller learns which hashes are still there.
|
|
640
|
+
failed.push(
|
|
641
|
+
...hashes.map((itemHash) => ({
|
|
642
|
+
itemHash,
|
|
643
|
+
error: error instanceof Error ? error.message : String(error),
|
|
644
|
+
})),
|
|
645
|
+
);
|
|
646
|
+
}
|
|
647
|
+
onProgress?.({
|
|
648
|
+
done: Math.min(i + batch, targets.length),
|
|
649
|
+
total: targets.length,
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
return { forgotten, kept: [...kept], rejected, pending, failed };
|
|
332
653
|
}
|
|
333
654
|
|
|
334
655
|
/** The reserved channel and aggregate key Aleph keeps permissions in. */
|
|
@@ -359,10 +680,25 @@ export const SECURITY = "security";
|
|
|
359
680
|
* @param {number} [args.now] - seconds
|
|
360
681
|
* @param {(payload: string) => Promise<string>} [args.hasher]
|
|
361
682
|
*/
|
|
362
|
-
export async function buildAuthorizationMessage({
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
683
|
+
export async function buildAuthorizationMessage({
|
|
684
|
+
owner,
|
|
685
|
+
authorizations,
|
|
686
|
+
now,
|
|
687
|
+
hasher = sha256Hex,
|
|
688
|
+
}) {
|
|
689
|
+
if (!owner)
|
|
690
|
+
throw new BackendError(
|
|
691
|
+
"INVALID_BACKEND",
|
|
692
|
+
"an authorization needs the owner's address",
|
|
693
|
+
);
|
|
694
|
+
if (
|
|
695
|
+
!Array.isArray(authorizations) ||
|
|
696
|
+
authorizations.some((a) => typeof a?.address !== "string" || !a.address)
|
|
697
|
+
) {
|
|
698
|
+
throw new BackendError(
|
|
699
|
+
"INVALID_BACKEND",
|
|
700
|
+
"every authorization needs the delegate's address",
|
|
701
|
+
);
|
|
366
702
|
}
|
|
367
703
|
const time = now ?? Date.now() / 1000;
|
|
368
704
|
const item_content = JSON.stringify({
|
|
@@ -413,18 +749,35 @@ export function createAlephAuthorizer(options = {}) {
|
|
|
413
749
|
const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
|
|
414
750
|
const hasher = options.hasher || sha256Hex;
|
|
415
751
|
const doFetch = options.fetch || globalThis.fetch;
|
|
416
|
-
if (!owner)
|
|
417
|
-
|
|
418
|
-
|
|
752
|
+
if (!owner)
|
|
753
|
+
throw new BackendError(
|
|
754
|
+
"INVALID_BACKEND",
|
|
755
|
+
"createAlephAuthorizer needs the owner's address",
|
|
756
|
+
);
|
|
757
|
+
if (typeof sign !== "function")
|
|
758
|
+
throw new BackendError(
|
|
759
|
+
"INVALID_BACKEND",
|
|
760
|
+
"createAlephAuthorizer needs a `sign` function",
|
|
761
|
+
);
|
|
762
|
+
if (typeof doFetch !== "function")
|
|
763
|
+
throw new BackendError(
|
|
764
|
+
"INVALID_BACKEND",
|
|
765
|
+
"createAlephAuthorizer needs fetch",
|
|
766
|
+
);
|
|
419
767
|
|
|
420
768
|
const same = (a, b) => String(a).toLowerCase() === String(b).toLowerCase();
|
|
421
769
|
|
|
422
770
|
/** @returns {Promise<AlephAuthorization[]>} */
|
|
423
771
|
async function read() {
|
|
424
|
-
const response = await doFetch(
|
|
772
|
+
const response = await doFetch(
|
|
773
|
+
`${apiHost}/api/v0/aggregates/${owner}.json?keys=${SECURITY}`,
|
|
774
|
+
);
|
|
425
775
|
if (response.status === 404) return [];
|
|
426
776
|
if (!response.ok) {
|
|
427
|
-
throw new BackendError(
|
|
777
|
+
throw new BackendError(
|
|
778
|
+
"UNSUPPORTED",
|
|
779
|
+
`Aleph did not answer the owner's permissions: ${response.status}`,
|
|
780
|
+
);
|
|
428
781
|
}
|
|
429
782
|
const body = await response.json().catch(() => ({}));
|
|
430
783
|
const list = body?.data?.[SECURITY]?.authorizations;
|
|
@@ -433,22 +786,37 @@ export function createAlephAuthorizer(options = {}) {
|
|
|
433
786
|
|
|
434
787
|
/** @param {AlephAuthorization[]} authorizations */
|
|
435
788
|
async function write(authorizations) {
|
|
436
|
-
const unsigned = await buildAuthorizationMessage({
|
|
789
|
+
const unsigned = await buildAuthorizationMessage({
|
|
790
|
+
owner,
|
|
791
|
+
authorizations,
|
|
792
|
+
hasher,
|
|
793
|
+
now: options.now?.(),
|
|
794
|
+
});
|
|
437
795
|
const signature = await sign(owner, signaturePayload(unsigned));
|
|
438
796
|
const response = await doFetch(`${apiHost}/api/v0/messages`, {
|
|
439
797
|
method: "POST",
|
|
440
798
|
headers: { "content-type": "application/json" },
|
|
441
799
|
body: JSON.stringify({
|
|
442
|
-
message: {
|
|
800
|
+
message: {
|
|
801
|
+
...unsigned,
|
|
802
|
+
signature: signature.startsWith("0x") ? signature : `0x${signature}`,
|
|
803
|
+
},
|
|
443
804
|
sync: true,
|
|
444
805
|
}),
|
|
445
806
|
});
|
|
446
807
|
if (!response.ok && response.status !== 202) {
|
|
447
808
|
const detail = await response.text().catch(() => "");
|
|
448
|
-
throw new BackendError(
|
|
809
|
+
throw new BackendError(
|
|
810
|
+
"UNSUPPORTED",
|
|
811
|
+
`Aleph refused the authorization: ${response.status} ${detail.slice(0, 200)}`,
|
|
812
|
+
);
|
|
449
813
|
}
|
|
450
814
|
const body = await response.json().catch(() => ({}));
|
|
451
|
-
return {
|
|
815
|
+
return {
|
|
816
|
+
itemHash: unsigned.item_hash,
|
|
817
|
+
status: body?.message_status ?? "pending",
|
|
818
|
+
authorizations,
|
|
819
|
+
};
|
|
452
820
|
}
|
|
453
821
|
|
|
454
822
|
return {
|
|
@@ -456,7 +824,9 @@ export function createAlephAuthorizer(options = {}) {
|
|
|
456
824
|
write,
|
|
457
825
|
/** Grant, or replace the grant of the same address. @param {AlephAuthorization} authorization */
|
|
458
826
|
async authorize(authorization) {
|
|
459
|
-
const others = (await read()).filter(
|
|
827
|
+
const others = (await read()).filter(
|
|
828
|
+
(a) => !same(a.address, authorization?.address),
|
|
829
|
+
);
|
|
460
830
|
return write([...others, authorization]);
|
|
461
831
|
},
|
|
462
832
|
/** Take one address's grant away; the others stay. @param {string} address */
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
* @module backends/encryption
|
|
40
40
|
*/
|
|
41
41
|
|
|
42
|
-
import { defineBackend, BackendError } from "./types.js";
|
|
42
|
+
import { defineBackend, BackendError, handleId } from "./types.js";
|
|
43
43
|
|
|
44
44
|
/** `OSBE` — orbitdb-storage-bridge, encrypted. */
|
|
45
45
|
// Not frozen: freezing a typed array throws, because its elements live in a
|
|
@@ -53,7 +53,8 @@ const HEADER_LENGTH = ENVELOPE_MAGIC.length + 2;
|
|
|
53
53
|
|
|
54
54
|
/** Do these bytes start with an envelope this module wrote? */
|
|
55
55
|
export function isEncrypted(bytes) {
|
|
56
|
-
if (!(bytes instanceof Uint8Array) || bytes.length < HEADER_LENGTH)
|
|
56
|
+
if (!(bytes instanceof Uint8Array) || bytes.length < HEADER_LENGTH)
|
|
57
|
+
return false;
|
|
57
58
|
return ENVELOPE_MAGIC.every((byte, index) => bytes[index] === byte);
|
|
58
59
|
}
|
|
59
60
|
|
|
@@ -66,10 +67,16 @@ export function isEncrypted(bytes) {
|
|
|
66
67
|
*/
|
|
67
68
|
export function wrapEnvelope(ciphertext, iv) {
|
|
68
69
|
if (!(ciphertext instanceof Uint8Array) || !(iv instanceof Uint8Array)) {
|
|
69
|
-
throw new BackendError(
|
|
70
|
+
throw new BackendError(
|
|
71
|
+
"INVALID_BACKEND",
|
|
72
|
+
"encrypt must return Uint8Array ciphertext and iv",
|
|
73
|
+
);
|
|
70
74
|
}
|
|
71
75
|
if (iv.length > 255) {
|
|
72
|
-
throw new BackendError(
|
|
76
|
+
throw new BackendError(
|
|
77
|
+
"INVALID_BACKEND",
|
|
78
|
+
`An IV of ${iv.length} bytes does not fit the envelope`,
|
|
79
|
+
);
|
|
73
80
|
}
|
|
74
81
|
const out = new Uint8Array(HEADER_LENGTH + iv.length + ciphertext.length);
|
|
75
82
|
out.set(ENVELOPE_MAGIC, 0);
|
|
@@ -88,7 +95,10 @@ export function wrapEnvelope(ciphertext, iv) {
|
|
|
88
95
|
*/
|
|
89
96
|
export function readEnvelope(bytes) {
|
|
90
97
|
if (!isEncrypted(bytes)) {
|
|
91
|
-
throw new BackendError(
|
|
98
|
+
throw new BackendError(
|
|
99
|
+
"INVALID_BACKEND",
|
|
100
|
+
"These bytes are not an encrypted backup",
|
|
101
|
+
);
|
|
92
102
|
}
|
|
93
103
|
const version = bytes[ENVELOPE_MAGIC.length];
|
|
94
104
|
if (version !== ENVELOPE_VERSION) {
|
|
@@ -111,15 +121,24 @@ export function readEnvelope(bytes) {
|
|
|
111
121
|
* @param {object} options
|
|
112
122
|
* @param {(plaintext: Uint8Array) => Promise<{ciphertext: Uint8Array, iv: Uint8Array}>} options.encrypt
|
|
113
123
|
* @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} options.decrypt
|
|
114
|
-
* @param {boolean} [options.allowPlaintextReads=
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
124
|
+
* @param {boolean} [options.allowPlaintextReads=false] - also hand back a
|
|
125
|
+
* blob that is not an envelope: a backup written before backups were
|
|
126
|
+
* encrypted, or with `dontEncrypt`. Off by default since 0.19.0: a caller
|
|
127
|
+
* that brings a key expects ciphertext, and whoever answers the fetch
|
|
128
|
+
* (gateway, service, network) could otherwise hand it a plaintext backup
|
|
129
|
+
* of *another* database, which it would open and join. Turn it on only for
|
|
130
|
+
* the one restore that needs it, and the acceptance is logged.
|
|
118
131
|
* @returns {import("./types.js").StorageBackend}
|
|
119
132
|
*/
|
|
120
|
-
export function withEncryption(
|
|
133
|
+
export function withEncryption(
|
|
134
|
+
backend,
|
|
135
|
+
{ encrypt, decrypt, allowPlaintextReads = false } = {},
|
|
136
|
+
) {
|
|
121
137
|
if (!backend || typeof backend.putBlob !== "function") {
|
|
122
|
-
throw new BackendError(
|
|
138
|
+
throw new BackendError(
|
|
139
|
+
"INVALID_BACKEND",
|
|
140
|
+
"withEncryption needs a backend to wrap",
|
|
141
|
+
);
|
|
123
142
|
}
|
|
124
143
|
if (typeof encrypt !== "function" || typeof decrypt !== "function") {
|
|
125
144
|
throw new BackendError(
|
|
@@ -149,10 +168,15 @@ export function withEncryption(backend, { encrypt, decrypt, allowPlaintextReads
|
|
|
149
168
|
async getBlob(handle) {
|
|
150
169
|
const bytes = await inner.getBlob(handle);
|
|
151
170
|
if (!isEncrypted(bytes)) {
|
|
152
|
-
if (allowPlaintextReads)
|
|
171
|
+
if (allowPlaintextReads) {
|
|
172
|
+
console.warn(
|
|
173
|
+
`[orbitdb-storage-bridge] accepting an unencrypted blob (${handleId(handle)}) because allowPlaintextReads is on`,
|
|
174
|
+
);
|
|
175
|
+
return bytes;
|
|
176
|
+
}
|
|
153
177
|
throw new BackendError(
|
|
154
178
|
"INVALID_BACKEND",
|
|
155
|
-
"This backup is not encrypted
|
|
179
|
+
"This backup is not encrypted. A sealed archive never holds plaintext; pass allowPlaintextReads: true only to read a backup written before encryption.",
|
|
156
180
|
);
|
|
157
181
|
}
|
|
158
182
|
const { ciphertext, iv } = readEnvelope(bytes);
|
|
@@ -185,24 +209,38 @@ export function withEncryption(backend, { encrypt, decrypt, allowPlaintextReads
|
|
|
185
209
|
* @param {(cid: string, options?: object) => Promise<Uint8Array>} fetchBytes
|
|
186
210
|
* @param {object} options
|
|
187
211
|
* @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} options.decrypt
|
|
188
|
-
* @param {boolean} [options.allowPlaintextReads=
|
|
212
|
+
* @param {boolean} [options.allowPlaintextReads=false] - see {@link withEncryption}
|
|
189
213
|
* @returns {(cid: string, options?: object) => Promise<Uint8Array>}
|
|
190
214
|
*/
|
|
191
|
-
export function decryptingFetch(
|
|
215
|
+
export function decryptingFetch(
|
|
216
|
+
fetchBytes,
|
|
217
|
+
{ decrypt, allowPlaintextReads = false } = {},
|
|
218
|
+
) {
|
|
192
219
|
if (typeof fetchBytes !== "function") {
|
|
193
|
-
throw new BackendError(
|
|
220
|
+
throw new BackendError(
|
|
221
|
+
"INVALID_BACKEND",
|
|
222
|
+
"decryptingFetch needs a fetcher to wrap",
|
|
223
|
+
);
|
|
194
224
|
}
|
|
195
225
|
if (typeof decrypt !== "function") {
|
|
196
|
-
throw new BackendError(
|
|
226
|
+
throw new BackendError(
|
|
227
|
+
"INVALID_BACKEND",
|
|
228
|
+
"decryptingFetch needs a decrypt function",
|
|
229
|
+
);
|
|
197
230
|
}
|
|
198
231
|
|
|
199
232
|
return async function fetchAndDecrypt(cid, options) {
|
|
200
233
|
const bytes = await fetchBytes(cid, options);
|
|
201
234
|
if (!isEncrypted(bytes)) {
|
|
202
|
-
if (allowPlaintextReads)
|
|
235
|
+
if (allowPlaintextReads) {
|
|
236
|
+
console.warn(
|
|
237
|
+
`[orbitdb-storage-bridge] accepting an unencrypted backup at ${cid} because allowPlaintextReads is on`,
|
|
238
|
+
);
|
|
239
|
+
return bytes;
|
|
240
|
+
}
|
|
203
241
|
throw new BackendError(
|
|
204
242
|
"INVALID_BACKEND",
|
|
205
|
-
`The backup at ${cid} is not encrypted
|
|
243
|
+
`The backup at ${cid} is not encrypted. A sealed backup never comes back as plaintext; pass allowPlaintextReads: true only to read a backup written before encryption.`,
|
|
206
244
|
);
|
|
207
245
|
}
|
|
208
246
|
const { ciphertext, iv } = readEnvelope(bytes);
|
package/lib/backends/types.js
CHANGED
|
@@ -40,6 +40,10 @@
|
|
|
40
40
|
* @property {string} backend - the `name` of the backend that issued it
|
|
41
41
|
* @property {string} [cid] - set when the id is, or carries, a content identifier
|
|
42
42
|
* @property {number} [size] - stored size in bytes, when the backend reports one
|
|
43
|
+
* @property {boolean} [retained] - whether the service keeps it: a backend whose upload is
|
|
44
|
+
* only ingest (Aleph) says false until `pinCid` has made it stick; absent means stored is kept
|
|
45
|
+
* @property {string} [status] - the service's own word for where a pin stands (Aleph: `processed`, `pending`, `rejected`)
|
|
46
|
+
* @property {string} [itemHash] - the message that keeps it (Aleph), to follow with `waitForMessage`
|
|
43
47
|
*/
|
|
44
48
|
|
|
45
49
|
/**
|
package/lib/dehydrate.js
CHANGED
|
@@ -83,7 +83,10 @@ export async function dehydrate({
|
|
|
83
83
|
|
|
84
84
|
const { backupDatabaseCAR } = await import("./backup-car.js");
|
|
85
85
|
const storeIn = encrypt
|
|
86
|
-
? (await import("./backends/encryption.js")).withEncryption(backend, {
|
|
86
|
+
? (await import("./backends/encryption.js")).withEncryption(backend, {
|
|
87
|
+
encrypt,
|
|
88
|
+
decrypt,
|
|
89
|
+
})
|
|
87
90
|
: backend;
|
|
88
91
|
|
|
89
92
|
const result = await backupDatabaseCAR(orbitdb, address, {
|
|
@@ -129,7 +132,8 @@ export async function dehydrate({
|
|
|
129
132
|
* @param {string[]} [params.endpoints]
|
|
130
133
|
* @param {Object} [params.open] Options for `orbitdb.open` — a node without a
|
|
131
134
|
* pubsub service needs `{ sync: false }`.
|
|
132
|
-
* @param {Object} [params.restore] Extra options for `restoreFromCID
|
|
135
|
+
* @param {Object} [params.restore] Extra options for `restoreFromCID`; plus
|
|
136
|
+
* `allowPlaintextReads: true` to accept an unencrypted backup with a key in hand.
|
|
133
137
|
* @returns {Promise<{db: Object, address: string, name: string, metadataCID: string, blocks: number, entries: number|null}>}
|
|
134
138
|
*/
|
|
135
139
|
export async function hydrate({
|
|
@@ -145,21 +149,28 @@ export async function hydrate({
|
|
|
145
149
|
const pointer = await resolvePointer({ privateKey, endpoints });
|
|
146
150
|
|
|
147
151
|
const { restoreFromCID } = await import("./restore-cid.js");
|
|
148
|
-
const { decryptingFetch, explainIfEncrypted } =
|
|
152
|
+
const { decryptingFetch, explainIfEncrypted } =
|
|
153
|
+
await import("./backends/encryption.js");
|
|
149
154
|
const { fetchFromGateways } = await import("./gateway-fetch.js");
|
|
150
155
|
|
|
151
156
|
// A restore does not read through a backend — it fetches from a gateway or
|
|
152
157
|
// from peers — so decryption belongs here, around the fetcher. Without a
|
|
153
158
|
// key, an encrypted backup is still recognised, and says so.
|
|
154
159
|
const fetchBytes = restore.fetchBytes ?? fetchFromGateways;
|
|
160
|
+
// Plaintext is refused with a key in hand (0.19.0); `restore.allowPlaintextReads`
|
|
161
|
+
// is the way to read a backup written before encryption, for that one restore.
|
|
155
162
|
const reader = decrypt
|
|
156
|
-
? decryptingFetch(fetchBytes, {
|
|
163
|
+
? decryptingFetch(fetchBytes, {
|
|
164
|
+
decrypt,
|
|
165
|
+
allowPlaintextReads: restore.allowPlaintextReads === true,
|
|
166
|
+
})
|
|
157
167
|
: explainIfEncrypted(fetchBytes);
|
|
168
|
+
const { allowPlaintextReads: _plain, ...restoreOptions } = restore;
|
|
158
169
|
|
|
159
170
|
const result = await restoreFromCID(orbitdb, {
|
|
160
171
|
metadataCID: pointer.cid,
|
|
161
172
|
open,
|
|
162
|
-
...
|
|
173
|
+
...restoreOptions,
|
|
163
174
|
fetchBytes: reader,
|
|
164
175
|
});
|
|
165
176
|
|
package/lib/gateway-fetch.js
CHANGED
|
@@ -30,6 +30,8 @@
|
|
|
30
30
|
* rotted unnoticed. The replacement is not a longer list: it is fetching from
|
|
31
31
|
* the providers that hold the blocks, over libp2p — see issue #112.
|
|
32
32
|
*/
|
|
33
|
+
|
|
34
|
+
import { CID } from "multiformats/cid";
|
|
33
35
|
export const DEFAULT_GATEWAYS = ["https://ipfs.aleph.cloud/ipfs"];
|
|
34
36
|
|
|
35
37
|
/**
|
|
@@ -74,11 +76,16 @@ export const MAX_BACKOFF_MS = 30_000;
|
|
|
74
76
|
* because the header is not always honest.
|
|
75
77
|
*/
|
|
76
78
|
const looksLikeAnErrorPage = (bytes, contentType = "") => {
|
|
77
|
-
if (contentType.includes("text/html") || contentType.includes("xhtml"))
|
|
79
|
+
if (contentType.includes("text/html") || contentType.includes("xhtml"))
|
|
80
|
+
return true;
|
|
78
81
|
const start = new TextDecoder("utf-8", { fatal: false })
|
|
79
82
|
.decode(bytes.subarray(0, Math.min(100, bytes.length)))
|
|
80
83
|
.trim();
|
|
81
|
-
return
|
|
84
|
+
return (
|
|
85
|
+
start.startsWith("<!DOCTYPE") ||
|
|
86
|
+
start.startsWith("<html") ||
|
|
87
|
+
start.startsWith("<?xml")
|
|
88
|
+
);
|
|
82
89
|
};
|
|
83
90
|
|
|
84
91
|
const waitFor = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
@@ -93,9 +100,13 @@ const backoffFor = (response, attempt) => {
|
|
|
93
100
|
const seconds = Number.parseInt(retryAfter, 10);
|
|
94
101
|
if (Number.isFinite(seconds)) return seconds * 1000;
|
|
95
102
|
const date = new Date(retryAfter);
|
|
96
|
-
if (!Number.isNaN(date.getTime()))
|
|
103
|
+
if (!Number.isNaN(date.getTime()))
|
|
104
|
+
return Math.max(0, date.getTime() - Date.now());
|
|
97
105
|
}
|
|
98
|
-
const reset = Number.parseInt(
|
|
106
|
+
const reset = Number.parseInt(
|
|
107
|
+
response.headers?.get?.("X-RateLimit-Reset") ?? "",
|
|
108
|
+
10,
|
|
109
|
+
);
|
|
99
110
|
if (Number.isFinite(reset)) return Math.max(0, reset * 1000 - Date.now());
|
|
100
111
|
return 2000 * (attempt + 1);
|
|
101
112
|
};
|
|
@@ -115,7 +126,45 @@ const backoffFor = (response, attempt) => {
|
|
|
115
126
|
* @param {typeof fetch} [options.fetchImpl] injectable for tests
|
|
116
127
|
* @returns {Promise<Uint8Array>}
|
|
117
128
|
*/
|
|
118
|
-
|
|
129
|
+
/**
|
|
130
|
+
* True unless the CID is a raw sha-256 block whose digest the bytes miss.
|
|
131
|
+
* Anything this cannot check (another codec, another hash, an unparsable
|
|
132
|
+
* string) passes: the check is for the case it can prove, not a gate.
|
|
133
|
+
*
|
|
134
|
+
* @param {string} cid
|
|
135
|
+
* @param {Uint8Array} bytes
|
|
136
|
+
*/
|
|
137
|
+
export async function matchesRawCid(cid, bytes) {
|
|
138
|
+
let parsed;
|
|
139
|
+
try {
|
|
140
|
+
parsed = CID.parse(String(cid));
|
|
141
|
+
} catch {
|
|
142
|
+
return true;
|
|
143
|
+
}
|
|
144
|
+
if (parsed.code !== 0x55 || parsed.multihash.code !== 0x12) return true;
|
|
145
|
+
const digest = new Uint8Array(
|
|
146
|
+
await globalThis.crypto.subtle.digest("SHA-256", bytes),
|
|
147
|
+
);
|
|
148
|
+
const expected = parsed.multihash.digest;
|
|
149
|
+
if (digest.length !== expected.length) return false;
|
|
150
|
+
for (let i = 0; i < digest.length; i++)
|
|
151
|
+
if (digest[i] !== expected[i]) return false;
|
|
152
|
+
return true;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export async function fetchFromGateways(
|
|
156
|
+
cid,
|
|
157
|
+
{
|
|
158
|
+
gateways = DEFAULT_GATEWAYS,
|
|
159
|
+
timeout = DEFAULT_TIMEOUT_MS,
|
|
160
|
+
signal = null,
|
|
161
|
+
log = SILENT,
|
|
162
|
+
accept = null,
|
|
163
|
+
maxBackoff = MAX_BACKOFF_MS,
|
|
164
|
+
sleep = waitFor,
|
|
165
|
+
fetchImpl = fetch,
|
|
166
|
+
} = {},
|
|
167
|
+
) {
|
|
119
168
|
let lastError = null;
|
|
120
169
|
|
|
121
170
|
for (const gateway of gateways) {
|
|
@@ -139,10 +188,14 @@ export async function fetchFromGateways(cid, { gateways = DEFAULT_GATEWAYS, time
|
|
|
139
188
|
if (response.status === 429 && attempt < MAX_ATTEMPTS_PER_GATEWAY - 1) {
|
|
140
189
|
const wait = backoffFor(response, attempt);
|
|
141
190
|
if (wait > maxBackoff) {
|
|
142
|
-
log.warn(
|
|
191
|
+
log.warn(
|
|
192
|
+
` ⚠️ ${gateway} wants ${Math.round(wait / 1000)}s — treating it as closed`,
|
|
193
|
+
);
|
|
143
194
|
break;
|
|
144
195
|
}
|
|
145
|
-
log.warn(
|
|
196
|
+
log.warn(
|
|
197
|
+
` ⚠️ ${gateway} rate-limited; waiting ${Math.round(wait / 1000)}s`,
|
|
198
|
+
);
|
|
146
199
|
await sleep(wait);
|
|
147
200
|
continue;
|
|
148
201
|
}
|
|
@@ -152,10 +205,24 @@ export async function fetchFromGateways(cid, { gateways = DEFAULT_GATEWAYS, time
|
|
|
152
205
|
}
|
|
153
206
|
|
|
154
207
|
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
155
|
-
if (
|
|
208
|
+
if (
|
|
209
|
+
looksLikeAnErrorPage(
|
|
210
|
+
bytes,
|
|
211
|
+
response.headers?.get?.("content-type") ?? "",
|
|
212
|
+
)
|
|
213
|
+
) {
|
|
156
214
|
log.warn(` ⚠️ ${gateway} returned an error page with a 200`);
|
|
157
215
|
break;
|
|
158
216
|
}
|
|
217
|
+
// What came back is what was asked for — provable when the CID names
|
|
218
|
+
// the bytes themselves (a raw block, sha-256). A UnixFS CID names a
|
|
219
|
+
// DAG the gateway assembled, and the chunking that made it is the
|
|
220
|
+
// service's; those bytes are trusted as far as the envelope around
|
|
221
|
+
// them goes, which is why a sealed backup refuses plaintext.
|
|
222
|
+
if (!(await matchesRawCid(cid, bytes))) {
|
|
223
|
+
log.warn(` ⚠️ ${gateway} served bytes that do not hash to ${cid}`);
|
|
224
|
+
break;
|
|
225
|
+
}
|
|
159
226
|
log.info(` ✅ ${bytes.length} bytes from ${gateway}`);
|
|
160
227
|
return bytes;
|
|
161
228
|
} catch (error) {
|
package/lib/live-backup.js
CHANGED
|
@@ -347,6 +347,16 @@ export function mergeIncrements({ base, increments = [] }) {
|
|
|
347
347
|
timestamp = Math.max(timestamp, Number(metadata.timestamp) || 0);
|
|
348
348
|
for (const d of metadata.databases) {
|
|
349
349
|
const known = databases.get(String(d.address)) ?? {};
|
|
350
|
+
// Every source's heads, not only the newest's: a join is a merge, a
|
|
351
|
+
// stale head costs nothing, and a branch only the older source saw —
|
|
352
|
+
// two devices shipping in parallel, or a store that renumbered — is
|
|
353
|
+
// kept instead of dropped.
|
|
354
|
+
const heads = [
|
|
355
|
+
...new Set([
|
|
356
|
+
...(known.heads ?? []),
|
|
357
|
+
...(Array.isArray(d.heads) ? d.heads : []),
|
|
358
|
+
]),
|
|
359
|
+
];
|
|
350
360
|
databases.set(String(d.address), {
|
|
351
361
|
...known,
|
|
352
362
|
address: String(d.address),
|
|
@@ -354,8 +364,7 @@ export function mergeIncrements({ base, increments = [] }) {
|
|
|
354
364
|
...(d.type ? { type: d.type } : {}),
|
|
355
365
|
...(d.manifestCID ? { manifestCID: d.manifestCID } : {}),
|
|
356
366
|
...(d.collection ? { collection: d.collection } : {}),
|
|
357
|
-
|
|
358
|
-
...(Array.isArray(d.heads) && d.heads.length ? { heads: d.heads } : {}),
|
|
367
|
+
...(heads.length ? { heads } : {}),
|
|
359
368
|
...(typeof d.entryCount === "number"
|
|
360
369
|
? { entryCount: d.entryCount }
|
|
361
370
|
: {}),
|
|
@@ -363,6 +372,17 @@ export function mergeIncrements({ base, increments = [] }) {
|
|
|
363
372
|
}
|
|
364
373
|
}
|
|
365
374
|
const described = [...databases.values()];
|
|
375
|
+
const sequences = ordered.map((i) => i.manifest?.sequence ?? 0);
|
|
376
|
+
// A chain with a hole restores the heads of the newer part onto entries
|
|
377
|
+
// nobody has: the join then searches the network for each missing block,
|
|
378
|
+
// thirty seconds at a time. Named here, so a caller can refuse up front.
|
|
379
|
+
const gaps = [];
|
|
380
|
+
const duplicates = [];
|
|
381
|
+
for (let i = 1; i < sequences.length; i++) {
|
|
382
|
+
if (sequences[i] === sequences[i - 1]) duplicates.push(sequences[i]);
|
|
383
|
+
for (let missing = sequences[i - 1] + 1; missing < sequences[i]; missing++)
|
|
384
|
+
gaps.push(missing);
|
|
385
|
+
}
|
|
366
386
|
return {
|
|
367
387
|
blocks,
|
|
368
388
|
metadata: {
|
|
@@ -375,10 +395,27 @@ export function mergeIncrements({ base, increments = [] }) {
|
|
|
375
395
|
: {}),
|
|
376
396
|
databases: described,
|
|
377
397
|
},
|
|
378
|
-
sequences
|
|
398
|
+
sequences,
|
|
399
|
+
gaps,
|
|
400
|
+
duplicates,
|
|
379
401
|
};
|
|
380
402
|
}
|
|
381
403
|
|
|
404
|
+
/** Refuse a chain with holes, unless the caller said it knows. */
|
|
405
|
+
function assertContiguous({ gaps, duplicates }, allowGaps) {
|
|
406
|
+
if (allowGaps) return;
|
|
407
|
+
if (gaps.length) {
|
|
408
|
+
throw new Error(
|
|
409
|
+
`The increments have a gap: sequence ${gaps.join(", ")} missing. Fetch them, or pass allowGaps: true to join what is there.`,
|
|
410
|
+
);
|
|
411
|
+
}
|
|
412
|
+
if (duplicates.length) {
|
|
413
|
+
throw new Error(
|
|
414
|
+
`Two increments carry the same sequence ${[...new Set(duplicates)].join(", ")}: the chain was written twice, or renumbered. Pass allowGaps: true to join them all.`,
|
|
415
|
+
);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
382
419
|
/**
|
|
383
420
|
* Put a base backup and its increments back into a node, through
|
|
384
421
|
* `restoreFromBlocks`: the union of their blocks, and each database joined at
|
|
@@ -391,13 +428,17 @@ export function mergeIncrements({ base, increments = [] }) {
|
|
|
391
428
|
* @param {object} [params.open] - options for `orbitdb.open` (`encryption`, `AccessController`, …)
|
|
392
429
|
* @param {object} [params.log]
|
|
393
430
|
* @param {Function} [params.onProgress]
|
|
431
|
+
* @param {boolean} [params.allowGaps=false] - join a chain with missing or
|
|
432
|
+
* duplicate sequence numbers instead of refusing it
|
|
394
433
|
* @returns {Promise<{ blocks: number, databases: object[], sequences: number[] }>}
|
|
395
434
|
*/
|
|
396
435
|
export async function restoreFromIncrements(
|
|
397
436
|
orbitdb,
|
|
398
|
-
{ base, increments = [], open, log, onProgress } = {},
|
|
437
|
+
{ base, increments = [], open, log, onProgress, allowGaps = false } = {},
|
|
399
438
|
) {
|
|
400
|
-
const
|
|
439
|
+
const merged = mergeIncrements({ base, increments });
|
|
440
|
+
assertContiguous(merged, allowGaps);
|
|
441
|
+
const { blocks, metadata, sequences } = merged;
|
|
401
442
|
const restored = await restoreFromBlocks(orbitdb, blocks, metadata, {
|
|
402
443
|
open,
|
|
403
444
|
log,
|
|
@@ -420,10 +461,19 @@ export async function restoreFromIncrements(
|
|
|
420
461
|
* @param {object} params
|
|
421
462
|
* @param {{ metadata?: object, manifest?: object, blocks: Map }} [params.base]
|
|
422
463
|
* @param {{ manifest: IncrementManifest, blocks: Map }[]} params.increments
|
|
423
|
-
* @
|
|
464
|
+
* @param {{ put: (cid: CID, bytes: Uint8Array) => Promise<unknown> }} [params.blockstore] -
|
|
465
|
+
* where the application's own (non-dag-cbor) blocks go, e.g. `orbitdb.ipfs.blockstore`;
|
|
466
|
+
* without it they are counted in `skippedRaw` and left out
|
|
467
|
+
* @param {boolean} [params.allowGaps=false]
|
|
468
|
+
* @returns {Promise<{ joined: number, heads: number, blocks: number, skippedRaw?: number }>}
|
|
424
469
|
*/
|
|
425
|
-
export async function joinIncrements(
|
|
426
|
-
|
|
470
|
+
export async function joinIncrements(
|
|
471
|
+
database,
|
|
472
|
+
{ base, increments = [], blockstore, allowGaps = false } = {},
|
|
473
|
+
) {
|
|
474
|
+
const merged = mergeIncrements({ base, increments });
|
|
475
|
+
assertContiguous(merged, allowGaps);
|
|
476
|
+
const { blocks, metadata } = merged;
|
|
427
477
|
const address = String(database.address);
|
|
428
478
|
const described = metadata.databases.find(
|
|
429
479
|
(d) => String(d.address) === address,
|
|
@@ -431,14 +481,21 @@ export async function joinIncrements(database, { base, increments = [] } = {}) {
|
|
|
431
481
|
if (!described)
|
|
432
482
|
throw new Error("These increments hold nothing for this database.");
|
|
433
483
|
|
|
484
|
+
let skippedRaw = 0;
|
|
434
485
|
for (const [name, { bytes }] of blocks) {
|
|
435
486
|
const cid = CID.parse(name);
|
|
487
|
+
if (cid.code !== DAG_CBOR) {
|
|
488
|
+
// The application's own blocks (receipt files): not the log's, and the
|
|
489
|
+
// log's storage refuses their base32 names. They go where Helia looks,
|
|
490
|
+
// when the caller says where that is.
|
|
491
|
+
if (blockstore) await blockstore.put(cid, bytes);
|
|
492
|
+
else skippedRaw++;
|
|
493
|
+
continue;
|
|
494
|
+
}
|
|
436
495
|
// OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
|
|
437
496
|
// its own names, and identities are looked up by theirs, so each block
|
|
438
497
|
// goes in under the name its reader will ask for.
|
|
439
|
-
|
|
440
|
-
cid.code === DAG_CBOR ? cid.toV1().toString(base58btc) : name;
|
|
441
|
-
await database.log.storage.put(stored, bytes);
|
|
498
|
+
await database.log.storage.put(cid.toV1().toString(base58btc), bytes);
|
|
442
499
|
}
|
|
443
500
|
|
|
444
501
|
let joined = 0;
|
|
@@ -447,7 +504,12 @@ export async function joinIncrements(database, { base, increments = [] } = {}) {
|
|
|
447
504
|
const entry = await database.log.get(name).catch(() => null);
|
|
448
505
|
if (entry && (await database.log.joinEntry(entry))) joined++;
|
|
449
506
|
}
|
|
450
|
-
return {
|
|
507
|
+
return {
|
|
508
|
+
joined,
|
|
509
|
+
heads: (described.heads ?? []).length,
|
|
510
|
+
blocks: blocks.size,
|
|
511
|
+
...(skippedRaw ? { skippedRaw } : {}),
|
|
512
|
+
};
|
|
451
513
|
}
|
|
452
514
|
|
|
453
515
|
/** A state with nothing shipped yet. */
|
|
@@ -481,8 +543,13 @@ export const emptyLiveBackupState = () => ({
|
|
|
481
543
|
* @param {(sequence: number, at: Date) => string} [params.name] - the blob's name
|
|
482
544
|
* @param {(shipped: { sequence: number, handle: object, kept?: object, entries: number, blocks: number, bytes: number, at: Date }) => void} [params.onShipped]
|
|
483
545
|
* @param {(error: Error) => void} [params.onError]
|
|
546
|
+
* @param {(kept: object, handle: object) => Promise<object>} [params.confirm] -
|
|
547
|
+
* follow a backend's answer before the increment counts as shipped: Aleph
|
|
548
|
+
* answers `pending` first and may reject later (no credit, no grant). Given
|
|
549
|
+
* `(kept, handle)`, return the settled answer; `status: "rejected"` fails the
|
|
550
|
+
* flush and the state stays, so the next one ships it all again.
|
|
484
551
|
* @param {() => Date} [params.now]
|
|
485
|
-
* @returns {{ watch: Function, unwatch: Function, start: Function, stop: Function,
|
|
552
|
+
* @returns {{ watch: Function, unwatch: Function, start: Function, stop: Function, close: Function,
|
|
486
553
|
* flush: () => Promise<object|null>, rebase: Function, status: () => object }}
|
|
487
554
|
*/
|
|
488
555
|
export function createLiveBackup({
|
|
@@ -498,6 +565,7 @@ export function createLiveBackup({
|
|
|
498
565
|
`increment-${String(sequence).padStart(6, "0")}-${at.toISOString().replace(/[:.]/g, "-")}.car`,
|
|
499
566
|
onShipped,
|
|
500
567
|
onError,
|
|
568
|
+
confirm: confirmKept,
|
|
501
569
|
now = () => new Date(),
|
|
502
570
|
} = {}) {
|
|
503
571
|
if (!backend || typeof backend.putBlob !== "function") {
|
|
@@ -564,6 +632,13 @@ export function createLiveBackup({
|
|
|
564
632
|
watched.delete(address);
|
|
565
633
|
};
|
|
566
634
|
|
|
635
|
+
/** Stop, and let go of the databases: no listener keeps counting after this. */
|
|
636
|
+
const close = async ({ flush: last = false } = {}) => {
|
|
637
|
+
await stop({ flush: last });
|
|
638
|
+
for (const w of watched.values()) w.off();
|
|
639
|
+
watched.clear();
|
|
640
|
+
};
|
|
641
|
+
|
|
567
642
|
if (Array.isArray(databases)) databases.forEach((d) => watch(d));
|
|
568
643
|
else
|
|
569
644
|
for (const [collection, d] of Object.entries(databases ?? {}))
|
|
@@ -572,8 +647,17 @@ export function createLiveBackup({
|
|
|
572
647
|
const load = async () => {
|
|
573
648
|
if (!loaded) {
|
|
574
649
|
const saved = await store.load();
|
|
575
|
-
if (saved
|
|
650
|
+
if (saved) {
|
|
651
|
+
// A state this build does not read must not be dropped: starting at 1
|
|
652
|
+
// again would ship everything twice under sequence numbers the chain
|
|
653
|
+
// already has.
|
|
654
|
+
if (saved.v !== LIVE_BACKUP_STATE_VERSION) {
|
|
655
|
+
throw new Error(
|
|
656
|
+
`The live backup's state is version ${saved.v}, and this build reads ${LIVE_BACKUP_STATE_VERSION}. Upgrade @le-space/orbitdb-storage-bridge, or clear the state and rebase.`,
|
|
657
|
+
);
|
|
658
|
+
}
|
|
576
659
|
state = { ...emptyLiveBackupState(), ...saved };
|
|
660
|
+
}
|
|
577
661
|
loaded = true;
|
|
578
662
|
}
|
|
579
663
|
for (const [address, w] of watched) {
|
|
@@ -592,6 +676,10 @@ export function createLiveBackup({
|
|
|
592
676
|
|
|
593
677
|
const ship = async () => {
|
|
594
678
|
await load();
|
|
679
|
+
// Counted before the walk: OrbitDB appends before it emits `update`, so
|
|
680
|
+
// every update counted here is in the log the walk is about to read, and
|
|
681
|
+
// one that fires during the walk stays pending for the next flush.
|
|
682
|
+
const countedPending = pending;
|
|
595
683
|
const own = typeof ownBlocks === "function" ? await ownBlocks() : ownBlocks;
|
|
596
684
|
const collected = await collectIncrement(watched, {
|
|
597
685
|
sequence: state.sequence + 1,
|
|
@@ -600,9 +688,8 @@ export function createLiveBackup({
|
|
|
600
688
|
blocks: own,
|
|
601
689
|
now,
|
|
602
690
|
});
|
|
603
|
-
const countedPending = pending;
|
|
604
691
|
if (!collected) {
|
|
605
|
-
pending = 0;
|
|
692
|
+
pending = Math.max(0, pending - countedPending);
|
|
606
693
|
return null;
|
|
607
694
|
}
|
|
608
695
|
const at = now();
|
|
@@ -618,13 +705,25 @@ export function createLiveBackup({
|
|
|
618
705
|
typeof backend.pinCid === "function"
|
|
619
706
|
) {
|
|
620
707
|
kept = await backend.pinCid(handle.id, { name: handle.name });
|
|
708
|
+
if (confirmKept) kept = await confirmKept(kept, handle);
|
|
709
|
+
if (kept?.status === "rejected") {
|
|
710
|
+
throw new Error(
|
|
711
|
+
`The store refused to keep increment ${collected.manifest.sequence}${kept.errorCode != null ? ` (${kept.errorCode})` : ""}: it was uploaded but is not kept.`,
|
|
712
|
+
);
|
|
713
|
+
}
|
|
621
714
|
}
|
|
622
715
|
|
|
623
|
-
// Only now, with the bytes out of the house, does the state move
|
|
624
|
-
|
|
625
|
-
|
|
716
|
+
// Only now, with the bytes out of the house, does the state move — and
|
|
717
|
+
// all of it at once, so a database unwatched during the upload cannot
|
|
718
|
+
// leave it half moved.
|
|
719
|
+
const moved = [];
|
|
626
720
|
for (const [address, { heads, hashes }] of collected.shipped) {
|
|
627
721
|
const w = watched.get(address);
|
|
722
|
+
if (w) moved.push({ address, w, heads, hashes });
|
|
723
|
+
}
|
|
724
|
+
state.sequence = collected.manifest.sequence;
|
|
725
|
+
state.previous = String(handle.id ?? handle.cid ?? "");
|
|
726
|
+
for (const { address, w, heads, hashes } of moved) {
|
|
628
727
|
hashes.forEach((hash) => w.shipped.add(hash));
|
|
629
728
|
w.shipped.heads = heads;
|
|
630
729
|
state.databases[address] = { heads };
|
|
@@ -665,7 +764,7 @@ export function createLiveBackup({
|
|
|
665
764
|
inflight = null;
|
|
666
765
|
if (again) {
|
|
667
766
|
again = false;
|
|
668
|
-
flush().catch(() => {});
|
|
767
|
+
if (running) flush().catch(() => {});
|
|
669
768
|
}
|
|
670
769
|
}
|
|
671
770
|
})();
|
|
@@ -683,6 +782,8 @@ export function createLiveBackup({
|
|
|
683
782
|
* @param {{ base?: string|null }} [params] - what the caller calls the full backup (its id, CID, …)
|
|
684
783
|
*/
|
|
685
784
|
const rebase = async ({ base } = {}) => {
|
|
785
|
+
// Not under a flush: ship() would write the pre-rebase state back.
|
|
786
|
+
if (inflight) await inflight.catch(() => {});
|
|
686
787
|
await load();
|
|
687
788
|
if (base !== undefined)
|
|
688
789
|
state = { ...emptyLiveBackupState(), base: base ?? null };
|
|
@@ -713,7 +814,11 @@ export function createLiveBackup({
|
|
|
713
814
|
if (timer) clearTimeout(timer);
|
|
714
815
|
if (settling) clearTimeout(settling);
|
|
715
816
|
timer = settling = null;
|
|
716
|
-
|
|
817
|
+
// A flush asked for during one runs after it; the caller gets its own.
|
|
818
|
+
if (last) {
|
|
819
|
+
if (inflight) await inflight.catch(() => {});
|
|
820
|
+
await flush();
|
|
821
|
+
}
|
|
717
822
|
};
|
|
718
823
|
|
|
719
824
|
const status = () => ({
|
|
@@ -728,7 +833,7 @@ export function createLiveBackup({
|
|
|
728
833
|
failures,
|
|
729
834
|
});
|
|
730
835
|
|
|
731
|
-
return { watch, unwatch, start, stop, flush, rebase, status };
|
|
836
|
+
return { watch, unwatch, start, stop, close, flush, rebase, status };
|
|
732
837
|
}
|
|
733
838
|
|
|
734
839
|
export default createLiveBackup;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@le-space/orbitdb-storage-bridge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.20.0",
|
|
4
4
|
"description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
|
|
5
5
|
"main": "lib/orbitdb-storacha-bridge.js",
|
|
6
6
|
"svelte": "dist/components/",
|