@le-space/orbitdb-storage-bridge 0.28.0 → 0.28.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/backends/aleph-pin.js +14 -3
- package/lib/live-backup.js +88 -38
- package/package.json +1 -1
|
@@ -358,12 +358,16 @@ export async function waitForMessage(itemHash, options = {}) {
|
|
|
358
358
|
* coming back later, e.g. to an increment that was still pending when its
|
|
359
359
|
* flush ended (`createLiveBackup().reconcile`).
|
|
360
360
|
*
|
|
361
|
+
* `unknown` right after sending is normal: a node answers 404 until the
|
|
362
|
+
* message has reached it. `reconcile` therefore takes `unknown` as refused
|
|
363
|
+
* only for an increment shipped more than `UNKNOWN_GRACE_MS` ago.
|
|
364
|
+
*
|
|
361
365
|
* @param {string} itemHash
|
|
362
366
|
* @param {object} [options]
|
|
363
367
|
* @param {string} [options.apiHost]
|
|
364
368
|
* @param {typeof fetch} [options.fetch]
|
|
365
369
|
* @returns {Promise<{ itemHash: string, status: string, errorCode?: number, details?: unknown }>}
|
|
366
|
-
* @throws when Aleph does not answer (anything but 200 or 404)
|
|
370
|
+
* @throws when Aleph does not answer (anything but 200 or 404), or answers 200 without a status
|
|
367
371
|
*/
|
|
368
372
|
export async function messageStatus(itemHash, options = {}) {
|
|
369
373
|
const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
|
|
@@ -382,10 +386,17 @@ export async function messageStatus(itemHash, options = {}) {
|
|
|
382
386
|
"UNSUPPORTED",
|
|
383
387
|
`Aleph did not answer: ${response.status}`,
|
|
384
388
|
);
|
|
385
|
-
|
|
389
|
+
// A body that says nothing is no answer – not "unknown", which a caller
|
|
390
|
+
// takes as refused and ships again.
|
|
391
|
+
const body = await response.json().catch(() => null);
|
|
392
|
+
if (typeof body?.status !== "string")
|
|
393
|
+
throw new BackendError(
|
|
394
|
+
"UNSUPPORTED",
|
|
395
|
+
`Aleph answered ${response.status} without a message status`,
|
|
396
|
+
);
|
|
386
397
|
return {
|
|
387
398
|
itemHash,
|
|
388
|
-
status:
|
|
399
|
+
status: body.status,
|
|
389
400
|
...(body?.error_code != null ? { errorCode: body.error_code } : {}),
|
|
390
401
|
...(body?.details != null ? { details: body.details } : {}),
|
|
391
402
|
};
|
package/lib/live-backup.js
CHANGED
|
@@ -563,6 +563,12 @@ export async function joinIncrements(
|
|
|
563
563
|
};
|
|
564
564
|
}
|
|
565
565
|
|
|
566
|
+
/**
|
|
567
|
+
* How long an increment may be unknown to the store before that counts as
|
|
568
|
+
* refused: Aleph answers 404 for a message that has not reached the node yet.
|
|
569
|
+
*/
|
|
570
|
+
export const UNKNOWN_GRACE_MS = 10 * 60_000;
|
|
571
|
+
|
|
566
572
|
/** Statuses that mean the store does not keep a blob. */
|
|
567
573
|
export const NOT_KEPT = Object.freeze(["rejected", "unknown", "forgotten"]);
|
|
568
574
|
|
|
@@ -681,7 +687,10 @@ export const emptyLiveBackupState = () => ({
|
|
|
681
687
|
* @param {() => Promise<Map<string, { bytes: Uint8Array }>>|Map} [params.blocks] - blocks of the
|
|
682
688
|
* application's own to carry in the next increment (receipt files, …)
|
|
683
689
|
* @param {boolean} [params.keep=true] - ask a backend whose handle says
|
|
684
|
-
* `retained: false` to keep the increment (`pinCid`)
|
|
690
|
+
* `retained: false` to keep the increment (`pinCid`). A handle that already
|
|
691
|
+
* carries the store's answer (`status`, as an Aleph upload with a signing
|
|
692
|
+
* pin does: the STORE went with the file) is not pinned a second time; its
|
|
693
|
+
* answer is the one followed and recorded
|
|
685
694
|
* @param {(sequence: number, at: Date) => string} [params.name] - the blob's name
|
|
686
695
|
* @param {(shipped: { sequence: number, handle: object, kept?: object, entries: number, blocks: number, bytes: number, at: Date }) => void} [params.onShipped]
|
|
687
696
|
* @param {(error: Error) => void} [params.onError]
|
|
@@ -723,6 +732,8 @@ export function createLiveBackup({
|
|
|
723
732
|
let pending = 0;
|
|
724
733
|
let inflight = null;
|
|
725
734
|
let again = false;
|
|
735
|
+
/** A reconcile applying its answers: a flush waits for it, or it would ship from a state about to move. */
|
|
736
|
+
let reconciling = null;
|
|
726
737
|
let timer = null;
|
|
727
738
|
let settling = null;
|
|
728
739
|
let lastShipped = null;
|
|
@@ -817,6 +828,7 @@ export function createLiveBackup({
|
|
|
817
828
|
};
|
|
818
829
|
|
|
819
830
|
const ship = async () => {
|
|
831
|
+
if (reconciling) await reconciling.catch(() => {});
|
|
820
832
|
await load();
|
|
821
833
|
// Counted before the walk: OrbitDB appends before it emits `update`, so
|
|
822
834
|
// every update counted here is in the log the walk is about to read, and
|
|
@@ -841,13 +853,27 @@ export function createLiveBackup({
|
|
|
841
853
|
contentType: "application/vnd.ipld.car",
|
|
842
854
|
});
|
|
843
855
|
let kept;
|
|
844
|
-
if (
|
|
856
|
+
if (typeof handle?.status === "string") {
|
|
857
|
+
// The upload carried its own keep request (a signed Aleph upload): a
|
|
858
|
+
// second one through pinCid would be paid twice, and would take the
|
|
859
|
+
// path whose STOREs the nodes could not fetch (2026-11).
|
|
860
|
+
kept = {
|
|
861
|
+
id: handle.id,
|
|
862
|
+
backend: handle.backend,
|
|
863
|
+
retained: handle.retained === true,
|
|
864
|
+
status: handle.status,
|
|
865
|
+
...(handle.itemHash ? { itemHash: handle.itemHash } : {}),
|
|
866
|
+
};
|
|
867
|
+
} else if (
|
|
845
868
|
keep &&
|
|
846
869
|
handle?.retained === false &&
|
|
847
870
|
typeof backend.pinCid === "function"
|
|
848
871
|
) {
|
|
849
872
|
kept = await backend.pinCid(handle.id, { name: handle.name });
|
|
850
|
-
|
|
873
|
+
}
|
|
874
|
+
if (kept) {
|
|
875
|
+
if (confirmKept && kept.status !== "processed")
|
|
876
|
+
kept = await confirmKept(kept, handle);
|
|
851
877
|
if (kept?.status === "rejected") {
|
|
852
878
|
throw new Error(
|
|
853
879
|
`The store refused to keep increment ${collected.manifest.sequence}${kept.errorCode != null ? ` (${kept.errorCode})` : ""}: it was uploaded but is not kept.`,
|
|
@@ -878,6 +904,9 @@ export function createLiveBackup({
|
|
|
878
904
|
sequence: state.sequence,
|
|
879
905
|
previous: state.previous,
|
|
880
906
|
databases: JSON.parse(JSON.stringify(state.databases)),
|
|
907
|
+
// When it went out: a store that does not know a message yet
|
|
908
|
+
// (Aleph's 404 before it has propagated) is not a refusal.
|
|
909
|
+
shippedAt: at.toISOString(),
|
|
881
910
|
},
|
|
882
911
|
state.previous,
|
|
883
912
|
kept?.status ?? "processed",
|
|
@@ -976,48 +1005,69 @@ export function createLiveBackup({
|
|
|
976
1005
|
const reconcile = async (statusOf) => {
|
|
977
1006
|
if (inflight) await inflight.catch(() => {});
|
|
978
1007
|
await load();
|
|
1008
|
+
// The answers first, which takes network time; a flush may run meanwhile.
|
|
979
1009
|
const answers = new Map();
|
|
1010
|
+
const asked = Date.now();
|
|
980
1011
|
for (const point of state.unconfirmed ?? []) {
|
|
981
|
-
|
|
1012
|
+
let answer = await statusOf(point.cid, point);
|
|
1013
|
+
// Not known yet is not refused while the message may still be on its way.
|
|
1014
|
+
const age = point.shippedAt ? asked - Date.parse(point.shippedAt) : Infinity;
|
|
1015
|
+
if (answer === "unknown" && age < UNKNOWN_GRACE_MS) answer = undefined;
|
|
1016
|
+
answers.set(point.cid, answer);
|
|
982
1017
|
}
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
if (
|
|
1018
|
+
// Then applied to the state as it is now, with no flush running and none
|
|
1019
|
+
// starting until it is done: a point shipped meanwhile has no answer, so
|
|
1020
|
+
// the walk stops before it – or, when an earlier one was refused, it goes
|
|
1021
|
+
// back with the rest.
|
|
1022
|
+
if (inflight) await inflight.catch(() => {});
|
|
1023
|
+
let release;
|
|
1024
|
+
let wentBack = false;
|
|
1025
|
+
reconciling = new Promise((resolve) => (release = resolve));
|
|
1026
|
+
try {
|
|
1027
|
+
const now_ = reconcileConfirmation(
|
|
1028
|
+
{ kept: state.kept ?? null, pending: state.unconfirmed ?? [] },
|
|
1029
|
+
(cid) => answers.get(cid),
|
|
1030
|
+
);
|
|
1031
|
+
if (!now_.changed)
|
|
1032
|
+
return {
|
|
1033
|
+
changed: false,
|
|
1034
|
+
rewound: false,
|
|
1035
|
+
gap: false,
|
|
1036
|
+
sequence: state.sequence,
|
|
1037
|
+
};
|
|
1038
|
+
state.kept = now_.kept;
|
|
1039
|
+
state.unconfirmed = now_.pending;
|
|
1040
|
+
if (now_.rewound) {
|
|
1041
|
+
const back = now_.rewind ?? {
|
|
1042
|
+
sequence: 0,
|
|
1043
|
+
previous: null,
|
|
1044
|
+
databases: {},
|
|
1045
|
+
};
|
|
1046
|
+
state.sequence = back.sequence;
|
|
1047
|
+
state.previous = back.previous;
|
|
1048
|
+
state.databases = JSON.parse(JSON.stringify(back.databases ?? {}));
|
|
1049
|
+
for (const [address, w] of watched) {
|
|
1050
|
+
const heads = state.databases[address]?.heads ?? [];
|
|
1051
|
+
w.shipped = await reachable(w.database, heads);
|
|
1052
|
+
w.shipped.heads = heads;
|
|
1053
|
+
}
|
|
1054
|
+
// Nothing announces what has to go again: count it, so a flush ships it.
|
|
1055
|
+
pending = Math.max(pending, 1);
|
|
1056
|
+
wentBack = true;
|
|
1057
|
+
}
|
|
1058
|
+
await save();
|
|
988
1059
|
return {
|
|
989
|
-
changed:
|
|
990
|
-
rewound:
|
|
991
|
-
gap:
|
|
1060
|
+
changed: true,
|
|
1061
|
+
rewound: now_.rewound,
|
|
1062
|
+
gap: now_.gap,
|
|
992
1063
|
sequence: state.sequence,
|
|
993
1064
|
};
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
previous: null,
|
|
1000
|
-
databases: {},
|
|
1001
|
-
};
|
|
1002
|
-
state.sequence = back.sequence;
|
|
1003
|
-
state.previous = back.previous;
|
|
1004
|
-
state.databases = JSON.parse(JSON.stringify(back.databases ?? {}));
|
|
1005
|
-
for (const [address, w] of watched) {
|
|
1006
|
-
const heads = state.databases[address]?.heads ?? [];
|
|
1007
|
-
w.shipped = await reachable(w.database, heads);
|
|
1008
|
-
w.shipped.heads = heads;
|
|
1009
|
-
}
|
|
1010
|
-
// Nothing announces what has to go again: count it, so a flush ships it.
|
|
1011
|
-
pending = Math.max(pending, 1);
|
|
1012
|
-
if (running && (after || every)) scheduleFlush();
|
|
1065
|
+
} finally {
|
|
1066
|
+
reconciling = null;
|
|
1067
|
+
release();
|
|
1068
|
+
// Only once the flush can see the state it is to ship from.
|
|
1069
|
+
if (wentBack && running && (after || every)) scheduleFlush();
|
|
1013
1070
|
}
|
|
1014
|
-
await save();
|
|
1015
|
-
return {
|
|
1016
|
-
changed: true,
|
|
1017
|
-
rewound: now_.rewound,
|
|
1018
|
-
gap: now_.gap,
|
|
1019
|
-
sequence: state.sequence,
|
|
1020
|
-
};
|
|
1021
1071
|
};
|
|
1022
1072
|
|
|
1023
1073
|
const start = async () => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@le-space/orbitdb-storage-bridge",
|
|
3
|
-
"version": "0.28.
|
|
3
|
+
"version": "0.28.1",
|
|
4
4
|
"description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
|
|
5
5
|
"main": "lib/orbitdb-storacha-bridge.js",
|
|
6
6
|
"type": "module",
|