@le-space/orbitdb-storage-bridge 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/lib/app-backup.js +290 -40
- package/lib/backends/aleph-pin.js +14 -3
- package/lib/backends/aleph-upload.js +95 -0
- package/lib/backends/aleph.js +45 -1
- package/lib/backends/encryption.js +14 -1
- package/lib/backends/frames.js +507 -0
- package/lib/backends/types.js +3 -0
- package/lib/gateway-fetch.js +221 -0
- package/lib/live-backup.js +101 -41
- package/lib/peer-fetch.js +115 -9
- package/lib/restore-cid.js +257 -22
- package/package.json +2 -1
package/lib/gateway-fetch.js
CHANGED
|
@@ -184,6 +184,227 @@ async function readBody(
|
|
|
184
184
|
return bytes;
|
|
185
185
|
}
|
|
186
186
|
|
|
187
|
+
/** How often one transfer may pick up again where a connection dropped it. */
|
|
188
|
+
export const MAX_RESUMES = 20;
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* The bytes behind a CID as they arrive, never all of them at once (#187).
|
|
192
|
+
*
|
|
193
|
+
* {@link fetchFromGateways} assembles the whole file before it returns, which
|
|
194
|
+
* is right for a block and for a manifest, and more than a phone's browser tab
|
|
195
|
+
* may hold for a backup of several hundred MB. This yields each chunk as the
|
|
196
|
+
* gateway sends it; what the caller does not keep is gone.
|
|
197
|
+
*
|
|
198
|
+
* A connection that drops after the first byte is not the end: the next
|
|
199
|
+
* request asks for the rest (`Range: bytes=<received>-`), from the same
|
|
200
|
+
* gateway and then the next. A gateway that ignores the range and sends the
|
|
201
|
+
* file from its start has those bytes passed over. A phone changes cells in
|
|
202
|
+
* the minutes a large file takes.
|
|
203
|
+
*
|
|
204
|
+
* **What is yielded is not yet checked.** The whole-file checks of
|
|
205
|
+
* `fetchFromGateways` can only speak at the end, and by then the bytes are
|
|
206
|
+
* with the caller: a file of another length than `expectedSize`, or one the
|
|
207
|
+
* `verifier` refuses, throws after the last chunk was yielded and before the
|
|
208
|
+
* iteration ends. So a caller acts on chunks only as far as each proves
|
|
209
|
+
* itself (a sealed frame that opens, a block that hashes to its CID), and
|
|
210
|
+
* treats the file as whole only once the iteration finished without an error.
|
|
211
|
+
*
|
|
212
|
+
* @param {string} cid
|
|
213
|
+
* @param {Object} [options] as for {@link fetchFromGateways}, except:
|
|
214
|
+
* @param {number} [options.timeout] idle time in ms. The clock stands still
|
|
215
|
+
* while the caller is busy with a chunk: only the gateway's silence counts.
|
|
216
|
+
* @param {number} [options.offset] start at this byte: the file from there on
|
|
217
|
+
* @param {((cid: string) => { write(chunk: Uint8Array): void | Promise<void>, done(): Promise<boolean> } | null) | null} [options.verifier]
|
|
218
|
+
* a check of the whole file that takes it in pieces – the Aleph driver
|
|
219
|
+
* passes `unixfsCidVerifier`. Not asked when the transfer started at an
|
|
220
|
+
* `offset`, for it never sees the start.
|
|
221
|
+
* @param {number} [options.maxResumes]
|
|
222
|
+
* @returns {AsyncGenerator<Uint8Array>}
|
|
223
|
+
*/
|
|
224
|
+
export async function* streamFromGateways(
|
|
225
|
+
cid,
|
|
226
|
+
{
|
|
227
|
+
gateways = DEFAULT_GATEWAYS,
|
|
228
|
+
timeout = DEFAULT_TIMEOUT_MS,
|
|
229
|
+
signal = null,
|
|
230
|
+
log = SILENT,
|
|
231
|
+
accept = null,
|
|
232
|
+
maxBackoff = MAX_BACKOFF_MS,
|
|
233
|
+
sleep = waitFor,
|
|
234
|
+
fetchImpl = fetch,
|
|
235
|
+
onProgress = null,
|
|
236
|
+
expectedSize = null,
|
|
237
|
+
verifier = null,
|
|
238
|
+
offset = 0,
|
|
239
|
+
maxResumes = MAX_RESUMES,
|
|
240
|
+
} = {},
|
|
241
|
+
) {
|
|
242
|
+
let lastError = null;
|
|
243
|
+
/** Bytes of the file handed on so far, counted from its start. */
|
|
244
|
+
let received = offset;
|
|
245
|
+
/** The file's length, once a gateway announced it. */
|
|
246
|
+
let total = 0;
|
|
247
|
+
let resumes = 0;
|
|
248
|
+
let complete = false;
|
|
249
|
+
const checking = offset === 0 && verifier ? verifier(cid) : null;
|
|
250
|
+
|
|
251
|
+
outer: for (const gateway of gateways) {
|
|
252
|
+
const url = `${gateway}/${cid}`;
|
|
253
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS_PER_GATEWAY; attempt++) {
|
|
254
|
+
const idle = new AbortController();
|
|
255
|
+
let clock = null;
|
|
256
|
+
const stop = () => {
|
|
257
|
+
if (clock) clearTimeout(clock);
|
|
258
|
+
clock = null;
|
|
259
|
+
};
|
|
260
|
+
const stillAlive = () => {
|
|
261
|
+
stop();
|
|
262
|
+
clock = setTimeout(
|
|
263
|
+
() =>
|
|
264
|
+
idle.abort(new Error(`no bytes from ${gateway} for ${timeout} ms`)),
|
|
265
|
+
timeout,
|
|
266
|
+
);
|
|
267
|
+
};
|
|
268
|
+
const startedAt = received;
|
|
269
|
+
let reader = null;
|
|
270
|
+
stillAlive();
|
|
271
|
+
try {
|
|
272
|
+
const headers = {
|
|
273
|
+
...(accept ? { Accept: accept } : {}),
|
|
274
|
+
...(received > 0 ? { Range: `bytes=${received}-` } : {}),
|
|
275
|
+
};
|
|
276
|
+
const response = await fetchImpl(url, {
|
|
277
|
+
signal: signal ?? idle.signal,
|
|
278
|
+
...(Object.keys(headers).length ? { headers } : {}),
|
|
279
|
+
});
|
|
280
|
+
stillAlive();
|
|
281
|
+
|
|
282
|
+
const sunset = response.headers?.get?.("Sunset");
|
|
283
|
+
if (sunset && !response.ok) {
|
|
284
|
+
log.warn(` ⚠️ ${gateway} is retired (Sunset: ${sunset})`);
|
|
285
|
+
break;
|
|
286
|
+
}
|
|
287
|
+
if (response.status === 429 && attempt < MAX_ATTEMPTS_PER_GATEWAY - 1) {
|
|
288
|
+
const wait = backoffFor(response, attempt);
|
|
289
|
+
if (wait > maxBackoff) {
|
|
290
|
+
log.warn(
|
|
291
|
+
` ⚠️ ${gateway} wants ${Math.round(wait / 1000)}s — treating it as closed`,
|
|
292
|
+
);
|
|
293
|
+
break;
|
|
294
|
+
}
|
|
295
|
+
stop();
|
|
296
|
+
await sleep(wait);
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
if (!response.ok) {
|
|
300
|
+
lastError = new Error(`${gateway} answered ${response.status}`);
|
|
301
|
+
log.debug(` ⚠️ ${lastError.message}`);
|
|
302
|
+
break;
|
|
303
|
+
}
|
|
304
|
+
if (!response.body?.getReader) {
|
|
305
|
+
throw new Error(`${gateway} answered without a body to read`);
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// How much of the answer to pass over: all of what is already with
|
|
309
|
+
// the caller, when the gateway sent the file from its start anyway.
|
|
310
|
+
let skip = 0;
|
|
311
|
+
if (response.status === 206) {
|
|
312
|
+
const range = /bytes (\d+)-\d*\/(\d+|\*)/.exec(
|
|
313
|
+
response.headers?.get?.("content-range") ?? "",
|
|
314
|
+
);
|
|
315
|
+
const from = range ? Number(range[1]) : received;
|
|
316
|
+
if (from > received) {
|
|
317
|
+
throw new Error(
|
|
318
|
+
`${gateway} answered from byte ${from}, asked from ${received}`,
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
skip = received - from;
|
|
322
|
+
if (range && range[2] !== "*") total = Number(range[2]);
|
|
323
|
+
} else {
|
|
324
|
+
skip = received;
|
|
325
|
+
const announced =
|
|
326
|
+
Number(response.headers?.get?.("content-length") ?? 0) || 0;
|
|
327
|
+
if (announced) total = announced;
|
|
328
|
+
}
|
|
329
|
+
const contentType = response.headers?.get?.("content-type") ?? "";
|
|
330
|
+
|
|
331
|
+
reader = response.body.getReader();
|
|
332
|
+
for (;;) {
|
|
333
|
+
const { done, value } = await reader.read();
|
|
334
|
+
if (signal?.aborted) throw signal.reason;
|
|
335
|
+
if (idle.signal.aborted) throw idle.signal.reason;
|
|
336
|
+
if (done) break;
|
|
337
|
+
let chunk = value;
|
|
338
|
+
if (skip > 0) {
|
|
339
|
+
const passed = Math.min(skip, chunk.length);
|
|
340
|
+
skip -= passed;
|
|
341
|
+
chunk = chunk.subarray(passed);
|
|
342
|
+
stillAlive();
|
|
343
|
+
if (!chunk.length) continue;
|
|
344
|
+
}
|
|
345
|
+
// An error page with a 200: only the file's first bytes can tell.
|
|
346
|
+
if (received === 0 && looksLikeAnErrorPage(chunk, contentType)) {
|
|
347
|
+
lastError = new Error(
|
|
348
|
+
`${gateway} returned an error page with a 200`,
|
|
349
|
+
);
|
|
350
|
+
log.warn(` ⚠️ ${lastError.message}`);
|
|
351
|
+
continue outer;
|
|
352
|
+
}
|
|
353
|
+
if (checking) await checking.write(chunk);
|
|
354
|
+
received += chunk.length;
|
|
355
|
+
onProgress?.(received, total || expectedSize || 0);
|
|
356
|
+
// The caller's time with a chunk is not the gateway's silence.
|
|
357
|
+
stop();
|
|
358
|
+
yield chunk;
|
|
359
|
+
stillAlive();
|
|
360
|
+
}
|
|
361
|
+
// A body that ended early without an error: ask for the rest.
|
|
362
|
+
if (total && received < total) {
|
|
363
|
+
throw new Error(
|
|
364
|
+
`${gateway} ended the answer at byte ${received} of ${total}`,
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
complete = true;
|
|
368
|
+
log.info(` ✅ ${received - offset} bytes from ${gateway}`);
|
|
369
|
+
break outer;
|
|
370
|
+
} catch (error) {
|
|
371
|
+
lastError = idle.signal.aborted ? idle.signal.reason : error;
|
|
372
|
+
log.debug(` ⚠️ ${gateway} failed: ${lastError.message}`);
|
|
373
|
+
if (signal?.aborted) break outer;
|
|
374
|
+
// It moved: this was a dropped connection, not a gateway that cannot
|
|
375
|
+
// serve the file, and the attempts start over, within a budget.
|
|
376
|
+
if (received > startedAt) {
|
|
377
|
+
if (++resumes > maxResumes) break outer;
|
|
378
|
+
attempt = -1;
|
|
379
|
+
}
|
|
380
|
+
} finally {
|
|
381
|
+
stop();
|
|
382
|
+
// Left early, by an error or by the caller: let the connection go.
|
|
383
|
+
if (!complete) await reader?.cancel?.().catch(() => {});
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
if (!complete) {
|
|
389
|
+
const error = new Error(
|
|
390
|
+
`Could not fetch ${cid} from any gateway` +
|
|
391
|
+
(received > offset ? ` (it stopped at byte ${received})` : "") +
|
|
392
|
+
(lastError ? `. Last error: ${lastError.message}` : ""),
|
|
393
|
+
);
|
|
394
|
+
// For a caller with another source: where that one may go on.
|
|
395
|
+
error.received = received;
|
|
396
|
+
throw error;
|
|
397
|
+
}
|
|
398
|
+
if (expectedSize != null && received !== expectedSize) {
|
|
399
|
+
throw new Error(
|
|
400
|
+
`The gateways served ${received} bytes of ${cid}, not the ${expectedSize} it has`,
|
|
401
|
+
);
|
|
402
|
+
}
|
|
403
|
+
if (checking && !(await checking.done())) {
|
|
404
|
+
throw new Error(`The gateways served bytes that are not ${cid}`);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
187
408
|
/**
|
|
188
409
|
* Fetch the bytes behind a CID, trying each gateway in turn.
|
|
189
410
|
*
|
package/lib/live-backup.js
CHANGED
|
@@ -346,8 +346,11 @@ const metadataOf = (source) => source?.metadata ?? source?.manifest?.metadata;
|
|
|
346
346
|
* their `sequence`; the base, if given, comes first.
|
|
347
347
|
*
|
|
348
348
|
* @param {object} params
|
|
349
|
-
*
|
|
350
|
-
*
|
|
349
|
+
* A source may come without its `blocks`: a restore that streamed them into
|
|
350
|
+
* the blockstore has only the manifests left, and merges those.
|
|
351
|
+
*
|
|
352
|
+
* @param {{ metadata?: object, manifest?: object, blocks?: Map }} [params.base]
|
|
353
|
+
* @param {{ manifest: IncrementManifest, blocks?: Map }[]} params.increments
|
|
351
354
|
* @returns {{ blocks: Map, metadata: object, sequences: number[] }}
|
|
352
355
|
*/
|
|
353
356
|
export function mergeIncrements({ base, increments = [] }) {
|
|
@@ -386,7 +389,12 @@ export function mergeIncrements({ base, increments = [] }) {
|
|
|
386
389
|
...(d.type ? { type: d.type } : {}),
|
|
387
390
|
...(d.manifestCID ? { manifestCID: d.manifestCID } : {}),
|
|
388
391
|
...(d.collection ? { collection: d.collection } : {}),
|
|
389
|
-
|
|
392
|
+
// Stated and empty is not the same as not stated: an empty database
|
|
393
|
+
// has no heads, and a reader that is told so does not go looking for
|
|
394
|
+
// them among the blocks (`restoreFromStoredBlocks` cannot).
|
|
395
|
+
...(heads.length || Array.isArray(d.heads) || Array.isArray(known.heads)
|
|
396
|
+
? { heads }
|
|
397
|
+
: {}),
|
|
390
398
|
...(typeof d.entryCount === "number"
|
|
391
399
|
? { entryCount: d.entryCount }
|
|
392
400
|
: {}),
|
|
@@ -563,6 +571,12 @@ export async function joinIncrements(
|
|
|
563
571
|
};
|
|
564
572
|
}
|
|
565
573
|
|
|
574
|
+
/**
|
|
575
|
+
* How long an increment may be unknown to the store before that counts as
|
|
576
|
+
* refused: Aleph answers 404 for a message that has not reached the node yet.
|
|
577
|
+
*/
|
|
578
|
+
export const UNKNOWN_GRACE_MS = 10 * 60_000;
|
|
579
|
+
|
|
566
580
|
/** Statuses that mean the store does not keep a blob. */
|
|
567
581
|
export const NOT_KEPT = Object.freeze(["rejected", "unknown", "forgotten"]);
|
|
568
582
|
|
|
@@ -681,7 +695,10 @@ export const emptyLiveBackupState = () => ({
|
|
|
681
695
|
* @param {() => Promise<Map<string, { bytes: Uint8Array }>>|Map} [params.blocks] - blocks of the
|
|
682
696
|
* application's own to carry in the next increment (receipt files, …)
|
|
683
697
|
* @param {boolean} [params.keep=true] - ask a backend whose handle says
|
|
684
|
-
* `retained: false` to keep the increment (`pinCid`)
|
|
698
|
+
* `retained: false` to keep the increment (`pinCid`). A handle that already
|
|
699
|
+
* carries the store's answer (`status`, as an Aleph upload with a signing
|
|
700
|
+
* pin does: the STORE went with the file) is not pinned a second time; its
|
|
701
|
+
* answer is the one followed and recorded
|
|
685
702
|
* @param {(sequence: number, at: Date) => string} [params.name] - the blob's name
|
|
686
703
|
* @param {(shipped: { sequence: number, handle: object, kept?: object, entries: number, blocks: number, bytes: number, at: Date }) => void} [params.onShipped]
|
|
687
704
|
* @param {(error: Error) => void} [params.onError]
|
|
@@ -723,6 +740,8 @@ export function createLiveBackup({
|
|
|
723
740
|
let pending = 0;
|
|
724
741
|
let inflight = null;
|
|
725
742
|
let again = false;
|
|
743
|
+
/** A reconcile applying its answers: a flush waits for it, or it would ship from a state about to move. */
|
|
744
|
+
let reconciling = null;
|
|
726
745
|
let timer = null;
|
|
727
746
|
let settling = null;
|
|
728
747
|
let lastShipped = null;
|
|
@@ -817,6 +836,7 @@ export function createLiveBackup({
|
|
|
817
836
|
};
|
|
818
837
|
|
|
819
838
|
const ship = async () => {
|
|
839
|
+
if (reconciling) await reconciling.catch(() => {});
|
|
820
840
|
await load();
|
|
821
841
|
// Counted before the walk: OrbitDB appends before it emits `update`, so
|
|
822
842
|
// every update counted here is in the log the walk is about to read, and
|
|
@@ -841,13 +861,27 @@ export function createLiveBackup({
|
|
|
841
861
|
contentType: "application/vnd.ipld.car",
|
|
842
862
|
});
|
|
843
863
|
let kept;
|
|
844
|
-
if (
|
|
864
|
+
if (typeof handle?.status === "string") {
|
|
865
|
+
// The upload carried its own keep request (a signed Aleph upload): a
|
|
866
|
+
// second one through pinCid would be paid twice, and would take the
|
|
867
|
+
// path whose STOREs the nodes could not fetch (2026-11).
|
|
868
|
+
kept = {
|
|
869
|
+
id: handle.id,
|
|
870
|
+
backend: handle.backend,
|
|
871
|
+
retained: handle.retained === true,
|
|
872
|
+
status: handle.status,
|
|
873
|
+
...(handle.itemHash ? { itemHash: handle.itemHash } : {}),
|
|
874
|
+
};
|
|
875
|
+
} else if (
|
|
845
876
|
keep &&
|
|
846
877
|
handle?.retained === false &&
|
|
847
878
|
typeof backend.pinCid === "function"
|
|
848
879
|
) {
|
|
849
880
|
kept = await backend.pinCid(handle.id, { name: handle.name });
|
|
850
|
-
|
|
881
|
+
}
|
|
882
|
+
if (kept) {
|
|
883
|
+
if (confirmKept && kept.status !== "processed")
|
|
884
|
+
kept = await confirmKept(kept, handle);
|
|
851
885
|
if (kept?.status === "rejected") {
|
|
852
886
|
throw new Error(
|
|
853
887
|
`The store refused to keep increment ${collected.manifest.sequence}${kept.errorCode != null ? ` (${kept.errorCode})` : ""}: it was uploaded but is not kept.`,
|
|
@@ -878,6 +912,9 @@ export function createLiveBackup({
|
|
|
878
912
|
sequence: state.sequence,
|
|
879
913
|
previous: state.previous,
|
|
880
914
|
databases: JSON.parse(JSON.stringify(state.databases)),
|
|
915
|
+
// When it went out: a store that does not know a message yet
|
|
916
|
+
// (Aleph's 404 before it has propagated) is not a refusal.
|
|
917
|
+
shippedAt: at.toISOString(),
|
|
881
918
|
},
|
|
882
919
|
state.previous,
|
|
883
920
|
kept?.status ?? "processed",
|
|
@@ -976,48 +1013,71 @@ export function createLiveBackup({
|
|
|
976
1013
|
const reconcile = async (statusOf) => {
|
|
977
1014
|
if (inflight) await inflight.catch(() => {});
|
|
978
1015
|
await load();
|
|
1016
|
+
// The answers first, which takes network time; a flush may run meanwhile.
|
|
979
1017
|
const answers = new Map();
|
|
1018
|
+
const asked = Date.now();
|
|
980
1019
|
for (const point of state.unconfirmed ?? []) {
|
|
981
|
-
|
|
1020
|
+
let answer = await statusOf(point.cid, point);
|
|
1021
|
+
// Not known yet is not refused while the message may still be on its way.
|
|
1022
|
+
const age = point.shippedAt
|
|
1023
|
+
? asked - Date.parse(point.shippedAt)
|
|
1024
|
+
: Infinity;
|
|
1025
|
+
if (answer === "unknown" && age < UNKNOWN_GRACE_MS) answer = undefined;
|
|
1026
|
+
answers.set(point.cid, answer);
|
|
982
1027
|
}
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
if (
|
|
1028
|
+
// Then applied to the state as it is now, with no flush running and none
|
|
1029
|
+
// starting until it is done: a point shipped meanwhile has no answer, so
|
|
1030
|
+
// the walk stops before it – or, when an earlier one was refused, it goes
|
|
1031
|
+
// back with the rest.
|
|
1032
|
+
if (inflight) await inflight.catch(() => {});
|
|
1033
|
+
let release;
|
|
1034
|
+
let wentBack = false;
|
|
1035
|
+
reconciling = new Promise((resolve) => (release = resolve));
|
|
1036
|
+
try {
|
|
1037
|
+
const now_ = reconcileConfirmation(
|
|
1038
|
+
{ kept: state.kept ?? null, pending: state.unconfirmed ?? [] },
|
|
1039
|
+
(cid) => answers.get(cid),
|
|
1040
|
+
);
|
|
1041
|
+
if (!now_.changed)
|
|
1042
|
+
return {
|
|
1043
|
+
changed: false,
|
|
1044
|
+
rewound: false,
|
|
1045
|
+
gap: false,
|
|
1046
|
+
sequence: state.sequence,
|
|
1047
|
+
};
|
|
1048
|
+
state.kept = now_.kept;
|
|
1049
|
+
state.unconfirmed = now_.pending;
|
|
1050
|
+
if (now_.rewound) {
|
|
1051
|
+
const back = now_.rewind ?? {
|
|
1052
|
+
sequence: 0,
|
|
1053
|
+
previous: null,
|
|
1054
|
+
databases: {},
|
|
1055
|
+
};
|
|
1056
|
+
state.sequence = back.sequence;
|
|
1057
|
+
state.previous = back.previous;
|
|
1058
|
+
state.databases = JSON.parse(JSON.stringify(back.databases ?? {}));
|
|
1059
|
+
for (const [address, w] of watched) {
|
|
1060
|
+
const heads = state.databases[address]?.heads ?? [];
|
|
1061
|
+
w.shipped = await reachable(w.database, heads);
|
|
1062
|
+
w.shipped.heads = heads;
|
|
1063
|
+
}
|
|
1064
|
+
// Nothing announces what has to go again: count it, so a flush ships it.
|
|
1065
|
+
pending = Math.max(pending, 1);
|
|
1066
|
+
wentBack = true;
|
|
1067
|
+
}
|
|
1068
|
+
await save();
|
|
988
1069
|
return {
|
|
989
|
-
changed:
|
|
990
|
-
rewound:
|
|
991
|
-
gap:
|
|
1070
|
+
changed: true,
|
|
1071
|
+
rewound: now_.rewound,
|
|
1072
|
+
gap: now_.gap,
|
|
992
1073
|
sequence: state.sequence,
|
|
993
1074
|
};
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
previous: null,
|
|
1000
|
-
databases: {},
|
|
1001
|
-
};
|
|
1002
|
-
state.sequence = back.sequence;
|
|
1003
|
-
state.previous = back.previous;
|
|
1004
|
-
state.databases = JSON.parse(JSON.stringify(back.databases ?? {}));
|
|
1005
|
-
for (const [address, w] of watched) {
|
|
1006
|
-
const heads = state.databases[address]?.heads ?? [];
|
|
1007
|
-
w.shipped = await reachable(w.database, heads);
|
|
1008
|
-
w.shipped.heads = heads;
|
|
1009
|
-
}
|
|
1010
|
-
// Nothing announces what has to go again: count it, so a flush ships it.
|
|
1011
|
-
pending = Math.max(pending, 1);
|
|
1012
|
-
if (running && (after || every)) scheduleFlush();
|
|
1075
|
+
} finally {
|
|
1076
|
+
reconciling = null;
|
|
1077
|
+
release();
|
|
1078
|
+
// Only once the flush can see the state it is to ship from.
|
|
1079
|
+
if (wentBack && running && (after || every)) scheduleFlush();
|
|
1013
1080
|
}
|
|
1014
|
-
await save();
|
|
1015
|
-
return {
|
|
1016
|
-
changed: true,
|
|
1017
|
-
rewound: now_.rewound,
|
|
1018
|
-
gap: now_.gap,
|
|
1019
|
-
sequence: state.sequence,
|
|
1020
|
-
};
|
|
1021
1081
|
};
|
|
1022
1082
|
|
|
1023
1083
|
const start = async () => {
|
package/lib/peer-fetch.js
CHANGED
|
@@ -301,7 +301,10 @@ function parseProviderRecords(body) {
|
|
|
301
301
|
* @param {(addr: string) => Promise<unknown>} [options.dial] - defaults to the node's
|
|
302
302
|
* @param {(cid: any, options?: Object) => AsyncIterable<Uint8Array>} [options.cat] -
|
|
303
303
|
* defaults to `@helia/unixfs`, which reads a chunked file as well as a single block
|
|
304
|
-
* @returns {(cid: string, options?: Object) => Promise<Uint8Array>
|
|
304
|
+
* @returns {((cid: string, options?: Object) => Promise<Uint8Array>) & {
|
|
305
|
+
* stream: (cid: string, options?: { offset?: number, signal?: AbortSignal, timeout?: number }) => AsyncGenerator<Uint8Array> }}
|
|
306
|
+
* the fetch, and beside it `stream`: the same file as it arrives, from
|
|
307
|
+
* `offset` on when given, for a file too large to hold
|
|
305
308
|
*/
|
|
306
309
|
export function createPeerFetch({
|
|
307
310
|
helia,
|
|
@@ -315,9 +318,8 @@ export function createPeerFetch({
|
|
|
315
318
|
throw new Error("createPeerFetch needs a Helia node, or both dial and cat");
|
|
316
319
|
}
|
|
317
320
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
options.signal ?? AbortSignal.timeout(options.timeout ?? timeout);
|
|
321
|
+
/** Dial whoever holds `cid`, each peer once; how many answered. */
|
|
322
|
+
const reach = async (cid, signal) => {
|
|
321
323
|
const addrs =
|
|
322
324
|
typeof providers === "function" ? await providers(cid) : [...providers];
|
|
323
325
|
|
|
@@ -348,17 +350,38 @@ export function createPeerFetch({
|
|
|
348
350
|
log.debug(` ⚠️ could not dial ${addr}: ${error.message}`);
|
|
349
351
|
}
|
|
350
352
|
}
|
|
351
|
-
|
|
352
|
-
if (dialed === 0) {
|
|
353
|
+
if (reached.size === 0) {
|
|
353
354
|
throw new Error(
|
|
354
355
|
`Could not dial any provider for ${cid} (tried ${addrs.length})`,
|
|
355
356
|
);
|
|
356
357
|
}
|
|
358
|
+
return reached.size;
|
|
359
|
+
};
|
|
357
360
|
|
|
361
|
+
/**
|
|
362
|
+
* The file as the peers send it, never all of it at once (#187). With
|
|
363
|
+
* `offset`, from that byte on: where another source left off.
|
|
364
|
+
*/
|
|
365
|
+
async function* streamOverPeers(cid, options = {}) {
|
|
366
|
+
const signal =
|
|
367
|
+
options.signal ?? AbortSignal.timeout(options.timeout ?? timeout);
|
|
368
|
+
const dialed = await reach(cid, signal);
|
|
358
369
|
const read = cat ?? (await unixfsCat(helia));
|
|
370
|
+
let total = 0;
|
|
371
|
+
for await (const chunk of read(cid, {
|
|
372
|
+
signal,
|
|
373
|
+
...(options.offset ? { offset: options.offset } : {}),
|
|
374
|
+
})) {
|
|
375
|
+
total += chunk.length;
|
|
376
|
+
yield chunk;
|
|
377
|
+
}
|
|
378
|
+
log.info(` ✅ ${total} bytes over libp2p from ${dialed} peer(s)`);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
async function fetchOverPeers(cid, options = {}) {
|
|
359
382
|
const parts = [];
|
|
360
383
|
let total = 0;
|
|
361
|
-
for await (const chunk of
|
|
384
|
+
for await (const chunk of streamOverPeers(cid, options)) {
|
|
362
385
|
parts.push(chunk);
|
|
363
386
|
total += chunk.length;
|
|
364
387
|
}
|
|
@@ -368,9 +391,10 @@ export function createPeerFetch({
|
|
|
368
391
|
bytes.set(part, at);
|
|
369
392
|
at += part.length;
|
|
370
393
|
}
|
|
371
|
-
log.info(` ✅ ${bytes.length} bytes over libp2p from ${dialed} peer(s)`);
|
|
372
394
|
return bytes;
|
|
373
|
-
}
|
|
395
|
+
}
|
|
396
|
+
fetchOverPeers.stream = streamOverPeers;
|
|
397
|
+
return fetchOverPeers;
|
|
374
398
|
}
|
|
375
399
|
|
|
376
400
|
/**
|
|
@@ -442,3 +466,85 @@ export function createGatewayFirstFetch({
|
|
|
442
466
|
}
|
|
443
467
|
};
|
|
444
468
|
}
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* {@link createGatewayFirstFetch} for a file that is handed on as it arrives
|
|
472
|
+
* (#187): the gateway first, and when it fails – at the start or after half
|
|
473
|
+
* the file – the peers go on **from the byte the gateway stopped at**. The
|
|
474
|
+
* caller sees one run of bytes.
|
|
475
|
+
*
|
|
476
|
+
* Whole-file checks are each path's own and cover what that path delivered;
|
|
477
|
+
* across a change of path only `size` is checked here. That is enough for
|
|
478
|
+
* what this is for: a sealed backup's frames authenticate themselves.
|
|
479
|
+
*
|
|
480
|
+
* @param {Object} options
|
|
481
|
+
* @param {(cid: string, options?: Object) => AsyncIterable<Uint8Array>} options.viaGateway
|
|
482
|
+
* e.g. the Aleph driver's `getBlobStream`; an error of its may carry
|
|
483
|
+
* `received`, and is otherwise taken to have stopped where the count stands
|
|
484
|
+
* @param {(cid: string, options?: Object) => AsyncIterable<Uint8Array>} options.viaPeers
|
|
485
|
+
* e.g. `createPeerFetch(…).stream`; given `offset`
|
|
486
|
+
* @param {number} [options.gatewayTimeout=3000] idle time before the peers start
|
|
487
|
+
* @param {(path: "gateway"|"peers", info: Object) => void} [options.onPath]
|
|
488
|
+
* @returns {(cid: string, options?: { size?: number }) => AsyncGenerator<Uint8Array>}
|
|
489
|
+
*/
|
|
490
|
+
export function createGatewayFirstStream({
|
|
491
|
+
viaGateway,
|
|
492
|
+
viaPeers,
|
|
493
|
+
gatewayTimeout = 3000,
|
|
494
|
+
onPath = () => {},
|
|
495
|
+
now = () => Date.now(),
|
|
496
|
+
} = {}) {
|
|
497
|
+
return async function* streamBytes(cid, options = {}) {
|
|
498
|
+
const started = now();
|
|
499
|
+
let received = 0;
|
|
500
|
+
let gatewayError = null;
|
|
501
|
+
try {
|
|
502
|
+
for await (const chunk of viaGateway(cid, {
|
|
503
|
+
...options,
|
|
504
|
+
timeout: gatewayTimeout,
|
|
505
|
+
})) {
|
|
506
|
+
received += chunk.length;
|
|
507
|
+
yield chunk;
|
|
508
|
+
}
|
|
509
|
+
} catch (error) {
|
|
510
|
+
gatewayError = error;
|
|
511
|
+
}
|
|
512
|
+
if (!gatewayError) {
|
|
513
|
+
onPath("gateway", { cid, ms: now() - started, bytes: received });
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
const gatewayMs = now() - started;
|
|
517
|
+
const peersStarted = now();
|
|
518
|
+
const fromGateway = received;
|
|
519
|
+
try {
|
|
520
|
+
for await (const chunk of viaPeers(cid, {
|
|
521
|
+
...options,
|
|
522
|
+
...(received ? { offset: received } : {}),
|
|
523
|
+
})) {
|
|
524
|
+
received += chunk.length;
|
|
525
|
+
yield chunk;
|
|
526
|
+
}
|
|
527
|
+
} catch (peerError) {
|
|
528
|
+
throw new Error(
|
|
529
|
+
`Could not fetch ${cid}: the gateway said "${gatewayError.message}" and the peers said "${peerError.message}"`,
|
|
530
|
+
{ cause: peerError },
|
|
531
|
+
);
|
|
532
|
+
}
|
|
533
|
+
if (
|
|
534
|
+
Number.isInteger(options.size) &&
|
|
535
|
+
options.size > 0 &&
|
|
536
|
+
received !== options.size
|
|
537
|
+
) {
|
|
538
|
+
throw new Error(
|
|
539
|
+
`${received} bytes of ${cid} arrived, not the ${options.size} it has`,
|
|
540
|
+
);
|
|
541
|
+
}
|
|
542
|
+
onPath("peers", {
|
|
543
|
+
cid,
|
|
544
|
+
ms: now() - peersStarted,
|
|
545
|
+
bytes: received,
|
|
546
|
+
...(fromGateway ? { resumedAt: fromGateway } : {}),
|
|
547
|
+
after: { path: "gateway", ms: gatewayMs, error: gatewayError.message },
|
|
548
|
+
});
|
|
549
|
+
};
|
|
550
|
+
}
|