@le-space/orbitdb-storage-bridge 0.28.1 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -184,6 +184,227 @@ async function readBody(
184
184
  return bytes;
185
185
  }
186
186
 
187
+ /** How often one transfer may pick up again where a connection dropped it. */
188
+ export const MAX_RESUMES = 20;
189
+
190
+ /**
191
+ * The bytes behind a CID as they arrive, never all of them at once (#187).
192
+ *
193
+ * {@link fetchFromGateways} assembles the whole file before it returns, which
194
+ * is right for a block and for a manifest, and more than a phone's browser tab
195
+ * may hold for a backup of several hundred MB. This yields each chunk as the
196
+ * gateway sends it; what the caller does not keep is gone.
197
+ *
198
+ * A connection that drops after the first byte is not the end: the next
199
+ * request asks for the rest (`Range: bytes=<received>-`), from the same
200
+ * gateway and then the next. A gateway that ignores the range and sends the
201
+ * file from its start has those bytes passed over. A phone changes cells in
202
+ * the minutes a large file takes.
203
+ *
204
+ * **What is yielded is not yet checked.** The whole-file checks of
205
+ * `fetchFromGateways` can only speak at the end, and by then the bytes are
206
+ * with the caller: a file of another length than `expectedSize`, or one the
207
+ * `verifier` refuses, throws after the last chunk was yielded and before the
208
+ * iteration ends. So a caller acts on chunks only as far as each proves
209
+ * itself (a sealed frame that opens, a block that hashes to its CID), and
210
+ * treats the file as whole only once the iteration finished without an error.
211
+ *
212
+ * @param {string} cid
213
+ * @param {Object} [options] as for {@link fetchFromGateways}, except:
214
+ * @param {number} [options.timeout] idle time in ms. The clock stands still
215
+ * while the caller is busy with a chunk: only the gateway's silence counts.
216
+ * @param {number} [options.offset] start at this byte: the file from there on
217
+ * @param {((cid: string) => { write(chunk: Uint8Array): void | Promise<void>, done(): Promise<boolean> } | null) | null} [options.verifier]
218
+ * a check of the whole file that takes it in pieces – the Aleph driver
219
+ * passes `unixfsCidVerifier`. Not asked when the transfer started at an
220
+ * `offset`, for it never sees the start.
221
+ * @param {number} [options.maxResumes]
222
+ * @returns {AsyncGenerator<Uint8Array>}
223
+ */
224
+ export async function* streamFromGateways(
225
+ cid,
226
+ {
227
+ gateways = DEFAULT_GATEWAYS,
228
+ timeout = DEFAULT_TIMEOUT_MS,
229
+ signal = null,
230
+ log = SILENT,
231
+ accept = null,
232
+ maxBackoff = MAX_BACKOFF_MS,
233
+ sleep = waitFor,
234
+ fetchImpl = fetch,
235
+ onProgress = null,
236
+ expectedSize = null,
237
+ verifier = null,
238
+ offset = 0,
239
+ maxResumes = MAX_RESUMES,
240
+ } = {},
241
+ ) {
242
+ let lastError = null;
243
+ /** Bytes of the file handed on so far, counted from its start. */
244
+ let received = offset;
245
+ /** The file's length, once a gateway announced it. */
246
+ let total = 0;
247
+ let resumes = 0;
248
+ let complete = false;
249
+ const checking = offset === 0 && verifier ? verifier(cid) : null;
250
+
251
+ outer: for (const gateway of gateways) {
252
+ const url = `${gateway}/${cid}`;
253
+ for (let attempt = 0; attempt < MAX_ATTEMPTS_PER_GATEWAY; attempt++) {
254
+ const idle = new AbortController();
255
+ let clock = null;
256
+ const stop = () => {
257
+ if (clock) clearTimeout(clock);
258
+ clock = null;
259
+ };
260
+ const stillAlive = () => {
261
+ stop();
262
+ clock = setTimeout(
263
+ () =>
264
+ idle.abort(new Error(`no bytes from ${gateway} for ${timeout} ms`)),
265
+ timeout,
266
+ );
267
+ };
268
+ const startedAt = received;
269
+ let reader = null;
270
+ stillAlive();
271
+ try {
272
+ const headers = {
273
+ ...(accept ? { Accept: accept } : {}),
274
+ ...(received > 0 ? { Range: `bytes=${received}-` } : {}),
275
+ };
276
+ const response = await fetchImpl(url, {
277
+ signal: signal ?? idle.signal,
278
+ ...(Object.keys(headers).length ? { headers } : {}),
279
+ });
280
+ stillAlive();
281
+
282
+ const sunset = response.headers?.get?.("Sunset");
283
+ if (sunset && !response.ok) {
284
+ log.warn(` ⚠️ ${gateway} is retired (Sunset: ${sunset})`);
285
+ break;
286
+ }
287
+ if (response.status === 429 && attempt < MAX_ATTEMPTS_PER_GATEWAY - 1) {
288
+ const wait = backoffFor(response, attempt);
289
+ if (wait > maxBackoff) {
290
+ log.warn(
291
+ ` ⚠️ ${gateway} wants ${Math.round(wait / 1000)}s — treating it as closed`,
292
+ );
293
+ break;
294
+ }
295
+ stop();
296
+ await sleep(wait);
297
+ continue;
298
+ }
299
+ if (!response.ok) {
300
+ lastError = new Error(`${gateway} answered ${response.status}`);
301
+ log.debug(` ⚠️ ${lastError.message}`);
302
+ break;
303
+ }
304
+ if (!response.body?.getReader) {
305
+ throw new Error(`${gateway} answered without a body to read`);
306
+ }
307
+
308
+ // How much of the answer to pass over: all of what is already with
309
+ // the caller, when the gateway sent the file from its start anyway.
310
+ let skip = 0;
311
+ if (response.status === 206) {
312
+ const range = /bytes (\d+)-\d*\/(\d+|\*)/.exec(
313
+ response.headers?.get?.("content-range") ?? "",
314
+ );
315
+ const from = range ? Number(range[1]) : received;
316
+ if (from > received) {
317
+ throw new Error(
318
+ `${gateway} answered from byte ${from}, asked from ${received}`,
319
+ );
320
+ }
321
+ skip = received - from;
322
+ if (range && range[2] !== "*") total = Number(range[2]);
323
+ } else {
324
+ skip = received;
325
+ const announced =
326
+ Number(response.headers?.get?.("content-length") ?? 0) || 0;
327
+ if (announced) total = announced;
328
+ }
329
+ const contentType = response.headers?.get?.("content-type") ?? "";
330
+
331
+ reader = response.body.getReader();
332
+ for (;;) {
333
+ const { done, value } = await reader.read();
334
+ if (signal?.aborted) throw signal.reason;
335
+ if (idle.signal.aborted) throw idle.signal.reason;
336
+ if (done) break;
337
+ let chunk = value;
338
+ if (skip > 0) {
339
+ const passed = Math.min(skip, chunk.length);
340
+ skip -= passed;
341
+ chunk = chunk.subarray(passed);
342
+ stillAlive();
343
+ if (!chunk.length) continue;
344
+ }
345
+ // An error page with a 200: only the file's first bytes can tell.
346
+ if (received === 0 && looksLikeAnErrorPage(chunk, contentType)) {
347
+ lastError = new Error(
348
+ `${gateway} returned an error page with a 200`,
349
+ );
350
+ log.warn(` ⚠️ ${lastError.message}`);
351
+ continue outer;
352
+ }
353
+ if (checking) await checking.write(chunk);
354
+ received += chunk.length;
355
+ onProgress?.(received, total || expectedSize || 0);
356
+ // The caller's time with a chunk is not the gateway's silence.
357
+ stop();
358
+ yield chunk;
359
+ stillAlive();
360
+ }
361
+ // A body that ended early without an error: ask for the rest.
362
+ if (total && received < total) {
363
+ throw new Error(
364
+ `${gateway} ended the answer at byte ${received} of ${total}`,
365
+ );
366
+ }
367
+ complete = true;
368
+ log.info(` ✅ ${received - offset} bytes from ${gateway}`);
369
+ break outer;
370
+ } catch (error) {
371
+ lastError = idle.signal.aborted ? idle.signal.reason : error;
372
+ log.debug(` ⚠️ ${gateway} failed: ${lastError.message}`);
373
+ if (signal?.aborted) break outer;
374
+ // It moved: this was a dropped connection, not a gateway that cannot
375
+ // serve the file, and the attempts start over, within a budget.
376
+ if (received > startedAt) {
377
+ if (++resumes > maxResumes) break outer;
378
+ attempt = -1;
379
+ }
380
+ } finally {
381
+ stop();
382
+ // Left early, by an error or by the caller: let the connection go.
383
+ if (!complete) await reader?.cancel?.().catch(() => {});
384
+ }
385
+ }
386
+ }
387
+
388
+ if (!complete) {
389
+ const error = new Error(
390
+ `Could not fetch ${cid} from any gateway` +
391
+ (received > offset ? ` (it stopped at byte ${received})` : "") +
392
+ (lastError ? `. Last error: ${lastError.message}` : ""),
393
+ );
394
+ // For a caller with another source: where that one may go on.
395
+ error.received = received;
396
+ throw error;
397
+ }
398
+ if (expectedSize != null && received !== expectedSize) {
399
+ throw new Error(
400
+ `The gateways served ${received} bytes of ${cid}, not the ${expectedSize} it has`,
401
+ );
402
+ }
403
+ if (checking && !(await checking.done())) {
404
+ throw new Error(`The gateways served bytes that are not ${cid}`);
405
+ }
406
+ }
407
+
187
408
  /**
188
409
  * Fetch the bytes behind a CID, trying each gateway in turn.
189
410
  *
@@ -346,8 +346,11 @@ const metadataOf = (source) => source?.metadata ?? source?.manifest?.metadata;
346
346
  * their `sequence`; the base, if given, comes first.
347
347
  *
348
348
  * @param {object} params
349
- * @param {{ metadata?: object, manifest?: object, blocks: Map }} [params.base]
350
- * @param {{ manifest: IncrementManifest, blocks: Map }[]} params.increments
349
+ * A source may come without its `blocks`: a restore that streamed them into
350
+ * the blockstore has only the manifests left, and merges those.
351
+ *
352
+ * @param {{ metadata?: object, manifest?: object, blocks?: Map }} [params.base]
353
+ * @param {{ manifest: IncrementManifest, blocks?: Map }[]} params.increments
351
354
  * @returns {{ blocks: Map, metadata: object, sequences: number[] }}
352
355
  */
353
356
  export function mergeIncrements({ base, increments = [] }) {
@@ -386,7 +389,12 @@ export function mergeIncrements({ base, increments = [] }) {
386
389
  ...(d.type ? { type: d.type } : {}),
387
390
  ...(d.manifestCID ? { manifestCID: d.manifestCID } : {}),
388
391
  ...(d.collection ? { collection: d.collection } : {}),
389
- ...(heads.length ? { heads } : {}),
392
+ // Stated and empty is not the same as not stated: an empty database
393
+ // has no heads, and a reader that is told so does not go looking for
394
+ // them among the blocks (`restoreFromStoredBlocks` cannot).
395
+ ...(heads.length || Array.isArray(d.heads) || Array.isArray(known.heads)
396
+ ? { heads }
397
+ : {}),
390
398
  ...(typeof d.entryCount === "number"
391
399
  ? { entryCount: d.entryCount }
392
400
  : {}),
@@ -1011,7 +1019,9 @@ export function createLiveBackup({
1011
1019
  for (const point of state.unconfirmed ?? []) {
1012
1020
  let answer = await statusOf(point.cid, point);
1013
1021
  // Not known yet is not refused while the message may still be on its way.
1014
- const age = point.shippedAt ? asked - Date.parse(point.shippedAt) : Infinity;
1022
+ const age = point.shippedAt
1023
+ ? asked - Date.parse(point.shippedAt)
1024
+ : Infinity;
1015
1025
  if (answer === "unknown" && age < UNKNOWN_GRACE_MS) answer = undefined;
1016
1026
  answers.set(point.cid, answer);
1017
1027
  }
package/lib/peer-fetch.js CHANGED
@@ -301,7 +301,10 @@ function parseProviderRecords(body) {
301
301
  * @param {(addr: string) => Promise<unknown>} [options.dial] - defaults to the node's
302
302
  * @param {(cid: any, options?: Object) => AsyncIterable<Uint8Array>} [options.cat] -
303
303
  * defaults to `@helia/unixfs`, which reads a chunked file as well as a single block
304
- * @returns {(cid: string, options?: Object) => Promise<Uint8Array>}
304
+ * @returns {((cid: string, options?: Object) => Promise<Uint8Array>) & {
305
+ * stream: (cid: string, options?: { offset?: number, signal?: AbortSignal, timeout?: number }) => AsyncGenerator<Uint8Array> }}
306
+ * the fetch, and beside it `stream`: the same file as it arrives, from
307
+ * `offset` on when given, for a file too large to hold
305
308
  */
306
309
  export function createPeerFetch({
307
310
  helia,
@@ -315,9 +318,8 @@ export function createPeerFetch({
315
318
  throw new Error("createPeerFetch needs a Helia node, or both dial and cat");
316
319
  }
317
320
 
318
- return async function fetchOverPeers(cid, options = {}) {
319
- const signal =
320
- options.signal ?? AbortSignal.timeout(options.timeout ?? timeout);
321
+ /** Dial whoever holds `cid`, each peer once; how many answered. */
322
+ const reach = async (cid, signal) => {
321
323
  const addrs =
322
324
  typeof providers === "function" ? await providers(cid) : [...providers];
323
325
 
@@ -348,17 +350,38 @@ export function createPeerFetch({
348
350
  log.debug(` ⚠️ could not dial ${addr}: ${error.message}`);
349
351
  }
350
352
  }
351
- const dialed = reached.size;
352
- if (dialed === 0) {
353
+ if (reached.size === 0) {
353
354
  throw new Error(
354
355
  `Could not dial any provider for ${cid} (tried ${addrs.length})`,
355
356
  );
356
357
  }
358
+ return reached.size;
359
+ };
357
360
 
361
+ /**
362
+ * The file as the peers send it, never all of it at once (#187). With
363
+ * `offset`, from that byte on: where another source left off.
364
+ */
365
+ async function* streamOverPeers(cid, options = {}) {
366
+ const signal =
367
+ options.signal ?? AbortSignal.timeout(options.timeout ?? timeout);
368
+ const dialed = await reach(cid, signal);
358
369
  const read = cat ?? (await unixfsCat(helia));
370
+ let total = 0;
371
+ for await (const chunk of read(cid, {
372
+ signal,
373
+ ...(options.offset ? { offset: options.offset } : {}),
374
+ })) {
375
+ total += chunk.length;
376
+ yield chunk;
377
+ }
378
+ log.info(` ✅ ${total} bytes over libp2p from ${dialed} peer(s)`);
379
+ }
380
+
381
+ async function fetchOverPeers(cid, options = {}) {
359
382
  const parts = [];
360
383
  let total = 0;
361
- for await (const chunk of read(cid, { signal })) {
384
+ for await (const chunk of streamOverPeers(cid, options)) {
362
385
  parts.push(chunk);
363
386
  total += chunk.length;
364
387
  }
@@ -368,9 +391,10 @@ export function createPeerFetch({
368
391
  bytes.set(part, at);
369
392
  at += part.length;
370
393
  }
371
- log.info(` ✅ ${bytes.length} bytes over libp2p from ${dialed} peer(s)`);
372
394
  return bytes;
373
- };
395
+ }
396
+ fetchOverPeers.stream = streamOverPeers;
397
+ return fetchOverPeers;
374
398
  }
375
399
 
376
400
  /**
@@ -442,3 +466,85 @@ export function createGatewayFirstFetch({
442
466
  }
443
467
  };
444
468
  }
469
+
470
+ /**
471
+ * {@link createGatewayFirstFetch} for a file that is handed on as it arrives
472
+ * (#187): the gateway first, and when it fails – at the start or after half
473
+ * the file – the peers go on **from the byte the gateway stopped at**. The
474
+ * caller sees one run of bytes.
475
+ *
476
+ * Whole-file checks are each path's own and cover what that path delivered;
477
+ * across a change of path only `size` is checked here. That is enough for
478
+ * what this is for: a sealed backup's frames authenticate themselves.
479
+ *
480
+ * @param {Object} options
481
+ * @param {(cid: string, options?: Object) => AsyncIterable<Uint8Array>} options.viaGateway
482
+ * e.g. the Aleph driver's `getBlobStream`; an error of its may carry
483
+ * `received`, and is otherwise taken to have stopped where the count stands
484
+ * @param {(cid: string, options?: Object) => AsyncIterable<Uint8Array>} options.viaPeers
485
+ * e.g. `createPeerFetch(…).stream`; given `offset`
486
+ * @param {number} [options.gatewayTimeout=3000] idle time before the peers start
487
+ * @param {(path: "gateway"|"peers", info: Object) => void} [options.onPath]
488
+ * @returns {(cid: string, options?: { size?: number }) => AsyncGenerator<Uint8Array>}
489
+ */
490
+ export function createGatewayFirstStream({
491
+ viaGateway,
492
+ viaPeers,
493
+ gatewayTimeout = 3000,
494
+ onPath = () => {},
495
+ now = () => Date.now(),
496
+ } = {}) {
497
+ return async function* streamBytes(cid, options = {}) {
498
+ const started = now();
499
+ let received = 0;
500
+ let gatewayError = null;
501
+ try {
502
+ for await (const chunk of viaGateway(cid, {
503
+ ...options,
504
+ timeout: gatewayTimeout,
505
+ })) {
506
+ received += chunk.length;
507
+ yield chunk;
508
+ }
509
+ } catch (error) {
510
+ gatewayError = error;
511
+ }
512
+ if (!gatewayError) {
513
+ onPath("gateway", { cid, ms: now() - started, bytes: received });
514
+ return;
515
+ }
516
+ const gatewayMs = now() - started;
517
+ const peersStarted = now();
518
+ const fromGateway = received;
519
+ try {
520
+ for await (const chunk of viaPeers(cid, {
521
+ ...options,
522
+ ...(received ? { offset: received } : {}),
523
+ })) {
524
+ received += chunk.length;
525
+ yield chunk;
526
+ }
527
+ } catch (peerError) {
528
+ throw new Error(
529
+ `Could not fetch ${cid}: the gateway said "${gatewayError.message}" and the peers said "${peerError.message}"`,
530
+ { cause: peerError },
531
+ );
532
+ }
533
+ if (
534
+ Number.isInteger(options.size) &&
535
+ options.size > 0 &&
536
+ received !== options.size
537
+ ) {
538
+ throw new Error(
539
+ `${received} bytes of ${cid} arrived, not the ${options.size} it has`,
540
+ );
541
+ }
542
+ onPath("peers", {
543
+ cid,
544
+ ms: now() - peersStarted,
545
+ bytes: received,
546
+ ...(fromGateway ? { resumedAt: fromGateway } : {}),
547
+ after: { path: "gateway", ms: gatewayMs, error: gatewayError.message },
548
+ });
549
+ };
550
+ }