@le-space/orbitdb-storage-bridge 0.28.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -184,6 +184,227 @@ async function readBody(
184
184
  return bytes;
185
185
  }
186
186
 
187
+ /** How often one transfer may pick up again where a connection dropped it. */
188
+ export const MAX_RESUMES = 20;
189
+
190
+ /**
191
+ * The bytes behind a CID as they arrive, never all of them at once (#187).
192
+ *
193
+ * {@link fetchFromGateways} assembles the whole file before it returns, which
194
+ * is right for a block and for a manifest, and more than a phone's browser tab
195
+ * may hold for a backup of several hundred MB. This yields each chunk as the
196
+ * gateway sends it; what the caller does not keep is gone.
197
+ *
198
+ * A connection that drops after the first byte is not the end: the next
199
+ * request asks for the rest (`Range: bytes=<received>-`), from the same
200
+ * gateway and then the next. A gateway that ignores the range and sends the
201
+ * file from its start has those bytes passed over. A phone changes cells in
202
+ * the minutes a large file takes.
203
+ *
204
+ * **What is yielded is not yet checked.** The whole-file checks of
205
+ * `fetchFromGateways` can only speak at the end, and by then the bytes are
206
+ * with the caller: a file of another length than `expectedSize`, or one the
207
+ * `verifier` refuses, throws after the last chunk was yielded and before the
208
+ * iteration ends. So a caller acts on chunks only as far as each proves
209
+ * itself (a sealed frame that opens, a block that hashes to its CID), and
210
+ * treats the file as whole only once the iteration finished without an error.
211
+ *
212
+ * @param {string} cid
213
+ * @param {Object} [options] as for {@link fetchFromGateways}, except:
214
+ * @param {number} [options.timeout] idle time in ms. The clock stands still
215
+ * while the caller is busy with a chunk: only the gateway's silence counts.
216
+ * @param {number} [options.offset] start at this byte: the file from there on
217
+ * @param {((cid: string) => { write(chunk: Uint8Array): void | Promise<void>, done(): Promise<boolean> } | null) | null} [options.verifier]
218
+ * a check of the whole file that takes it in pieces – the Aleph driver
219
+ * passes `unixfsCidVerifier`. Not asked when the transfer started at an
220
+ * `offset`, for it never sees the start.
221
+ * @param {number} [options.maxResumes]
222
+ * @returns {AsyncGenerator<Uint8Array>}
223
+ */
224
+ export async function* streamFromGateways(
225
+ cid,
226
+ {
227
+ gateways = DEFAULT_GATEWAYS,
228
+ timeout = DEFAULT_TIMEOUT_MS,
229
+ signal = null,
230
+ log = SILENT,
231
+ accept = null,
232
+ maxBackoff = MAX_BACKOFF_MS,
233
+ sleep = waitFor,
234
+ fetchImpl = fetch,
235
+ onProgress = null,
236
+ expectedSize = null,
237
+ verifier = null,
238
+ offset = 0,
239
+ maxResumes = MAX_RESUMES,
240
+ } = {},
241
+ ) {
242
+ let lastError = null;
243
+ /** Bytes of the file handed on so far, counted from its start. */
244
+ let received = offset;
245
+ /** The file's length, once a gateway announced it. */
246
+ let total = 0;
247
+ let resumes = 0;
248
+ let complete = false;
249
+ const checking = offset === 0 && verifier ? verifier(cid) : null;
250
+
251
+ outer: for (const gateway of gateways) {
252
+ const url = `${gateway}/${cid}`;
253
+ for (let attempt = 0; attempt < MAX_ATTEMPTS_PER_GATEWAY; attempt++) {
254
+ const idle = new AbortController();
255
+ let clock = null;
256
+ const stop = () => {
257
+ if (clock) clearTimeout(clock);
258
+ clock = null;
259
+ };
260
+ const stillAlive = () => {
261
+ stop();
262
+ clock = setTimeout(
263
+ () =>
264
+ idle.abort(new Error(`no bytes from ${gateway} for ${timeout} ms`)),
265
+ timeout,
266
+ );
267
+ };
268
+ const startedAt = received;
269
+ let reader = null;
270
+ stillAlive();
271
+ try {
272
+ const headers = {
273
+ ...(accept ? { Accept: accept } : {}),
274
+ ...(received > 0 ? { Range: `bytes=${received}-` } : {}),
275
+ };
276
+ const response = await fetchImpl(url, {
277
+ signal: signal ?? idle.signal,
278
+ ...(Object.keys(headers).length ? { headers } : {}),
279
+ });
280
+ stillAlive();
281
+
282
+ const sunset = response.headers?.get?.("Sunset");
283
+ if (sunset && !response.ok) {
284
+ log.warn(` ⚠️ ${gateway} is retired (Sunset: ${sunset})`);
285
+ break;
286
+ }
287
+ if (response.status === 429 && attempt < MAX_ATTEMPTS_PER_GATEWAY - 1) {
288
+ const wait = backoffFor(response, attempt);
289
+ if (wait > maxBackoff) {
290
+ log.warn(
291
+ ` ⚠️ ${gateway} wants ${Math.round(wait / 1000)}s — treating it as closed`,
292
+ );
293
+ break;
294
+ }
295
+ stop();
296
+ await sleep(wait);
297
+ continue;
298
+ }
299
+ if (!response.ok) {
300
+ lastError = new Error(`${gateway} answered ${response.status}`);
301
+ log.debug(` ⚠️ ${lastError.message}`);
302
+ break;
303
+ }
304
+ if (!response.body?.getReader) {
305
+ throw new Error(`${gateway} answered without a body to read`);
306
+ }
307
+
308
+ // How much of the answer to pass over: all of what is already with
309
+ // the caller, when the gateway sent the file from its start anyway.
310
+ let skip = 0;
311
+ if (response.status === 206) {
312
+ const range = /bytes (\d+)-\d*\/(\d+|\*)/.exec(
313
+ response.headers?.get?.("content-range") ?? "",
314
+ );
315
+ const from = range ? Number(range[1]) : received;
316
+ if (from > received) {
317
+ throw new Error(
318
+ `${gateway} answered from byte ${from}, asked from ${received}`,
319
+ );
320
+ }
321
+ skip = received - from;
322
+ if (range && range[2] !== "*") total = Number(range[2]);
323
+ } else {
324
+ skip = received;
325
+ const announced =
326
+ Number(response.headers?.get?.("content-length") ?? 0) || 0;
327
+ if (announced) total = announced;
328
+ }
329
+ const contentType = response.headers?.get?.("content-type") ?? "";
330
+
331
+ reader = response.body.getReader();
332
+ for (;;) {
333
+ const { done, value } = await reader.read();
334
+ if (signal?.aborted) throw signal.reason;
335
+ if (idle.signal.aborted) throw idle.signal.reason;
336
+ if (done) break;
337
+ let chunk = value;
338
+ if (skip > 0) {
339
+ const passed = Math.min(skip, chunk.length);
340
+ skip -= passed;
341
+ chunk = chunk.subarray(passed);
342
+ stillAlive();
343
+ if (!chunk.length) continue;
344
+ }
345
+ // An error page with a 200: only the file's first bytes can tell.
346
+ if (received === 0 && looksLikeAnErrorPage(chunk, contentType)) {
347
+ lastError = new Error(
348
+ `${gateway} returned an error page with a 200`,
349
+ );
350
+ log.warn(` ⚠️ ${lastError.message}`);
351
+ continue outer;
352
+ }
353
+ if (checking) await checking.write(chunk);
354
+ received += chunk.length;
355
+ onProgress?.(received, total || expectedSize || 0);
356
+ // The caller's time with a chunk is not the gateway's silence.
357
+ stop();
358
+ yield chunk;
359
+ stillAlive();
360
+ }
361
+ // A body that ended early without an error: ask for the rest.
362
+ if (total && received < total) {
363
+ throw new Error(
364
+ `${gateway} ended the answer at byte ${received} of ${total}`,
365
+ );
366
+ }
367
+ complete = true;
368
+ log.info(` ✅ ${received - offset} bytes from ${gateway}`);
369
+ break outer;
370
+ } catch (error) {
371
+ lastError = idle.signal.aborted ? idle.signal.reason : error;
372
+ log.debug(` ⚠️ ${gateway} failed: ${lastError.message}`);
373
+ if (signal?.aborted) break outer;
374
+ // It moved: this was a dropped connection, not a gateway that cannot
375
+ // serve the file, and the attempts start over, within a budget.
376
+ if (received > startedAt) {
377
+ if (++resumes > maxResumes) break outer;
378
+ attempt = -1;
379
+ }
380
+ } finally {
381
+ stop();
382
+ // Left early, by an error or by the caller: let the connection go.
383
+ if (!complete) await reader?.cancel?.().catch(() => {});
384
+ }
385
+ }
386
+ }
387
+
388
+ if (!complete) {
389
+ const error = new Error(
390
+ `Could not fetch ${cid} from any gateway` +
391
+ (received > offset ? ` (it stopped at byte ${received})` : "") +
392
+ (lastError ? `. Last error: ${lastError.message}` : ""),
393
+ );
394
+ // For a caller with another source: where that one may go on.
395
+ error.received = received;
396
+ throw error;
397
+ }
398
+ if (expectedSize != null && received !== expectedSize) {
399
+ throw new Error(
400
+ `The gateways served ${received} bytes of ${cid}, not the ${expectedSize} it has`,
401
+ );
402
+ }
403
+ if (checking && !(await checking.done())) {
404
+ throw new Error(`The gateways served bytes that are not ${cid}`);
405
+ }
406
+ }
407
+
187
408
  /**
188
409
  * Fetch the bytes behind a CID, trying each gateway in turn.
189
410
  *
@@ -346,8 +346,11 @@ const metadataOf = (source) => source?.metadata ?? source?.manifest?.metadata;
346
346
  * their `sequence`; the base, if given, comes first.
347
347
  *
348
348
  * @param {object} params
349
- * @param {{ metadata?: object, manifest?: object, blocks: Map }} [params.base]
350
- * @param {{ manifest: IncrementManifest, blocks: Map }[]} params.increments
349
+ * A source may come without its `blocks`: a restore that streamed them into
350
+ * the blockstore has only the manifests left, and merges those.
351
+ *
352
+ * @param {{ metadata?: object, manifest?: object, blocks?: Map }} [params.base]
353
+ * @param {{ manifest: IncrementManifest, blocks?: Map }[]} params.increments
351
354
  * @returns {{ blocks: Map, metadata: object, sequences: number[] }}
352
355
  */
353
356
  export function mergeIncrements({ base, increments = [] }) {
@@ -386,7 +389,12 @@ export function mergeIncrements({ base, increments = [] }) {
386
389
  ...(d.type ? { type: d.type } : {}),
387
390
  ...(d.manifestCID ? { manifestCID: d.manifestCID } : {}),
388
391
  ...(d.collection ? { collection: d.collection } : {}),
389
- ...(heads.length ? { heads } : {}),
392
+ // Stated and empty is not the same as not stated: an empty database
393
+ // has no heads, and a reader that is told so does not go looking for
394
+ // them among the blocks (`restoreFromStoredBlocks` cannot).
395
+ ...(heads.length || Array.isArray(d.heads) || Array.isArray(known.heads)
396
+ ? { heads }
397
+ : {}),
390
398
  ...(typeof d.entryCount === "number"
391
399
  ? { entryCount: d.entryCount }
392
400
  : {}),
@@ -563,6 +571,12 @@ export async function joinIncrements(
563
571
  };
564
572
  }
565
573
 
574
+ /**
575
+ * How long an increment may be unknown to the store before that counts as
576
+ * refused: Aleph answers 404 for a message that has not reached the node yet.
577
+ */
578
+ export const UNKNOWN_GRACE_MS = 10 * 60_000;
579
+
566
580
  /** Statuses that mean the store does not keep a blob. */
567
581
  export const NOT_KEPT = Object.freeze(["rejected", "unknown", "forgotten"]);
568
582
 
@@ -681,7 +695,10 @@ export const emptyLiveBackupState = () => ({
681
695
  * @param {() => Promise<Map<string, { bytes: Uint8Array }>>|Map} [params.blocks] - blocks of the
682
696
  * application's own to carry in the next increment (receipt files, …)
683
697
  * @param {boolean} [params.keep=true] - ask a backend whose handle says
684
- * `retained: false` to keep the increment (`pinCid`)
698
+ * `retained: false` to keep the increment (`pinCid`). A handle that already
699
+ * carries the store's answer (`status`, as an Aleph upload with a signing
700
+ * pin does: the STORE went with the file) is not pinned a second time; its
701
+ * answer is the one followed and recorded
685
702
  * @param {(sequence: number, at: Date) => string} [params.name] - the blob's name
686
703
  * @param {(shipped: { sequence: number, handle: object, kept?: object, entries: number, blocks: number, bytes: number, at: Date }) => void} [params.onShipped]
687
704
  * @param {(error: Error) => void} [params.onError]
@@ -723,6 +740,8 @@ export function createLiveBackup({
723
740
  let pending = 0;
724
741
  let inflight = null;
725
742
  let again = false;
743
+ /** A reconcile applying its answers: a flush waits for it, or it would ship from a state about to move. */
744
+ let reconciling = null;
726
745
  let timer = null;
727
746
  let settling = null;
728
747
  let lastShipped = null;
@@ -817,6 +836,7 @@ export function createLiveBackup({
817
836
  };
818
837
 
819
838
  const ship = async () => {
839
+ if (reconciling) await reconciling.catch(() => {});
820
840
  await load();
821
841
  // Counted before the walk: OrbitDB appends before it emits `update`, so
822
842
  // every update counted here is in the log the walk is about to read, and
@@ -841,13 +861,27 @@ export function createLiveBackup({
841
861
  contentType: "application/vnd.ipld.car",
842
862
  });
843
863
  let kept;
844
- if (
864
+ if (typeof handle?.status === "string") {
865
+ // The upload carried its own keep request (a signed Aleph upload): a
866
+ // second one through pinCid would be paid twice, and would take the
867
+ // path whose STOREs the nodes could not fetch (2026-11).
868
+ kept = {
869
+ id: handle.id,
870
+ backend: handle.backend,
871
+ retained: handle.retained === true,
872
+ status: handle.status,
873
+ ...(handle.itemHash ? { itemHash: handle.itemHash } : {}),
874
+ };
875
+ } else if (
845
876
  keep &&
846
877
  handle?.retained === false &&
847
878
  typeof backend.pinCid === "function"
848
879
  ) {
849
880
  kept = await backend.pinCid(handle.id, { name: handle.name });
850
- if (confirmKept) kept = await confirmKept(kept, handle);
881
+ }
882
+ if (kept) {
883
+ if (confirmKept && kept.status !== "processed")
884
+ kept = await confirmKept(kept, handle);
851
885
  if (kept?.status === "rejected") {
852
886
  throw new Error(
853
887
  `The store refused to keep increment ${collected.manifest.sequence}${kept.errorCode != null ? ` (${kept.errorCode})` : ""}: it was uploaded but is not kept.`,
@@ -878,6 +912,9 @@ export function createLiveBackup({
878
912
  sequence: state.sequence,
879
913
  previous: state.previous,
880
914
  databases: JSON.parse(JSON.stringify(state.databases)),
915
+ // When it went out: a store that does not know a message yet
916
+ // (Aleph's 404 before it has propagated) is not a refusal.
917
+ shippedAt: at.toISOString(),
881
918
  },
882
919
  state.previous,
883
920
  kept?.status ?? "processed",
@@ -976,48 +1013,71 @@ export function createLiveBackup({
976
1013
  const reconcile = async (statusOf) => {
977
1014
  if (inflight) await inflight.catch(() => {});
978
1015
  await load();
1016
+ // The answers first, which takes network time; a flush may run meanwhile.
979
1017
  const answers = new Map();
1018
+ const asked = Date.now();
980
1019
  for (const point of state.unconfirmed ?? []) {
981
- answers.set(point.cid, await statusOf(point.cid, point));
1020
+ let answer = await statusOf(point.cid, point);
1021
+ // Not known yet is not refused while the message may still be on its way.
1022
+ const age = point.shippedAt
1023
+ ? asked - Date.parse(point.shippedAt)
1024
+ : Infinity;
1025
+ if (answer === "unknown" && age < UNKNOWN_GRACE_MS) answer = undefined;
1026
+ answers.set(point.cid, answer);
982
1027
  }
983
- const now_ = reconcileConfirmation(
984
- { kept: state.kept ?? null, pending: state.unconfirmed ?? [] },
985
- (cid) => answers.get(cid),
986
- );
987
- if (!now_.changed)
1028
+ // Then applied to the state as it is now, with no flush running and none
1029
+ // starting until it is done: a point shipped meanwhile has no answer, so
1030
+ // the walk stops before it – or, when an earlier one was refused, it goes
1031
+ // back with the rest.
1032
+ if (inflight) await inflight.catch(() => {});
1033
+ let release;
1034
+ let wentBack = false;
1035
+ reconciling = new Promise((resolve) => (release = resolve));
1036
+ try {
1037
+ const now_ = reconcileConfirmation(
1038
+ { kept: state.kept ?? null, pending: state.unconfirmed ?? [] },
1039
+ (cid) => answers.get(cid),
1040
+ );
1041
+ if (!now_.changed)
1042
+ return {
1043
+ changed: false,
1044
+ rewound: false,
1045
+ gap: false,
1046
+ sequence: state.sequence,
1047
+ };
1048
+ state.kept = now_.kept;
1049
+ state.unconfirmed = now_.pending;
1050
+ if (now_.rewound) {
1051
+ const back = now_.rewind ?? {
1052
+ sequence: 0,
1053
+ previous: null,
1054
+ databases: {},
1055
+ };
1056
+ state.sequence = back.sequence;
1057
+ state.previous = back.previous;
1058
+ state.databases = JSON.parse(JSON.stringify(back.databases ?? {}));
1059
+ for (const [address, w] of watched) {
1060
+ const heads = state.databases[address]?.heads ?? [];
1061
+ w.shipped = await reachable(w.database, heads);
1062
+ w.shipped.heads = heads;
1063
+ }
1064
+ // Nothing announces what has to go again: count it, so a flush ships it.
1065
+ pending = Math.max(pending, 1);
1066
+ wentBack = true;
1067
+ }
1068
+ await save();
988
1069
  return {
989
- changed: false,
990
- rewound: false,
991
- gap: false,
1070
+ changed: true,
1071
+ rewound: now_.rewound,
1072
+ gap: now_.gap,
992
1073
  sequence: state.sequence,
993
1074
  };
994
- state.kept = now_.kept;
995
- state.unconfirmed = now_.pending;
996
- if (now_.rewound) {
997
- const back = now_.rewind ?? {
998
- sequence: 0,
999
- previous: null,
1000
- databases: {},
1001
- };
1002
- state.sequence = back.sequence;
1003
- state.previous = back.previous;
1004
- state.databases = JSON.parse(JSON.stringify(back.databases ?? {}));
1005
- for (const [address, w] of watched) {
1006
- const heads = state.databases[address]?.heads ?? [];
1007
- w.shipped = await reachable(w.database, heads);
1008
- w.shipped.heads = heads;
1009
- }
1010
- // Nothing announces what has to go again: count it, so a flush ships it.
1011
- pending = Math.max(pending, 1);
1012
- if (running && (after || every)) scheduleFlush();
1075
+ } finally {
1076
+ reconciling = null;
1077
+ release();
1078
+ // Only once the flush can see the state it is to ship from.
1079
+ if (wentBack && running && (after || every)) scheduleFlush();
1013
1080
  }
1014
- await save();
1015
- return {
1016
- changed: true,
1017
- rewound: now_.rewound,
1018
- gap: now_.gap,
1019
- sequence: state.sequence,
1020
- };
1021
1081
  };
1022
1082
 
1023
1083
  const start = async () => {
package/lib/peer-fetch.js CHANGED
@@ -301,7 +301,10 @@ function parseProviderRecords(body) {
301
301
  * @param {(addr: string) => Promise<unknown>} [options.dial] - defaults to the node's
302
302
  * @param {(cid: any, options?: Object) => AsyncIterable<Uint8Array>} [options.cat] -
303
303
  * defaults to `@helia/unixfs`, which reads a chunked file as well as a single block
304
- * @returns {(cid: string, options?: Object) => Promise<Uint8Array>}
304
+ * @returns {((cid: string, options?: Object) => Promise<Uint8Array>) & {
305
+ * stream: (cid: string, options?: { offset?: number, signal?: AbortSignal, timeout?: number }) => AsyncGenerator<Uint8Array> }}
306
+ * the fetch, and beside it `stream`: the same file as it arrives, from
307
+ * `offset` on when given, for a file too large to hold
305
308
  */
306
309
  export function createPeerFetch({
307
310
  helia,
@@ -315,9 +318,8 @@ export function createPeerFetch({
315
318
  throw new Error("createPeerFetch needs a Helia node, or both dial and cat");
316
319
  }
317
320
 
318
- return async function fetchOverPeers(cid, options = {}) {
319
- const signal =
320
- options.signal ?? AbortSignal.timeout(options.timeout ?? timeout);
321
+ /** Dial whoever holds `cid`, each peer once; how many answered. */
322
+ const reach = async (cid, signal) => {
321
323
  const addrs =
322
324
  typeof providers === "function" ? await providers(cid) : [...providers];
323
325
 
@@ -348,17 +350,38 @@ export function createPeerFetch({
348
350
  log.debug(` ⚠️ could not dial ${addr}: ${error.message}`);
349
351
  }
350
352
  }
351
- const dialed = reached.size;
352
- if (dialed === 0) {
353
+ if (reached.size === 0) {
353
354
  throw new Error(
354
355
  `Could not dial any provider for ${cid} (tried ${addrs.length})`,
355
356
  );
356
357
  }
358
+ return reached.size;
359
+ };
357
360
 
361
+ /**
362
+ * The file as the peers send it, never all of it at once (#187). With
363
+ * `offset`, from that byte on: where another source left off.
364
+ */
365
+ async function* streamOverPeers(cid, options = {}) {
366
+ const signal =
367
+ options.signal ?? AbortSignal.timeout(options.timeout ?? timeout);
368
+ const dialed = await reach(cid, signal);
358
369
  const read = cat ?? (await unixfsCat(helia));
370
+ let total = 0;
371
+ for await (const chunk of read(cid, {
372
+ signal,
373
+ ...(options.offset ? { offset: options.offset } : {}),
374
+ })) {
375
+ total += chunk.length;
376
+ yield chunk;
377
+ }
378
+ log.info(` ✅ ${total} bytes over libp2p from ${dialed} peer(s)`);
379
+ }
380
+
381
+ async function fetchOverPeers(cid, options = {}) {
359
382
  const parts = [];
360
383
  let total = 0;
361
- for await (const chunk of read(cid, { signal })) {
384
+ for await (const chunk of streamOverPeers(cid, options)) {
362
385
  parts.push(chunk);
363
386
  total += chunk.length;
364
387
  }
@@ -368,9 +391,10 @@ export function createPeerFetch({
368
391
  bytes.set(part, at);
369
392
  at += part.length;
370
393
  }
371
- log.info(` ✅ ${bytes.length} bytes over libp2p from ${dialed} peer(s)`);
372
394
  return bytes;
373
- };
395
+ }
396
+ fetchOverPeers.stream = streamOverPeers;
397
+ return fetchOverPeers;
374
398
  }
375
399
 
376
400
  /**
@@ -442,3 +466,85 @@ export function createGatewayFirstFetch({
442
466
  }
443
467
  };
444
468
  }
469
+
470
+ /**
471
+ * {@link createGatewayFirstFetch} for a file that is handed on as it arrives
472
+ * (#187): the gateway first, and when it fails – at the start or after half
473
+ * the file – the peers go on **from the byte the gateway stopped at**. The
474
+ * caller sees one run of bytes.
475
+ *
476
+ * Whole-file checks are each path's own and cover what that path delivered;
477
+ * across a change of path only `size` is checked here. That is enough for
478
+ * what this is for: a sealed backup's frames authenticate themselves.
479
+ *
480
+ * @param {Object} options
481
+ * @param {(cid: string, options?: Object) => AsyncIterable<Uint8Array>} options.viaGateway
482
+ * e.g. the Aleph driver's `getBlobStream`; an error of its may carry
483
+ * `received`, and is otherwise taken to have stopped where the count stands
484
+ * @param {(cid: string, options?: Object) => AsyncIterable<Uint8Array>} options.viaPeers
485
+ * e.g. `createPeerFetch(…).stream`; given `offset`
486
+ * @param {number} [options.gatewayTimeout=3000] idle time before the peers start
487
+ * @param {(path: "gateway"|"peers", info: Object) => void} [options.onPath]
488
+ * @returns {(cid: string, options?: { size?: number }) => AsyncGenerator<Uint8Array>}
489
+ */
490
+ export function createGatewayFirstStream({
491
+ viaGateway,
492
+ viaPeers,
493
+ gatewayTimeout = 3000,
494
+ onPath = () => {},
495
+ now = () => Date.now(),
496
+ } = {}) {
497
+ return async function* streamBytes(cid, options = {}) {
498
+ const started = now();
499
+ let received = 0;
500
+ let gatewayError = null;
501
+ try {
502
+ for await (const chunk of viaGateway(cid, {
503
+ ...options,
504
+ timeout: gatewayTimeout,
505
+ })) {
506
+ received += chunk.length;
507
+ yield chunk;
508
+ }
509
+ } catch (error) {
510
+ gatewayError = error;
511
+ }
512
+ if (!gatewayError) {
513
+ onPath("gateway", { cid, ms: now() - started, bytes: received });
514
+ return;
515
+ }
516
+ const gatewayMs = now() - started;
517
+ const peersStarted = now();
518
+ const fromGateway = received;
519
+ try {
520
+ for await (const chunk of viaPeers(cid, {
521
+ ...options,
522
+ ...(received ? { offset: received } : {}),
523
+ })) {
524
+ received += chunk.length;
525
+ yield chunk;
526
+ }
527
+ } catch (peerError) {
528
+ throw new Error(
529
+ `Could not fetch ${cid}: the gateway said "${gatewayError.message}" and the peers said "${peerError.message}"`,
530
+ { cause: peerError },
531
+ );
532
+ }
533
+ if (
534
+ Number.isInteger(options.size) &&
535
+ options.size > 0 &&
536
+ received !== options.size
537
+ ) {
538
+ throw new Error(
539
+ `${received} bytes of ${cid} arrived, not the ${options.size} it has`,
540
+ );
541
+ }
542
+ onPath("peers", {
543
+ cid,
544
+ ms: now() - peersStarted,
545
+ bytes: received,
546
+ ...(fromGateway ? { resumedAt: fromGateway } : {}),
547
+ after: { path: "gateway", ms: gatewayMs, error: gatewayError.message },
548
+ });
549
+ };
550
+ }