@instantdb/platform 1.0.64 → 1.0.65-branch-disable-prefetch.32771708048.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,17 +1,17 @@
1
1
 
2
- > @instantdb/platform@1.0.64 build /home/runner/work/instant/instant/client/packages/platform
2
+ > @instantdb/platform@1.0.65-branch-disable-prefetch.32771708048.1 build /home/runner/work/instant/instant/client/packages/platform
3
3
  > rm -rf dist; npm run build:tshy && npm run build:standalone && npm run check-exports
4
4
 
5
5
  npm warn Unknown env config "verify-deps-before-run". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
6
6
  npm warn Unknown user config "always-auth". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
7
7
 
8
- > @instantdb/platform@1.0.64 build:tshy
8
+ > @instantdb/platform@1.0.65-branch-disable-prefetch.32771708048.1 build:tshy
9
9
  > tshy
10
10
 
11
11
  npm warn Unknown env config "verify-deps-before-run". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
12
12
  npm warn Unknown user config "always-auth". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
13
13
 
14
- > @instantdb/platform@1.0.64 build:standalone
14
+ > @instantdb/platform@1.0.65-branch-disable-prefetch.32771708048.1 build:standalone
15
15
  > vite build
16
16
 
17
17
  vite v5.4.14 building for production...
@@ -19,17 +19,17 @@ transforming...
19
19
  ✓ 88 modules transformed.
20
20
  rendering chunks...
21
21
  computing gzip size...
22
- dist/standalone/index.umd.cjs 370.53 kB │ gzip: 102.56 kB
23
- dist/standalone/index.js 496.65 kB │ gzip: 119.99 kB
24
- ✓ built in 1.71s
22
+ dist/standalone/index.umd.cjs 371.73 kB │ gzip: 103.04 kB
23
+ dist/standalone/index.js 498.62 kB │ gzip: 120.59 kB
24
+ ✓ built in 3.37s
25
25
  npm warn Unknown env config "verify-deps-before-run". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
26
26
  npm warn Unknown user config "always-auth". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
27
27
 
28
- > @instantdb/platform@1.0.64 check-exports
28
+ > @instantdb/platform@1.0.65-branch-disable-prefetch.32771708048.1 check-exports
29
29
  > attw --pack .
30
30
 
31
31
 
32
- @instantdb/platform v1.0.64
32
+ @instantdb/platform v1.0.65-branch-disable-prefetch.32771708048.1
33
33
 
34
34
  Build tools:
35
35
  - @arethetypeswrong/cli@^0.18.5
@@ -168,6 +168,235 @@ describe('downloadBackupArchive', () => {
168
168
  await expect(promise).rejects.toMatchObject({ name: 'AbortError' });
169
169
  });
170
170
 
171
+ test('fetches each entry body only when the writer reaches it, never ahead', async () => {
172
+ const files = [
173
+ { name: 'config.json', size: 1 },
174
+ { name: 'entities/a.jsonl', size: 1 },
175
+ { name: 'entities/b.jsonl', size: 1 },
176
+ ];
177
+ const started: string[] = [];
178
+
179
+ const manager = {
180
+ listFiles: async () => files,
181
+ getFileUrl,
182
+ streamStorageFiles: async function* () {},
183
+ } as any;
184
+ // Records the moment a body is fetched. Downloads are strictly sequential,
185
+ // so this fires only when the writer reaches the entry — never ahead.
186
+ const trackingFetch = async (url: string) => {
187
+ started.push(url);
188
+ return bodyOf(`body:${url}`);
189
+ };
190
+
191
+ // Block the encoder on the first entry so we can observe whether any later
192
+ // entry's body is pulled ahead while it sits there. A body downloaded ahead
193
+ // of the encoder — the browser-fetch bug that buffered whole entities before
194
+ // their turn — would show up in `started` while the first write is blocked.
195
+ let releaseFirst!: () => void;
196
+ const firstReleased = new Promise<void>((resolve) => {
197
+ releaseFirst = resolve;
198
+ });
199
+ let signalFirstAdd!: () => void;
200
+ const firstAddBegan = new Promise<void>((resolve) => {
201
+ signalFirstAdd = resolve;
202
+ });
203
+ let addCount = 0;
204
+ const createWriter = async (sink: WritableStream<Uint8Array>) => {
205
+ const w = sink.getWriter();
206
+ return {
207
+ add: async (_name: string, input: ReadableStream<Uint8Array>) => {
208
+ if (addCount++ === 0) {
209
+ signalFirstAdd();
210
+ await firstReleased;
211
+ }
212
+ for await (const _chunk of input) {
213
+ // drain
214
+ }
215
+ await w.write(new Uint8Array([0]));
216
+ },
217
+ close: () => w.close(),
218
+ };
219
+ };
220
+
221
+ const done = downloadBackupArchive({
222
+ manager,
223
+ backup,
224
+ fetchBody: trackingFetch,
225
+ sink: nullSink(),
226
+ createWriter,
227
+ });
228
+
229
+ // While the encoder is blocked on config.json, only its body has been
230
+ // fetched — nothing is pulled ahead.
231
+ await firstAddBegan;
232
+ expect(started).toEqual(['config.json']);
233
+
234
+ releaseFirst();
235
+ await done;
236
+
237
+ // Once unblocked the remaining entries fetch in order, still one at a time.
238
+ expect(started).toEqual([
239
+ 'config.json',
240
+ 'entities/a.jsonl',
241
+ 'entities/b.jsonl',
242
+ ]);
243
+ });
244
+
245
+ test('retries a transient failure opening an entry body', async () => {
246
+ const names: string[] = [];
247
+ const manager = {
248
+ listFiles,
249
+ getFileUrl,
250
+ streamStorageFiles: async function* () {
251
+ yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' };
252
+ },
253
+ } as any;
254
+
255
+ // The storage blob's first open fails once, then succeeds.
256
+ let storageAttempts = 0;
257
+ const flakyFetch = async (url: string) => {
258
+ if (url === 'loc-1-url') {
259
+ storageAttempts++;
260
+ if (storageAttempts === 1) throw new Error('ECONNRESET');
261
+ }
262
+ return bodyOf(`body:${url}`);
263
+ };
264
+
265
+ const result = await downloadBackupArchive({
266
+ manager,
267
+ backup,
268
+ fetchBody: flakyFetch,
269
+ sink: nullSink(),
270
+ createWriter: makeWriter(names),
271
+ retry: { attempts: 3, delayMs: 0 },
272
+ });
273
+
274
+ expect(storageAttempts).toBe(2);
275
+ expect(names).toEqual([
276
+ 'config.json',
277
+ 'entities/todos.jsonl',
278
+ 'files/loc-1',
279
+ ]);
280
+ expect(result.files).toBe(1);
281
+ });
282
+
283
+ test('retries a body that connects but errors on its first read', async () => {
284
+ const names: string[] = [];
285
+ const manager = {
286
+ listFiles,
287
+ getFileUrl,
288
+ streamStorageFiles: async function* () {
289
+ yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' };
290
+ },
291
+ } as any;
292
+
293
+ // fetchBody resolves (connection established) but the body errors on its
294
+ // first read the first time — the shape of a reset idle connection.
295
+ let storageAttempts = 0;
296
+ const flakyFetch = async (url: string) => {
297
+ if (url === 'loc-1-url') {
298
+ storageAttempts++;
299
+ if (storageAttempts === 1) {
300
+ return new ReadableStream<Uint8Array>({
301
+ pull(controller) {
302
+ controller.error(new Error('ECONNRESET'));
303
+ },
304
+ });
305
+ }
306
+ }
307
+ return bodyOf(`body:${url}`);
308
+ };
309
+
310
+ const result = await downloadBackupArchive({
311
+ manager,
312
+ backup,
313
+ fetchBody: flakyFetch,
314
+ sink: nullSink(),
315
+ createWriter: makeWriter(names),
316
+ retry: { attempts: 3, delayMs: 0 },
317
+ });
318
+
319
+ expect(storageAttempts).toBe(2);
320
+ expect(names).toEqual([
321
+ 'config.json',
322
+ 'entities/todos.jsonl',
323
+ 'files/loc-1',
324
+ ]);
325
+ expect(result.files).toBe(1);
326
+ });
327
+
328
+ test('does not retry once the body has yielded a chunk', async () => {
329
+ const manager = {
330
+ listFiles,
331
+ getFileUrl,
332
+ streamStorageFiles: async function* () {
333
+ yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' };
334
+ },
335
+ } as any;
336
+
337
+ // The body delivers one chunk on the first read, then errors on the next —
338
+ // a mid-stream failure. Once bytes are flowing the entry can't be
339
+ // restarted, so this must not retry.
340
+ let storageAttempts = 0;
341
+ const fetchBody = async (url: string) => {
342
+ if (url === 'loc-1-url') {
343
+ storageAttempts++;
344
+ let phase = 0;
345
+ return new ReadableStream<Uint8Array>({
346
+ pull(controller) {
347
+ if (phase++ === 0) {
348
+ controller.enqueue(new TextEncoder().encode('partial'));
349
+ } else {
350
+ controller.error(new Error('mid-stream reset'));
351
+ }
352
+ },
353
+ });
354
+ }
355
+ return bodyOf(`body:${url}`);
356
+ };
357
+
358
+ await expect(
359
+ downloadBackupArchive({
360
+ manager,
361
+ backup,
362
+ fetchBody,
363
+ sink: nullSink(),
364
+ createWriter: makeWriter([]),
365
+ retry: { attempts: 3, delayMs: 0 },
366
+ }),
367
+ ).rejects.toThrow();
368
+ expect(storageAttempts).toBe(1);
369
+ });
370
+
371
+ test('gives up after exhausting retries and names the failing entry', async () => {
372
+ const manager = {
373
+ listFiles,
374
+ getFileUrl,
375
+ streamStorageFiles: async function* () {
376
+ yield { locationId: 'loc-9', path: null, url: 'bad-url' };
377
+ },
378
+ } as any;
379
+
380
+ let attempts = 0;
381
+ await expect(
382
+ downloadBackupArchive({
383
+ manager,
384
+ backup,
385
+ fetchBody: async (url: string) => {
386
+ if (url === 'bad-url') {
387
+ attempts++;
388
+ throw new Error('HTTP 500');
389
+ }
390
+ return bodyOf('x');
391
+ },
392
+ sink: nullSink(),
393
+ createWriter: makeWriter([]),
394
+ retry: { attempts: 3, delayMs: 0 },
395
+ }),
396
+ ).rejects.toThrow('Couldn\'t download storage file "loc-9" (HTTP 500).');
397
+ expect(attempts).toBe(3);
398
+ });
399
+
171
400
  test('names a pathless storage file by locationId when its download fails', async () => {
172
401
  const manager = {
173
402
  listFiles,
@@ -187,6 +416,7 @@ describe('downloadBackupArchive', () => {
187
416
  },
188
417
  sink: nullSink(),
189
418
  createWriter: makeWriter([]),
419
+ retry: { attempts: 1 },
190
420
  }),
191
421
  ).rejects.toThrow('Couldn\'t download storage file "loc-9" (HTTP 500).');
192
422
  });
@@ -54,6 +54,21 @@ export type DownloadBackupArchiveOpts = {
54
54
  createWriter: (sink: WritableStream<Uint8Array>, signal: AbortSignal) => Promise<BackupArchiveWriter>;
55
55
  signal?: AbortSignal;
56
56
  onProgress?: (progress: BackupDownloadProgress) => void;
57
+ /**
58
+ * Retry policy for *opening* an entry's body (fetching its presigned URL and
59
+ * getting a live response).
60
+ * `attempts` is the total number of tries (default {@link DEFAULT_FETCH_ATTEMPTS});
61
+ * `delayMs` is the base for exponential backoff (default {@link DEFAULT_RETRY_DELAY_MS}).
62
+ *
63
+ * This only covers failures before the writer starts consuming the body.
64
+ * Once bytes have been written into the archive entry there's no way to
65
+ * restart it without HTTP range/resume support, so a mid-stream failure
66
+ * still fails the download.
67
+ */
68
+ retry?: {
69
+ attempts?: number;
70
+ delayMs?: number;
71
+ };
57
72
  };
58
73
  /**
59
74
  * Downloads a backup into a single archive written to `opts.sink`: entries
@@ -1 +1 @@
1
- {"version":3,"file":"backupDownload.d.ts","sourceRoot":"","sources":["../../src/backupDownload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EAET,cAAc,EACf,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,sBAAsB,GAAG;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1B,QAAQ,EAAE,MAAM,CAAC;IAIjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAG1B,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CACD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,IAAI,EAAE;QAAE,WAAW,EAAE,IAAI,CAAA;KAAE,GAC1B,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,MAAM,EAAE,SAAS,CAAC;IAClB;;;;;;OAMG;IACH,SAAS,EAAE,CACT,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,WAAW,KAChB,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;IACzC;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;IACjC;;;OAGG;IACH,YAAY,EAAE,CACZ,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,EAChC,MAAM,EAAE,WAAW,KAChB,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,sBAAsB,KAAK,IAAI,CAAC;CACzD,CAAC;AAQF;;;;;;;;;;;;GAYG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,yBAAyB,GAAG;IAChC,OAAO,EAAE,IAAI,CACX,cAAc,EACd,WAAW,GAAG,YAAY,GAAG,oBAAoB,CAClD,CAAC;CACH,GACA,OAAO,CAAC,oBAAoB,CAAC,CAiQ/B"}
1
+ {"version":3,"file":"backupDownload.d.ts","sourceRoot":"","sources":["../../src/backupDownload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EAET,cAAc,EACf,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,sBAAsB,GAAG;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1B,QAAQ,EAAE,MAAM,CAAC;IAIjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAG1B,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CACD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,IAAI,EAAE;QAAE,WAAW,EAAE,IAAI,CAAA;KAAE,GAC1B,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,MAAM,EAAE,SAAS,CAAC;IAClB;;;;;;OAMG;IACH,SAAS,EAAE,CACT,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,WAAW,KAChB,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;IACzC;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;IACjC;;;OAGG;IACH,YAAY,EAAE,CACZ,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,EAChC,MAAM,EAAE,WAAW,KAChB,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACxD;;;;;;;;;;OAUG;IACH,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACjD,CAAC;AAiIF;;;;;;;;;;;;GAYG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,yBAAyB,GAAG;IAChC,OAAO,EAAE,IAAI,CACX,cAAc,EACd,WAAW,GAAG,YAAY,GAAG,oBAAoB,CAClD,CAAC;CACH,GACA,OAAO,CAAC,oBAAoB,CAAC,CAuR/B"}
@@ -1,8 +1,93 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.downloadBackupArchive = downloadBackupArchive;
4
+ const DEFAULT_FETCH_ATTEMPTS = 3;
5
+ const DEFAULT_RETRY_DELAY_MS = 500;
6
+ const MAX_RETRY_DELAY_MS = 5000;
7
+ // Normalize caller-supplied numeric options so NaN/Infinity/non-integers can't
8
+ // alter retry counts or pipeline bounds — fall back to the default instead.
9
+ const finitePositiveInt = (v, fallback) => v != null && Number.isInteger(v) && v > 0 ? v : fallback;
10
+ const finiteNonNegative = (v, fallback) => v != null && Number.isFinite(v) && v >= 0 ? v : fallback;
4
11
  const isAbortError = (e) => e?.name === 'AbortError';
5
12
  const errorMessage = (e) => e instanceof Error ? e.message : String(e);
13
+ // A cancellable sleep: resolves after `ms`, or rejects if the signal aborts
14
+ // first so backoff between retries doesn't outlive a cancelled download.
15
+ const delay = (ms, signal) => new Promise((resolve, reject) => {
16
+ const onAbort = () => {
17
+ clearTimeout(timer);
18
+ reject(signal.reason ?? new DOMException('Aborted', 'AbortError'));
19
+ };
20
+ const timer = setTimeout(() => {
21
+ signal.removeEventListener('abort', onAbort);
22
+ resolve();
23
+ }, ms);
24
+ signal.addEventListener('abort', onAbort, { once: true });
25
+ });
26
+ // Re-emits an already-read first chunk, then streams the rest from `reader`.
27
+ // Past that first chunk read errors propagate to the consumer unchanged — by
28
+ // then bytes are in the archive entry and it can't be restarted.
29
+ function replayFrom(first, reader) {
30
+ let replayed = false;
31
+ return new ReadableStream({
32
+ async pull(controller) {
33
+ if (!replayed) {
34
+ replayed = true;
35
+ if (first.done) {
36
+ controller.close();
37
+ return;
38
+ }
39
+ controller.enqueue(first.value);
40
+ return;
41
+ }
42
+ const { done, value } = await reader.read();
43
+ if (done)
44
+ controller.close();
45
+ else
46
+ controller.enqueue(value);
47
+ },
48
+ async cancel(reason) {
49
+ await reader.cancel(reason);
50
+ },
51
+ });
52
+ }
53
+ /**
54
+ * Opens a body via `open`, retrying on transient failure with abortable
55
+ * exponential backoff. `open` is re-invoked from scratch each attempt, so an
56
+ * entity file re-mints its presigned URL. An abort propagates immediately; any
57
+ * other final failure is wrapped by `describe` into a user-facing message
58
+ * naming the entry.
59
+ *
60
+ * The first chunk is read inside the retry scope, so a body that connects but
61
+ * fails on its first read is re-fetched too, since nothing has been written
62
+ * to the archive yet. Only failures once bytes are flowing are treated as
63
+ * unrecoverable.
64
+ */
65
+ async function openWithRetry(open, opts) {
66
+ let lastError;
67
+ for (let attempt = 1; attempt <= opts.attempts; attempt++) {
68
+ opts.signal.throwIfAborted();
69
+ let reader;
70
+ try {
71
+ const body = await open();
72
+ reader = body.getReader();
73
+ const first = await reader.read();
74
+ return replayFrom(first, reader);
75
+ }
76
+ catch (e) {
77
+ // Release the failed connection before retrying (or giving up).
78
+ if (reader)
79
+ reader.cancel().catch(() => { });
80
+ if (isAbortError(e))
81
+ throw e;
82
+ lastError = e;
83
+ if (attempt < opts.attempts) {
84
+ const backoff = Math.min(opts.delayMs * 2 ** (attempt - 1), MAX_RETRY_DELAY_MS);
85
+ await delay(backoff, opts.signal);
86
+ }
87
+ }
88
+ }
89
+ throw new Error(opts.describe(lastError));
90
+ }
6
91
  /**
7
92
  * Downloads a backup into a single archive written to `opts.sink`: entries
8
93
  * in the canonical restore order (`config.json`, then the
@@ -41,6 +126,8 @@ async function downloadBackupArchive(opts) {
41
126
  const bytesTotal = backup.uncompressedSize != null
42
127
  ? backup.uncompressedSize + (backup.filesSize ?? 0)
43
128
  : null;
129
+ const retryAttempts = finitePositiveInt(opts.retry?.attempts, DEFAULT_FETCH_ATTEMPTS);
130
+ const retryDelayMs = finiteNonNegative(opts.retry?.delayMs, DEFAULT_RETRY_DELAY_MS);
44
131
  const tick = () => onProgress?.({
45
132
  entitiesCompleted,
46
133
  entitiesTotal,
@@ -119,7 +206,14 @@ async function downloadBackupArchive(opts) {
119
206
  notify();
120
207
  }
121
208
  })();
122
- const entries = (async function* () {
209
+ // Entry write order is significant for restore: config.json first, then the
210
+ // entities/*.jsonl shards, then files/<locationId>. In particular ALL entity
211
+ // files must be written before ANY storage file. listFiles returns the
212
+ // entity files in write order; this generator yields thunks for them in
213
+ // order, then drains the storage queue. The consumer calls each thunk in turn
214
+ // and writes it to completion before the next, so bodies download one at a
215
+ // time and never ahead of the encoder.
216
+ const thunks = (async function* () {
123
217
  const files = await manager.listFiles(backup.id, { signal });
124
218
  if (files.length === 0) {
125
219
  throw new Error('No files found for this backup.');
@@ -134,30 +228,32 @@ async function downloadBackupArchive(opts) {
134
228
  entitiesTotal = files.filter((f) => f.name !== 'config.json').length;
135
229
  tick();
136
230
  for (const f of files) {
137
- currentEntity = f.name;
138
- tick();
139
- const url = await manager.getFileUrl(backup.id, f.name, { signal });
140
- let body;
141
- try {
142
- body = await fetchBody(url, signal);
143
- }
144
- catch (e) {
145
- if (isAbortError(e))
146
- throw e;
147
- throw new Error(`Failed to fetch ${f.name}: ${errorMessage(e)}.`);
148
- }
149
- yield {
150
- name: f.name,
151
- input: countBytes(body),
152
- onAdded: () => {
153
- if (f.name !== 'config.json')
154
- entitiesCompleted++;
155
- tick();
156
- },
231
+ yield async () => {
232
+ const body = await openWithRetry(async () => {
233
+ const url = await manager.getFileUrl(backup.id, f.name, { signal });
234
+ return fetchBody(url, signal);
235
+ }, {
236
+ signal,
237
+ attempts: retryAttempts,
238
+ delayMs: retryDelayMs,
239
+ describe: (e) => `Failed to fetch ${f.name}: ${errorMessage(e)}.`,
240
+ });
241
+ return {
242
+ name: f.name,
243
+ input: countBytes(body),
244
+ onWriting: () => {
245
+ currentEntity = f.name;
246
+ currentFile = '';
247
+ tick();
248
+ },
249
+ onAdded: () => {
250
+ if (f.name !== 'config.json')
251
+ entitiesCompleted++;
252
+ tick();
253
+ },
254
+ };
157
255
  };
158
256
  }
159
- currentEntity = '';
160
- tick();
161
257
  while (true) {
162
258
  if (storageError)
163
259
  throw storageError;
@@ -173,25 +269,28 @@ async function downloadBackupArchive(opts) {
173
269
  queueHead++;
174
270
  }
175
271
  if (file) {
176
- const label = file.path || file.locationId;
177
- currentFile = label;
178
- tick();
179
- let body;
180
- try {
181
- body = await fetchBody(file.url, signal);
182
- }
183
- catch (e) {
184
- if (isAbortError(e))
185
- throw e;
186
- throw new Error(`Couldn't download storage file "${label}" (${errorMessage(e)}).`);
187
- }
188
- yield {
189
- name: `files/${file.locationId}`,
190
- input: countBytes(body),
191
- onAdded: () => {
192
- filesCompleted++;
193
- tick();
194
- },
272
+ const storageFile = file;
273
+ const label = storageFile.path || storageFile.locationId;
274
+ yield async () => {
275
+ const body = await openWithRetry(() => fetchBody(storageFile.url, signal), {
276
+ signal,
277
+ attempts: retryAttempts,
278
+ delayMs: retryDelayMs,
279
+ describe: (e) => `Couldn't download storage file "${label}" (${errorMessage(e)}).`,
280
+ });
281
+ return {
282
+ name: `files/${storageFile.locationId}`,
283
+ input: countBytes(body),
284
+ onWriting: () => {
285
+ currentEntity = '';
286
+ currentFile = label;
287
+ tick();
288
+ },
289
+ onAdded: () => {
290
+ filesCompleted++;
291
+ tick();
292
+ },
293
+ };
195
294
  };
196
295
  }
197
296
  else if (storageDone) {
@@ -203,8 +302,6 @@ async function downloadBackupArchive(opts) {
203
302
  });
204
303
  }
205
304
  }
206
- currentFile = '';
207
- tick();
208
305
  if (storageError)
209
306
  throw storageError;
210
307
  })();
@@ -229,12 +326,17 @@ async function downloadBackupArchive(opts) {
229
326
  },
230
327
  });
231
328
  const writer = await createWriter(countingSink, signal);
232
- for await (const entry of entries) {
329
+ for await (const thunk of thunks) {
330
+ const entry = await thunk();
331
+ entry.onWriting();
233
332
  await writer.add(entry.name, entry.input, {
234
333
  lastModDate: backup.backupAt,
235
334
  });
236
335
  entry.onAdded();
237
336
  }
337
+ currentEntity = '';
338
+ currentFile = '';
339
+ tick();
238
340
  // A caller abort that lands after the last entry lets the generator
239
341
  // finish cleanly; don't close and return a complete-looking archive.
240
342
  signal.throwIfAborted();