@instantdb/platform 1.0.64 → 1.0.65-branch-disable-prefetch.32771708048.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +8 -8
- package/__tests__/src/backupDownload.test.ts +230 -0
- package/dist/commonjs/backupDownload.d.ts +15 -0
- package/dist/commonjs/backupDownload.d.ts.map +1 -1
- package/dist/commonjs/backupDownload.js +147 -45
- package/dist/commonjs/backupDownload.js.map +1 -1
- package/dist/esm/backupDownload.d.ts +15 -0
- package/dist/esm/backupDownload.d.ts.map +1 -1
- package/dist/esm/backupDownload.js +147 -45
- package/dist/esm/backupDownload.js.map +1 -1
- package/dist/standalone/index.js +1343 -1272
- package/dist/standalone/index.umd.cjs +23 -23
- package/package.json +4 -4
- package/src/backupDownload.ts +205 -50
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
|
|
2
|
-
> @instantdb/platform@1.0.
|
|
2
|
+
> @instantdb/platform@1.0.65-branch-disable-prefetch.32771708048.1 build /home/runner/work/instant/instant/client/packages/platform
|
|
3
3
|
> rm -rf dist; npm run build:tshy && npm run build:standalone && npm run check-exports
|
|
4
4
|
|
|
5
5
|
npm warn Unknown env config "verify-deps-before-run". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
|
|
6
6
|
npm warn Unknown user config "always-auth". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
|
|
7
7
|
|
|
8
|
-
> @instantdb/platform@1.0.
|
|
8
|
+
> @instantdb/platform@1.0.65-branch-disable-prefetch.32771708048.1 build:tshy
|
|
9
9
|
> tshy
|
|
10
10
|
|
|
11
11
|
npm warn Unknown env config "verify-deps-before-run". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
|
|
12
12
|
npm warn Unknown user config "always-auth". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
|
|
13
13
|
|
|
14
|
-
> @instantdb/platform@1.0.
|
|
14
|
+
> @instantdb/platform@1.0.65-branch-disable-prefetch.32771708048.1 build:standalone
|
|
15
15
|
> vite build
|
|
16
16
|
|
|
17
17
|
[36mvite v5.4.14 [32mbuilding for production...[36m[39m
|
|
@@ -19,17 +19,17 @@ transforming...
|
|
|
19
19
|
[32m✓[39m 88 modules transformed.
|
|
20
20
|
rendering chunks...
|
|
21
21
|
computing gzip size...
|
|
22
|
-
[2mdist/standalone/[22m[36mindex.umd.cjs [39m[1m[
|
|
23
|
-
[2mdist/standalone/[22m[36mindex.js [39m[1m[
|
|
24
|
-
[32m✓ built in
|
|
22
|
+
[2mdist/standalone/[22m[36mindex.umd.cjs [39m[1m[2m371.73 kB[22m[1m[22m[2m │ gzip: 103.04 kB[22m
|
|
23
|
+
[2mdist/standalone/[22m[36mindex.js [39m[1m[2m498.62 kB[22m[1m[22m[2m │ gzip: 120.59 kB[22m
|
|
24
|
+
[32m✓ built in 3.37s[39m
|
|
25
25
|
npm warn Unknown env config "verify-deps-before-run". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
|
|
26
26
|
npm warn Unknown user config "always-auth". This will stop working in the next major version of npm. See `npm help npmrc` for supported config options.
|
|
27
27
|
|
|
28
|
-
> @instantdb/platform@1.0.
|
|
28
|
+
> @instantdb/platform@1.0.65-branch-disable-prefetch.32771708048.1 check-exports
|
|
29
29
|
> attw --pack .
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
@instantdb/platform v1.0.
|
|
32
|
+
@instantdb/platform v1.0.65-branch-disable-prefetch.32771708048.1
|
|
33
33
|
|
|
34
34
|
Build tools:
|
|
35
35
|
- @arethetypeswrong/cli@^0.18.5
|
|
@@ -168,6 +168,235 @@ describe('downloadBackupArchive', () => {
|
|
|
168
168
|
await expect(promise).rejects.toMatchObject({ name: 'AbortError' });
|
|
169
169
|
});
|
|
170
170
|
|
|
171
|
+
test('fetches each entry body only when the writer reaches it, never ahead', async () => {
|
|
172
|
+
const files = [
|
|
173
|
+
{ name: 'config.json', size: 1 },
|
|
174
|
+
{ name: 'entities/a.jsonl', size: 1 },
|
|
175
|
+
{ name: 'entities/b.jsonl', size: 1 },
|
|
176
|
+
];
|
|
177
|
+
const started: string[] = [];
|
|
178
|
+
|
|
179
|
+
const manager = {
|
|
180
|
+
listFiles: async () => files,
|
|
181
|
+
getFileUrl,
|
|
182
|
+
streamStorageFiles: async function* () {},
|
|
183
|
+
} as any;
|
|
184
|
+
// Records the moment a body is fetched. Downloads are strictly sequential,
|
|
185
|
+
// so this fires only when the writer reaches the entry — never ahead.
|
|
186
|
+
const trackingFetch = async (url: string) => {
|
|
187
|
+
started.push(url);
|
|
188
|
+
return bodyOf(`body:${url}`);
|
|
189
|
+
};
|
|
190
|
+
|
|
191
|
+
// Block the encoder on the first entry so we can observe whether any later
|
|
192
|
+
// entry's body is pulled ahead while it sits there. A body downloaded ahead
|
|
193
|
+
// of the encoder — the browser-fetch bug that buffered whole entities before
|
|
194
|
+
// their turn — would show up in `started` while the first write is blocked.
|
|
195
|
+
let releaseFirst!: () => void;
|
|
196
|
+
const firstReleased = new Promise<void>((resolve) => {
|
|
197
|
+
releaseFirst = resolve;
|
|
198
|
+
});
|
|
199
|
+
let signalFirstAdd!: () => void;
|
|
200
|
+
const firstAddBegan = new Promise<void>((resolve) => {
|
|
201
|
+
signalFirstAdd = resolve;
|
|
202
|
+
});
|
|
203
|
+
let addCount = 0;
|
|
204
|
+
const createWriter = async (sink: WritableStream<Uint8Array>) => {
|
|
205
|
+
const w = sink.getWriter();
|
|
206
|
+
return {
|
|
207
|
+
add: async (_name: string, input: ReadableStream<Uint8Array>) => {
|
|
208
|
+
if (addCount++ === 0) {
|
|
209
|
+
signalFirstAdd();
|
|
210
|
+
await firstReleased;
|
|
211
|
+
}
|
|
212
|
+
for await (const _chunk of input) {
|
|
213
|
+
// drain
|
|
214
|
+
}
|
|
215
|
+
await w.write(new Uint8Array([0]));
|
|
216
|
+
},
|
|
217
|
+
close: () => w.close(),
|
|
218
|
+
};
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
const done = downloadBackupArchive({
|
|
222
|
+
manager,
|
|
223
|
+
backup,
|
|
224
|
+
fetchBody: trackingFetch,
|
|
225
|
+
sink: nullSink(),
|
|
226
|
+
createWriter,
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
// While the encoder is blocked on config.json, only its body has been
|
|
230
|
+
// fetched — nothing is pulled ahead.
|
|
231
|
+
await firstAddBegan;
|
|
232
|
+
expect(started).toEqual(['config.json']);
|
|
233
|
+
|
|
234
|
+
releaseFirst();
|
|
235
|
+
await done;
|
|
236
|
+
|
|
237
|
+
// Once unblocked the remaining entries fetch in order, still one at a time.
|
|
238
|
+
expect(started).toEqual([
|
|
239
|
+
'config.json',
|
|
240
|
+
'entities/a.jsonl',
|
|
241
|
+
'entities/b.jsonl',
|
|
242
|
+
]);
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
test('retries a transient failure opening an entry body', async () => {
|
|
246
|
+
const names: string[] = [];
|
|
247
|
+
const manager = {
|
|
248
|
+
listFiles,
|
|
249
|
+
getFileUrl,
|
|
250
|
+
streamStorageFiles: async function* () {
|
|
251
|
+
yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' };
|
|
252
|
+
},
|
|
253
|
+
} as any;
|
|
254
|
+
|
|
255
|
+
// The storage blob's first open fails once, then succeeds.
|
|
256
|
+
let storageAttempts = 0;
|
|
257
|
+
const flakyFetch = async (url: string) => {
|
|
258
|
+
if (url === 'loc-1-url') {
|
|
259
|
+
storageAttempts++;
|
|
260
|
+
if (storageAttempts === 1) throw new Error('ECONNRESET');
|
|
261
|
+
}
|
|
262
|
+
return bodyOf(`body:${url}`);
|
|
263
|
+
};
|
|
264
|
+
|
|
265
|
+
const result = await downloadBackupArchive({
|
|
266
|
+
manager,
|
|
267
|
+
backup,
|
|
268
|
+
fetchBody: flakyFetch,
|
|
269
|
+
sink: nullSink(),
|
|
270
|
+
createWriter: makeWriter(names),
|
|
271
|
+
retry: { attempts: 3, delayMs: 0 },
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
expect(storageAttempts).toBe(2);
|
|
275
|
+
expect(names).toEqual([
|
|
276
|
+
'config.json',
|
|
277
|
+
'entities/todos.jsonl',
|
|
278
|
+
'files/loc-1',
|
|
279
|
+
]);
|
|
280
|
+
expect(result.files).toBe(1);
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
test('retries a body that connects but errors on its first read', async () => {
|
|
284
|
+
const names: string[] = [];
|
|
285
|
+
const manager = {
|
|
286
|
+
listFiles,
|
|
287
|
+
getFileUrl,
|
|
288
|
+
streamStorageFiles: async function* () {
|
|
289
|
+
yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' };
|
|
290
|
+
},
|
|
291
|
+
} as any;
|
|
292
|
+
|
|
293
|
+
// fetchBody resolves (connection established) but the body errors on its
|
|
294
|
+
// first read the first time — the shape of a reset idle connection.
|
|
295
|
+
let storageAttempts = 0;
|
|
296
|
+
const flakyFetch = async (url: string) => {
|
|
297
|
+
if (url === 'loc-1-url') {
|
|
298
|
+
storageAttempts++;
|
|
299
|
+
if (storageAttempts === 1) {
|
|
300
|
+
return new ReadableStream<Uint8Array>({
|
|
301
|
+
pull(controller) {
|
|
302
|
+
controller.error(new Error('ECONNRESET'));
|
|
303
|
+
},
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
return bodyOf(`body:${url}`);
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
const result = await downloadBackupArchive({
|
|
311
|
+
manager,
|
|
312
|
+
backup,
|
|
313
|
+
fetchBody: flakyFetch,
|
|
314
|
+
sink: nullSink(),
|
|
315
|
+
createWriter: makeWriter(names),
|
|
316
|
+
retry: { attempts: 3, delayMs: 0 },
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
expect(storageAttempts).toBe(2);
|
|
320
|
+
expect(names).toEqual([
|
|
321
|
+
'config.json',
|
|
322
|
+
'entities/todos.jsonl',
|
|
323
|
+
'files/loc-1',
|
|
324
|
+
]);
|
|
325
|
+
expect(result.files).toBe(1);
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
test('does not retry once the body has yielded a chunk', async () => {
|
|
329
|
+
const manager = {
|
|
330
|
+
listFiles,
|
|
331
|
+
getFileUrl,
|
|
332
|
+
streamStorageFiles: async function* () {
|
|
333
|
+
yield { locationId: 'loc-1', path: 'a.png', url: 'loc-1-url' };
|
|
334
|
+
},
|
|
335
|
+
} as any;
|
|
336
|
+
|
|
337
|
+
// The body delivers one chunk on the first read, then errors on the next —
|
|
338
|
+
// a mid-stream failure. Once bytes are flowing the entry can't be
|
|
339
|
+
// restarted, so this must not retry.
|
|
340
|
+
let storageAttempts = 0;
|
|
341
|
+
const fetchBody = async (url: string) => {
|
|
342
|
+
if (url === 'loc-1-url') {
|
|
343
|
+
storageAttempts++;
|
|
344
|
+
let phase = 0;
|
|
345
|
+
return new ReadableStream<Uint8Array>({
|
|
346
|
+
pull(controller) {
|
|
347
|
+
if (phase++ === 0) {
|
|
348
|
+
controller.enqueue(new TextEncoder().encode('partial'));
|
|
349
|
+
} else {
|
|
350
|
+
controller.error(new Error('mid-stream reset'));
|
|
351
|
+
}
|
|
352
|
+
},
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
return bodyOf(`body:${url}`);
|
|
356
|
+
};
|
|
357
|
+
|
|
358
|
+
await expect(
|
|
359
|
+
downloadBackupArchive({
|
|
360
|
+
manager,
|
|
361
|
+
backup,
|
|
362
|
+
fetchBody,
|
|
363
|
+
sink: nullSink(),
|
|
364
|
+
createWriter: makeWriter([]),
|
|
365
|
+
retry: { attempts: 3, delayMs: 0 },
|
|
366
|
+
}),
|
|
367
|
+
).rejects.toThrow();
|
|
368
|
+
expect(storageAttempts).toBe(1);
|
|
369
|
+
});
|
|
370
|
+
|
|
371
|
+
test('gives up after exhausting retries and names the failing entry', async () => {
|
|
372
|
+
const manager = {
|
|
373
|
+
listFiles,
|
|
374
|
+
getFileUrl,
|
|
375
|
+
streamStorageFiles: async function* () {
|
|
376
|
+
yield { locationId: 'loc-9', path: null, url: 'bad-url' };
|
|
377
|
+
},
|
|
378
|
+
} as any;
|
|
379
|
+
|
|
380
|
+
let attempts = 0;
|
|
381
|
+
await expect(
|
|
382
|
+
downloadBackupArchive({
|
|
383
|
+
manager,
|
|
384
|
+
backup,
|
|
385
|
+
fetchBody: async (url: string) => {
|
|
386
|
+
if (url === 'bad-url') {
|
|
387
|
+
attempts++;
|
|
388
|
+
throw new Error('HTTP 500');
|
|
389
|
+
}
|
|
390
|
+
return bodyOf('x');
|
|
391
|
+
},
|
|
392
|
+
sink: nullSink(),
|
|
393
|
+
createWriter: makeWriter([]),
|
|
394
|
+
retry: { attempts: 3, delayMs: 0 },
|
|
395
|
+
}),
|
|
396
|
+
).rejects.toThrow('Couldn\'t download storage file "loc-9" (HTTP 500).');
|
|
397
|
+
expect(attempts).toBe(3);
|
|
398
|
+
});
|
|
399
|
+
|
|
171
400
|
test('names a pathless storage file by locationId when its download fails', async () => {
|
|
172
401
|
const manager = {
|
|
173
402
|
listFiles,
|
|
@@ -187,6 +416,7 @@ describe('downloadBackupArchive', () => {
|
|
|
187
416
|
},
|
|
188
417
|
sink: nullSink(),
|
|
189
418
|
createWriter: makeWriter([]),
|
|
419
|
+
retry: { attempts: 1 },
|
|
190
420
|
}),
|
|
191
421
|
).rejects.toThrow('Couldn\'t download storage file "loc-9" (HTTP 500).');
|
|
192
422
|
});
|
|
@@ -54,6 +54,21 @@ export type DownloadBackupArchiveOpts = {
|
|
|
54
54
|
createWriter: (sink: WritableStream<Uint8Array>, signal: AbortSignal) => Promise<BackupArchiveWriter>;
|
|
55
55
|
signal?: AbortSignal;
|
|
56
56
|
onProgress?: (progress: BackupDownloadProgress) => void;
|
|
57
|
+
/**
|
|
58
|
+
* Retry policy for *opening* an entry's body (fetching its presigned URL and
|
|
59
|
+
* getting a live response).
|
|
60
|
+
* `attempts` is the total number of tries (default {@link DEFAULT_FETCH_ATTEMPTS});
|
|
61
|
+
* `delayMs` is the base for exponential backoff (default {@link DEFAULT_RETRY_DELAY_MS}).
|
|
62
|
+
*
|
|
63
|
+
* This only covers failures before the writer starts consuming the body.
|
|
64
|
+
* Once bytes have been written into the archive entry there's no way to
|
|
65
|
+
* restart it without HTTP range/resume support, so a mid-stream failure
|
|
66
|
+
* still fails the download.
|
|
67
|
+
*/
|
|
68
|
+
retry?: {
|
|
69
|
+
attempts?: number;
|
|
70
|
+
delayMs?: number;
|
|
71
|
+
};
|
|
57
72
|
};
|
|
58
73
|
/**
|
|
59
74
|
* Downloads a backup into a single archive written to `opts.sink`: entries
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backupDownload.d.ts","sourceRoot":"","sources":["../../src/backupDownload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EAET,cAAc,EACf,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,sBAAsB,GAAG;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1B,QAAQ,EAAE,MAAM,CAAC;IAIjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAG1B,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CACD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,IAAI,EAAE;QAAE,WAAW,EAAE,IAAI,CAAA;KAAE,GAC1B,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,MAAM,EAAE,SAAS,CAAC;IAClB;;;;;;OAMG;IACH,SAAS,EAAE,CACT,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,WAAW,KAChB,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;IACzC;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;IACjC;;;OAGG;IACH,YAAY,EAAE,CACZ,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,EAChC,MAAM,EAAE,WAAW,KAChB,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,sBAAsB,KAAK,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"backupDownload.d.ts","sourceRoot":"","sources":["../../src/backupDownload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EAET,cAAc,EACf,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,sBAAsB,GAAG;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1B,QAAQ,EAAE,MAAM,CAAC;IAIjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAG1B,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CACD,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,IAAI,EAAE;QAAE,WAAW,EAAE,IAAI,CAAA;KAAE,GAC1B,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,MAAM,EAAE,SAAS,CAAC;IAClB;;;;;;OAMG;IACH,SAAS,EAAE,CACT,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,WAAW,KAChB,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;IACzC;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;IACjC;;;OAGG;IACH,YAAY,EAAE,CACZ,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,EAChC,MAAM,EAAE,WAAW,KAChB,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACxD;;;;;;;;;;OAUG;IACH,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACjD,CAAC;AAiIF;;;;;;;;;;;;GAYG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,yBAAyB,GAAG;IAChC,OAAO,EAAE,IAAI,CACX,cAAc,EACd,WAAW,GAAG,YAAY,GAAG,oBAAoB,CAClD,CAAC;CACH,GACA,OAAO,CAAC,oBAAoB,CAAC,CAuR/B"}
|
|
@@ -1,8 +1,93 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.downloadBackupArchive = downloadBackupArchive;
|
|
4
|
+
const DEFAULT_FETCH_ATTEMPTS = 3;
|
|
5
|
+
const DEFAULT_RETRY_DELAY_MS = 500;
|
|
6
|
+
const MAX_RETRY_DELAY_MS = 5000;
|
|
7
|
+
// Normalize caller-supplied numeric options so NaN/Infinity/non-integers can't
|
|
8
|
+
// alter retry counts or pipeline bounds — fall back to the default instead.
|
|
9
|
+
const finitePositiveInt = (v, fallback) => v != null && Number.isInteger(v) && v > 0 ? v : fallback;
|
|
10
|
+
const finiteNonNegative = (v, fallback) => v != null && Number.isFinite(v) && v >= 0 ? v : fallback;
|
|
4
11
|
const isAbortError = (e) => e?.name === 'AbortError';
|
|
5
12
|
const errorMessage = (e) => e instanceof Error ? e.message : String(e);
|
|
13
|
+
// A cancellable sleep: resolves after `ms`, or rejects if the signal aborts
|
|
14
|
+
// first so backoff between retries doesn't outlive a cancelled download.
|
|
15
|
+
const delay = (ms, signal) => new Promise((resolve, reject) => {
|
|
16
|
+
const onAbort = () => {
|
|
17
|
+
clearTimeout(timer);
|
|
18
|
+
reject(signal.reason ?? new DOMException('Aborted', 'AbortError'));
|
|
19
|
+
};
|
|
20
|
+
const timer = setTimeout(() => {
|
|
21
|
+
signal.removeEventListener('abort', onAbort);
|
|
22
|
+
resolve();
|
|
23
|
+
}, ms);
|
|
24
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
25
|
+
});
|
|
26
|
+
// Re-emits an already-read first chunk, then streams the rest from `reader`.
|
|
27
|
+
// Past that first chunk read errors propagate to the consumer unchanged — by
|
|
28
|
+
// then bytes are in the archive entry and it can't be restarted.
|
|
29
|
+
function replayFrom(first, reader) {
|
|
30
|
+
let replayed = false;
|
|
31
|
+
return new ReadableStream({
|
|
32
|
+
async pull(controller) {
|
|
33
|
+
if (!replayed) {
|
|
34
|
+
replayed = true;
|
|
35
|
+
if (first.done) {
|
|
36
|
+
controller.close();
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
controller.enqueue(first.value);
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
const { done, value } = await reader.read();
|
|
43
|
+
if (done)
|
|
44
|
+
controller.close();
|
|
45
|
+
else
|
|
46
|
+
controller.enqueue(value);
|
|
47
|
+
},
|
|
48
|
+
async cancel(reason) {
|
|
49
|
+
await reader.cancel(reason);
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Opens a body via `open`, retrying on transient failure with abortable
|
|
55
|
+
* exponential backoff. `open` is re-invoked from scratch each attempt, so an
|
|
56
|
+
* entity file re-mints its presigned URL. An abort propagates immediately; any
|
|
57
|
+
* other final failure is wrapped by `describe` into a user-facing message
|
|
58
|
+
* naming the entry.
|
|
59
|
+
*
|
|
60
|
+
* The first chunk is read inside the retry scope, so a body that connects but
|
|
61
|
+
* fails on its first read is re-fetched too, since nothing has been written
|
|
62
|
+
* to the archive yet. Only failures once bytes are flowing are treated as
|
|
63
|
+
* unrecoverable.
|
|
64
|
+
*/
|
|
65
|
+
async function openWithRetry(open, opts) {
|
|
66
|
+
let lastError;
|
|
67
|
+
for (let attempt = 1; attempt <= opts.attempts; attempt++) {
|
|
68
|
+
opts.signal.throwIfAborted();
|
|
69
|
+
let reader;
|
|
70
|
+
try {
|
|
71
|
+
const body = await open();
|
|
72
|
+
reader = body.getReader();
|
|
73
|
+
const first = await reader.read();
|
|
74
|
+
return replayFrom(first, reader);
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
// Release the failed connection before retrying (or giving up).
|
|
78
|
+
if (reader)
|
|
79
|
+
reader.cancel().catch(() => { });
|
|
80
|
+
if (isAbortError(e))
|
|
81
|
+
throw e;
|
|
82
|
+
lastError = e;
|
|
83
|
+
if (attempt < opts.attempts) {
|
|
84
|
+
const backoff = Math.min(opts.delayMs * 2 ** (attempt - 1), MAX_RETRY_DELAY_MS);
|
|
85
|
+
await delay(backoff, opts.signal);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
throw new Error(opts.describe(lastError));
|
|
90
|
+
}
|
|
6
91
|
/**
|
|
7
92
|
* Downloads a backup into a single archive written to `opts.sink`: entries
|
|
8
93
|
* in the canonical restore order (`config.json`, then the
|
|
@@ -41,6 +126,8 @@ async function downloadBackupArchive(opts) {
|
|
|
41
126
|
const bytesTotal = backup.uncompressedSize != null
|
|
42
127
|
? backup.uncompressedSize + (backup.filesSize ?? 0)
|
|
43
128
|
: null;
|
|
129
|
+
const retryAttempts = finitePositiveInt(opts.retry?.attempts, DEFAULT_FETCH_ATTEMPTS);
|
|
130
|
+
const retryDelayMs = finiteNonNegative(opts.retry?.delayMs, DEFAULT_RETRY_DELAY_MS);
|
|
44
131
|
const tick = () => onProgress?.({
|
|
45
132
|
entitiesCompleted,
|
|
46
133
|
entitiesTotal,
|
|
@@ -119,7 +206,14 @@ async function downloadBackupArchive(opts) {
|
|
|
119
206
|
notify();
|
|
120
207
|
}
|
|
121
208
|
})();
|
|
122
|
-
|
|
209
|
+
// Entry write order is significant for restore: config.json first, then the
|
|
210
|
+
// entities/*.jsonl shards, then files/<locationId>. In particular ALL entity
|
|
211
|
+
// files must be written before ANY storage file. listFiles returns the
|
|
212
|
+
// entity files in write order; this generator yields thunks for them in
|
|
213
|
+
// order, then drains the storage queue. The consumer calls each thunk in turn
|
|
214
|
+
// and writes it to completion before the next, so bodies download one at a
|
|
215
|
+
// time and never ahead of the encoder.
|
|
216
|
+
const thunks = (async function* () {
|
|
123
217
|
const files = await manager.listFiles(backup.id, { signal });
|
|
124
218
|
if (files.length === 0) {
|
|
125
219
|
throw new Error('No files found for this backup.');
|
|
@@ -134,30 +228,32 @@ async function downloadBackupArchive(opts) {
|
|
|
134
228
|
entitiesTotal = files.filter((f) => f.name !== 'config.json').length;
|
|
135
229
|
tick();
|
|
136
230
|
for (const f of files) {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
231
|
+
yield async () => {
|
|
232
|
+
const body = await openWithRetry(async () => {
|
|
233
|
+
const url = await manager.getFileUrl(backup.id, f.name, { signal });
|
|
234
|
+
return fetchBody(url, signal);
|
|
235
|
+
}, {
|
|
236
|
+
signal,
|
|
237
|
+
attempts: retryAttempts,
|
|
238
|
+
delayMs: retryDelayMs,
|
|
239
|
+
describe: (e) => `Failed to fetch ${f.name}: ${errorMessage(e)}.`,
|
|
240
|
+
});
|
|
241
|
+
return {
|
|
242
|
+
name: f.name,
|
|
243
|
+
input: countBytes(body),
|
|
244
|
+
onWriting: () => {
|
|
245
|
+
currentEntity = f.name;
|
|
246
|
+
currentFile = '';
|
|
247
|
+
tick();
|
|
248
|
+
},
|
|
249
|
+
onAdded: () => {
|
|
250
|
+
if (f.name !== 'config.json')
|
|
251
|
+
entitiesCompleted++;
|
|
252
|
+
tick();
|
|
253
|
+
},
|
|
254
|
+
};
|
|
157
255
|
};
|
|
158
256
|
}
|
|
159
|
-
currentEntity = '';
|
|
160
|
-
tick();
|
|
161
257
|
while (true) {
|
|
162
258
|
if (storageError)
|
|
163
259
|
throw storageError;
|
|
@@ -173,25 +269,28 @@ async function downloadBackupArchive(opts) {
|
|
|
173
269
|
queueHead++;
|
|
174
270
|
}
|
|
175
271
|
if (file) {
|
|
176
|
-
const
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
272
|
+
const storageFile = file;
|
|
273
|
+
const label = storageFile.path || storageFile.locationId;
|
|
274
|
+
yield async () => {
|
|
275
|
+
const body = await openWithRetry(() => fetchBody(storageFile.url, signal), {
|
|
276
|
+
signal,
|
|
277
|
+
attempts: retryAttempts,
|
|
278
|
+
delayMs: retryDelayMs,
|
|
279
|
+
describe: (e) => `Couldn't download storage file "${label}" (${errorMessage(e)}).`,
|
|
280
|
+
});
|
|
281
|
+
return {
|
|
282
|
+
name: `files/${storageFile.locationId}`,
|
|
283
|
+
input: countBytes(body),
|
|
284
|
+
onWriting: () => {
|
|
285
|
+
currentEntity = '';
|
|
286
|
+
currentFile = label;
|
|
287
|
+
tick();
|
|
288
|
+
},
|
|
289
|
+
onAdded: () => {
|
|
290
|
+
filesCompleted++;
|
|
291
|
+
tick();
|
|
292
|
+
},
|
|
293
|
+
};
|
|
195
294
|
};
|
|
196
295
|
}
|
|
197
296
|
else if (storageDone) {
|
|
@@ -203,8 +302,6 @@ async function downloadBackupArchive(opts) {
|
|
|
203
302
|
});
|
|
204
303
|
}
|
|
205
304
|
}
|
|
206
|
-
currentFile = '';
|
|
207
|
-
tick();
|
|
208
305
|
if (storageError)
|
|
209
306
|
throw storageError;
|
|
210
307
|
})();
|
|
@@ -229,12 +326,17 @@ async function downloadBackupArchive(opts) {
|
|
|
229
326
|
},
|
|
230
327
|
});
|
|
231
328
|
const writer = await createWriter(countingSink, signal);
|
|
232
|
-
for await (const
|
|
329
|
+
for await (const thunk of thunks) {
|
|
330
|
+
const entry = await thunk();
|
|
331
|
+
entry.onWriting();
|
|
233
332
|
await writer.add(entry.name, entry.input, {
|
|
234
333
|
lastModDate: backup.backupAt,
|
|
235
334
|
});
|
|
236
335
|
entry.onAdded();
|
|
237
336
|
}
|
|
337
|
+
currentEntity = '';
|
|
338
|
+
currentFile = '';
|
|
339
|
+
tick();
|
|
238
340
|
// A caller abort that lands after the last entry lets the generator
|
|
239
341
|
// finish cleanly; don't close and return a complete-looking archive.
|
|
240
342
|
signal.throwIfAborted();
|