@terreno/syncdb 57.14.0 → 57.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +200 -80
- package/dist/client.js.map +1 -1
- package/dist/client.test.js +86 -0
- package/dist/client.test.js.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.d.ts +4 -3
- package/dist/persisters/encryptedIndexedDbPersister.d.ts.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.js +21 -16
- package/dist/persisters/encryptedIndexedDbPersister.js.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.test.js +40 -1
- package/dist/persisters/encryptedIndexedDbPersister.test.js.map +1 -1
- package/dist/sync/bootstrap.d.ts +34 -3
- package/dist/sync/bootstrap.d.ts.map +1 -1
- package/dist/sync/bootstrap.js +224 -71
- package/dist/sync/bootstrap.js.map +1 -1
- package/dist/sync/bootstrap.test.js +210 -2
- package/dist/sync/bootstrap.test.js.map +1 -1
- package/dist/sync/httpChannel.d.ts +14 -1
- package/dist/sync/httpChannel.d.ts.map +1 -1
- package/dist/sync/httpChannel.js +34 -0
- package/dist/sync/httpChannel.js.map +1 -1
- package/dist/sync/httpChannel.test.js +54 -0
- package/dist/sync/httpChannel.test.js.map +1 -1
- package/dist/types.d.ts +14 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/client.test.ts +111 -1
- package/src/client.ts +232 -86
- package/src/persisters/encryptedIndexedDbPersister.test.ts +44 -1
- package/src/persisters/encryptedIndexedDbPersister.ts +24 -17
- package/src/sync/bootstrap.test.ts +269 -3
- package/src/sync/bootstrap.ts +336 -82
- package/src/sync/httpChannel.test.ts +67 -0
- package/src/sync/httpChannel.ts +50 -0
- package/src/types.ts +12 -0
package/src/sync/bootstrap.ts
CHANGED
|
@@ -33,7 +33,12 @@
|
|
|
33
33
|
*/
|
|
34
34
|
|
|
35
35
|
import type {SyncStore} from "../storage/store";
|
|
36
|
-
import type {
|
|
36
|
+
import type {
|
|
37
|
+
SyncSnapshotBatchRequest,
|
|
38
|
+
SyncSnapshotBatchResponse,
|
|
39
|
+
SyncSnapshotEntity,
|
|
40
|
+
SyncSnapshotResponse,
|
|
41
|
+
} from "../types";
|
|
37
42
|
import {
|
|
38
43
|
getCursor,
|
|
39
44
|
getSnapshotCursor,
|
|
@@ -102,116 +107,365 @@ const applySnapshotEntity = ({
|
|
|
102
107
|
return true;
|
|
103
108
|
};
|
|
104
109
|
|
|
110
|
+
/** Entities written per store transaction before yielding to the host event loop. */
|
|
111
|
+
const DEFAULT_APPLY_CHUNK_SIZE = 500;
|
|
112
|
+
|
|
113
|
+
/** Most streams named in one `POST /sync/snapshot/batch` request (server cap). */
|
|
114
|
+
const MAX_SNAPSHOT_BATCH_STREAMS = 500;
|
|
115
|
+
|
|
116
|
+
const defaultYieldToHost = (): Promise<void> =>
|
|
117
|
+
new Promise((resolve) => {
|
|
118
|
+
setTimeout(resolve, 0);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
/** Per-stream paging state shared by single-stream and batch bootstrap. */
|
|
122
|
+
interface StreamPagingState {
|
|
123
|
+
stream: string;
|
|
124
|
+
collection: string;
|
|
125
|
+
cursor: number;
|
|
126
|
+
legacyCursor?: string;
|
|
127
|
+
hasMore: boolean;
|
|
128
|
+
retentionChecked: boolean;
|
|
129
|
+
/** True when the server itself reported the stream head was reached. */
|
|
130
|
+
reachedHead: boolean;
|
|
131
|
+
}
|
|
132
|
+
|
|
105
133
|
/**
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
* re-bootstrap. Idempotent and incremental — safe to call for both initial bootstrap and
|
|
109
|
-
* reconcile.
|
|
134
|
+
* What one fetched page means for the local store. Planning only touches paging state,
|
|
135
|
+
* so the next request can go out before this page is written.
|
|
110
136
|
*/
|
|
111
|
-
|
|
137
|
+
interface PagePlan {
|
|
138
|
+
stream: string;
|
|
139
|
+
collection: string;
|
|
140
|
+
/** C7: purge the stream (and drop this page) before anything else is written. */
|
|
141
|
+
purge: boolean;
|
|
142
|
+
entities: SyncSnapshotEntity[];
|
|
143
|
+
/** Seq both cursors advance to once the entities land. */
|
|
144
|
+
advanceTo?: number;
|
|
145
|
+
/** True when this is the stream's last page and the server reported its head. */
|
|
146
|
+
completesStream: boolean;
|
|
147
|
+
progress: Omit<BootstrapProgress, "applied">;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const initialPagingState = ({
|
|
112
151
|
store,
|
|
113
|
-
channel,
|
|
114
152
|
stream,
|
|
115
153
|
collection,
|
|
116
|
-
limit,
|
|
117
|
-
onProgress,
|
|
118
|
-
now,
|
|
119
154
|
}: {
|
|
120
155
|
store: SyncStore;
|
|
121
|
-
channel: Pick<HttpChannel, "fetchSnapshotPage">;
|
|
122
156
|
stream: string;
|
|
123
157
|
collection: string;
|
|
124
|
-
|
|
125
|
-
onProgress?: (progress: BootstrapProgress) => void;
|
|
126
|
-
now?: () => string;
|
|
127
|
-
}): Promise<void> => {
|
|
158
|
+
}): StreamPagingState => {
|
|
128
159
|
// Until a snapshot pass has reached the stream head, resume from snapshot progress
|
|
129
160
|
// (`snapshotSeq`) rather than the applied-seq cursor: a live delta can push the latter
|
|
130
161
|
// to the head mid-bootstrap, and resuming there would skip every unpaged seq below it
|
|
131
162
|
// forever. Once bootstrapped, the applied-seq cursor IS the correct catch-up point.
|
|
132
163
|
const bootstrapped = isStreamBootstrapped({store, stream});
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
164
|
+
return {
|
|
165
|
+
collection,
|
|
166
|
+
cursor: bootstrapped ? getCursor({store, stream}) : getSnapshotCursor({store, stream}),
|
|
167
|
+
hasMore: true,
|
|
168
|
+
legacyCursor: undefined,
|
|
169
|
+
reachedHead: false,
|
|
170
|
+
retentionChecked: false,
|
|
171
|
+
stream,
|
|
172
|
+
};
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Advance `state` past one fetched page (C7 retention check, C3 legacy stratum, C1
|
|
177
|
+
* frontier clamp) and describe the writes that page needs.
|
|
178
|
+
*/
|
|
179
|
+
const planSnapshotPage = ({
|
|
180
|
+
state,
|
|
181
|
+
page,
|
|
182
|
+
}: {
|
|
183
|
+
state: StreamPagingState;
|
|
184
|
+
page: SyncSnapshotResponse;
|
|
185
|
+
}): PagePlan => {
|
|
186
|
+
const {collection, stream} = state;
|
|
187
|
+
const base = {collection, completesStream: false, entities: page.entities, stream};
|
|
188
|
+
// C7: a stored cursor below the retained floor means compacted tombstones were
|
|
189
|
+
// missed — purge and re-bootstrap this stream from 0 exactly once.
|
|
190
|
+
if (!state.retentionChecked) {
|
|
191
|
+
state.retentionChecked = true;
|
|
192
|
+
if (state.cursor > 0 && state.cursor < page.oldestRetainedSeq) {
|
|
193
|
+
state.cursor = 0;
|
|
194
|
+
state.legacyCursor = undefined;
|
|
195
|
+
return {
|
|
196
|
+
...base,
|
|
197
|
+
entities: [],
|
|
198
|
+
progress: {collection, cursor: 0, fetched: 0, hasMore: true, stream},
|
|
199
|
+
purge: true,
|
|
200
|
+
};
|
|
154
201
|
}
|
|
202
|
+
}
|
|
155
203
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
return count;
|
|
167
|
-
});
|
|
168
|
-
const advanced = page.legacyCursor !== legacyCursor;
|
|
169
|
-
legacyCursor = page.legacyCursor;
|
|
170
|
-
hasMore = advanced;
|
|
171
|
-
onProgress?.({
|
|
172
|
-
applied,
|
|
204
|
+
if (page.legacyCursor !== undefined) {
|
|
205
|
+
// C3: still draining the seq-0 stratum — echo the token, cursor stays 0.
|
|
206
|
+
const advanced = page.legacyCursor !== state.legacyCursor;
|
|
207
|
+
state.legacyCursor = page.legacyCursor;
|
|
208
|
+
state.hasMore = advanced;
|
|
209
|
+
return {
|
|
210
|
+
...base,
|
|
211
|
+
progress: {
|
|
173
212
|
collection,
|
|
174
213
|
cursor: 0,
|
|
175
214
|
fetched: page.entities.length,
|
|
176
|
-
hasMore,
|
|
215
|
+
hasMore: state.hasMore,
|
|
177
216
|
stream,
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
217
|
+
},
|
|
218
|
+
purge: false,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
// Legacy stratum done (or never present) — proceed by seq.
|
|
222
|
+
state.legacyCursor = undefined;
|
|
223
|
+
|
|
224
|
+
// C1: never advance past the stable frontier; the server already clamped page.cursor.
|
|
225
|
+
const clampedCursor = Math.min(page.cursor, page.frontierSeq);
|
|
226
|
+
const madeProgress = clampedCursor > state.cursor;
|
|
227
|
+
// Guard against a server reporting hasMore without advancing (would loop forever).
|
|
228
|
+
state.hasMore = page.hasMore && madeProgress;
|
|
229
|
+
// Only the server saying "no more pages" proves the head was reached; exiting via the
|
|
230
|
+
// guard above means the pass is incomplete and must resume from snapshot progress.
|
|
231
|
+
state.reachedHead = !page.hasMore;
|
|
232
|
+
state.cursor = Math.max(state.cursor, clampedCursor);
|
|
233
|
+
return {
|
|
234
|
+
...base,
|
|
235
|
+
advanceTo: madeProgress ? clampedCursor : undefined,
|
|
236
|
+
completesStream: !state.hasMore && state.reachedHead,
|
|
237
|
+
progress: {
|
|
238
|
+
collection,
|
|
239
|
+
cursor: state.cursor,
|
|
240
|
+
fetched: page.entities.length,
|
|
241
|
+
hasMore: state.hasMore,
|
|
242
|
+
stream,
|
|
243
|
+
},
|
|
244
|
+
purge: false,
|
|
245
|
+
};
|
|
246
|
+
};
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Write a planned page. Entities land in transactions of `chunkSize` with a yield to the
|
|
250
|
+
* host between them, so a large page never blocks the UI thread in one piece. The cursor
|
|
251
|
+
* advance rides in the last chunk's transaction: if the app dies mid-page, the page is
|
|
252
|
+
* re-fetched, and the seq guards make re-applying it a no-op.
|
|
253
|
+
*
|
|
254
|
+
* Returns false when `isSuperseded` fired during a yield: a sign-out or user switch may
|
|
255
|
+
* have wiped the store, so the rest of the page must not be written into it.
|
|
256
|
+
*/
|
|
257
|
+
const writePagePlan = async ({
|
|
258
|
+
store,
|
|
259
|
+
plan,
|
|
260
|
+
chunkSize,
|
|
261
|
+
yieldToHost,
|
|
262
|
+
isSuperseded,
|
|
263
|
+
onProgress,
|
|
264
|
+
now,
|
|
265
|
+
}: {
|
|
266
|
+
store: SyncStore;
|
|
267
|
+
plan: PagePlan;
|
|
268
|
+
chunkSize: number;
|
|
269
|
+
yieldToHost: () => Promise<void>;
|
|
270
|
+
isSuperseded?: () => boolean;
|
|
271
|
+
onProgress?: (progress: BootstrapProgress) => void;
|
|
272
|
+
now?: () => string;
|
|
273
|
+
}): Promise<boolean> => {
|
|
274
|
+
const {collection, stream, entities, advanceTo} = plan;
|
|
275
|
+
if (plan.purge) {
|
|
276
|
+
store.purgeStream({stream});
|
|
277
|
+
store.addKnownStream({collection, stream});
|
|
278
|
+
return true;
|
|
279
|
+
}
|
|
280
|
+
let applied = 0;
|
|
281
|
+
let offset = 0;
|
|
282
|
+
do {
|
|
283
|
+
const chunk = entities.slice(offset, offset + chunkSize);
|
|
284
|
+
offset += chunkSize;
|
|
285
|
+
const isLastChunk = offset >= entities.length;
|
|
286
|
+
// E4: one transaction per chunk, so TinyBase fires listeners (and the autosave
|
|
287
|
+
// persister's did-finish-transaction write) once per chunk rather than once per row.
|
|
288
|
+
applied += store.raw.transaction(() => {
|
|
193
289
|
let count = 0;
|
|
194
|
-
for (const entity of
|
|
290
|
+
for (const entity of chunk) {
|
|
195
291
|
if (applySnapshotEntity({collection, entity, store, stream})) {
|
|
196
292
|
count += 1;
|
|
197
293
|
}
|
|
198
294
|
}
|
|
199
|
-
if (
|
|
200
|
-
setCursor({now, seq:
|
|
201
|
-
setSnapshotCursor({seq:
|
|
295
|
+
if (isLastChunk && advanceTo !== undefined) {
|
|
296
|
+
setCursor({now, seq: advanceTo, store, stream});
|
|
297
|
+
setSnapshotCursor({seq: advanceTo, store, stream});
|
|
202
298
|
}
|
|
203
299
|
return count;
|
|
204
300
|
});
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
301
|
+
if (!isLastChunk) {
|
|
302
|
+
await yieldToHost();
|
|
303
|
+
if (isSuperseded?.()) {
|
|
304
|
+
return false;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
} while (offset < entities.length);
|
|
308
|
+
if (plan.completesStream) {
|
|
309
|
+
markStreamBootstrapped({store, stream});
|
|
212
310
|
}
|
|
311
|
+
onProgress?.({...plan.progress, applied});
|
|
312
|
+
return true;
|
|
313
|
+
};
|
|
213
314
|
|
|
214
|
-
|
|
215
|
-
|
|
315
|
+
/**
|
|
316
|
+
* Page a single stream from its current cursor to the server's head. Handles the C3
|
|
317
|
+
* legacy stratum (echoing `legacyCursor`), C1 frontier clamping, and C7 retention-gap
|
|
318
|
+
* re-bootstrap.
|
|
319
|
+
*/
|
|
320
|
+
export const bootstrapStream = async ({
|
|
321
|
+
store,
|
|
322
|
+
channel,
|
|
323
|
+
stream,
|
|
324
|
+
collection,
|
|
325
|
+
limit,
|
|
326
|
+
onProgress,
|
|
327
|
+
now,
|
|
328
|
+
isSuperseded,
|
|
329
|
+
applyChunkSize = DEFAULT_APPLY_CHUNK_SIZE,
|
|
330
|
+
yieldToHost = defaultYieldToHost,
|
|
331
|
+
}: {
|
|
332
|
+
store: SyncStore;
|
|
333
|
+
channel: Pick<HttpChannel, "fetchSnapshotPage">;
|
|
334
|
+
stream: string;
|
|
335
|
+
collection: string;
|
|
336
|
+
limit?: number;
|
|
337
|
+
onProgress?: (progress: BootstrapProgress) => void;
|
|
338
|
+
now?: () => string;
|
|
339
|
+
/** Checked after every response and every chunk yield; true stops the pass unwritten. */
|
|
340
|
+
isSuperseded?: () => boolean;
|
|
341
|
+
applyChunkSize?: number;
|
|
342
|
+
yieldToHost?: () => Promise<void>;
|
|
343
|
+
}): Promise<void> => {
|
|
344
|
+
const state = initialPagingState({collection, store, stream});
|
|
345
|
+
while (state.hasMore) {
|
|
346
|
+
const page = await channel.fetchSnapshotPage({
|
|
347
|
+
cursor: state.cursor,
|
|
348
|
+
legacyCursor: state.legacyCursor,
|
|
349
|
+
limit,
|
|
350
|
+
stream,
|
|
351
|
+
});
|
|
352
|
+
if (isSuperseded?.()) {
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
const plan = planSnapshotPage({page, state});
|
|
356
|
+
const written = await writePagePlan({
|
|
357
|
+
chunkSize: applyChunkSize,
|
|
358
|
+
isSuperseded,
|
|
359
|
+
now,
|
|
360
|
+
onProgress,
|
|
361
|
+
plan,
|
|
362
|
+
store,
|
|
363
|
+
yieldToHost,
|
|
364
|
+
});
|
|
365
|
+
if (!written) {
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
};
|
|
370
|
+
|
|
371
|
+
/** Fetches one page for each requested stream in a single round trip. */
|
|
372
|
+
type FetchSnapshotBatch = (request: SyncSnapshotBatchRequest) => Promise<SyncSnapshotBatchResponse>;
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* Page many streams through `POST /sync/snapshot/batch`: one round trip per round
|
|
376
|
+
* instead of one per stream per page. `limit` is the server's entity budget for the
|
|
377
|
+
* whole response, shared across the requested streams. Rounds are pipelined: the next
|
|
378
|
+
* request leaves as soon as a response arrives (its cursors come from the response,
|
|
379
|
+
* not the store), and the current round is written while it is in flight.
|
|
380
|
+
*
|
|
381
|
+
* Returns false when `isSuperseded` fired and the pass stopped early.
|
|
382
|
+
*/
|
|
383
|
+
export const bootstrapStreamsBatch = async ({
|
|
384
|
+
store,
|
|
385
|
+
fetchSnapshotBatch,
|
|
386
|
+
streams,
|
|
387
|
+
limit,
|
|
388
|
+
onProgress,
|
|
389
|
+
now,
|
|
390
|
+
isSuperseded,
|
|
391
|
+
applyChunkSize = DEFAULT_APPLY_CHUNK_SIZE,
|
|
392
|
+
yieldToHost = defaultYieldToHost,
|
|
393
|
+
}: {
|
|
394
|
+
store: SyncStore;
|
|
395
|
+
fetchSnapshotBatch: FetchSnapshotBatch;
|
|
396
|
+
streams: {stream: string; collection: string}[];
|
|
397
|
+
limit?: number;
|
|
398
|
+
onProgress?: (progress: BootstrapProgress) => void;
|
|
399
|
+
now?: () => string;
|
|
400
|
+
/** Checked after every response and every written page; true stops the pass. */
|
|
401
|
+
isSuperseded?: () => boolean;
|
|
402
|
+
applyChunkSize?: number;
|
|
403
|
+
yieldToHost?: () => Promise<void>;
|
|
404
|
+
}): Promise<boolean> => {
|
|
405
|
+
const states = new Map<string, StreamPagingState>();
|
|
406
|
+
for (const {stream, collection} of streams) {
|
|
407
|
+
states.set(stream, initialPagingState({collection, store, stream}));
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
const requestRound = (): Promise<SyncSnapshotBatchResponse> | undefined => {
|
|
411
|
+
const pending = [...states.values()]
|
|
412
|
+
.filter((state) => state.hasMore)
|
|
413
|
+
.slice(0, MAX_SNAPSHOT_BATCH_STREAMS);
|
|
414
|
+
if (pending.length === 0) {
|
|
415
|
+
return undefined;
|
|
416
|
+
}
|
|
417
|
+
return fetchSnapshotBatch({
|
|
418
|
+
limit,
|
|
419
|
+
streams: pending.map((state) => ({
|
|
420
|
+
cursor: state.cursor,
|
|
421
|
+
legacyCursor: state.legacyCursor,
|
|
422
|
+
stream: state.stream,
|
|
423
|
+
})),
|
|
424
|
+
}).then((response) => {
|
|
425
|
+
const answered = new Set(response.pages.map((page) => page.stream));
|
|
426
|
+
for (const state of pending) {
|
|
427
|
+
if (!answered.has(state.stream)) {
|
|
428
|
+
throw new Error(`Snapshot batch response is missing stream ${state.stream}`);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
return response;
|
|
432
|
+
});
|
|
433
|
+
};
|
|
434
|
+
|
|
435
|
+
let inFlight = requestRound();
|
|
436
|
+
try {
|
|
437
|
+
while (inFlight !== undefined) {
|
|
438
|
+
const {pages} = await inFlight;
|
|
439
|
+
inFlight = undefined;
|
|
440
|
+
if (isSuperseded?.()) {
|
|
441
|
+
return false;
|
|
442
|
+
}
|
|
443
|
+
const plans: PagePlan[] = [];
|
|
444
|
+
for (const page of pages) {
|
|
445
|
+
const state = states.get(page.stream);
|
|
446
|
+
if (state) {
|
|
447
|
+
plans.push(planSnapshotPage({page, state}));
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
inFlight = requestRound();
|
|
451
|
+
for (const plan of plans) {
|
|
452
|
+
const written = await writePagePlan({
|
|
453
|
+
chunkSize: applyChunkSize,
|
|
454
|
+
isSuperseded,
|
|
455
|
+
now,
|
|
456
|
+
onProgress,
|
|
457
|
+
plan,
|
|
458
|
+
store,
|
|
459
|
+
yieldToHost,
|
|
460
|
+
});
|
|
461
|
+
if (!written || isSuperseded?.()) {
|
|
462
|
+
return false;
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
return true;
|
|
467
|
+
} finally {
|
|
468
|
+
// An early exit abandons the prefetched round; never leave its rejection unhandled.
|
|
469
|
+
inFlight?.catch(() => {});
|
|
216
470
|
}
|
|
217
471
|
};
|
|
@@ -308,6 +308,73 @@ describe("createHttpChannel", () => {
|
|
|
308
308
|
});
|
|
309
309
|
});
|
|
310
310
|
|
|
311
|
+
describe("fetchSnapshotBatch", () => {
|
|
312
|
+
const batch = {
|
|
313
|
+
limit: 2000,
|
|
314
|
+
streams: [
|
|
315
|
+
{cursor: 0, stream: "todos|owner:u1"},
|
|
316
|
+
{cursor: 4, legacyCursor: "abc", stream: "notes|owner:u1"},
|
|
317
|
+
],
|
|
318
|
+
};
|
|
319
|
+
const page = {
|
|
320
|
+
cursor: 5,
|
|
321
|
+
entities: [],
|
|
322
|
+
frontierSeq: 5,
|
|
323
|
+
hasMore: false,
|
|
324
|
+
oldestRetainedSeq: 0,
|
|
325
|
+
stream: "todos|owner:u1",
|
|
326
|
+
};
|
|
327
|
+
|
|
328
|
+
it("POSTs the stream list with the bearer token and returns the pages", async () => {
|
|
329
|
+
const {fetchImpl, requests} = makeFetch(() => json({pages: [page]}));
|
|
330
|
+
const channel = createHttpChannel({authProvider, baseUrl: "http://api", fetchImpl});
|
|
331
|
+
await expect(channel.fetchSnapshotBatch?.(batch)).resolves.toEqual({
|
|
332
|
+
pages: [page],
|
|
333
|
+
type: "pages",
|
|
334
|
+
});
|
|
335
|
+
expect(requests[0]?.input).toBe("http://api/sync/snapshot/batch");
|
|
336
|
+
expect(requests[0]?.init?.method).toBe("POST");
|
|
337
|
+
expect(JSON.parse(String(requests[0]?.init?.body))).toEqual(batch);
|
|
338
|
+
expect(headersOf(requests[0]).Authorization).toBe("Bearer token-123");
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
it("maps a missing-route 404 to unsupported", async () => {
|
|
342
|
+
const {fetchImpl} = makeFetch(
|
|
343
|
+
() => new Response("Cannot POST /sync/snapshot/batch", {status: 404})
|
|
344
|
+
);
|
|
345
|
+
const channel = createHttpChannel({authProvider, baseUrl: "http://api", fetchImpl});
|
|
346
|
+
await expect(channel.fetchSnapshotBatch?.(batch)).resolves.toEqual({type: "unsupported"});
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
it("rejects an APIError 404 instead of reporting the route unsupported", async () => {
|
|
350
|
+
const {fetchImpl} = makeFetch(() =>
|
|
351
|
+
json({status: 404, title: "Unknown sync collection: notes"}, 404)
|
|
352
|
+
);
|
|
353
|
+
const channel = createHttpChannel({authProvider, baseUrl: "http://api", fetchImpl});
|
|
354
|
+
await expect(channel.fetchSnapshotBatch?.(batch)).rejects.toThrow(
|
|
355
|
+
"status 404: Unknown sync collection: notes"
|
|
356
|
+
);
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
it("rejects 401 with AuthRequiredError", async () => {
|
|
360
|
+
const {fetchImpl} = makeFetch(() => json({}, 401));
|
|
361
|
+
const channel = createHttpChannel({authProvider, baseUrl: "http://api", fetchImpl});
|
|
362
|
+
await expect(channel.fetchSnapshotBatch?.(batch)).rejects.toBeInstanceOf(AuthRequiredError);
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
it("rejects other error statuses", async () => {
|
|
366
|
+
const {fetchImpl} = makeFetch(() => json({}, 403));
|
|
367
|
+
const channel = createHttpChannel({authProvider, baseUrl: "http://api", fetchImpl});
|
|
368
|
+
await expect(channel.fetchSnapshotBatch?.(batch)).rejects.toThrow("status 403");
|
|
369
|
+
});
|
|
370
|
+
|
|
371
|
+
it("rejects when pages is missing", async () => {
|
|
372
|
+
const {fetchImpl} = makeFetch(() => json({}));
|
|
373
|
+
const channel = createHttpChannel({authProvider, baseUrl: "http://api", fetchImpl});
|
|
374
|
+
await expect(channel.fetchSnapshotBatch?.(batch)).rejects.toThrow("missing pages");
|
|
375
|
+
});
|
|
376
|
+
});
|
|
377
|
+
|
|
311
378
|
it("uses global fetch when no fetchImpl is provided (network error path)", async () => {
|
|
312
379
|
const channel = createHttpChannel({
|
|
313
380
|
authProvider,
|
package/src/sync/httpChannel.ts
CHANGED
|
@@ -7,6 +7,8 @@ import type {
|
|
|
7
7
|
SyncMutateBatchRequest,
|
|
8
8
|
SyncMutateRequest,
|
|
9
9
|
SyncNack,
|
|
10
|
+
SyncSnapshotBatchRequest,
|
|
11
|
+
SyncSnapshotBatchResponse,
|
|
10
12
|
SyncSnapshotResponse,
|
|
11
13
|
SyncStreamInfo,
|
|
12
14
|
} from "../types";
|
|
@@ -35,12 +37,23 @@ export interface FetchSnapshotPageArgs {
|
|
|
35
37
|
legacyCursor?: string;
|
|
36
38
|
}
|
|
37
39
|
|
|
40
|
+
/** Outcome of a snapshot batch request; `unsupported` means an older server (404). */
|
|
41
|
+
type FetchSnapshotBatchResult =
|
|
42
|
+
| ({type: "pages"} & SyncSnapshotBatchResponse)
|
|
43
|
+
| {type: "unsupported"};
|
|
44
|
+
|
|
38
45
|
/**
|
|
39
46
|
* HTTP side of the sync protocol: snapshot paging for bootstrap/reconcile and
|
|
40
47
|
* the `POST /sync/mutate` fallback used when the socket is unavailable.
|
|
41
48
|
*/
|
|
42
49
|
export interface HttpChannel {
|
|
43
50
|
fetchSnapshotPage: (args: FetchSnapshotPageArgs) => Promise<SyncSnapshotResponse>;
|
|
51
|
+
/**
|
|
52
|
+
* POST many streams' page requests to `/sync/snapshot/batch` in one round trip;
|
|
53
|
+
* resolves `{type: "unsupported"}` on 404 (older server with no batch route), 401
|
|
54
|
+
* rejects with {@link AuthRequiredError}, anything else non-2xx rejects.
|
|
55
|
+
*/
|
|
56
|
+
fetchSnapshotBatch?: (request: SyncSnapshotBatchRequest) => Promise<FetchSnapshotBatchResult>;
|
|
44
57
|
/**
|
|
45
58
|
* Point lookup for entity repair: fetch canonical server state for specific ids
|
|
46
59
|
* (`GET /sync/entities`). Used when deltas were skipped while an entity was
|
|
@@ -212,6 +225,16 @@ export interface HttpChannelConfig {
|
|
|
212
225
|
organizationIdProvider?: () => string | undefined;
|
|
213
226
|
}
|
|
214
227
|
|
|
228
|
+
/** The `title` of an @terreno/api APIError response body, or undefined for any other body. */
|
|
229
|
+
const readErrorTitle = async (response: Response): Promise<string | undefined> => {
|
|
230
|
+
try {
|
|
231
|
+
const body = (await response.json()) as {title?: unknown} | null;
|
|
232
|
+
return typeof body?.title === "string" ? body.title : undefined;
|
|
233
|
+
} catch {
|
|
234
|
+
return undefined;
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
|
|
215
238
|
/** Create an {@link HttpChannel} speaking @terreno/api's `/sync/*` routes. */
|
|
216
239
|
export const createHttpChannel = ({
|
|
217
240
|
baseUrl,
|
|
@@ -257,6 +280,32 @@ export const createHttpChannel = ({
|
|
|
257
280
|
return (await response.json()) as SyncSnapshotResponse;
|
|
258
281
|
};
|
|
259
282
|
|
|
283
|
+
const fetchSnapshotBatch = async (
|
|
284
|
+
batch: SyncSnapshotBatchRequest
|
|
285
|
+
): Promise<FetchSnapshotBatchResult> => {
|
|
286
|
+
const response = await request("/sync/snapshot/batch", {
|
|
287
|
+
body: JSON.stringify(batch),
|
|
288
|
+
method: "POST",
|
|
289
|
+
});
|
|
290
|
+
if (response.status === 404) {
|
|
291
|
+
// A missing route (older server) is a bare 404. An APIError 404 (e.g. an unknown
|
|
292
|
+
// collection) has a JSON `title` and is a real failure, not a reason to stop batching.
|
|
293
|
+
const title = await readErrorTitle(response);
|
|
294
|
+
if (title === undefined) {
|
|
295
|
+
return {type: "unsupported"};
|
|
296
|
+
}
|
|
297
|
+
throw new Error(`Snapshot batch request failed with status 404: ${title}`);
|
|
298
|
+
}
|
|
299
|
+
if (!response.ok) {
|
|
300
|
+
throw new Error(`Snapshot batch request failed with status ${response.status}`);
|
|
301
|
+
}
|
|
302
|
+
const body = (await response.json()) as Partial<SyncSnapshotBatchResponse>;
|
|
303
|
+
if (!Array.isArray(body.pages)) {
|
|
304
|
+
throw new Error("Snapshot batch response is missing pages");
|
|
305
|
+
}
|
|
306
|
+
return {pages: body.pages, type: "pages"};
|
|
307
|
+
};
|
|
308
|
+
|
|
260
309
|
const fetchEntities = async ({
|
|
261
310
|
collection,
|
|
262
311
|
ids,
|
|
@@ -359,6 +408,7 @@ export const createHttpChannel = ({
|
|
|
359
408
|
fetchEntities,
|
|
360
409
|
fetchKeyMaterial,
|
|
361
410
|
fetchList,
|
|
411
|
+
fetchSnapshotBatch,
|
|
362
412
|
fetchSnapshotPage,
|
|
363
413
|
fetchStreams,
|
|
364
414
|
sendMutation,
|
package/src/types.ts
CHANGED
|
@@ -179,6 +179,18 @@ export interface SyncSnapshotResponse {
|
|
|
179
179
|
legacyCursor?: string;
|
|
180
180
|
}
|
|
181
181
|
|
|
182
|
+
/** Body of `POST /sync/snapshot/batch`: one page request per stream. */
|
|
183
|
+
export interface SyncSnapshotBatchRequest {
|
|
184
|
+
streams: {stream: string; cursor: number; legacyCursor?: string}[];
|
|
185
|
+
/** Entity budget for the whole response, shared across `streams`; server default when omitted. */
|
|
186
|
+
limit?: number;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Response of `POST /sync/snapshot/batch`: one page per requested stream. */
|
|
190
|
+
export interface SyncSnapshotBatchResponse {
|
|
191
|
+
pages: SyncSnapshotResponse[];
|
|
192
|
+
}
|
|
193
|
+
|
|
182
194
|
/** Response shape of a `modelRouter` list endpoint (`GET /{collection}`), used by query windows. */
|
|
183
195
|
export interface ListResponse {
|
|
184
196
|
data: unknown[];
|