@terreno/syncdb 57.12.0 → 57.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,OAAO,EACL,YAAY,EACZ,6BAA6B,EAC7B,0CAA0C,EAC1C,iCAAiC,EACjC,iCAAiC,EACjC,8BAA8B,EAC9B,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,EAC1B,KAAK,UAAU,EACf,KAAK,MAAM,EACX,KAAK,YAAY,GAClB,MAAM,UAAU,CAAC;AAIlB,YAAY,EACV,YAAY,EACZ,0BAA0B,EAC1B,cAAc,EACd,YAAY,EACZ,OAAO,EACP,oBAAoB,EACpB,YAAY,EACZ,SAAS,EACT,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,EACrB,iBAAiB,EACjB,qBAAqB,EACrB,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,oBAAoB,EACpB,UAAU,EACV,cAAc,EACd,cAAc,GACf,MAAM,SAAS,CAAC;AAMjB,YAAY,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAClD,OAAO,EACL,wBAAwB,EACxB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,WAAW,GACjB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAC,aAAa,EAAE,QAAQ,EAAC,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAC,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,cAAc,EAAC,MAAM,eAAe,CAAC;AASlF,OAAO,EAAC,aAAa,EAAC,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAC,kBAAkB,EAAE,KAAK,MAAM,EAAC,MAAM,oBAAoB,CAAC;AACnE,YAAY,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAC,YAAY,EAAE,KAAK,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAI7C,OAAO,EACL,KAAK,wBAAwB,EAC7B,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,KAAK,yBAAyB,EAC9B,2BAA2B,GAC5B,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EACzB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AAItB,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,qBAAqB,EACrB,2BAA2B,GAC5B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAC,WAAW,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAI9D,OAAO,EAAC,6BAA6B,EAAC,MAAM,sCAAsC,CAAC;AACnF,OAAO,EAAC,iCAAiC,EAAC,MAAM,0CAA0C,CAAC;AAC3F,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAC,6BAA6B,EAAE,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AAIxF,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,KAAK,SAAS,EACd,KAAK,qBAAqB,EAC1B,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,gBAAgB,EAChB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAC,qBAAqB,EAAE,KAAK,qBAAqB,EAAC,MAAM,wBAAwB,CAAC;AACzF,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,eAAe,GACrB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAC,YAAY,EAAC,MAAM,SAAS,CAAC;AAM1C,YAAY,EACV,kBAAkB,EAClB,cAAc,EACd,kBAAkB,EAClB,YAAY,EACZ,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,cAAc,GACf,MAAM,kBAAkB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,OAAO,EACL,YAAY,EACZ,6BAA6B,EAC7B,0CAA0C,EAC1C,4BAA4B,EAC5B,iCAAiC,EACjC,iCAAiC,EACjC,8BAA8B,EAC9B,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,EAC1B,KAAK,UAAU,EACf,KAAK,MAAM,EACX,KAAK,YAAY,GAClB,MAAM,UAAU,CAAC;AAIlB,YAAY,EACV,YAAY,EACZ,0BAA0B,EAC1B,cAAc,EACd,YAAY,EACZ,OAAO,EACP,oBAAoB,EACpB,YAAY,EACZ,SAAS,EACT,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,EACrB,iBAAiB,EACjB,qBAAqB,EACrB,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,oBAAoB,EACpB,UAAU,EACV,cAAc,EACd,cAAc,GACf,MAAM,SAAS,CAAC;AAMjB,YAAY,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAClD,OAAO,EACL,wBAAwB,EACxB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,WAAW,GACjB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAC,aAAa,EAAE,QAAQ,EAAC,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAC,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,cAAc,EAAC,MAAM,eAAe,CAAC;AASlF,OAAO,EAAC,aAAa,EAAC,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAC,kBAAkB,EAAE,KAAK,MAAM,EAAC,MAAM,oBAAoB,CAAC;AACnE,YAAY,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAC,YAAY,EAAE,KAAK,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAI7C,OAAO,EACL,KAAK,wBAAwB,EAC7B,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,KAAK,yBAAyB,EAC9B,2BAA2B,GAC5B,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EACzB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AAItB,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,qBAAqB,EACrB,2BAA2B,GAC5B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAC,WAAW,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAI9D,OAAO,EAAC,6BAA6B,EAAC,MAAM,sCAAsC,CAAC;AACnF,OAAO,EAAC,iCAAiC,EAAC,MAAM,0CAA0C,CAAC;AAC3F,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAC,6BAA6B,EAAE,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AAIxF,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,KAAK,SAAS,EACd,KAAK,qBAAqB,EAC1B,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,gBAAgB,EAChB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAC,qBAAqB,EAAE,KAAK,qBAAqB,EAAC,MAAM,wBAAwB,CAAC;AACzF,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,eAAe,GACrB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAC,YAAY,EAAC,MAAM,SAAS,CAAC;AAM1C,YAAY,EACV,kBAAkB,EAClB,cAAc,EACd,kBAAkB,EAClB,YAAY,EACZ,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,cAAc,GACf,MAAM,kBAAkB,CAAC"}
package/dist/index.js CHANGED
@@ -23,7 +23,7 @@
23
23
  * `createFakeTransport`, kept out of production bundles.
24
24
  */
25
25
  // --- Client ---------------------------------------------------------------
26
- export { createSyncDb, DEFAULT_RECONCILE_INTERVAL_MS, DEFAULT_SEQ_JUMP_RECONCILE_MIN_INTERVAL_MS, DEFAULT_START_AUTH_RETRY_ATTEMPTS, DEFAULT_START_AUTH_RETRY_DELAY_MS, DEFAULT_TOMBSTONE_RETENTION_MS, } from "./client";
26
+ export { createSyncDb, DEFAULT_RECONCILE_INTERVAL_MS, DEFAULT_SEQ_JUMP_RECONCILE_MIN_INTERVAL_MS, DEFAULT_SNAPSHOT_CONCURRENCY, DEFAULT_START_AUTH_RETRY_ATTEMPTS, DEFAULT_START_AUTH_RETRY_DELAY_MS, DEFAULT_TOMBSTONE_RETENTION_MS, } from "./client";
27
27
  export { DEFAULT_WINDOW_PAGE_SIZE, } from "./query/queryWindows";
28
28
  export { compileWhere } from "./query/where";
29
29
  // --- Local store and outbox (read surface) -------------------------------
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,6EAA6E;AAE7E,OAAO,EACL,YAAY,EACZ,6BAA6B,EAC7B,0CAA0C,EAC1C,iCAAiC,EACjC,iCAAiC,EACjC,8BAA8B,GAM/B,MAAM,UAAU,CAAC;AAiClB,OAAO,EACL,wBAAwB,GAKzB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,YAAY,EAAwC,MAAM,eAAe,CAAC;AAElF,4EAA4E;AAC5E,sEAAsE;AACtE,gFAAgF;AAChF,8EAA8E;AAC9E,kFAAkF;AAClF,gDAAgD;AAEhD,OAAO,EAAC,aAAa,EAAC,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAC,kBAAkB,EAAc,MAAM,oBAAoB,CAAC;AAEnE,OAAO,EAAC,YAAY,EAAkB,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAE7C,6EAA6E;AAE7E,OAAO,EAEL,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,2BAA2B,GAC5B,MAAM,8BAA8B,CAAC;AAUtC,4EAA4E;AAE5E,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,qBAAqB,EACrB,2BAA2B,GAC5B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,uBAAuB,CAAC;AAG/B,6EAA6E;AAE7E,OAAO,EAAC,6BAA6B,EAAC,MAAM,sCAAsC,CAAC;AACnF,OAAO,EAAC,iCAAiC,EAAC,MAAM,0CAA0C,CAAC;AAC3F,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,8BAA8B,CAAC;AAGtC,4EAA4E;AAE5E,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EAKjB,gBAAgB,EAChB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAC,qBAAqB,EAA6B,MAAM,wBAAwB,CAAC;AACzF,OAAO,EACL,kBAAkB,EAClB,2BAA2B,GAK5B,MAAM,kBAAkB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,6EAA6E;AAE7E,OAAO,EACL,YAAY,EACZ,6BAA6B,EAC7B,0CAA0C,EAC1C,4BAA4B,EAC5B,iCAAiC,EACjC,iCAAiC,EACjC,8BAA8B,GAM/B,MAAM,UAAU,CAAC;AAiClB,OAAO,EACL,wBAAwB,GAKzB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,YAAY,EAAwC,MAAM,eAAe,CAAC;AAElF,4EAA4E;AAC5E,sEAAsE;AACtE,gFAAgF;AAChF,8EAA8E;AAC9E,kFAAkF;AAClF,gDAAgD;AAEhD,OAAO,EAAC,aAAa,EAAC,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAC,kBAAkB,EAAc,MAAM,oBAAoB,CAAC;AAEnE,OAAO,EAAC,YAAY,EAAkB,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAE7C,6EAA6E;AAE7E,OAAO,EAEL,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,2BAA2B,GAC5B,MAAM,8BAA8B,CAAC;AAUtC,4EAA4E;AAE5E,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,qBAAqB,EACrB,2BAA2B,GAC5B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,uBAAuB,CAAC;AAG/B,6EAA6E;AAE7E,OAAO,EAAC,6BAA6B,EAAC,MAAM,sCAAsC,CAAC;AACnF,OAAO,EAAC,iCAAiC,EAAC,MAAM,0CAA0C,CAAC;AAC3F,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,8BAA8B,CAAC;AAGtC,4EAA4E;AAE5E,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EAKjB,gBAAgB,EAChB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAC,qBAAqB,EAA6B,MAAM,wBAAwB,CAAC;AACzF,OAAO,EACL,kBAAkB,EAClB,2BAA2B,GAK5B,MAAM,kBAAkB,CAAC"}
package/package.json CHANGED
@@ -92,5 +92,5 @@
92
92
  "test:coverage": "bun run ../scripts/check-coverage.ts"
93
93
  },
94
94
  "types": "dist/index.d.ts",
95
- "version": "57.12.0"
95
+ "version": "57.14.0"
96
96
  }
@@ -3305,3 +3305,206 @@ describe("createSyncDb", () => {
3305
3305
  });
3306
3306
  });
3307
3307
  });
3308
+
3309
+ describe("snapshot paging concurrency", () => {
3310
+ /**
3311
+ * Wrap the harness channel so every snapshot page waits on a gate the test releases,
3312
+ * and record how many pages are in flight overall and per stream.
3313
+ */
3314
+ const trackSnapshotPages = (
3315
+ harness: Harness
3316
+ ): {
3317
+ fetched: string[];
3318
+ maxInFlight: () => number;
3319
+ maxInFlightPerStream: () => number;
3320
+ pending: () => number;
3321
+ releaseAll: () => void;
3322
+ } => {
3323
+ const fetched: string[] = [];
3324
+ const inFlight = new Map<string, number>();
3325
+ const gates: (() => void)[] = [];
3326
+ let total = 0;
3327
+ let maxTotal = 0;
3328
+ let maxPerStream = 0;
3329
+ const fetchSnapshotPage = harness.http.channel.fetchSnapshotPage;
3330
+ harness.http.channel.fetchSnapshotPage = async (args) => {
3331
+ fetched.push(args.stream);
3332
+ total += 1;
3333
+ maxTotal = Math.max(maxTotal, total);
3334
+ const streamCount = (inFlight.get(args.stream) ?? 0) + 1;
3335
+ inFlight.set(args.stream, streamCount);
3336
+ maxPerStream = Math.max(maxPerStream, streamCount);
3337
+ await new Promise<void>((resolve) => gates.push(resolve));
3338
+ total -= 1;
3339
+ inFlight.set(args.stream, (inFlight.get(args.stream) ?? 1) - 1);
3340
+ return fetchSnapshotPage(args);
3341
+ };
3342
+ return {
3343
+ fetched,
3344
+ maxInFlight: () => maxTotal,
3345
+ maxInFlightPerStream: () => maxPerStream,
3346
+ pending: () => gates.length,
3347
+ releaseAll: () => {
3348
+ for (const release of gates.splice(0)) {
3349
+ release();
3350
+ }
3351
+ },
3352
+ };
3353
+ };
3354
+
3355
+ /** Release gated pages until `done` settles. */
3356
+ const drain = async <T>(
3357
+ pages: ReturnType<typeof trackSnapshotPages>,
3358
+ done: Promise<T>
3359
+ ): Promise<T> => {
3360
+ let settled = false;
3361
+ const result = done.finally(() => {
3362
+ settled = true;
3363
+ });
3364
+ for (let attempt = 0; attempt < 100 && !settled; attempt += 1) {
3365
+ pages.releaseAll();
3366
+ await flush();
3367
+ }
3368
+ return result;
3369
+ };
3370
+
3371
+ const manyStreams = (count: number): SyncStreamInfo[] =>
3372
+ Array.from({length: count}, (_, index) => ({
3373
+ collection: "todos",
3374
+ stream: `todos|tenant:w${index}`,
3375
+ }));
3376
+
3377
+ const isBootstrapped = (client: ReturnType<typeof createSyncDb>, stream: string): boolean =>
3378
+ client.store.raw.getCell(CURSORS_TABLE, stream, "bootstrapped") === true;
3379
+
3380
+ it("pages streams side by side, up to snapshotConcurrency at once", async () => {
3381
+ const harness = makeHarness({snapshotConcurrency: 3});
3382
+ harness.http.state.streams = manyStreams(10);
3383
+ const pages = trackSnapshotPages(harness);
3384
+ const client = createSyncDb(harness.config);
3385
+ await client.start();
3386
+
3387
+ await waitUntil(() => pages.pending() === 3);
3388
+ await flush();
3389
+ expect(pages.pending()).toBe(3);
3390
+ await drain(pages, client.reconcile());
3391
+
3392
+ expect(pages.maxInFlight()).toBe(3);
3393
+ for (const {stream} of harness.http.state.streams) {
3394
+ expect(isBootstrapped(client, stream)).toBe(true);
3395
+ }
3396
+ await client.stop();
3397
+ });
3398
+
3399
+ it("pages one stream at a time when snapshotConcurrency is 1", async () => {
3400
+ const harness = makeHarness({snapshotConcurrency: 1});
3401
+ harness.http.state.streams = manyStreams(4);
3402
+ const pages = trackSnapshotPages(harness);
3403
+ const client = createSyncDb(harness.config);
3404
+ await client.start();
3405
+ await drain(pages, client.reconcile());
3406
+
3407
+ expect(pages.maxInFlight()).toBe(1);
3408
+ expect(pages.fetched).toHaveLength(4);
3409
+ await client.stop();
3410
+ });
3411
+
3412
+ it("queues a subscribe catch-up behind an in-flight pass on the same stream", async () => {
3413
+ const harness = makeHarness();
3414
+ const pages = trackSnapshotPages(harness);
3415
+ const client = createSyncDb(harness.config);
3416
+ await client.start();
3417
+ await waitUntil(() => pages.pending() === 1);
3418
+
3419
+ harness.transport.confirmSubscribed({collection: "todos", streams: [DEFAULT_STREAM]});
3420
+ harness.transport.confirmSubscribed({collection: "todos", streams: [DEFAULT_STREAM]});
3421
+ await flush();
3422
+ // The catch-up must not page beside the reconcile pass that began before the join.
3423
+ expect(pages.pending()).toBe(1);
3424
+ await drain(pages, client.reconcile());
3425
+ await waitUntil(() => pages.fetched.length === 2);
3426
+ pages.releaseAll();
3427
+ await flush();
3428
+
3429
+ // One startup pass, then one fresh pass shared by both confirmations.
3430
+ expect(pages.fetched).toEqual([DEFAULT_STREAM, DEFAULT_STREAM]);
3431
+ expect(pages.maxInFlightPerStream()).toBe(1);
3432
+ await client.stop();
3433
+ });
3434
+
3435
+ it("drops a queued catch-up when the client stops before it starts", async () => {
3436
+ const harness = makeHarness();
3437
+ const pages = trackSnapshotPages(harness);
3438
+ const client = createSyncDb(harness.config);
3439
+ await client.start();
3440
+ await waitUntil(() => pages.pending() === 1);
3441
+ harness.transport.confirmSubscribed({collection: "todos", streams: [DEFAULT_STREAM]});
3442
+ await flush();
3443
+
3444
+ const stopped = client.stop();
3445
+ for (let attempt = 0; attempt < 10; attempt += 1) {
3446
+ pages.releaseAll();
3447
+ await flush();
3448
+ }
3449
+ await stopped;
3450
+
3451
+ // Only the startup pass paged; the catch-up queued behind it belonged to the
3452
+ // stopped lifecycle.
3453
+ expect(pages.fetched).toEqual([DEFAULT_STREAM]);
3454
+ });
3455
+
3456
+ it("forceResync waits for an in-flight pass before purging the stream", async () => {
3457
+ const harness = makeHarness();
3458
+ harness.http.state.pages[DEFAULT_STREAM] = {
3459
+ cursor: 3,
3460
+ entities: [{data: {title: "kept"}, deleted: false, id: "t1", seq: 3}],
3461
+ frontierSeq: 3,
3462
+ hasMore: false,
3463
+ oldestRetainedSeq: 0,
3464
+ stream: DEFAULT_STREAM,
3465
+ };
3466
+ const pages = trackSnapshotPages(harness);
3467
+ const client = createSyncDb(harness.config);
3468
+ await client.start();
3469
+ await waitUntil(() => pages.pending() === 1);
3470
+
3471
+ const resync = client.forceResync();
3472
+ await flush();
3473
+ await flush();
3474
+ // The purge-and-repage must not start while the startup pass is still paging.
3475
+ expect(pages.pending()).toBe(1);
3476
+ const result = await drain(pages, resync);
3477
+
3478
+ assert.equal(result.ok, true);
3479
+ expect(pages.maxInFlightPerStream()).toBe(1);
3480
+ expect(client.store.getEntity({collection: "todos", id: "t1"})?.data).toEqual({
3481
+ title: "kept",
3482
+ });
3483
+ expect(isBootstrapped(client, DEFAULT_STREAM)).toBe(true);
3484
+ await client.stop();
3485
+ });
3486
+
3487
+ it("stops taking new streams after one fails", async () => {
3488
+ const harness = makeHarness({snapshotConcurrency: 1});
3489
+ harness.http.state.streams = manyStreams(3);
3490
+ const failing = "todos|tenant:w1";
3491
+ const fetched: string[] = [];
3492
+ const fetchSnapshotPage = harness.http.channel.fetchSnapshotPage;
3493
+ harness.http.channel.fetchSnapshotPage = async (args) => {
3494
+ fetched.push(args.stream);
3495
+ if (args.stream === failing) {
3496
+ throw new Error("snapshot failed");
3497
+ }
3498
+ return fetchSnapshotPage(args);
3499
+ };
3500
+ const client = createSyncDb(harness.config);
3501
+
3502
+ // start() fires a reconcile of its own; this call shares that pass.
3503
+ await client.start();
3504
+ await expect(client.reconcile()).rejects.toThrow("snapshot failed");
3505
+
3506
+ expect(fetched).toEqual(["todos|tenant:w0", failing]);
3507
+ expect(isBootstrapped(client, "todos|tenant:w2")).toBe(false);
3508
+ await client.stop();
3509
+ });
3510
+ });
package/src/client.ts CHANGED
@@ -43,6 +43,13 @@ export const DEFAULT_RECONCILE_INTERVAL_MS = 5 * 60_000;
43
43
  /** Minimum interval between seq-jump-triggered reconciles per stream (30s). */
44
44
  export const DEFAULT_SEQ_JUMP_RECONCILE_MIN_INTERVAL_MS = 30_000;
45
45
 
46
+ /**
47
+ * Streams snapshot-paged at once by reconcile, subscribe catch-up, force resync and
48
+ * wipe re-bootstrap. Every stream costs at least one `GET /sync/snapshot` round trip,
49
+ * so paging them one at a time made cold start scale with collections × scopes.
50
+ */
51
+ export const DEFAULT_SNAPSHOT_CONCURRENCY = 6;
52
+
46
53
  /** E5: default local tombstone retention window (90 days), matching the server's (C7). */
47
54
  export const DEFAULT_TOMBSTONE_RETENTION_MS = 90 * 24 * 60 * 60 * 1_000;
48
55
 
@@ -136,6 +143,12 @@ export interface SyncDbConfig {
136
143
  wipeOnSignOut?: boolean;
137
144
  /** Max mutations per batched drain send (B3, default 50; server caps at 100). */
138
145
  batchSize?: number;
146
+ /**
147
+ * Max streams snapshot-paged at once (default {@link DEFAULT_SNAPSHOT_CONCURRENCY}).
148
+ * Each stream still pages its own pages in order; this bounds how many streams page
149
+ * side by side. `1` restores one-stream-at-a-time paging.
150
+ */
151
+ snapshotConcurrency?: number;
139
152
  /**
140
153
  * B4: when true, a conflict halts the ENTIRE drain instead of just blocking
141
154
  * its entity. Default false (per-entity blocking — see README "Conflict
@@ -427,6 +440,17 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
427
440
  | {generation: number; promise: Promise<void>; userId: string | undefined}
428
441
  | undefined;
429
442
  const lastSeqJumpReconcileAt = new Map<string, number>();
443
+ /**
444
+ * The latest snapshot pass per stream (single-flight). Two passes paging the same
445
+ * stream at once double the requests, and a purge in one (force resync, retention
446
+ * gap, leave) lets the other write its high cursor back and mark the stream
447
+ * bootstrapped over rows it never paged. Tagged with generation + user like
448
+ * `inFlightReconcile`, so a pass from a superseded lifecycle is never joined.
449
+ */
450
+ const streamPasses = new Map<
451
+ string,
452
+ {generation: number; promise: Promise<void>; started: boolean; userId: string | undefined}
453
+ >();
430
454
  const statusListeners = new Set<() => void>();
431
455
  // A4: auth-pause state. `authPaused` gates reconcile/replay/timer triggers
432
456
  // (INV-2 — nothing else may unpause except a same-user auth change).
@@ -487,6 +511,105 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
487
511
  }
488
512
  };
489
513
 
514
+ /**
515
+ * Run a snapshot pass for one stream, single-flight per stream.
516
+ *
517
+ * - `join`: share the stream's latest pass when one exists (reconcile — any pass to
518
+ * the head will do).
519
+ * - `fresh`: the pass must start after this call (subscribe catch-up — a pass that
520
+ * began before the room join may have missed a write). Joins a queued pass that has
521
+ * not started yet, otherwise queues behind the running one.
522
+ * - `exclusive`: always queue behind whatever is in flight (force resync and wipe
523
+ * re-bootstrap purge first, which must never overlap another pass's paging).
524
+ */
525
+ const runStreamPass = ({
526
+ mode,
527
+ run,
528
+ stream,
529
+ }: {
530
+ mode: "join" | "fresh" | "exclusive";
531
+ run: () => Promise<void>;
532
+ stream: string;
533
+ }): Promise<void> => {
534
+ const existing = streamPasses.get(stream);
535
+ const current =
536
+ existing && existing.generation === generation && existing.userId === currentUserId
537
+ ? existing
538
+ : undefined;
539
+ if (current && (mode === "join" || (mode === "fresh" && !current.started))) {
540
+ return current.promise;
541
+ }
542
+ const entry = {generation, promise: Promise.resolve(), started: false, userId: currentUserId};
543
+ const previous = existing?.promise;
544
+ entry.promise = (async (): Promise<void> => {
545
+ if (previous) {
546
+ // The previous pass's failure belongs to its own caller.
547
+ await previous.catch(() => undefined);
548
+ }
549
+ // A stop()/start() or user switch while this pass was queued supersedes it:
550
+ // paging now would write into a store that belongs to a newer lifecycle.
551
+ if (entry.generation !== generation || entry.userId !== currentUserId) {
552
+ return;
553
+ }
554
+ entry.started = true;
555
+ await run();
556
+ })().finally(() => {
557
+ if (streamPasses.get(stream) === entry) {
558
+ streamPasses.delete(stream);
559
+ }
560
+ });
561
+ streamPasses.set(stream, entry);
562
+ return entry.promise;
563
+ };
564
+
565
+ /**
566
+ * Run `worker` over `items` with at most `snapshotConcurrency` in flight. Stops taking
567
+ * new items once `isSuperseded()` or any worker throws, waits for the in-flight ones,
568
+ * then rethrows — `AuthRequiredError` first, so callers still auth-pause on a 401 that
569
+ * raced another failure.
570
+ */
571
+ const runSnapshotPool = async <T>({
572
+ isSuperseded,
573
+ items,
574
+ worker,
575
+ }: {
576
+ isSuperseded: () => boolean;
577
+ items: T[];
578
+ worker: (item: T) => Promise<void>;
579
+ }): Promise<void> => {
580
+ const limit = Math.max(
581
+ 1,
582
+ Math.floor(config.snapshotConcurrency ?? DEFAULT_SNAPSHOT_CONCURRENCY)
583
+ );
584
+ let next = 0;
585
+ let authError: AuthRequiredError | undefined;
586
+ let firstError: unknown;
587
+ let failed = false;
588
+ const runWorker = async (): Promise<void> => {
589
+ while (next < items.length && !failed && !isSuperseded()) {
590
+ const item = items[next] as T;
591
+ next += 1;
592
+ try {
593
+ await worker(item);
594
+ } catch (error) {
595
+ failed = true;
596
+ if (error instanceof AuthRequiredError) {
597
+ authError ??= error;
598
+ } else if (firstError === undefined) {
599
+ firstError = error;
600
+ }
601
+ }
602
+ }
603
+ };
604
+ await Promise.all(Array.from({length: Math.min(limit, items.length)}, runWorker));
605
+ if (authError) {
606
+ throw authError;
607
+ }
608
+ if (failed) {
609
+ throw firstError;
610
+ }
611
+ };
612
+
490
613
  const notifyStatusChange = (): void => {
491
614
  for (const listener of statusListeners) {
492
615
  listener();
@@ -759,14 +882,21 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
759
882
  // lifecycle/user while awaiting discovery.
760
883
  return;
761
884
  }
762
- for (const [stream, collection] of streams) {
763
- if (skipSnapshotPaging({collection, stream})) {
764
- continue;
765
- }
766
- await bootstrapStream({channel: httpChannel, collection, store, stream});
767
- if (isSuperseded()) {
768
- return;
769
- }
885
+ const channel = httpChannel;
886
+ await runSnapshotPool({
887
+ isSuperseded,
888
+ items: [...streams].filter(
889
+ ([stream, collection]) => !skipSnapshotPaging({collection, stream})
890
+ ),
891
+ worker: ([stream, collection]) =>
892
+ runStreamPass({
893
+ mode: "join",
894
+ run: () => bootstrapStream({channel, collection, store, stream}),
895
+ stream,
896
+ }),
897
+ });
898
+ if (isSuperseded()) {
899
+ return;
770
900
  }
771
901
  // E5: client-side compaction runs only after a successful per-stream reconcile —
772
902
  // reconcile just proved the local store caught up with the server via real network
@@ -874,15 +1004,21 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
874
1004
  const isSuperseded = (): boolean => generation !== myGeneration || currentUserId !== myUserId;
875
1005
  addSyncing(1);
876
1006
  try {
877
- for (const stream of streams) {
878
- if (isSuperseded()) {
879
- return;
880
- }
881
- if (skipSnapshotPaging({collection, mode, stream})) {
882
- continue;
883
- }
884
- store.addKnownStream({collection, stream});
885
- await bootstrapStream({channel: httpChannel, collection, store, stream});
1007
+ const channel = httpChannel;
1008
+ await runSnapshotPool({
1009
+ isSuperseded,
1010
+ items: streams.filter((stream) => !skipSnapshotPaging({collection, mode, stream})),
1011
+ worker: (stream) => {
1012
+ store.addKnownStream({collection, stream});
1013
+ return runStreamPass({
1014
+ mode: "fresh",
1015
+ run: () => bootstrapStream({channel, collection, store, stream}),
1016
+ stream,
1017
+ });
1018
+ },
1019
+ });
1020
+ if (isSuperseded()) {
1021
+ return;
886
1022
  }
887
1023
  debugLog?.record({
888
1024
  detail: {collection, streams},
@@ -1017,19 +1153,30 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
1017
1153
  for (const collection of new Set(streamInfos.map((info) => info.collection))) {
1018
1154
  purged += store.purgeUnknownStreamEntities({collection});
1019
1155
  }
1020
- for (const {stream, collection} of streamInfos) {
1021
- if (isQueryCollection(collection)) {
1022
- // Query collections have no snapshot to rebuild from; their retained
1023
- // windows are refetched below instead.
1024
- continue;
1025
- }
1026
- purged += store.purgeStream({stream});
1027
- store.addKnownStream({collection, stream});
1028
- await bootstrapStream({channel: httpChannel, collection, store, stream});
1029
- await waitForLifecycleIdle();
1030
- if (isSuperseded()) {
1031
- return {ok: false, purged, reason: "superseded", repaired, streams: streamInfos.length};
1032
- }
1156
+ const channel = httpChannel;
1157
+ await runSnapshotPool({
1158
+ isSuperseded,
1159
+ // Query collections have no snapshot to rebuild from; their retained windows
1160
+ // are refetched below instead.
1161
+ items: streamInfos.filter(({collection}) => !isQueryCollection(collection)),
1162
+ worker: async ({stream, collection}) => {
1163
+ await runStreamPass({
1164
+ mode: "exclusive",
1165
+ run: async () => {
1166
+ if (isSuperseded()) {
1167
+ return;
1168
+ }
1169
+ purged += store.purgeStream({stream});
1170
+ store.addKnownStream({collection, stream});
1171
+ await bootstrapStream({channel, collection, store, stream});
1172
+ },
1173
+ stream,
1174
+ });
1175
+ await waitForLifecycleIdle();
1176
+ },
1177
+ });
1178
+ if (isSuperseded()) {
1179
+ return {ok: false, purged, reason: "superseded", repaired, streams: streamInfos.length};
1033
1180
  }
1034
1181
  repaired = await repairAllMarkedEntities();
1035
1182
  await queryWindows.refetchRetained();
@@ -1278,9 +1425,24 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
1278
1425
  // INV-2 auth-pause case (returns undefined on a 401) — skip bootstrap then.
1279
1426
  const streams = await syncStreams();
1280
1427
  if (streams) {
1281
- for (const [stream, collection] of streams) {
1282
- await bootstrapStream({channel: httpChannel, collection, store, stream});
1283
- }
1428
+ const channel = httpChannel;
1429
+ await runSnapshotPool({
1430
+ isSuperseded: () => false,
1431
+ items: [...streams],
1432
+ worker: ([stream, collection]) =>
1433
+ runStreamPass({
1434
+ mode: "exclusive",
1435
+ // A pass that was paging when the wipe ran can write its cursor and
1436
+ // `bootstrapped` mark into the wiped store before this one starts. Purge
1437
+ // again so this pass pages from 0 instead of resuming from that cursor.
1438
+ run: async () => {
1439
+ store.purgeStream({stream});
1440
+ store.addKnownStream({collection, stream});
1441
+ await bootstrapStream({channel, collection, store, stream});
1442
+ },
1443
+ stream,
1444
+ }),
1445
+ });
1284
1446
  }
1285
1447
  }
1286
1448
  };
package/src/index.ts CHANGED
@@ -29,6 +29,7 @@ export {
29
29
  createSyncDb,
30
30
  DEFAULT_RECONCILE_INTERVAL_MS,
31
31
  DEFAULT_SEQ_JUMP_RECONCILE_MIN_INTERVAL_MS,
32
+ DEFAULT_SNAPSHOT_CONCURRENCY,
32
33
  DEFAULT_START_AUTH_RETRY_ATTEMPTS,
33
34
  DEFAULT_START_AUTH_RETRY_DELAY_MS,
34
35
  DEFAULT_TOMBSTONE_RETENTION_MS,