@terreno/syncdb 57.13.0 → 57.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +6 -3
  2. package/dist/client.d.ts +12 -0
  3. package/dist/client.d.ts.map +1 -1
  4. package/dist/client.js +304 -56
  5. package/dist/client.js.map +1 -1
  6. package/dist/client.test.js +255 -0
  7. package/dist/client.test.js.map +1 -1
  8. package/dist/index.d.ts +1 -1
  9. package/dist/index.d.ts.map +1 -1
  10. package/dist/index.js +1 -1
  11. package/dist/index.js.map +1 -1
  12. package/dist/persisters/encryptedIndexedDbPersister.d.ts +4 -3
  13. package/dist/persisters/encryptedIndexedDbPersister.d.ts.map +1 -1
  14. package/dist/persisters/encryptedIndexedDbPersister.js +21 -16
  15. package/dist/persisters/encryptedIndexedDbPersister.js.map +1 -1
  16. package/dist/persisters/encryptedIndexedDbPersister.test.js +40 -1
  17. package/dist/persisters/encryptedIndexedDbPersister.test.js.map +1 -1
  18. package/dist/sync/bootstrap.d.ts +34 -3
  19. package/dist/sync/bootstrap.d.ts.map +1 -1
  20. package/dist/sync/bootstrap.js +224 -71
  21. package/dist/sync/bootstrap.js.map +1 -1
  22. package/dist/sync/bootstrap.test.js +210 -2
  23. package/dist/sync/bootstrap.test.js.map +1 -1
  24. package/dist/sync/httpChannel.d.ts +14 -1
  25. package/dist/sync/httpChannel.d.ts.map +1 -1
  26. package/dist/sync/httpChannel.js +34 -0
  27. package/dist/sync/httpChannel.js.map +1 -1
  28. package/dist/sync/httpChannel.test.js +54 -0
  29. package/dist/sync/httpChannel.test.js.map +1 -1
  30. package/dist/types.d.ts +14 -0
  31. package/dist/types.d.ts.map +1 -1
  32. package/package.json +1 -1
  33. package/src/client.test.ts +314 -1
  34. package/src/client.ts +362 -54
  35. package/src/index.ts +1 -0
  36. package/src/persisters/encryptedIndexedDbPersister.test.ts +44 -1
  37. package/src/persisters/encryptedIndexedDbPersister.ts +24 -17
  38. package/src/sync/bootstrap.test.ts +269 -3
  39. package/src/sync/bootstrap.ts +336 -82
  40. package/src/sync/httpChannel.test.ts +67 -0
  41. package/src/sync/httpChannel.ts +50 -0
  42. package/src/types.ts +12 -0
@@ -15,7 +15,13 @@ import {CURSORS_TABLE} from "./storage/types";
15
15
  import {AuthRequiredError, type HttpChannel} from "./sync/httpChannel";
16
16
  import type {SyncTransport} from "./sync/transport";
17
17
  import {createFakeTransport, type FakeTransport} from "./testing/fakeTransport";
18
- import type {AuthProvider, SyncDelta, SyncSnapshotResponse, SyncStreamInfo} from "./types";
18
+ import type {
19
+ AuthProvider,
20
+ SyncDelta,
21
+ SyncSnapshotBatchRequest,
22
+ SyncSnapshotResponse,
23
+ SyncStreamInfo,
24
+ } from "./types";
19
25
 
20
26
  let nameCounter = 0;
21
27
  const uniqueName = (): string => {
@@ -580,6 +586,110 @@ describe("createSyncDb", () => {
580
586
  });
581
587
  });
582
588
 
589
+ describe("snapshot batch", () => {
590
+ const NOTES_STREAM = "notes|owner:u1";
591
+
592
+ /** Harness whose channel also serves `POST /sync/snapshot/batch` from `state.pages`. */
593
+ const makeBatchHarness = (): Harness & {
594
+ batchRequests: SyncSnapshotBatchRequest[];
595
+ setBatchUnsupported: (unsupported: boolean) => void;
596
+ gateNextBatch: () => () => void;
597
+ } => {
598
+ const harness = makeHarness({collections: ["todos", "notes"]});
599
+ harness.http.state.streams = [
600
+ {collection: "todos", stream: DEFAULT_STREAM},
601
+ {collection: "notes", stream: NOTES_STREAM},
602
+ ];
603
+ const batchRequests: SyncSnapshotBatchRequest[] = [];
604
+ let isUnsupported = false;
605
+ let gate: Promise<void> | undefined;
606
+ harness.http.channel.fetchSnapshotBatch = async (request) => {
607
+ batchRequests.push(request);
608
+ const pending = gate;
609
+ gate = undefined;
610
+ await pending;
611
+ if (isUnsupported) {
612
+ return {type: "unsupported"};
613
+ }
614
+ return {
615
+ pages: request.streams.map(
616
+ ({stream}) => harness.http.state.pages[stream] ?? emptyPage(stream)
617
+ ),
618
+ type: "pages",
619
+ };
620
+ };
621
+ return {
622
+ ...harness,
623
+ batchRequests,
624
+ gateNextBatch: (): (() => void) => {
625
+ let release = (): void => {};
626
+ gate = new Promise<void>((resolve) => {
627
+ release = (): void => resolve();
628
+ });
629
+ return release;
630
+ },
631
+ setBatchUnsupported: (unsupported: boolean): void => {
632
+ isUnsupported = unsupported;
633
+ },
634
+ };
635
+ };
636
+
637
+ it("bootstraps every stream with one batch request instead of one request per stream", async () => {
638
+ const harness = makeBatchHarness();
639
+ harness.http.state.pages[NOTES_STREAM] = {
640
+ ...emptyPage(NOTES_STREAM, 3),
641
+ entities: [{data: {body: "hello"}, deleted: false, id: "n1", seq: 3}],
642
+ };
643
+ const client = createSyncDb(harness.config);
644
+ await client.start();
645
+ await waitUntil(() => client.store.getEntity({collection: "notes", id: "n1"}) !== undefined);
646
+
647
+ expect(harness.batchRequests).toHaveLength(1);
648
+ expect(harness.batchRequests[0]?.streams.map(({stream}) => stream)).toEqual([
649
+ DEFAULT_STREAM,
650
+ NOTES_STREAM,
651
+ ]);
652
+ expect(harness.batchRequests[0]?.limit).toBe(5000);
653
+ expect(harness.http.state.fetchCount).toBe(0);
654
+ await client.stop();
655
+ });
656
+
657
+ it("falls back to per-stream paging when the server has no batch route, and remembers", async () => {
658
+ const harness = makeBatchHarness();
659
+ harness.setBatchUnsupported(true);
660
+ const client = createSyncDb(harness.config);
661
+ await client.start();
662
+ await waitUntil(() => harness.http.state.fetchCount === 2);
663
+
664
+ await client.reconcile();
665
+
666
+ expect(harness.batchRequests).toHaveLength(1);
667
+ expect(harness.http.state.fetchCount).toBe(4);
668
+ await client.stop();
669
+ });
670
+
671
+ it("pages confirmations that arrive during a catch-up together in the next request", async () => {
672
+ const harness = makeBatchHarness();
673
+ const client = createSyncDb(harness.config);
674
+ await client.start();
675
+ await flush();
676
+ harness.batchRequests.length = 0;
677
+
678
+ const release = harness.gateNextBatch();
679
+ harness.transport.confirmSubscribed({collection: "todos", streams: [DEFAULT_STREAM]});
680
+ harness.transport.confirmSubscribed({collection: "notes", streams: [NOTES_STREAM]});
681
+ harness.transport.confirmSubscribed({collection: "notes", streams: ["notes|owner:shared"]});
682
+ release();
683
+ await waitUntil(() => harness.batchRequests.length === 2);
684
+ await flush();
685
+
686
+ expect(
687
+ harness.batchRequests.map((request) => request.streams.map(({stream}) => stream))
688
+ ).toEqual([[DEFAULT_STREAM], [NOTES_STREAM, "notes|owner:shared"]]);
689
+ await client.stop();
690
+ });
691
+ });
692
+
583
693
  describe("subscribe confirmation catch-up", () => {
584
694
  it("pages a confirmed stream, catching a write the startup snapshot missed", async () => {
585
695
  const harness = makeHarness();
@@ -3305,3 +3415,206 @@ describe("createSyncDb", () => {
3305
3415
  });
3306
3416
  });
3307
3417
  });
3418
+
3419
+ describe("snapshot paging concurrency", () => {
3420
+ /**
3421
+ * Wrap the harness channel so every snapshot page waits on a gate the test releases,
3422
+ * and record how many pages are in flight overall and per stream.
3423
+ */
3424
+ const trackSnapshotPages = (
3425
+ harness: Harness
3426
+ ): {
3427
+ fetched: string[];
3428
+ maxInFlight: () => number;
3429
+ maxInFlightPerStream: () => number;
3430
+ pending: () => number;
3431
+ releaseAll: () => void;
3432
+ } => {
3433
+ const fetched: string[] = [];
3434
+ const inFlight = new Map<string, number>();
3435
+ const gates: (() => void)[] = [];
3436
+ let total = 0;
3437
+ let maxTotal = 0;
3438
+ let maxPerStream = 0;
3439
+ const fetchSnapshotPage = harness.http.channel.fetchSnapshotPage;
3440
+ harness.http.channel.fetchSnapshotPage = async (args) => {
3441
+ fetched.push(args.stream);
3442
+ total += 1;
3443
+ maxTotal = Math.max(maxTotal, total);
3444
+ const streamCount = (inFlight.get(args.stream) ?? 0) + 1;
3445
+ inFlight.set(args.stream, streamCount);
3446
+ maxPerStream = Math.max(maxPerStream, streamCount);
3447
+ await new Promise<void>((resolve) => gates.push(resolve));
3448
+ total -= 1;
3449
+ inFlight.set(args.stream, (inFlight.get(args.stream) ?? 1) - 1);
3450
+ return fetchSnapshotPage(args);
3451
+ };
3452
+ return {
3453
+ fetched,
3454
+ maxInFlight: () => maxTotal,
3455
+ maxInFlightPerStream: () => maxPerStream,
3456
+ pending: () => gates.length,
3457
+ releaseAll: () => {
3458
+ for (const release of gates.splice(0)) {
3459
+ release();
3460
+ }
3461
+ },
3462
+ };
3463
+ };
3464
+
3465
+ /** Release gated pages until `done` settles. */
3466
+ const drain = async <T>(
3467
+ pages: ReturnType<typeof trackSnapshotPages>,
3468
+ done: Promise<T>
3469
+ ): Promise<T> => {
3470
+ let settled = false;
3471
+ const result = done.finally(() => {
3472
+ settled = true;
3473
+ });
3474
+ for (let attempt = 0; attempt < 100 && !settled; attempt += 1) {
3475
+ pages.releaseAll();
3476
+ await flush();
3477
+ }
3478
+ return result;
3479
+ };
3480
+
3481
+ const manyStreams = (count: number): SyncStreamInfo[] =>
3482
+ Array.from({length: count}, (_, index) => ({
3483
+ collection: "todos",
3484
+ stream: `todos|tenant:w${index}`,
3485
+ }));
3486
+
3487
+ const isBootstrapped = (client: ReturnType<typeof createSyncDb>, stream: string): boolean =>
3488
+ client.store.raw.getCell(CURSORS_TABLE, stream, "bootstrapped") === true;
3489
+
3490
+ it("pages streams side by side, up to snapshotConcurrency at once", async () => {
3491
+ const harness = makeHarness({snapshotConcurrency: 3});
3492
+ harness.http.state.streams = manyStreams(10);
3493
+ const pages = trackSnapshotPages(harness);
3494
+ const client = createSyncDb(harness.config);
3495
+ await client.start();
3496
+
3497
+ await waitUntil(() => pages.pending() === 3);
3498
+ await flush();
3499
+ expect(pages.pending()).toBe(3);
3500
+ await drain(pages, client.reconcile());
3501
+
3502
+ expect(pages.maxInFlight()).toBe(3);
3503
+ for (const {stream} of harness.http.state.streams) {
3504
+ expect(isBootstrapped(client, stream)).toBe(true);
3505
+ }
3506
+ await client.stop();
3507
+ });
3508
+
3509
+ it("pages one stream at a time when snapshotConcurrency is 1", async () => {
3510
+ const harness = makeHarness({snapshotConcurrency: 1});
3511
+ harness.http.state.streams = manyStreams(4);
3512
+ const pages = trackSnapshotPages(harness);
3513
+ const client = createSyncDb(harness.config);
3514
+ await client.start();
3515
+ await drain(pages, client.reconcile());
3516
+
3517
+ expect(pages.maxInFlight()).toBe(1);
3518
+ expect(pages.fetched).toHaveLength(4);
3519
+ await client.stop();
3520
+ });
3521
+
3522
+ it("queues a subscribe catch-up behind an in-flight pass on the same stream", async () => {
3523
+ const harness = makeHarness();
3524
+ const pages = trackSnapshotPages(harness);
3525
+ const client = createSyncDb(harness.config);
3526
+ await client.start();
3527
+ await waitUntil(() => pages.pending() === 1);
3528
+
3529
+ harness.transport.confirmSubscribed({collection: "todos", streams: [DEFAULT_STREAM]});
3530
+ harness.transport.confirmSubscribed({collection: "todos", streams: [DEFAULT_STREAM]});
3531
+ await flush();
3532
+ // The catch-up must not page beside the reconcile pass that began before the join.
3533
+ expect(pages.pending()).toBe(1);
3534
+ await drain(pages, client.reconcile());
3535
+ await waitUntil(() => pages.fetched.length === 2);
3536
+ pages.releaseAll();
3537
+ await flush();
3538
+
3539
+ // One startup pass, then one fresh pass shared by both confirmations.
3540
+ expect(pages.fetched).toEqual([DEFAULT_STREAM, DEFAULT_STREAM]);
3541
+ expect(pages.maxInFlightPerStream()).toBe(1);
3542
+ await client.stop();
3543
+ });
3544
+
3545
+ it("drops a queued catch-up when the client stops before it starts", async () => {
3546
+ const harness = makeHarness();
3547
+ const pages = trackSnapshotPages(harness);
3548
+ const client = createSyncDb(harness.config);
3549
+ await client.start();
3550
+ await waitUntil(() => pages.pending() === 1);
3551
+ harness.transport.confirmSubscribed({collection: "todos", streams: [DEFAULT_STREAM]});
3552
+ await flush();
3553
+
3554
+ const stopped = client.stop();
3555
+ for (let attempt = 0; attempt < 10; attempt += 1) {
3556
+ pages.releaseAll();
3557
+ await flush();
3558
+ }
3559
+ await stopped;
3560
+
3561
+ // Only the startup pass paged; the catch-up queued behind it belonged to the
3562
+ // stopped lifecycle.
3563
+ expect(pages.fetched).toEqual([DEFAULT_STREAM]);
3564
+ });
3565
+
3566
+ it("forceResync waits for an in-flight pass before purging the stream", async () => {
3567
+ const harness = makeHarness();
3568
+ harness.http.state.pages[DEFAULT_STREAM] = {
3569
+ cursor: 3,
3570
+ entities: [{data: {title: "kept"}, deleted: false, id: "t1", seq: 3}],
3571
+ frontierSeq: 3,
3572
+ hasMore: false,
3573
+ oldestRetainedSeq: 0,
3574
+ stream: DEFAULT_STREAM,
3575
+ };
3576
+ const pages = trackSnapshotPages(harness);
3577
+ const client = createSyncDb(harness.config);
3578
+ await client.start();
3579
+ await waitUntil(() => pages.pending() === 1);
3580
+
3581
+ const resync = client.forceResync();
3582
+ await flush();
3583
+ await flush();
3584
+ // The purge-and-repage must not start while the startup pass is still paging.
3585
+ expect(pages.pending()).toBe(1);
3586
+ const result = await drain(pages, resync);
3587
+
3588
+ assert.equal(result.ok, true);
3589
+ expect(pages.maxInFlightPerStream()).toBe(1);
3590
+ expect(client.store.getEntity({collection: "todos", id: "t1"})?.data).toEqual({
3591
+ title: "kept",
3592
+ });
3593
+ expect(isBootstrapped(client, DEFAULT_STREAM)).toBe(true);
3594
+ await client.stop();
3595
+ });
3596
+
3597
+ it("stops taking new streams after one fails", async () => {
3598
+ const harness = makeHarness({snapshotConcurrency: 1});
3599
+ harness.http.state.streams = manyStreams(3);
3600
+ const failing = "todos|tenant:w1";
3601
+ const fetched: string[] = [];
3602
+ const fetchSnapshotPage = harness.http.channel.fetchSnapshotPage;
3603
+ harness.http.channel.fetchSnapshotPage = async (args) => {
3604
+ fetched.push(args.stream);
3605
+ if (args.stream === failing) {
3606
+ throw new Error("snapshot failed");
3607
+ }
3608
+ return fetchSnapshotPage(args);
3609
+ };
3610
+ const client = createSyncDb(harness.config);
3611
+
3612
+ // start() fires a reconcile of its own; this call shares that pass.
3613
+ await client.start();
3614
+ await expect(client.reconcile()).rejects.toThrow("snapshot failed");
3615
+
3616
+ expect(fetched).toEqual(["todos|tenant:w0", failing]);
3617
+ expect(isBootstrapped(client, "todos|tenant:w2")).toBe(false);
3618
+ await client.stop();
3619
+ });
3620
+ });