@terreno/syncdb 57.13.0 → 57.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +6 -3
  2. package/dist/client.d.ts +12 -0
  3. package/dist/client.d.ts.map +1 -1
  4. package/dist/client.js +304 -56
  5. package/dist/client.js.map +1 -1
  6. package/dist/client.test.js +255 -0
  7. package/dist/client.test.js.map +1 -1
  8. package/dist/index.d.ts +1 -1
  9. package/dist/index.d.ts.map +1 -1
  10. package/dist/index.js +1 -1
  11. package/dist/index.js.map +1 -1
  12. package/dist/persisters/encryptedIndexedDbPersister.d.ts +4 -3
  13. package/dist/persisters/encryptedIndexedDbPersister.d.ts.map +1 -1
  14. package/dist/persisters/encryptedIndexedDbPersister.js +21 -16
  15. package/dist/persisters/encryptedIndexedDbPersister.js.map +1 -1
  16. package/dist/persisters/encryptedIndexedDbPersister.test.js +40 -1
  17. package/dist/persisters/encryptedIndexedDbPersister.test.js.map +1 -1
  18. package/dist/sync/bootstrap.d.ts +34 -3
  19. package/dist/sync/bootstrap.d.ts.map +1 -1
  20. package/dist/sync/bootstrap.js +224 -71
  21. package/dist/sync/bootstrap.js.map +1 -1
  22. package/dist/sync/bootstrap.test.js +210 -2
  23. package/dist/sync/bootstrap.test.js.map +1 -1
  24. package/dist/sync/httpChannel.d.ts +14 -1
  25. package/dist/sync/httpChannel.d.ts.map +1 -1
  26. package/dist/sync/httpChannel.js +34 -0
  27. package/dist/sync/httpChannel.js.map +1 -1
  28. package/dist/sync/httpChannel.test.js +54 -0
  29. package/dist/sync/httpChannel.test.js.map +1 -1
  30. package/dist/types.d.ts +14 -0
  31. package/dist/types.d.ts.map +1 -1
  32. package/package.json +1 -1
  33. package/src/client.test.ts +314 -1
  34. package/src/client.ts +362 -54
  35. package/src/index.ts +1 -0
  36. package/src/persisters/encryptedIndexedDbPersister.test.ts +44 -1
  37. package/src/persisters/encryptedIndexedDbPersister.ts +24 -17
  38. package/src/sync/bootstrap.test.ts +269 -3
  39. package/src/sync/bootstrap.ts +336 -82
  40. package/src/sync/httpChannel.test.ts +67 -0
  41. package/src/sync/httpChannel.ts +50 -0
  42. package/src/types.ts +12 -0
@@ -1,4 +1,4 @@
1
- import type {MergeableContent, MergeableStore} from "tinybase";
1
+ import type {ContentHashes, MergeableContent, MergeableStore} from "tinybase";
2
2
  import {createCustomPersister, type Persister, Persists} from "tinybase/persisters";
3
3
 
4
4
  import type {PayloadCodec} from "../crypto/types";
@@ -11,6 +11,9 @@ const RECORD_KEY = "content";
11
11
  /** Default trailing debounce applied to saves (autosave can fire per transaction). */
12
12
  const DEFAULT_SAVE_DEBOUNCE_MS = 500;
13
13
 
14
+ const sameHashes = (a: ContentHashes, b: ContentHashes | undefined): boolean =>
15
+ b !== undefined && a[0] === b[0] && a[1] === b[1];
16
+
14
17
  interface PendingSave {
15
18
  promise: Promise<void>;
16
19
  resolve: () => void;
@@ -35,9 +38,10 @@ interface PendingSave {
35
38
  *
36
39
  * Saves are debounced with a trailing edge (default 500ms) since TinyBase's
37
40
  * autosave fires on every transaction: each write waits out the debounce
38
- * window, and follow-up saves whose serialized content matches the last
39
- * written snapshot are skipped, coalescing a burst of transactions into a
40
- * single encode + IndexedDB put.
41
+ * window, and follow-up saves whose content hashes match the last written
42
+ * snapshot are skipped, coalescing a burst of transactions into a single
43
+ * encode + IndexedDB put. The store is serialized only when a write actually
44
+ * happens, never per transaction.
41
45
  */
42
46
  export const createEncryptedIndexedDbPersister = ({
43
47
  store,
@@ -68,7 +72,9 @@ export const createEncryptedIndexedDbPersister = ({
68
72
  /** Test-only override for the IndexedDB write (default: the real `idbSet`). */
69
73
  idbSetImpl?: typeof idbSet;
70
74
  }): Persister<Persists.MergeableStoreOnly> & SyncPersister => {
71
- let lastWrittenJson: string | undefined;
75
+ // Content hashes of the last snapshot that reached IndexedDB. TinyBase keeps
76
+ // them incrementally, so comparing is O(1) where re-serializing is O(store).
77
+ let lastWrittenHashes: ContentHashes | undefined;
72
78
  let pendingSave: PendingSave | undefined;
73
79
  // E3(e): the debounce timer scheduling `flushPendingSave` — tracked so
74
80
  // `destroy()` can cancel it. Without this, a pending debounced save would
@@ -81,7 +87,7 @@ export const createEncryptedIndexedDbPersister = ({
81
87
  // a newer snapshot's write.
82
88
  let writeChain: Promise<void> = Promise.resolve();
83
89
 
84
- const writeJson = (json: string): Promise<void> => {
90
+ const writeJson = (json: string, hashes: ContentHashes): Promise<void> => {
85
91
  const write = writeChain.then(async () => {
86
92
  // E3(e): destroy() may have run while this write was already chained
87
93
  // behind an earlier one — never let it land after destroy() completes.
@@ -90,7 +96,7 @@ export const createEncryptedIndexedDbPersister = ({
90
96
  }
91
97
  const payload = await codec.encode(json);
92
98
  await idbSetImpl({databaseName, key: RECORD_KEY, value: payload});
93
- lastWrittenJson = json;
99
+ lastWrittenHashes = hashes;
94
100
  });
95
101
  // Keep the chain alive after a failed write; the failure still propagates
96
102
  // to the caller through `write`.
@@ -107,7 +113,8 @@ export const createEncryptedIndexedDbPersister = ({
107
113
  try {
108
114
  // Trailing edge: snapshot the store at flush time so a burst of
109
115
  // transactions during the debounce window lands as one fresh write.
110
- await writeJson(JSON.stringify(store.getMergeableContent()));
116
+ const hashes = store.getMergeableContentHashes();
117
+ await writeJson(JSON.stringify(store.getMergeableContent()), hashes);
111
118
  pending.resolve();
112
119
  } catch (error) {
113
120
  pending.reject(error);
@@ -126,12 +133,12 @@ export const createEncryptedIndexedDbPersister = ({
126
133
  return;
127
134
  }
128
135
 
129
- const json = JSON.stringify(store.getMergeableContent());
130
- if (json === lastWrittenJson) {
136
+ const hashes = store.getMergeableContentHashes();
137
+ if (sameHashes(hashes, lastWrittenHashes)) {
131
138
  return;
132
139
  }
133
140
  try {
134
- await writeJson(json);
141
+ await writeJson(JSON.stringify(store.getMergeableContent()), hashes);
135
142
  } catch (error) {
136
143
  onSaveFailure?.(error);
137
144
  throw error;
@@ -168,20 +175,20 @@ export const createEncryptedIndexedDbPersister = ({
168
175
  };
169
176
 
170
177
  const setPersisted = async (getContent: () => MergeableContent): Promise<void> => {
171
- const json = JSON.stringify(getContent());
172
178
  if (saveDebounceMs <= 0) {
173
179
  try {
174
- await writeJson(json);
180
+ await writeJson(JSON.stringify(getContent()), store.getMergeableContentHashes());
175
181
  } catch (error) {
176
182
  onSaveFailure?.(error);
177
183
  throw error;
178
184
  }
179
185
  return;
180
186
  }
181
- // TinyBase serializes setPersisted calls, so per-transaction autosaves
182
- // arrive one at a time; skipping saves whose content already matches the
183
- // last written snapshot is what coalesces a rapid burst into one write.
184
- if (json === lastWrittenJson && !pendingSave) {
187
+ // TinyBase queues one setPersisted per transaction and runs them one at a
188
+ // time, so a burst of N transactions arrives here N times, mostly after the
189
+ // write that already captured them. Skipping saves whose content already
190
+ // matches the last written snapshot is what coalesces the burst into one write.
191
+ if (!pendingSave && sameHashes(store.getMergeableContentHashes(), lastWrittenHashes)) {
185
192
  return;
186
193
  }
187
194
  if (!pendingSave) {
@@ -1,9 +1,16 @@
1
1
  import {describe, expect, it} from "bun:test";
2
2
 
3
3
  import {createSyncStore, type SyncStore} from "../storage/store";
4
- import type {SyncSnapshotResponse} from "../types";
5
- import {type BootstrapProgress, bootstrapStream} from "./bootstrap";
6
- import {getCursor, markStreamBootstrapped, setCursor, setSnapshotCursor} from "./cursor";
4
+ import type {SyncSnapshotBatchRequest, SyncSnapshotResponse} from "../types";
5
+ import {type BootstrapProgress, bootstrapStream, bootstrapStreamsBatch} from "./bootstrap";
6
+ import {
7
+ getCursor,
8
+ getSnapshotCursor,
9
+ isStreamBootstrapped,
10
+ markStreamBootstrapped,
11
+ setCursor,
12
+ setSnapshotCursor,
13
+ } from "./cursor";
7
14
  import {applyDelta} from "./deltaApplier";
8
15
  import type {FetchSnapshotPageArgs} from "./httpChannel";
9
16
 
@@ -414,6 +421,40 @@ describe("bootstrapStream", () => {
414
421
  expect(second.calls.map((call) => call.cursor)).toEqual([40]);
415
422
  });
416
423
 
424
+ it("writes nothing when superseded while a page was in flight", async () => {
425
+ const store = makeStore();
426
+ const channel = makeChannel({0: page({cursor: 1, entities: entitiesFor(1, 1)})});
427
+ await bootstrapStream({
428
+ channel,
429
+ collection: "todos",
430
+ isSuperseded: () => true,
431
+ store,
432
+ stream: STREAM,
433
+ });
434
+ expect(store.getEntity({collection: "todos", id: "e1"})).toBeUndefined();
435
+ expect(getCursor({store, stream: STREAM})).toBe(0);
436
+ });
437
+
438
+ it("stops mid-page when superseded during a chunk yield", async () => {
439
+ const store = makeStore();
440
+ const channel = makeChannel({0: page({cursor: 4, entities: entitiesFor(1, 4)})});
441
+ let superseded = false;
442
+ await bootstrapStream({
443
+ applyChunkSize: 2,
444
+ channel,
445
+ collection: "todos",
446
+ isSuperseded: () => superseded,
447
+ store,
448
+ stream: STREAM,
449
+ yieldToHost: async () => {
450
+ superseded = true;
451
+ },
452
+ });
453
+ expect(store.getEntity({collection: "todos", id: "e2"})).toBeDefined();
454
+ expect(store.getEntity({collection: "todos", id: "e3"})).toBeUndefined();
455
+ expect(getCursor({store, stream: STREAM})).toBe(0);
456
+ });
457
+
417
458
  it("reports progress per page and forwards the limit", async () => {
418
459
  const store = makeStore();
419
460
  store.upsertEntity({
@@ -484,3 +525,228 @@ describe("bootstrapStream", () => {
484
525
  expect(store.listEntities({collection: "todos"})).toHaveLength(5);
485
526
  });
486
527
  });
528
+
529
+ const NOTES_STREAM = "notes|owner:u1";
530
+
531
+ /** Entities with seqs `from..to` (inclusive) for a snapshot page. */
532
+ const entitiesFor = (from: number, to: number): SyncSnapshotResponse["entities"] =>
533
+ Array.from({length: to - from + 1}, (_unused, index) => ({
534
+ data: {title: `row ${from + index}`},
535
+ deleted: false,
536
+ id: `e${from + index}`,
537
+ seq: from + index,
538
+ }));
539
+
540
+ /**
541
+ * Batch fetch stub: each stream's page comes from `pageFor(stream, cursor)`; every
542
+ * request is recorded along with the order of fetches relative to other events.
543
+ */
544
+ const makeBatchFetch = (
545
+ pageFor: (args: {stream: string; cursor: number; legacyCursor?: string}) => SyncSnapshotResponse,
546
+ events: string[] = []
547
+ ): {
548
+ fetchSnapshotBatch: (
549
+ request: SyncSnapshotBatchRequest
550
+ ) => Promise<{pages: SyncSnapshotResponse[]}>;
551
+ requests: SyncSnapshotBatchRequest[];
552
+ } => {
553
+ const requests: SyncSnapshotBatchRequest[] = [];
554
+ return {
555
+ fetchSnapshotBatch: async (request) => {
556
+ requests.push(request);
557
+ events.push(`fetch:${request.streams.map((item) => item.stream).join(",")}`);
558
+ return {pages: request.streams.map(pageFor)};
559
+ },
560
+ requests,
561
+ };
562
+ };
563
+
564
+ describe("bootstrapStreamsBatch", () => {
565
+ const streams = [
566
+ {collection: "todos", stream: STREAM},
567
+ {collection: "notes", stream: NOTES_STREAM},
568
+ ];
569
+
570
+ it("pages every stream to its head, re-requesting only streams with more", async () => {
571
+ const store = makeStore();
572
+ const {fetchSnapshotBatch, requests} = makeBatchFetch(({stream, cursor}) => {
573
+ if (stream === NOTES_STREAM) {
574
+ return page({cursor: 1, entities: entitiesFor(1, 1), stream});
575
+ }
576
+ return cursor === 0
577
+ ? page({cursor: 2, entities: entitiesFor(1, 2), frontierSeq: 3, hasMore: true})
578
+ : page({cursor: 3, entities: entitiesFor(3, 3)});
579
+ });
580
+
581
+ const completed = await bootstrapStreamsBatch({
582
+ fetchSnapshotBatch,
583
+ limit: 2000,
584
+ store,
585
+ streams,
586
+ });
587
+
588
+ expect(completed).toBe(true);
589
+ expect(requests).toEqual([
590
+ {
591
+ limit: 2000,
592
+ streams: [
593
+ {cursor: 0, legacyCursor: undefined, stream: STREAM},
594
+ {cursor: 0, legacyCursor: undefined, stream: NOTES_STREAM},
595
+ ],
596
+ },
597
+ {limit: 2000, streams: [{cursor: 2, legacyCursor: undefined, stream: STREAM}]},
598
+ ]);
599
+ expect(store.getEntity({collection: "todos", id: "e3"})?.stream).toBe(STREAM);
600
+ expect(store.getEntity({collection: "notes", id: "e1"})?.stream).toBe(NOTES_STREAM);
601
+ expect(getCursor({store, stream: STREAM})).toBe(3);
602
+ expect(getCursor({store, stream: NOTES_STREAM})).toBe(1);
603
+ expect(isStreamBootstrapped({store, stream: STREAM})).toBe(true);
604
+ expect(isStreamBootstrapped({store, stream: NOTES_STREAM})).toBe(true);
605
+ });
606
+
607
+ it("sends the next round before writing the current one", async () => {
608
+ const store = makeStore();
609
+ const events: string[] = [];
610
+ const {fetchSnapshotBatch} = makeBatchFetch(
611
+ ({cursor}) =>
612
+ cursor === 0
613
+ ? page({cursor: 1, entities: entitiesFor(1, 1), frontierSeq: 2, hasMore: true})
614
+ : page({cursor: 2, entities: entitiesFor(2, 2)}),
615
+ events
616
+ );
617
+
618
+ await bootstrapStreamsBatch({
619
+ fetchSnapshotBatch,
620
+ onProgress: (progress) => events.push(`write:${progress.cursor}`),
621
+ store,
622
+ streams: [{collection: "todos", stream: STREAM}],
623
+ });
624
+
625
+ expect(events).toEqual([`fetch:${STREAM}`, `fetch:${STREAM}`, "write:1", "write:2"]);
626
+ });
627
+
628
+ it("writes a large page in chunks, yielding between them, and advances the cursor last", async () => {
629
+ const store = makeStore();
630
+ const {fetchSnapshotBatch} = makeBatchFetch(() =>
631
+ page({cursor: 5, entities: entitiesFor(1, 5)})
632
+ );
633
+ let transactions = 0;
634
+ store.raw.addDidFinishTransactionListener(() => {
635
+ transactions += 1;
636
+ });
637
+ const cursorsAtYield: number[] = [];
638
+
639
+ await bootstrapStreamsBatch({
640
+ applyChunkSize: 2,
641
+ fetchSnapshotBatch,
642
+ store,
643
+ streams: [{collection: "todos", stream: STREAM}],
644
+ yieldToHost: async () => {
645
+ cursorsAtYield.push(getCursor({store, stream: STREAM}));
646
+ },
647
+ });
648
+
649
+ // Chunks of 2, 2, 1: two yields, and the cursor only moves in the final chunk.
650
+ expect(cursorsAtYield).toEqual([0, 0]);
651
+ expect(transactions).toBeGreaterThanOrEqual(3);
652
+ expect(getCursor({store, stream: STREAM})).toBe(5);
653
+ expect(getSnapshotCursor({store, stream: STREAM})).toBe(5);
654
+ expect(store.getEntity({collection: "todos", id: "e5"})?.seq).toBe(5);
655
+ });
656
+
657
+ it("stops without writing when superseded while a round was in flight", async () => {
658
+ const store = makeStore();
659
+ const {fetchSnapshotBatch} = makeBatchFetch(() =>
660
+ page({cursor: 1, entities: entitiesFor(1, 1)})
661
+ );
662
+
663
+ const completed = await bootstrapStreamsBatch({
664
+ fetchSnapshotBatch,
665
+ isSuperseded: () => true,
666
+ store,
667
+ streams: [{collection: "todos", stream: STREAM}],
668
+ });
669
+
670
+ expect(completed).toBe(false);
671
+ expect(store.getEntity({collection: "todos", id: "e1"})).toBeUndefined();
672
+ expect(getCursor({store, stream: STREAM})).toBe(0);
673
+ });
674
+
675
+ it("stops mid-page when superseded during a chunk yield, leaving the cursor unmoved", async () => {
676
+ const store = makeStore();
677
+ const {fetchSnapshotBatch} = makeBatchFetch(() =>
678
+ page({cursor: 5, entities: entitiesFor(1, 5)})
679
+ );
680
+ let superseded = false;
681
+
682
+ const completed = await bootstrapStreamsBatch({
683
+ applyChunkSize: 2,
684
+ fetchSnapshotBatch,
685
+ isSuperseded: () => superseded,
686
+ store,
687
+ streams: [{collection: "todos", stream: STREAM}],
688
+ yieldToHost: async () => {
689
+ // A sign-out or user switch wipes the store while the page is being written.
690
+ superseded = true;
691
+ },
692
+ });
693
+
694
+ expect(completed).toBe(false);
695
+ expect(store.getEntity({collection: "todos", id: "e2"})).toBeDefined();
696
+ expect(store.getEntity({collection: "todos", id: "e3"})).toBeUndefined();
697
+ expect(getCursor({store, stream: STREAM})).toBe(0);
698
+ expect(isStreamBootstrapped({store, stream: STREAM})).toBe(false);
699
+ });
700
+
701
+ it("C7: purges a stream whose cursor fell below the retained floor and re-pages it from 0", async () => {
702
+ const store = makeStore();
703
+ seedBootstrappedStream({seq: 4, store});
704
+ store.upsertEntity({
705
+ collection: "todos",
706
+ data: {title: "stale"},
707
+ id: "gone",
708
+ seq: 2,
709
+ stream: STREAM,
710
+ });
711
+ const {fetchSnapshotBatch, requests} = makeBatchFetch(({cursor}) =>
712
+ cursor === 4
713
+ ? page({cursor: 9, entities: [], oldestRetainedSeq: 6})
714
+ : page({cursor: 9, entities: entitiesFor(9, 9), oldestRetainedSeq: 6})
715
+ );
716
+
717
+ await bootstrapStreamsBatch({
718
+ fetchSnapshotBatch,
719
+ store,
720
+ streams: [{collection: "todos", stream: STREAM}],
721
+ });
722
+
723
+ expect(requests.map((request) => request.streams[0]?.cursor)).toEqual([4, 0]);
724
+ expect(store.getEntity({collection: "todos", id: "gone"})).toBeUndefined();
725
+ expect(store.getEntity({collection: "todos", id: "e9"})?.seq).toBe(9);
726
+ expect(getCursor({store, stream: STREAM})).toBe(9);
727
+ });
728
+
729
+ it("rejects a response that leaves out a requested stream", async () => {
730
+ const store = makeStore();
731
+ await expect(
732
+ bootstrapStreamsBatch({
733
+ fetchSnapshotBatch: async () => ({pages: [page({cursor: 0})]}),
734
+ store,
735
+ streams,
736
+ })
737
+ ).rejects.toThrow(`missing stream ${NOTES_STREAM}`);
738
+ });
739
+
740
+ it("caps each request at the server's 500-stream limit", async () => {
741
+ const store = makeStore();
742
+ const many = Array.from({length: 501}, (_unused, index) => ({
743
+ collection: "todos",
744
+ stream: `todos|owner:u${index}`,
745
+ }));
746
+ const {fetchSnapshotBatch, requests} = makeBatchFetch(({stream}) => page({cursor: 0, stream}));
747
+
748
+ await bootstrapStreamsBatch({fetchSnapshotBatch, store, streams: many});
749
+
750
+ expect(requests.map((request) => request.streams.length)).toEqual([500, 1]);
751
+ });
752
+ });