@terreno/syncdb 57.14.0 → 57.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +200 -80
- package/dist/client.js.map +1 -1
- package/dist/client.test.js +86 -0
- package/dist/client.test.js.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.d.ts +4 -3
- package/dist/persisters/encryptedIndexedDbPersister.d.ts.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.js +21 -16
- package/dist/persisters/encryptedIndexedDbPersister.js.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.test.js +40 -1
- package/dist/persisters/encryptedIndexedDbPersister.test.js.map +1 -1
- package/dist/sync/bootstrap.d.ts +34 -3
- package/dist/sync/bootstrap.d.ts.map +1 -1
- package/dist/sync/bootstrap.js +224 -71
- package/dist/sync/bootstrap.js.map +1 -1
- package/dist/sync/bootstrap.test.js +210 -2
- package/dist/sync/bootstrap.test.js.map +1 -1
- package/dist/sync/httpChannel.d.ts +14 -1
- package/dist/sync/httpChannel.d.ts.map +1 -1
- package/dist/sync/httpChannel.js +34 -0
- package/dist/sync/httpChannel.js.map +1 -1
- package/dist/sync/httpChannel.test.js +54 -0
- package/dist/sync/httpChannel.test.js.map +1 -1
- package/dist/types.d.ts +14 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/client.test.ts +111 -1
- package/src/client.ts +232 -86
- package/src/persisters/encryptedIndexedDbPersister.test.ts +44 -1
- package/src/persisters/encryptedIndexedDbPersister.ts +24 -17
- package/src/sync/bootstrap.test.ts +269 -3
- package/src/sync/bootstrap.ts +336 -82
- package/src/sync/httpChannel.test.ts +67 -0
- package/src/sync/httpChannel.ts +50 -0
- package/src/types.ts +12 -0
package/src/client.ts
CHANGED
|
@@ -16,7 +16,7 @@ import {SYNC_SCHEMA_VERSION} from "./storage/schema";
|
|
|
16
16
|
import {createSyncStore, type SyncStore} from "./storage/store";
|
|
17
17
|
import {CURSORS_TABLE, KNOWN_STREAMS_TABLE} from "./storage/types";
|
|
18
18
|
import {wipeLocalData} from "./storage/wipe";
|
|
19
|
-
import {bootstrapStream} from "./sync/bootstrap";
|
|
19
|
+
import {bootstrapStream, bootstrapStreamsBatch} from "./sync/bootstrap";
|
|
20
20
|
import {getAllCursors} from "./sync/cursor";
|
|
21
21
|
import {applyDelta} from "./sync/deltaApplier";
|
|
22
22
|
import {repairMarkedEntities} from "./sync/entityRepair";
|
|
@@ -53,6 +53,15 @@ export const DEFAULT_SNAPSHOT_CONCURRENCY = 6;
|
|
|
53
53
|
/** E5: default local tombstone retention window (90 days), matching the server's (C7). */
|
|
54
54
|
export const DEFAULT_TOMBSTONE_RETENTION_MS = 90 * 24 * 60 * 60 * 1_000;
|
|
55
55
|
|
|
56
|
+
/**
|
|
57
|
+
* Entities one `POST /sync/snapshot/batch` response may carry, shared across its streams.
|
|
58
|
+
* Bounds both the response size (about 2 MB of typical rows) and the work per round.
|
|
59
|
+
*/
|
|
60
|
+
const SNAPSHOT_BATCH_LIMIT = 5_000;
|
|
61
|
+
|
|
62
|
+
/** The server has no `POST /sync/snapshot/batch` route (404). */
|
|
63
|
+
class SnapshotBatchUnsupportedError extends Error {}
|
|
64
|
+
|
|
56
65
|
/**
|
|
57
66
|
* `start()` auth-resolution retry defaults. A host app only calls `start()` once it
|
|
58
67
|
* already believes the user is authenticated (e.g. right after its own login flow
|
|
@@ -458,6 +467,9 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
458
467
|
// refresh has already run for the CURRENT pause episode.
|
|
459
468
|
let authPaused = false;
|
|
460
469
|
let refreshAttempted = false;
|
|
470
|
+
// Set once the server answers 404 to the snapshot batch route; per-stream paging is
|
|
471
|
+
// used for the rest of this client's life.
|
|
472
|
+
let snapshotBatchUnsupported = false;
|
|
461
473
|
// E1: lifecycle serialization. `generation` bumps on every start()/stop() so
|
|
462
474
|
// an in-flight operation that resumes after an await can detect it has been
|
|
463
475
|
// superseded (a rapid stop()-then-start() for a new user) and abort instead
|
|
@@ -512,7 +524,9 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
512
524
|
};
|
|
513
525
|
|
|
514
526
|
/**
|
|
515
|
-
* Run a snapshot pass
|
|
527
|
+
* Run a snapshot pass over `streams`, single-flight per stream. Each stream is either
|
|
528
|
+
* joined to its latest pass or owned by this one; `run` receives the owned streams once
|
|
529
|
+
* every pass it queued behind has settled.
|
|
516
530
|
*
|
|
517
531
|
* - `join`: share the stream's latest pass when one exists (reconcile — any pass to
|
|
518
532
|
* the head will do).
|
|
@@ -522,46 +536,71 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
522
536
|
* - `exclusive`: always queue behind whatever is in flight (force resync and wipe
|
|
523
537
|
* re-bootstrap purge first, which must never overlap another pass's paging).
|
|
524
538
|
*/
|
|
525
|
-
const
|
|
539
|
+
const runStreamsPass = ({
|
|
526
540
|
mode,
|
|
527
541
|
run,
|
|
528
|
-
|
|
542
|
+
streams,
|
|
529
543
|
}: {
|
|
530
544
|
mode: "join" | "fresh" | "exclusive";
|
|
531
|
-
run: () => Promise<void>;
|
|
532
|
-
|
|
545
|
+
run: (owned: string[]) => Promise<void>;
|
|
546
|
+
streams: string[];
|
|
533
547
|
}): Promise<void> => {
|
|
534
|
-
const
|
|
535
|
-
const
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
548
|
+
const joined: Promise<void>[] = [];
|
|
549
|
+
const owned: string[] = [];
|
|
550
|
+
const previous: Promise<void>[] = [];
|
|
551
|
+
for (const stream of streams) {
|
|
552
|
+
const existing = streamPasses.get(stream);
|
|
553
|
+
const current =
|
|
554
|
+
existing && existing.generation === generation && existing.userId === currentUserId
|
|
555
|
+
? existing
|
|
556
|
+
: undefined;
|
|
557
|
+
if (current && (mode === "join" || (mode === "fresh" && !current.started))) {
|
|
558
|
+
joined.push(current.promise);
|
|
559
|
+
continue;
|
|
560
|
+
}
|
|
561
|
+
owned.push(stream);
|
|
562
|
+
if (existing) {
|
|
563
|
+
previous.push(existing.promise);
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
if (owned.length === 0) {
|
|
567
|
+
return Promise.all(joined).then(() => undefined);
|
|
541
568
|
}
|
|
542
569
|
const entry = {generation, promise: Promise.resolve(), started: false, userId: currentUserId};
|
|
543
|
-
const previous = existing?.promise;
|
|
544
570
|
entry.promise = (async (): Promise<void> => {
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
await previous.catch(() => undefined);
|
|
548
|
-
}
|
|
571
|
+
// The previous passes' failures belong to their own callers.
|
|
572
|
+
await Promise.all(previous.map((pass) => pass.catch(() => undefined)));
|
|
549
573
|
// A stop()/start() or user switch while this pass was queued supersedes it:
|
|
550
574
|
// paging now would write into a store that belongs to a newer lifecycle.
|
|
551
575
|
if (entry.generation !== generation || entry.userId !== currentUserId) {
|
|
552
576
|
return;
|
|
553
577
|
}
|
|
554
578
|
entry.started = true;
|
|
555
|
-
await run();
|
|
579
|
+
await run(owned);
|
|
556
580
|
})().finally(() => {
|
|
557
|
-
|
|
558
|
-
streamPasses.
|
|
581
|
+
for (const stream of owned) {
|
|
582
|
+
if (streamPasses.get(stream) === entry) {
|
|
583
|
+
streamPasses.delete(stream);
|
|
584
|
+
}
|
|
559
585
|
}
|
|
560
586
|
});
|
|
561
|
-
|
|
562
|
-
|
|
587
|
+
for (const stream of owned) {
|
|
588
|
+
streamPasses.set(stream, entry);
|
|
589
|
+
}
|
|
590
|
+
return Promise.all([entry.promise, ...joined]).then(() => undefined);
|
|
563
591
|
};
|
|
564
592
|
|
|
593
|
+
/** {@link runStreamsPass} for a single stream. */
|
|
594
|
+
const runStreamPass = ({
|
|
595
|
+
mode,
|
|
596
|
+
run,
|
|
597
|
+
stream,
|
|
598
|
+
}: {
|
|
599
|
+
mode: "join" | "fresh" | "exclusive";
|
|
600
|
+
run: () => Promise<void>;
|
|
601
|
+
stream: string;
|
|
602
|
+
}): Promise<void> => runStreamsPass({mode, run: () => run(), streams: [stream]});
|
|
603
|
+
|
|
565
604
|
/**
|
|
566
605
|
* Run `worker` over `items` with at most `snapshotConcurrency` in flight. Stops taking
|
|
567
606
|
* new items once `isSuperseded()` or any worker throws, waits for the in-flight ones,
|
|
@@ -850,6 +889,84 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
850
889
|
return serverSet;
|
|
851
890
|
};
|
|
852
891
|
|
|
892
|
+
/**
|
|
893
|
+
* Page `streams` to their heads as one single-flight pass (see {@link runStreamsPass}).
|
|
894
|
+
* One `POST /sync/snapshot/batch` round trip per round covers every owned stream;
|
|
895
|
+
* against a server without that route, falls back to `GET /sync/snapshot` per stream,
|
|
896
|
+
* `snapshotConcurrency` streams at a time.
|
|
897
|
+
*/
|
|
898
|
+
const pageStreams = async ({
|
|
899
|
+
streams,
|
|
900
|
+
mode,
|
|
901
|
+
isSuperseded,
|
|
902
|
+
purgeFirst = false,
|
|
903
|
+
}: {
|
|
904
|
+
streams: {stream: string; collection: string}[];
|
|
905
|
+
mode: "join" | "fresh" | "exclusive";
|
|
906
|
+
isSuperseded: () => boolean;
|
|
907
|
+
/** Purge each owned stream before paging it from 0 (wipe re-bootstrap). */
|
|
908
|
+
purgeFirst?: boolean;
|
|
909
|
+
}): Promise<void> => {
|
|
910
|
+
const channel = httpChannel;
|
|
911
|
+
if (!channel || streams.length === 0) {
|
|
912
|
+
return;
|
|
913
|
+
}
|
|
914
|
+
const collections = new Map(streams.map(({stream, collection}) => [stream, collection]));
|
|
915
|
+
await runStreamsPass({
|
|
916
|
+
mode,
|
|
917
|
+
run: async (owned) => {
|
|
918
|
+
const ownedStreams = owned.map((stream) => ({
|
|
919
|
+
collection: collections.get(stream) as string,
|
|
920
|
+
stream,
|
|
921
|
+
}));
|
|
922
|
+
if (purgeFirst) {
|
|
923
|
+
for (const {stream, collection} of ownedStreams) {
|
|
924
|
+
store.purgeStream({stream});
|
|
925
|
+
store.addKnownStream({collection, stream});
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
const fetchSnapshotBatch = channel.fetchSnapshotBatch;
|
|
929
|
+
if (fetchSnapshotBatch && !snapshotBatchUnsupported) {
|
|
930
|
+
try {
|
|
931
|
+
await bootstrapStreamsBatch({
|
|
932
|
+
fetchSnapshotBatch: async (request) => {
|
|
933
|
+
const result = await fetchSnapshotBatch(request);
|
|
934
|
+
if (result.type === "unsupported") {
|
|
935
|
+
throw new SnapshotBatchUnsupportedError();
|
|
936
|
+
}
|
|
937
|
+
return result;
|
|
938
|
+
},
|
|
939
|
+
isSuperseded,
|
|
940
|
+
limit: SNAPSHOT_BATCH_LIMIT,
|
|
941
|
+
store,
|
|
942
|
+
streams: ownedStreams,
|
|
943
|
+
});
|
|
944
|
+
return;
|
|
945
|
+
} catch (error) {
|
|
946
|
+
if (!(error instanceof SnapshotBatchUnsupportedError)) {
|
|
947
|
+
throw error;
|
|
948
|
+
}
|
|
949
|
+
// Every round before the 404 is already written, so per-stream paging
|
|
950
|
+
// resumes from the store's cursors without losing anything.
|
|
951
|
+
snapshotBatchUnsupported = true;
|
|
952
|
+
debugLog?.record({
|
|
953
|
+
direction: "system",
|
|
954
|
+
label: "snapshot batch unsupported by server; paging per stream",
|
|
955
|
+
type: "reconcile",
|
|
956
|
+
});
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
await runSnapshotPool({
|
|
960
|
+
isSuperseded,
|
|
961
|
+
items: ownedStreams,
|
|
962
|
+
worker: ({stream, collection}) =>
|
|
963
|
+
bootstrapStream({channel, collection, isSuperseded, store, stream}),
|
|
964
|
+
});
|
|
965
|
+
},
|
|
966
|
+
streams: streams.map(({stream}) => stream),
|
|
967
|
+
});
|
|
968
|
+
};
|
|
969
|
+
|
|
853
970
|
const runReconcile = async (): Promise<void> => {
|
|
854
971
|
// Simulated offline severs all network activity, including the HTTP
|
|
855
972
|
// channel; an auth pause stands down every network trigger (INV-2) until
|
|
@@ -882,18 +999,12 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
882
999
|
// lifecycle/user while awaiting discovery.
|
|
883
1000
|
return;
|
|
884
1001
|
}
|
|
885
|
-
|
|
886
|
-
await runSnapshotPool({
|
|
1002
|
+
await pageStreams({
|
|
887
1003
|
isSuperseded,
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
runStreamPass({
|
|
893
|
-
mode: "join",
|
|
894
|
-
run: () => bootstrapStream({channel, collection, store, stream}),
|
|
895
|
-
stream,
|
|
896
|
-
}),
|
|
1004
|
+
mode: "join",
|
|
1005
|
+
streams: [...streams]
|
|
1006
|
+
.filter(([stream, collection]) => !skipSnapshotPaging({collection, stream}))
|
|
1007
|
+
.map(([stream, collection]) => ({collection, stream})),
|
|
897
1008
|
});
|
|
898
1009
|
if (isSuperseded()) {
|
|
899
1010
|
return;
|
|
@@ -962,6 +1073,67 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
962
1073
|
return promise;
|
|
963
1074
|
};
|
|
964
1075
|
|
|
1076
|
+
// Streams confirmed by `sync:subscribed` and not yet paged, tagged with the lifecycle
|
|
1077
|
+
// and user that confirmed them. One confirmation arrives per collection; those landing
|
|
1078
|
+
// while a catch-up request is in flight are paged together in the next one.
|
|
1079
|
+
const pendingCatchUp = new Map<
|
|
1080
|
+
string,
|
|
1081
|
+
{collection: string; generation: number; userId: string | undefined}
|
|
1082
|
+
>();
|
|
1083
|
+
let isCatchUpDraining = false;
|
|
1084
|
+
|
|
1085
|
+
const drainCatchUp = async (): Promise<void> => {
|
|
1086
|
+
if (isCatchUpDraining) {
|
|
1087
|
+
return;
|
|
1088
|
+
}
|
|
1089
|
+
isCatchUpDraining = true;
|
|
1090
|
+
addSyncing(1);
|
|
1091
|
+
try {
|
|
1092
|
+
while (pendingCatchUp.size > 0) {
|
|
1093
|
+
const myGeneration = generation;
|
|
1094
|
+
const myUserId = currentUserId;
|
|
1095
|
+
const isSuperseded = (): boolean =>
|
|
1096
|
+
generation !== myGeneration || currentUserId !== myUserId;
|
|
1097
|
+
const streams: {stream: string; collection: string}[] = [];
|
|
1098
|
+
for (const [stream, pending] of pendingCatchUp) {
|
|
1099
|
+
if (pending.generation === myGeneration && pending.userId === myUserId) {
|
|
1100
|
+
streams.push({collection: pending.collection, stream});
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
pendingCatchUp.clear();
|
|
1104
|
+
if (!httpChannel || simulatedOffline || authPaused) {
|
|
1105
|
+
return;
|
|
1106
|
+
}
|
|
1107
|
+
if (streams.length === 0) {
|
|
1108
|
+
continue;
|
|
1109
|
+
}
|
|
1110
|
+
await pageStreams({isSuperseded, mode: "fresh", streams});
|
|
1111
|
+
if (isSuperseded()) {
|
|
1112
|
+
return;
|
|
1113
|
+
}
|
|
1114
|
+
debugLog?.record({
|
|
1115
|
+
detail: {streams: streams.map(({stream}) => stream)},
|
|
1116
|
+
direction: "system",
|
|
1117
|
+
label: `subscribe catch-up: ${streams.map(({stream}) => stream).join(", ")}`,
|
|
1118
|
+
type: "reconcile",
|
|
1119
|
+
});
|
|
1120
|
+
}
|
|
1121
|
+
} catch (error) {
|
|
1122
|
+
if (error instanceof AuthRequiredError) {
|
|
1123
|
+
setAuthPaused(true);
|
|
1124
|
+
return;
|
|
1125
|
+
}
|
|
1126
|
+
throw error;
|
|
1127
|
+
} finally {
|
|
1128
|
+
isCatchUpDraining = false;
|
|
1129
|
+
addSyncing(-1);
|
|
1130
|
+
// Confirmations for a newer lifecycle can queue while a superseded pass unwinds.
|
|
1131
|
+
if (pendingCatchUp.size > 0) {
|
|
1132
|
+
void drainCatchUp().catch(warn("subscribe catch-up failed"));
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
};
|
|
1136
|
+
|
|
965
1137
|
/**
|
|
966
1138
|
* Catch up the streams the server just confirmed this socket joined
|
|
967
1139
|
* (`sync:subscribed`).
|
|
@@ -978,8 +1150,9 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
978
1150
|
* confirmed stream from its cursor here makes snapshot + delta coverage contiguous.
|
|
979
1151
|
* It deliberately does NOT go through `reconcile()`: that call coalesces onto a pass
|
|
980
1152
|
* which may have started before the join (exactly the pass that could have missed the
|
|
981
|
-
* write).
|
|
982
|
-
*
|
|
1153
|
+
* write). For the same reason a confirmation arriving while a catch-up request is in
|
|
1154
|
+
* flight waits for the next request rather than joining that one. Paging is
|
|
1155
|
+
* idempotent and cursor-guarded, so running it beside an in-flight reconcile is safe.
|
|
983
1156
|
*/
|
|
984
1157
|
const catchUpSubscribedStreams = async ({
|
|
985
1158
|
collection,
|
|
@@ -999,42 +1172,24 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
999
1172
|
if (mode === "window" || isWindowCollection(collection)) {
|
|
1000
1173
|
return;
|
|
1001
1174
|
}
|
|
1002
|
-
const
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
addSyncing(1);
|
|
1006
|
-
try {
|
|
1007
|
-
const channel = httpChannel;
|
|
1008
|
-
await runSnapshotPool({
|
|
1009
|
-
isSuperseded,
|
|
1010
|
-
items: streams.filter((stream) => !skipSnapshotPaging({collection, mode, stream})),
|
|
1011
|
-
worker: (stream) => {
|
|
1012
|
-
store.addKnownStream({collection, stream});
|
|
1013
|
-
return runStreamPass({
|
|
1014
|
-
mode: "fresh",
|
|
1015
|
-
run: () => bootstrapStream({channel, collection, store, stream}),
|
|
1016
|
-
stream,
|
|
1017
|
-
});
|
|
1018
|
-
},
|
|
1019
|
-
});
|
|
1020
|
-
if (isSuperseded()) {
|
|
1021
|
-
return;
|
|
1175
|
+
for (const stream of streams) {
|
|
1176
|
+
if (skipSnapshotPaging({collection, mode, stream})) {
|
|
1177
|
+
continue;
|
|
1022
1178
|
}
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1179
|
+
store.addKnownStream({collection, stream});
|
|
1180
|
+
// A queued pass that has not started yet already starts after this join.
|
|
1181
|
+
const queued = streamPasses.get(stream);
|
|
1182
|
+
if (
|
|
1183
|
+
queued &&
|
|
1184
|
+
!queued.started &&
|
|
1185
|
+
queued.generation === generation &&
|
|
1186
|
+
queued.userId === currentUserId
|
|
1187
|
+
) {
|
|
1188
|
+
continue;
|
|
1033
1189
|
}
|
|
1034
|
-
|
|
1035
|
-
} finally {
|
|
1036
|
-
addSyncing(-1);
|
|
1190
|
+
pendingCatchUp.set(stream, {collection, generation, userId: currentUserId});
|
|
1037
1191
|
}
|
|
1192
|
+
await drainCatchUp();
|
|
1038
1193
|
};
|
|
1039
1194
|
|
|
1040
1195
|
/**
|
|
@@ -1168,7 +1323,7 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
1168
1323
|
}
|
|
1169
1324
|
purged += store.purgeStream({stream});
|
|
1170
1325
|
store.addKnownStream({collection, stream});
|
|
1171
|
-
await bootstrapStream({channel, collection, store, stream});
|
|
1326
|
+
await bootstrapStream({channel, collection, isSuperseded, store, stream});
|
|
1172
1327
|
},
|
|
1173
1328
|
stream,
|
|
1174
1329
|
});
|
|
@@ -1425,23 +1580,14 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
1425
1580
|
// INV-2 auth-pause case (returns undefined on a 401) — skip bootstrap then.
|
|
1426
1581
|
const streams = await syncStreams();
|
|
1427
1582
|
if (streams) {
|
|
1428
|
-
|
|
1429
|
-
await runSnapshotPool({
|
|
1583
|
+
await pageStreams({
|
|
1430
1584
|
isSuperseded: () => false,
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
// again so this pass pages from 0 instead of resuming from that cursor.
|
|
1438
|
-
run: async () => {
|
|
1439
|
-
store.purgeStream({stream});
|
|
1440
|
-
store.addKnownStream({collection, stream});
|
|
1441
|
-
await bootstrapStream({channel, collection, store, stream});
|
|
1442
|
-
},
|
|
1443
|
-
stream,
|
|
1444
|
-
}),
|
|
1585
|
+
mode: "exclusive",
|
|
1586
|
+
// A pass that was paging when the wipe ran can write its cursor and
|
|
1587
|
+
// `bootstrapped` mark into the wiped store before this one starts. Purge
|
|
1588
|
+
// again so this pass pages from 0 instead of resuming from that cursor.
|
|
1589
|
+
purgeFirst: true,
|
|
1590
|
+
streams: [...streams].map(([stream, collection]) => ({collection, stream})),
|
|
1445
1591
|
});
|
|
1446
1592
|
}
|
|
1447
1593
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import "fake-indexeddb/auto";
|
|
2
2
|
|
|
3
|
-
import {describe, expect, it, mock} from "bun:test";
|
|
3
|
+
import {describe, expect, it, mock, spyOn} from "bun:test";
|
|
4
4
|
import {assert} from "chai";
|
|
5
5
|
|
|
6
6
|
import {createAesGcmCodec} from "../crypto/aesGcmCodec";
|
|
@@ -304,6 +304,49 @@ describe("createEncryptedIndexedDbPersister", () => {
|
|
|
304
304
|
expect(target.getEntity({collection: "todos", id: "t1"})?.data).toEqual({title: "three"});
|
|
305
305
|
});
|
|
306
306
|
|
|
307
|
+
it("serializes the store once per write, not once per queued autosave", async () => {
|
|
308
|
+
const databaseName = uniqueDbName();
|
|
309
|
+
let encodeCount = 0;
|
|
310
|
+
const countingCodec: PayloadCodec = {
|
|
311
|
+
decode: identityCodec.decode,
|
|
312
|
+
encode: (plaintext) => {
|
|
313
|
+
encodeCount += 1;
|
|
314
|
+
return identityCodec.encode(plaintext);
|
|
315
|
+
},
|
|
316
|
+
};
|
|
317
|
+
const source = makeStore();
|
|
318
|
+
const persister = createEncryptedIndexedDbPersister({
|
|
319
|
+
codec: countingCodec,
|
|
320
|
+
databaseName,
|
|
321
|
+
saveDebounceMs: 20,
|
|
322
|
+
store: source.raw,
|
|
323
|
+
});
|
|
324
|
+
await persister.startAutoSave();
|
|
325
|
+
const stringify = spyOn(JSON, "stringify");
|
|
326
|
+
// Each upsert is its own transaction, so TinyBase queues one autosave per row.
|
|
327
|
+
for (let i = 0; i < 50; i++) {
|
|
328
|
+
source.upsertEntity({collection: "todos", data: {title: `row ${i}`}, id: `t${i}`});
|
|
329
|
+
}
|
|
330
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
331
|
+
const storeSerializations = stringify.mock.calls.filter(
|
|
332
|
+
([value]) => Array.isArray(value) && value.length === 2
|
|
333
|
+
).length;
|
|
334
|
+
stringify.mockRestore();
|
|
335
|
+
// One write captures the whole burst; the 49 queued saves behind it are no-ops.
|
|
336
|
+
expect(encodeCount).toBe(2);
|
|
337
|
+
expect(storeSerializations).toBe(1);
|
|
338
|
+
await persister.destroy();
|
|
339
|
+
|
|
340
|
+
const target = makeStore();
|
|
341
|
+
await createEncryptedIndexedDbPersister({
|
|
342
|
+
codec: countingCodec,
|
|
343
|
+
databaseName,
|
|
344
|
+
saveDebounceMs: 0,
|
|
345
|
+
store: target.raw,
|
|
346
|
+
}).load();
|
|
347
|
+
expect(target.getEntity({collection: "todos", id: "t49"})?.data).toEqual({title: "row 49"});
|
|
348
|
+
});
|
|
349
|
+
|
|
307
350
|
it("flush() persists the latest store state without waiting for the save debounce", async () => {
|
|
308
351
|
const databaseName = uniqueDbName();
|
|
309
352
|
const codec = createAesGcmCodec({key: await generateKey()});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {MergeableContent, MergeableStore} from "tinybase";
|
|
1
|
+
import type {ContentHashes, MergeableContent, MergeableStore} from "tinybase";
|
|
2
2
|
import {createCustomPersister, type Persister, Persists} from "tinybase/persisters";
|
|
3
3
|
|
|
4
4
|
import type {PayloadCodec} from "../crypto/types";
|
|
@@ -11,6 +11,9 @@ const RECORD_KEY = "content";
|
|
|
11
11
|
/** Default trailing debounce applied to saves (autosave can fire per transaction). */
|
|
12
12
|
const DEFAULT_SAVE_DEBOUNCE_MS = 500;
|
|
13
13
|
|
|
14
|
+
const sameHashes = (a: ContentHashes, b: ContentHashes | undefined): boolean =>
|
|
15
|
+
b !== undefined && a[0] === b[0] && a[1] === b[1];
|
|
16
|
+
|
|
14
17
|
interface PendingSave {
|
|
15
18
|
promise: Promise<void>;
|
|
16
19
|
resolve: () => void;
|
|
@@ -35,9 +38,10 @@ interface PendingSave {
|
|
|
35
38
|
*
|
|
36
39
|
* Saves are debounced with a trailing edge (default 500ms) since TinyBase's
|
|
37
40
|
* autosave fires on every transaction: each write waits out the debounce
|
|
38
|
-
* window, and follow-up saves whose
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
+
* window, and follow-up saves whose content hashes match the last written
|
|
42
|
+
* snapshot are skipped, coalescing a burst of transactions into a single
|
|
43
|
+
* encode + IndexedDB put. The store is serialized only when a write actually
|
|
44
|
+
* happens, never per transaction.
|
|
41
45
|
*/
|
|
42
46
|
export const createEncryptedIndexedDbPersister = ({
|
|
43
47
|
store,
|
|
@@ -68,7 +72,9 @@ export const createEncryptedIndexedDbPersister = ({
|
|
|
68
72
|
/** Test-only override for the IndexedDB write (default: the real `idbSet`). */
|
|
69
73
|
idbSetImpl?: typeof idbSet;
|
|
70
74
|
}): Persister<Persists.MergeableStoreOnly> & SyncPersister => {
|
|
71
|
-
|
|
75
|
+
// Content hashes of the last snapshot that reached IndexedDB. TinyBase keeps
|
|
76
|
+
// them incrementally, so comparing is O(1) where re-serializing is O(store).
|
|
77
|
+
let lastWrittenHashes: ContentHashes | undefined;
|
|
72
78
|
let pendingSave: PendingSave | undefined;
|
|
73
79
|
// E3(e): the debounce timer scheduling `flushPendingSave` — tracked so
|
|
74
80
|
// `destroy()` can cancel it. Without this, a pending debounced save would
|
|
@@ -81,7 +87,7 @@ export const createEncryptedIndexedDbPersister = ({
|
|
|
81
87
|
// a newer snapshot's write.
|
|
82
88
|
let writeChain: Promise<void> = Promise.resolve();
|
|
83
89
|
|
|
84
|
-
const writeJson = (json: string): Promise<void> => {
|
|
90
|
+
const writeJson = (json: string, hashes: ContentHashes): Promise<void> => {
|
|
85
91
|
const write = writeChain.then(async () => {
|
|
86
92
|
// E3(e): destroy() may have run while this write was already chained
|
|
87
93
|
// behind an earlier one — never let it land after destroy() completes.
|
|
@@ -90,7 +96,7 @@ export const createEncryptedIndexedDbPersister = ({
|
|
|
90
96
|
}
|
|
91
97
|
const payload = await codec.encode(json);
|
|
92
98
|
await idbSetImpl({databaseName, key: RECORD_KEY, value: payload});
|
|
93
|
-
|
|
99
|
+
lastWrittenHashes = hashes;
|
|
94
100
|
});
|
|
95
101
|
// Keep the chain alive after a failed write; the failure still propagates
|
|
96
102
|
// to the caller through `write`.
|
|
@@ -107,7 +113,8 @@ export const createEncryptedIndexedDbPersister = ({
|
|
|
107
113
|
try {
|
|
108
114
|
// Trailing edge: snapshot the store at flush time so a burst of
|
|
109
115
|
// transactions during the debounce window lands as one fresh write.
|
|
110
|
-
|
|
116
|
+
const hashes = store.getMergeableContentHashes();
|
|
117
|
+
await writeJson(JSON.stringify(store.getMergeableContent()), hashes);
|
|
111
118
|
pending.resolve();
|
|
112
119
|
} catch (error) {
|
|
113
120
|
pending.reject(error);
|
|
@@ -126,12 +133,12 @@ export const createEncryptedIndexedDbPersister = ({
|
|
|
126
133
|
return;
|
|
127
134
|
}
|
|
128
135
|
|
|
129
|
-
const
|
|
130
|
-
if (
|
|
136
|
+
const hashes = store.getMergeableContentHashes();
|
|
137
|
+
if (sameHashes(hashes, lastWrittenHashes)) {
|
|
131
138
|
return;
|
|
132
139
|
}
|
|
133
140
|
try {
|
|
134
|
-
await writeJson(
|
|
141
|
+
await writeJson(JSON.stringify(store.getMergeableContent()), hashes);
|
|
135
142
|
} catch (error) {
|
|
136
143
|
onSaveFailure?.(error);
|
|
137
144
|
throw error;
|
|
@@ -168,20 +175,20 @@ export const createEncryptedIndexedDbPersister = ({
|
|
|
168
175
|
};
|
|
169
176
|
|
|
170
177
|
const setPersisted = async (getContent: () => MergeableContent): Promise<void> => {
|
|
171
|
-
const json = JSON.stringify(getContent());
|
|
172
178
|
if (saveDebounceMs <= 0) {
|
|
173
179
|
try {
|
|
174
|
-
await writeJson(
|
|
180
|
+
await writeJson(JSON.stringify(getContent()), store.getMergeableContentHashes());
|
|
175
181
|
} catch (error) {
|
|
176
182
|
onSaveFailure?.(error);
|
|
177
183
|
throw error;
|
|
178
184
|
}
|
|
179
185
|
return;
|
|
180
186
|
}
|
|
181
|
-
// TinyBase
|
|
182
|
-
//
|
|
183
|
-
//
|
|
184
|
-
|
|
187
|
+
// TinyBase queues one setPersisted per transaction and runs them one at a
|
|
188
|
+
// time, so a burst of N transactions arrives here N times, mostly after the
|
|
189
|
+
// write that already captured them. Skipping saves whose content already
|
|
190
|
+
// matches the last written snapshot is what coalesces the burst into one write.
|
|
191
|
+
if (!pendingSave && sameHashes(store.getMergeableContentHashes(), lastWrittenHashes)) {
|
|
185
192
|
return;
|
|
186
193
|
}
|
|
187
194
|
if (!pendingSave) {
|