@terreno/syncdb 57.13.0 → 57.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/dist/client.d.ts +12 -0
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +304 -56
- package/dist/client.js.map +1 -1
- package/dist/client.test.js +255 -0
- package/dist/client.test.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.d.ts +4 -3
- package/dist/persisters/encryptedIndexedDbPersister.d.ts.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.js +21 -16
- package/dist/persisters/encryptedIndexedDbPersister.js.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.test.js +40 -1
- package/dist/persisters/encryptedIndexedDbPersister.test.js.map +1 -1
- package/dist/sync/bootstrap.d.ts +34 -3
- package/dist/sync/bootstrap.d.ts.map +1 -1
- package/dist/sync/bootstrap.js +224 -71
- package/dist/sync/bootstrap.js.map +1 -1
- package/dist/sync/bootstrap.test.js +210 -2
- package/dist/sync/bootstrap.test.js.map +1 -1
- package/dist/sync/httpChannel.d.ts +14 -1
- package/dist/sync/httpChannel.d.ts.map +1 -1
- package/dist/sync/httpChannel.js +34 -0
- package/dist/sync/httpChannel.js.map +1 -1
- package/dist/sync/httpChannel.test.js +54 -0
- package/dist/sync/httpChannel.test.js.map +1 -1
- package/dist/types.d.ts +14 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/client.test.ts +314 -1
- package/src/client.ts +362 -54
- package/src/index.ts +1 -0
- package/src/persisters/encryptedIndexedDbPersister.test.ts +44 -1
- package/src/persisters/encryptedIndexedDbPersister.ts +24 -17
- package/src/sync/bootstrap.test.ts +269 -3
- package/src/sync/bootstrap.ts +336 -82
- package/src/sync/httpChannel.test.ts +67 -0
- package/src/sync/httpChannel.ts +50 -0
- package/src/types.ts +12 -0
package/src/client.ts
CHANGED
|
@@ -16,7 +16,7 @@ import {SYNC_SCHEMA_VERSION} from "./storage/schema";
|
|
|
16
16
|
import {createSyncStore, type SyncStore} from "./storage/store";
|
|
17
17
|
import {CURSORS_TABLE, KNOWN_STREAMS_TABLE} from "./storage/types";
|
|
18
18
|
import {wipeLocalData} from "./storage/wipe";
|
|
19
|
-
import {bootstrapStream} from "./sync/bootstrap";
|
|
19
|
+
import {bootstrapStream, bootstrapStreamsBatch} from "./sync/bootstrap";
|
|
20
20
|
import {getAllCursors} from "./sync/cursor";
|
|
21
21
|
import {applyDelta} from "./sync/deltaApplier";
|
|
22
22
|
import {repairMarkedEntities} from "./sync/entityRepair";
|
|
@@ -43,9 +43,25 @@ export const DEFAULT_RECONCILE_INTERVAL_MS = 5 * 60_000;
|
|
|
43
43
|
/** Minimum interval between seq-jump-triggered reconciles per stream (30s). */
|
|
44
44
|
export const DEFAULT_SEQ_JUMP_RECONCILE_MIN_INTERVAL_MS = 30_000;
|
|
45
45
|
|
|
46
|
+
/**
|
|
47
|
+
* Streams snapshot-paged at once by reconcile, subscribe catch-up, force resync and
|
|
48
|
+
* wipe re-bootstrap. Every stream costs at least one `GET /sync/snapshot` round trip,
|
|
49
|
+
* so paging them one at a time made cold start scale with collections × scopes.
|
|
50
|
+
*/
|
|
51
|
+
export const DEFAULT_SNAPSHOT_CONCURRENCY = 6;
|
|
52
|
+
|
|
46
53
|
/** E5: default local tombstone retention window (90 days), matching the server's (C7). */
|
|
47
54
|
export const DEFAULT_TOMBSTONE_RETENTION_MS = 90 * 24 * 60 * 60 * 1_000;
|
|
48
55
|
|
|
56
|
+
/**
|
|
57
|
+
* Entities one `POST /sync/snapshot/batch` response may carry, shared across its streams.
|
|
58
|
+
* Bounds both the response size (about 2 MB of typical rows) and the work per round.
|
|
59
|
+
*/
|
|
60
|
+
const SNAPSHOT_BATCH_LIMIT = 5_000;
|
|
61
|
+
|
|
62
|
+
/** The server has no `POST /sync/snapshot/batch` route (404). */
|
|
63
|
+
class SnapshotBatchUnsupportedError extends Error {}
|
|
64
|
+
|
|
49
65
|
/**
|
|
50
66
|
* `start()` auth-resolution retry defaults. A host app only calls `start()` once it
|
|
51
67
|
* already believes the user is authenticated (e.g. right after its own login flow
|
|
@@ -136,6 +152,12 @@ export interface SyncDbConfig {
|
|
|
136
152
|
wipeOnSignOut?: boolean;
|
|
137
153
|
/** Max mutations per batched drain send (B3, default 50; server caps at 100). */
|
|
138
154
|
batchSize?: number;
|
|
155
|
+
/**
|
|
156
|
+
* Max streams snapshot-paged at once (default {@link DEFAULT_SNAPSHOT_CONCURRENCY}).
|
|
157
|
+
* Each stream still pages its own pages in order; this bounds how many streams page
|
|
158
|
+
* side by side. `1` restores one-stream-at-a-time paging.
|
|
159
|
+
*/
|
|
160
|
+
snapshotConcurrency?: number;
|
|
139
161
|
/**
|
|
140
162
|
* B4: when true, a conflict halts the ENTIRE drain instead of just blocking
|
|
141
163
|
* its entity. Default false (per-entity blocking — see README "Conflict
|
|
@@ -427,6 +449,17 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
427
449
|
| {generation: number; promise: Promise<void>; userId: string | undefined}
|
|
428
450
|
| undefined;
|
|
429
451
|
const lastSeqJumpReconcileAt = new Map<string, number>();
|
|
452
|
+
/**
|
|
453
|
+
* The latest snapshot pass per stream (single-flight). Two passes paging the same
|
|
454
|
+
* stream at once double the requests, and a purge in one (force resync, retention
|
|
455
|
+
* gap, leave) lets the other write its high cursor back and mark the stream
|
|
456
|
+
* bootstrapped over rows it never paged. Tagged with generation + user like
|
|
457
|
+
* `inFlightReconcile`, so a pass from a superseded lifecycle is never joined.
|
|
458
|
+
*/
|
|
459
|
+
const streamPasses = new Map<
|
|
460
|
+
string,
|
|
461
|
+
{generation: number; promise: Promise<void>; started: boolean; userId: string | undefined}
|
|
462
|
+
>();
|
|
430
463
|
const statusListeners = new Set<() => void>();
|
|
431
464
|
// A4: auth-pause state. `authPaused` gates reconcile/replay/timer triggers
|
|
432
465
|
// (INV-2 — nothing else may unpause except a same-user auth change).
|
|
@@ -434,6 +467,9 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
434
467
|
// refresh has already run for the CURRENT pause episode.
|
|
435
468
|
let authPaused = false;
|
|
436
469
|
let refreshAttempted = false;
|
|
470
|
+
// Set once the server answers 404 to the snapshot batch route; per-stream paging is
|
|
471
|
+
// used for the rest of this client's life.
|
|
472
|
+
let snapshotBatchUnsupported = false;
|
|
437
473
|
// E1: lifecycle serialization. `generation` bumps on every start()/stop() so
|
|
438
474
|
// an in-flight operation that resumes after an await can detect it has been
|
|
439
475
|
// superseded (a rapid stop()-then-start() for a new user) and abort instead
|
|
@@ -487,6 +523,132 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
487
523
|
}
|
|
488
524
|
};
|
|
489
525
|
|
|
526
|
+
/**
|
|
527
|
+
* Run a snapshot pass over `streams`, single-flight per stream. Each stream is either
|
|
528
|
+
* joined to its latest pass or owned by this one; `run` receives the owned streams once
|
|
529
|
+
* every pass it queued behind has settled.
|
|
530
|
+
*
|
|
531
|
+
* - `join`: share the stream's latest pass when one exists (reconcile — any pass to
|
|
532
|
+
* the head will do).
|
|
533
|
+
* - `fresh`: the pass must start after this call (subscribe catch-up — a pass that
|
|
534
|
+
* began before the room join may have missed a write). Joins a queued pass that has
|
|
535
|
+
* not started yet, otherwise queues behind the running one.
|
|
536
|
+
* - `exclusive`: always queue behind whatever is in flight (force resync and wipe
|
|
537
|
+
* re-bootstrap purge first, which must never overlap another pass's paging).
|
|
538
|
+
*/
|
|
539
|
+
const runStreamsPass = ({
|
|
540
|
+
mode,
|
|
541
|
+
run,
|
|
542
|
+
streams,
|
|
543
|
+
}: {
|
|
544
|
+
mode: "join" | "fresh" | "exclusive";
|
|
545
|
+
run: (owned: string[]) => Promise<void>;
|
|
546
|
+
streams: string[];
|
|
547
|
+
}): Promise<void> => {
|
|
548
|
+
const joined: Promise<void>[] = [];
|
|
549
|
+
const owned: string[] = [];
|
|
550
|
+
const previous: Promise<void>[] = [];
|
|
551
|
+
for (const stream of streams) {
|
|
552
|
+
const existing = streamPasses.get(stream);
|
|
553
|
+
const current =
|
|
554
|
+
existing && existing.generation === generation && existing.userId === currentUserId
|
|
555
|
+
? existing
|
|
556
|
+
: undefined;
|
|
557
|
+
if (current && (mode === "join" || (mode === "fresh" && !current.started))) {
|
|
558
|
+
joined.push(current.promise);
|
|
559
|
+
continue;
|
|
560
|
+
}
|
|
561
|
+
owned.push(stream);
|
|
562
|
+
if (existing) {
|
|
563
|
+
previous.push(existing.promise);
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
if (owned.length === 0) {
|
|
567
|
+
return Promise.all(joined).then(() => undefined);
|
|
568
|
+
}
|
|
569
|
+
const entry = {generation, promise: Promise.resolve(), started: false, userId: currentUserId};
|
|
570
|
+
entry.promise = (async (): Promise<void> => {
|
|
571
|
+
// The previous passes' failures belong to their own callers.
|
|
572
|
+
await Promise.all(previous.map((pass) => pass.catch(() => undefined)));
|
|
573
|
+
// A stop()/start() or user switch while this pass was queued supersedes it:
|
|
574
|
+
// paging now would write into a store that belongs to a newer lifecycle.
|
|
575
|
+
if (entry.generation !== generation || entry.userId !== currentUserId) {
|
|
576
|
+
return;
|
|
577
|
+
}
|
|
578
|
+
entry.started = true;
|
|
579
|
+
await run(owned);
|
|
580
|
+
})().finally(() => {
|
|
581
|
+
for (const stream of owned) {
|
|
582
|
+
if (streamPasses.get(stream) === entry) {
|
|
583
|
+
streamPasses.delete(stream);
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
});
|
|
587
|
+
for (const stream of owned) {
|
|
588
|
+
streamPasses.set(stream, entry);
|
|
589
|
+
}
|
|
590
|
+
return Promise.all([entry.promise, ...joined]).then(() => undefined);
|
|
591
|
+
};
|
|
592
|
+
|
|
593
|
+
/** {@link runStreamsPass} for a single stream. */
|
|
594
|
+
const runStreamPass = ({
|
|
595
|
+
mode,
|
|
596
|
+
run,
|
|
597
|
+
stream,
|
|
598
|
+
}: {
|
|
599
|
+
mode: "join" | "fresh" | "exclusive";
|
|
600
|
+
run: () => Promise<void>;
|
|
601
|
+
stream: string;
|
|
602
|
+
}): Promise<void> => runStreamsPass({mode, run: () => run(), streams: [stream]});
|
|
603
|
+
|
|
604
|
+
/**
|
|
605
|
+
* Run `worker` over `items` with at most `snapshotConcurrency` in flight. Stops taking
|
|
606
|
+
* new items once `isSuperseded()` or any worker throws, waits for the in-flight ones,
|
|
607
|
+
* then rethrows — `AuthRequiredError` first, so callers still auth-pause on a 401 that
|
|
608
|
+
* raced another failure.
|
|
609
|
+
*/
|
|
610
|
+
const runSnapshotPool = async <T>({
|
|
611
|
+
isSuperseded,
|
|
612
|
+
items,
|
|
613
|
+
worker,
|
|
614
|
+
}: {
|
|
615
|
+
isSuperseded: () => boolean;
|
|
616
|
+
items: T[];
|
|
617
|
+
worker: (item: T) => Promise<void>;
|
|
618
|
+
}): Promise<void> => {
|
|
619
|
+
const limit = Math.max(
|
|
620
|
+
1,
|
|
621
|
+
Math.floor(config.snapshotConcurrency ?? DEFAULT_SNAPSHOT_CONCURRENCY)
|
|
622
|
+
);
|
|
623
|
+
let next = 0;
|
|
624
|
+
let authError: AuthRequiredError | undefined;
|
|
625
|
+
let firstError: unknown;
|
|
626
|
+
let failed = false;
|
|
627
|
+
const runWorker = async (): Promise<void> => {
|
|
628
|
+
while (next < items.length && !failed && !isSuperseded()) {
|
|
629
|
+
const item = items[next] as T;
|
|
630
|
+
next += 1;
|
|
631
|
+
try {
|
|
632
|
+
await worker(item);
|
|
633
|
+
} catch (error) {
|
|
634
|
+
failed = true;
|
|
635
|
+
if (error instanceof AuthRequiredError) {
|
|
636
|
+
authError ??= error;
|
|
637
|
+
} else if (firstError === undefined) {
|
|
638
|
+
firstError = error;
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
};
|
|
643
|
+
await Promise.all(Array.from({length: Math.min(limit, items.length)}, runWorker));
|
|
644
|
+
if (authError) {
|
|
645
|
+
throw authError;
|
|
646
|
+
}
|
|
647
|
+
if (failed) {
|
|
648
|
+
throw firstError;
|
|
649
|
+
}
|
|
650
|
+
};
|
|
651
|
+
|
|
490
652
|
const notifyStatusChange = (): void => {
|
|
491
653
|
for (const listener of statusListeners) {
|
|
492
654
|
listener();
|
|
@@ -727,6 +889,84 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
727
889
|
return serverSet;
|
|
728
890
|
};
|
|
729
891
|
|
|
892
|
+
/**
|
|
893
|
+
* Page `streams` to their heads as one single-flight pass (see {@link runStreamsPass}).
|
|
894
|
+
* One `POST /sync/snapshot/batch` round trip per round covers every owned stream;
|
|
895
|
+
* against a server without that route, falls back to `GET /sync/snapshot` per stream,
|
|
896
|
+
* `snapshotConcurrency` streams at a time.
|
|
897
|
+
*/
|
|
898
|
+
const pageStreams = async ({
|
|
899
|
+
streams,
|
|
900
|
+
mode,
|
|
901
|
+
isSuperseded,
|
|
902
|
+
purgeFirst = false,
|
|
903
|
+
}: {
|
|
904
|
+
streams: {stream: string; collection: string}[];
|
|
905
|
+
mode: "join" | "fresh" | "exclusive";
|
|
906
|
+
isSuperseded: () => boolean;
|
|
907
|
+
/** Purge each owned stream before paging it from 0 (wipe re-bootstrap). */
|
|
908
|
+
purgeFirst?: boolean;
|
|
909
|
+
}): Promise<void> => {
|
|
910
|
+
const channel = httpChannel;
|
|
911
|
+
if (!channel || streams.length === 0) {
|
|
912
|
+
return;
|
|
913
|
+
}
|
|
914
|
+
const collections = new Map(streams.map(({stream, collection}) => [stream, collection]));
|
|
915
|
+
await runStreamsPass({
|
|
916
|
+
mode,
|
|
917
|
+
run: async (owned) => {
|
|
918
|
+
const ownedStreams = owned.map((stream) => ({
|
|
919
|
+
collection: collections.get(stream) as string,
|
|
920
|
+
stream,
|
|
921
|
+
}));
|
|
922
|
+
if (purgeFirst) {
|
|
923
|
+
for (const {stream, collection} of ownedStreams) {
|
|
924
|
+
store.purgeStream({stream});
|
|
925
|
+
store.addKnownStream({collection, stream});
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
const fetchSnapshotBatch = channel.fetchSnapshotBatch;
|
|
929
|
+
if (fetchSnapshotBatch && !snapshotBatchUnsupported) {
|
|
930
|
+
try {
|
|
931
|
+
await bootstrapStreamsBatch({
|
|
932
|
+
fetchSnapshotBatch: async (request) => {
|
|
933
|
+
const result = await fetchSnapshotBatch(request);
|
|
934
|
+
if (result.type === "unsupported") {
|
|
935
|
+
throw new SnapshotBatchUnsupportedError();
|
|
936
|
+
}
|
|
937
|
+
return result;
|
|
938
|
+
},
|
|
939
|
+
isSuperseded,
|
|
940
|
+
limit: SNAPSHOT_BATCH_LIMIT,
|
|
941
|
+
store,
|
|
942
|
+
streams: ownedStreams,
|
|
943
|
+
});
|
|
944
|
+
return;
|
|
945
|
+
} catch (error) {
|
|
946
|
+
if (!(error instanceof SnapshotBatchUnsupportedError)) {
|
|
947
|
+
throw error;
|
|
948
|
+
}
|
|
949
|
+
// Every round before the 404 is already written, so per-stream paging
|
|
950
|
+
// resumes from the store's cursors without losing anything.
|
|
951
|
+
snapshotBatchUnsupported = true;
|
|
952
|
+
debugLog?.record({
|
|
953
|
+
direction: "system",
|
|
954
|
+
label: "snapshot batch unsupported by server; paging per stream",
|
|
955
|
+
type: "reconcile",
|
|
956
|
+
});
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
await runSnapshotPool({
|
|
960
|
+
isSuperseded,
|
|
961
|
+
items: ownedStreams,
|
|
962
|
+
worker: ({stream, collection}) =>
|
|
963
|
+
bootstrapStream({channel, collection, isSuperseded, store, stream}),
|
|
964
|
+
});
|
|
965
|
+
},
|
|
966
|
+
streams: streams.map(({stream}) => stream),
|
|
967
|
+
});
|
|
968
|
+
};
|
|
969
|
+
|
|
730
970
|
const runReconcile = async (): Promise<void> => {
|
|
731
971
|
// Simulated offline severs all network activity, including the HTTP
|
|
732
972
|
// channel; an auth pause stands down every network trigger (INV-2) until
|
|
@@ -759,14 +999,15 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
759
999
|
// lifecycle/user while awaiting discovery.
|
|
760
1000
|
return;
|
|
761
1001
|
}
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
1002
|
+
await pageStreams({
|
|
1003
|
+
isSuperseded,
|
|
1004
|
+
mode: "join",
|
|
1005
|
+
streams: [...streams]
|
|
1006
|
+
.filter(([stream, collection]) => !skipSnapshotPaging({collection, stream}))
|
|
1007
|
+
.map(([stream, collection]) => ({collection, stream})),
|
|
1008
|
+
});
|
|
1009
|
+
if (isSuperseded()) {
|
|
1010
|
+
return;
|
|
770
1011
|
}
|
|
771
1012
|
// E5: client-side compaction runs only after a successful per-stream reconcile —
|
|
772
1013
|
// reconcile just proved the local store caught up with the server via real network
|
|
@@ -832,6 +1073,67 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
832
1073
|
return promise;
|
|
833
1074
|
};
|
|
834
1075
|
|
|
1076
|
+
// Streams confirmed by `sync:subscribed` and not yet paged, tagged with the lifecycle
|
|
1077
|
+
// and user that confirmed them. One confirmation arrives per collection; those landing
|
|
1078
|
+
// while a catch-up request is in flight are paged together in the next one.
|
|
1079
|
+
const pendingCatchUp = new Map<
|
|
1080
|
+
string,
|
|
1081
|
+
{collection: string; generation: number; userId: string | undefined}
|
|
1082
|
+
>();
|
|
1083
|
+
let isCatchUpDraining = false;
|
|
1084
|
+
|
|
1085
|
+
const drainCatchUp = async (): Promise<void> => {
|
|
1086
|
+
if (isCatchUpDraining) {
|
|
1087
|
+
return;
|
|
1088
|
+
}
|
|
1089
|
+
isCatchUpDraining = true;
|
|
1090
|
+
addSyncing(1);
|
|
1091
|
+
try {
|
|
1092
|
+
while (pendingCatchUp.size > 0) {
|
|
1093
|
+
const myGeneration = generation;
|
|
1094
|
+
const myUserId = currentUserId;
|
|
1095
|
+
const isSuperseded = (): boolean =>
|
|
1096
|
+
generation !== myGeneration || currentUserId !== myUserId;
|
|
1097
|
+
const streams: {stream: string; collection: string}[] = [];
|
|
1098
|
+
for (const [stream, pending] of pendingCatchUp) {
|
|
1099
|
+
if (pending.generation === myGeneration && pending.userId === myUserId) {
|
|
1100
|
+
streams.push({collection: pending.collection, stream});
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
pendingCatchUp.clear();
|
|
1104
|
+
if (!httpChannel || simulatedOffline || authPaused) {
|
|
1105
|
+
return;
|
|
1106
|
+
}
|
|
1107
|
+
if (streams.length === 0) {
|
|
1108
|
+
continue;
|
|
1109
|
+
}
|
|
1110
|
+
await pageStreams({isSuperseded, mode: "fresh", streams});
|
|
1111
|
+
if (isSuperseded()) {
|
|
1112
|
+
return;
|
|
1113
|
+
}
|
|
1114
|
+
debugLog?.record({
|
|
1115
|
+
detail: {streams: streams.map(({stream}) => stream)},
|
|
1116
|
+
direction: "system",
|
|
1117
|
+
label: `subscribe catch-up: ${streams.map(({stream}) => stream).join(", ")}`,
|
|
1118
|
+
type: "reconcile",
|
|
1119
|
+
});
|
|
1120
|
+
}
|
|
1121
|
+
} catch (error) {
|
|
1122
|
+
if (error instanceof AuthRequiredError) {
|
|
1123
|
+
setAuthPaused(true);
|
|
1124
|
+
return;
|
|
1125
|
+
}
|
|
1126
|
+
throw error;
|
|
1127
|
+
} finally {
|
|
1128
|
+
isCatchUpDraining = false;
|
|
1129
|
+
addSyncing(-1);
|
|
1130
|
+
// Confirmations for a newer lifecycle can queue while a superseded pass unwinds.
|
|
1131
|
+
if (pendingCatchUp.size > 0) {
|
|
1132
|
+
void drainCatchUp().catch(warn("subscribe catch-up failed"));
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
};
|
|
1136
|
+
|
|
835
1137
|
/**
|
|
836
1138
|
* Catch up the streams the server just confirmed this socket joined
|
|
837
1139
|
* (`sync:subscribed`).
|
|
@@ -848,8 +1150,9 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
848
1150
|
* confirmed stream from its cursor here makes snapshot + delta coverage contiguous.
|
|
849
1151
|
* It deliberately does NOT go through `reconcile()`: that call coalesces onto a pass
|
|
850
1152
|
* which may have started before the join (exactly the pass that could have missed the
|
|
851
|
-
* write).
|
|
852
|
-
*
|
|
1153
|
+
* write). For the same reason a confirmation arriving while a catch-up request is in
|
|
1154
|
+
* flight waits for the next request rather than joining that one. Paging is
|
|
1155
|
+
* idempotent and cursor-guarded, so running it beside an in-flight reconcile is safe.
|
|
853
1156
|
*/
|
|
854
1157
|
const catchUpSubscribedStreams = async ({
|
|
855
1158
|
collection,
|
|
@@ -869,36 +1172,24 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
869
1172
|
if (mode === "window" || isWindowCollection(collection)) {
|
|
870
1173
|
return;
|
|
871
1174
|
}
|
|
872
|
-
const
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
addSyncing(1);
|
|
876
|
-
try {
|
|
877
|
-
for (const stream of streams) {
|
|
878
|
-
if (isSuperseded()) {
|
|
879
|
-
return;
|
|
880
|
-
}
|
|
881
|
-
if (skipSnapshotPaging({collection, mode, stream})) {
|
|
882
|
-
continue;
|
|
883
|
-
}
|
|
884
|
-
store.addKnownStream({collection, stream});
|
|
885
|
-
await bootstrapStream({channel: httpChannel, collection, store, stream});
|
|
1175
|
+
for (const stream of streams) {
|
|
1176
|
+
if (skipSnapshotPaging({collection, mode, stream})) {
|
|
1177
|
+
continue;
|
|
886
1178
|
}
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
1179
|
+
store.addKnownStream({collection, stream});
|
|
1180
|
+
// A queued pass that has not started yet already starts after this join.
|
|
1181
|
+
const queued = streamPasses.get(stream);
|
|
1182
|
+
if (
|
|
1183
|
+
queued &&
|
|
1184
|
+
!queued.started &&
|
|
1185
|
+
queued.generation === generation &&
|
|
1186
|
+
queued.userId === currentUserId
|
|
1187
|
+
) {
|
|
1188
|
+
continue;
|
|
897
1189
|
}
|
|
898
|
-
|
|
899
|
-
} finally {
|
|
900
|
-
addSyncing(-1);
|
|
1190
|
+
pendingCatchUp.set(stream, {collection, generation, userId: currentUserId});
|
|
901
1191
|
}
|
|
1192
|
+
await drainCatchUp();
|
|
902
1193
|
};
|
|
903
1194
|
|
|
904
1195
|
/**
|
|
@@ -1017,19 +1308,30 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
1017
1308
|
for (const collection of new Set(streamInfos.map((info) => info.collection))) {
|
|
1018
1309
|
purged += store.purgeUnknownStreamEntities({collection});
|
|
1019
1310
|
}
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1311
|
+
const channel = httpChannel;
|
|
1312
|
+
await runSnapshotPool({
|
|
1313
|
+
isSuperseded,
|
|
1314
|
+
// Query collections have no snapshot to rebuild from; their retained windows
|
|
1315
|
+
// are refetched below instead.
|
|
1316
|
+
items: streamInfos.filter(({collection}) => !isQueryCollection(collection)),
|
|
1317
|
+
worker: async ({stream, collection}) => {
|
|
1318
|
+
await runStreamPass({
|
|
1319
|
+
mode: "exclusive",
|
|
1320
|
+
run: async () => {
|
|
1321
|
+
if (isSuperseded()) {
|
|
1322
|
+
return;
|
|
1323
|
+
}
|
|
1324
|
+
purged += store.purgeStream({stream});
|
|
1325
|
+
store.addKnownStream({collection, stream});
|
|
1326
|
+
await bootstrapStream({channel, collection, isSuperseded, store, stream});
|
|
1327
|
+
},
|
|
1328
|
+
stream,
|
|
1329
|
+
});
|
|
1330
|
+
await waitForLifecycleIdle();
|
|
1331
|
+
},
|
|
1332
|
+
});
|
|
1333
|
+
if (isSuperseded()) {
|
|
1334
|
+
return {ok: false, purged, reason: "superseded", repaired, streams: streamInfos.length};
|
|
1033
1335
|
}
|
|
1034
1336
|
repaired = await repairAllMarkedEntities();
|
|
1035
1337
|
await queryWindows.refetchRetained();
|
|
@@ -1278,9 +1580,15 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
1278
1580
|
// INV-2 auth-pause case (returns undefined on a 401) — skip bootstrap then.
|
|
1279
1581
|
const streams = await syncStreams();
|
|
1280
1582
|
if (streams) {
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1583
|
+
await pageStreams({
|
|
1584
|
+
isSuperseded: () => false,
|
|
1585
|
+
mode: "exclusive",
|
|
1586
|
+
// A pass that was paging when the wipe ran can write its cursor and
|
|
1587
|
+
// `bootstrapped` mark into the wiped store before this one starts. Purge
|
|
1588
|
+
// again so this pass pages from 0 instead of resuming from that cursor.
|
|
1589
|
+
purgeFirst: true,
|
|
1590
|
+
streams: [...streams].map(([stream, collection]) => ({collection, stream})),
|
|
1591
|
+
});
|
|
1284
1592
|
}
|
|
1285
1593
|
}
|
|
1286
1594
|
};
|
package/src/index.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import "fake-indexeddb/auto";
|
|
2
2
|
|
|
3
|
-
import {describe, expect, it, mock} from "bun:test";
|
|
3
|
+
import {describe, expect, it, mock, spyOn} from "bun:test";
|
|
4
4
|
import {assert} from "chai";
|
|
5
5
|
|
|
6
6
|
import {createAesGcmCodec} from "../crypto/aesGcmCodec";
|
|
@@ -304,6 +304,49 @@ describe("createEncryptedIndexedDbPersister", () => {
|
|
|
304
304
|
expect(target.getEntity({collection: "todos", id: "t1"})?.data).toEqual({title: "three"});
|
|
305
305
|
});
|
|
306
306
|
|
|
307
|
+
it("serializes the store once per write, not once per queued autosave", async () => {
|
|
308
|
+
const databaseName = uniqueDbName();
|
|
309
|
+
let encodeCount = 0;
|
|
310
|
+
const countingCodec: PayloadCodec = {
|
|
311
|
+
decode: identityCodec.decode,
|
|
312
|
+
encode: (plaintext) => {
|
|
313
|
+
encodeCount += 1;
|
|
314
|
+
return identityCodec.encode(plaintext);
|
|
315
|
+
},
|
|
316
|
+
};
|
|
317
|
+
const source = makeStore();
|
|
318
|
+
const persister = createEncryptedIndexedDbPersister({
|
|
319
|
+
codec: countingCodec,
|
|
320
|
+
databaseName,
|
|
321
|
+
saveDebounceMs: 20,
|
|
322
|
+
store: source.raw,
|
|
323
|
+
});
|
|
324
|
+
await persister.startAutoSave();
|
|
325
|
+
const stringify = spyOn(JSON, "stringify");
|
|
326
|
+
// Each upsert is its own transaction, so TinyBase queues one autosave per row.
|
|
327
|
+
for (let i = 0; i < 50; i++) {
|
|
328
|
+
source.upsertEntity({collection: "todos", data: {title: `row ${i}`}, id: `t${i}`});
|
|
329
|
+
}
|
|
330
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
331
|
+
const storeSerializations = stringify.mock.calls.filter(
|
|
332
|
+
([value]) => Array.isArray(value) && value.length === 2
|
|
333
|
+
).length;
|
|
334
|
+
stringify.mockRestore();
|
|
335
|
+
// One write captures the whole burst; the 49 queued saves behind it are no-ops.
|
|
336
|
+
expect(encodeCount).toBe(2);
|
|
337
|
+
expect(storeSerializations).toBe(1);
|
|
338
|
+
await persister.destroy();
|
|
339
|
+
|
|
340
|
+
const target = makeStore();
|
|
341
|
+
await createEncryptedIndexedDbPersister({
|
|
342
|
+
codec: countingCodec,
|
|
343
|
+
databaseName,
|
|
344
|
+
saveDebounceMs: 0,
|
|
345
|
+
store: target.raw,
|
|
346
|
+
}).load();
|
|
347
|
+
expect(target.getEntity({collection: "todos", id: "t49"})?.data).toEqual({title: "row 49"});
|
|
348
|
+
});
|
|
349
|
+
|
|
307
350
|
it("flush() persists the latest store state without waiting for the save debounce", async () => {
|
|
308
351
|
const databaseName = uniqueDbName();
|
|
309
352
|
const codec = createAesGcmCodec({key: await generateKey()});
|