@pylonsync/sync 0.22.50 → 0.22.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +98 -0
- package/dist/test-harness/env.d.ts +2 -0
- package/dist/test-harness/server.d.ts +20 -0
- package/dist/types.d.ts +10 -0
- package/package.json +1 -1
- package/src/first-sync-requests.test.ts +147 -0
- package/src/index.ts +342 -65
- package/src/session-before-pull.test.ts +51 -0
- package/src/session-dedupe.test.ts +77 -0
- package/src/snapshot-catch-up.test.ts +101 -0
- package/src/test-harness/env.ts +3 -0
- package/src/test-harness/server.ts +54 -0
- package/src/test-harness/transport.ts +9 -0
- package/src/types.ts +10 -0
package/src/index.ts
CHANGED
|
@@ -89,6 +89,25 @@ export {
|
|
|
89
89
|
/** The store channel the outbox notifies on. `store.subscribe(cb,
|
|
90
90
|
* OUTBOX_CHANNEL)` hears outbox changes and no entity changes. */
|
|
91
91
|
export const OUTBOX_CHANNEL = "$outbox";
|
|
92
|
+
|
|
93
|
+
/** The store channel `syncProgress()` changes notify on. */
|
|
94
|
+
export const SYNC_PROGRESS_CHANNEL = "$sync-progress";
|
|
95
|
+
|
|
96
|
+
/** How far the current pull has got. See `SyncEngine.syncProgress()`. */
|
|
97
|
+
export interface SyncProgress {
|
|
98
|
+
/** A pull is running. */
|
|
99
|
+
pulling: boolean;
|
|
100
|
+
/** The pull is a snapshot of current rows (first sync, or a far-behind
|
|
101
|
+
* catch-up) rather than the change log. */
|
|
102
|
+
snapshot: boolean;
|
|
103
|
+
/** Rows received by this pull so far. */
|
|
104
|
+
rows: number;
|
|
105
|
+
/** Pages received by this pull so far. */
|
|
106
|
+
pages: number;
|
|
107
|
+
/** A pull has completed against the server since the last replica
|
|
108
|
+
* reset (`isSynced()`). */
|
|
109
|
+
synced: boolean;
|
|
110
|
+
}
|
|
92
111
|
export { generateId } from "./ids";
|
|
93
112
|
export {
|
|
94
113
|
uploadBytes,
|
|
@@ -205,6 +224,19 @@ export interface SyncEngineConfig {
|
|
|
205
224
|
* becomes visible).
|
|
206
225
|
*/
|
|
207
226
|
reconcileOnVisibility?: boolean;
|
|
227
|
+
/**
|
|
228
|
+
* The automatic sweep (on connect and when the tab becomes visible)
|
|
229
|
+
* downloads every replicated row to catch changes the change log does not
|
|
230
|
+
* carry. It runs only when the app was offline (the connection dropped,
|
|
231
|
+
* or it started from a cached replica) and the replica last matched the
|
|
232
|
+
* server (a complete snapshot, or a full sweep) at least this long ago;
|
|
233
|
+
* or after a snapshot that could not remove deleted rows itself. It
|
|
234
|
+
* never runs right after a complete snapshot or merely because the tab
|
|
235
|
+
* came to the foreground. Server hints (`sync-reconcile`) reconcile at
|
|
236
|
+
* once, and an explicit `reconcile()` call always runs. Default 300000
|
|
237
|
+
* (5 minutes).
|
|
238
|
+
*/
|
|
239
|
+
autoReconcileIntervalMs?: number;
|
|
208
240
|
/**
|
|
209
241
|
* Keep the local replica across ORG switches (same user). Default
|
|
210
242
|
* `true`: a tenant flip wipes the replica and re-bootstraps, because
|
|
@@ -350,10 +382,46 @@ export class SyncEngine {
|
|
|
350
382
|
return this._synced;
|
|
351
383
|
}
|
|
352
384
|
|
|
385
|
+
private _progress: SyncProgress = {
|
|
386
|
+
pulling: false,
|
|
387
|
+
snapshot: false,
|
|
388
|
+
rows: 0,
|
|
389
|
+
pages: 0,
|
|
390
|
+
synced: false,
|
|
391
|
+
};
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* How far the current pull has got: whether one is running, whether it
|
|
395
|
+
* is a snapshot, and the rows and pages received. Apps render from the
|
|
396
|
+
* session as soon as it resolves and show this while the replica fills.
|
|
397
|
+
* A new object on every change, so it works as a `useSyncExternalStore`
|
|
398
|
+
* snapshot. Follower tabs report `pulling: false` (the leader pulls).
|
|
399
|
+
*/
|
|
400
|
+
syncProgress(): SyncProgress {
|
|
401
|
+
return this._progress;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
private setProgress(patch: Partial<SyncProgress>): void {
|
|
405
|
+
const next = { ...this._progress, ...patch, synced: this._synced };
|
|
406
|
+
const prev = this._progress;
|
|
407
|
+
if (
|
|
408
|
+
next.pulling === prev.pulling &&
|
|
409
|
+
next.snapshot === prev.snapshot &&
|
|
410
|
+
next.rows === prev.rows &&
|
|
411
|
+
next.pages === prev.pages &&
|
|
412
|
+
next.synced === prev.synced
|
|
413
|
+
) {
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
416
|
+
this._progress = next;
|
|
417
|
+
this.store.notify(SYNC_PROGRESS_CHANNEL);
|
|
418
|
+
}
|
|
419
|
+
|
|
353
420
|
/** Flip `_synced` true (idempotent), notify, and tell follower tabs. */
|
|
354
421
|
private markSynced(): void {
|
|
355
422
|
if (this._synced) return;
|
|
356
423
|
this._synced = true;
|
|
424
|
+
this.setProgress({});
|
|
357
425
|
this.store.notify();
|
|
358
426
|
if (this.isMultiTabLeader) this.broadcastToTabs({ type: "synced" });
|
|
359
427
|
}
|
|
@@ -981,6 +1049,9 @@ export class SyncEngine {
|
|
|
981
1049
|
// whose pull-from-cursor misses an offline server-side delete
|
|
982
1050
|
// depends on that reconcile pass to catch the ghost row.
|
|
983
1051
|
this._hadCachedReplica = hadCache;
|
|
1052
|
+
// Rows from an earlier run: the server may have changed them while
|
|
1053
|
+
// this app was closed, which counts as offline.
|
|
1054
|
+
if (hadCache) this.offlineSinceMatch = true;
|
|
984
1055
|
// Which identity the hydrated rows belong to. Compared against the
|
|
985
1056
|
// resolved session in `guardReplicaIdentity` before the first pull
|
|
986
1057
|
// so a reload-time account switch drops the prior user's rows.
|
|
@@ -1492,7 +1563,7 @@ export class SyncEngine {
|
|
|
1492
1563
|
if (!this.isMultiTabLeader) return;
|
|
1493
1564
|
if (this.observedEntities.has(entity)) return;
|
|
1494
1565
|
this.observedEntities.add(entity);
|
|
1495
|
-
if (this.isHydrated() && this.store.list(entity).length === 0) {
|
|
1566
|
+
if (this.isHydrated() && this.store.list(entity).length === 0 && this.emptyMayBeStale()) {
|
|
1496
1567
|
void this.reconcile([entity]);
|
|
1497
1568
|
}
|
|
1498
1569
|
},
|
|
@@ -1641,7 +1712,7 @@ export class SyncEngine {
|
|
|
1641
1712
|
// doesn't duplicate the cursor-catch-up work pull is already
|
|
1642
1713
|
// doing. They're serial via the queue, not concurrent.
|
|
1643
1714
|
void this.pull();
|
|
1644
|
-
void this.reconcile();
|
|
1715
|
+
if (this.autoReconcileDue()) void this.reconcile();
|
|
1645
1716
|
};
|
|
1646
1717
|
document.addEventListener("visibilitychange", this.visibilityHandler);
|
|
1647
1718
|
}
|
|
@@ -1736,6 +1807,12 @@ export class SyncEngine {
|
|
|
1736
1807
|
* push whose current token differs holds owned writes: the token may
|
|
1737
1808
|
* belong to someone else. `undefined` until a session is committed. */
|
|
1738
1809
|
private sessionToken: string | null | undefined = undefined;
|
|
1810
|
+
/** The `/api/auth/me` request in flight, and the token it was sent with.
|
|
1811
|
+
* `refreshSessionIfStale` joins it instead of sending another. */
|
|
1812
|
+
private sessionFetch: {
|
|
1813
|
+
token: string | null;
|
|
1814
|
+
promise: Promise<{ session: ResolvedSession; token: string | null } | null>;
|
|
1815
|
+
} | null = null;
|
|
1739
1816
|
private identityRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
|
1740
1817
|
/** Delayed push / pull retries scheduled through `later()`. `stop()`
|
|
1741
1818
|
* clears them, so a stopped engine makes no further requests. */
|
|
@@ -2059,6 +2136,7 @@ export class SyncEngine {
|
|
|
2059
2136
|
// did nothing: an org switch dropped straight to the new org's empty list.
|
|
2060
2137
|
this._initialSyncSettled = false;
|
|
2061
2138
|
this._synced = false;
|
|
2139
|
+
this.setProgress({});
|
|
2062
2140
|
this.armInitialSyncFallback();
|
|
2063
2141
|
this.store.clearAll();
|
|
2064
2142
|
// Disk is about to be wiped + re-pulled from 0, so any prior
|
|
@@ -2267,6 +2345,7 @@ export class SyncEngine {
|
|
|
2267
2345
|
// bootstrap reconcile (the snapshot path already returned every
|
|
2268
2346
|
// policy-visible row, per-entity refetch right after is waste).
|
|
2269
2347
|
const startedFromZero = this.cursor.last_seq === 0;
|
|
2348
|
+
this.setProgress({ pulling: true, snapshot: false, rows: 0, pages: 0 });
|
|
2270
2349
|
// Open the live-event hold for the pull's whole (possibly multi-page)
|
|
2271
2350
|
// duration so a racing WS frame can't leapfrog the cursor — past the
|
|
2272
2351
|
// snapshot's rows on a from-zero pull, or past the not-yet-pulled gap
|
|
@@ -2308,6 +2387,20 @@ export class SyncEngine {
|
|
|
2308
2387
|
let hasMore = false;
|
|
2309
2388
|
let firstPass = true;
|
|
2310
2389
|
let nextSince = this.cursor.last_seq;
|
|
2390
|
+
// Adaptive page size: a small first page paints quickly, then each
|
|
2391
|
+
// full page doubles the next one up to the server's maximum, so a
|
|
2392
|
+
// large catch-up costs a few round trips instead of hundreds.
|
|
2393
|
+
let pageRows = SyncEngine.FIRST_PULL_PAGE_ROWS;
|
|
2394
|
+
// Ids per entity seen in a snapshot (a pull from 0, or the server's
|
|
2395
|
+
// answer to a far-behind cursor). When the snapshot completes, local
|
|
2396
|
+
// rows it did not send are deleted: a snapshot has no tombstones.
|
|
2397
|
+
let snapshotSeen: Map<string, Set<string>> | null = null;
|
|
2398
|
+
// A snapshot replaced rows this replica already held (a catch-up, or a
|
|
2399
|
+
// pull from 0 over a cached replica), and whether its last page said
|
|
2400
|
+
// which entities it covers. Without that list the deletes can't be
|
|
2401
|
+
// swept here, and the next automatic reconcile has to.
|
|
2402
|
+
let replacedRows = startedFromZero && this._hadCachedReplica;
|
|
2403
|
+
let snapshotCompleted = false;
|
|
2311
2404
|
while (firstPass || snapshotAfter || hasMore) {
|
|
2312
2405
|
firstPass = false;
|
|
2313
2406
|
// `snapshot_after` is an OPAQUE cursor the server already URL-encoded
|
|
@@ -2318,17 +2411,44 @@ export class SyncEngine {
|
|
|
2318
2411
|
// restart the snapshot from row 0 — an infinite re-snapshot loop for any
|
|
2319
2412
|
// table larger than one page (SNAPSHOT_BATCH_LIMIT rows). `since` is a
|
|
2320
2413
|
// plain integer, so it's safe to inline.
|
|
2321
|
-
|
|
2414
|
+
// `snapshot_ok=1`: a far-behind cursor gets a snapshot of current
|
|
2415
|
+
// rows (applied in place, then the log resumes from its seq)
|
|
2416
|
+
// instead of a 410 that wipes the replica.
|
|
2417
|
+
let query = `since=${nextSince}&limit=${pageRows}&snapshot_ok=1`;
|
|
2322
2418
|
if (snapshotAfter) {
|
|
2323
2419
|
query += `&snapshot_after=${snapshotAfter}`;
|
|
2324
2420
|
}
|
|
2325
|
-
const resp = await this.request<
|
|
2326
|
-
PullResponse & { snapshot_after?: string | null }
|
|
2327
|
-
>("GET", `/api/sync/pull?${query}`);
|
|
2421
|
+
const resp = await this.request<PullResponse>("GET", `/api/sync/pull?${query}`);
|
|
2328
2422
|
if (pendingApply) await pendingApply;
|
|
2329
2423
|
pendingApply = this.enqueueApply(resp.changes, resp.cursor, {
|
|
2330
2424
|
isPull: true,
|
|
2331
2425
|
});
|
|
2426
|
+
if (resp.snapshot && !startedFromZero) replacedRows = true;
|
|
2427
|
+
if (resp.snapshot) {
|
|
2428
|
+
snapshotSeen ??= new Map();
|
|
2429
|
+
for (const ev of resp.changes) {
|
|
2430
|
+
let ids = snapshotSeen.get(ev.entity);
|
|
2431
|
+
if (!ids) snapshotSeen.set(ev.entity, (ids = new Set()));
|
|
2432
|
+
ids.add(ev.row_id);
|
|
2433
|
+
}
|
|
2434
|
+
if (resp.snapshot_entities) {
|
|
2435
|
+
snapshotCompleted = true;
|
|
2436
|
+
const entities = resp.snapshot_entities;
|
|
2437
|
+
const seen = snapshotSeen;
|
|
2438
|
+
const tombstoneSeq = resp.cursor.last_seq;
|
|
2439
|
+
snapshotSeen = null;
|
|
2440
|
+
const applied = pendingApply;
|
|
2441
|
+
pendingApply = applied.then(() => this.sweepAfterSnapshot(entities, seen, tombstoneSeq));
|
|
2442
|
+
}
|
|
2443
|
+
}
|
|
2444
|
+
if (resp.changes.length >= pageRows) {
|
|
2445
|
+
pageRows = Math.min(pageRows * 2, SyncEngine.MAX_PULL_PAGE_ROWS);
|
|
2446
|
+
}
|
|
2447
|
+
this.setProgress({
|
|
2448
|
+
snapshot: resp.snapshot === true || this._progress.snapshot,
|
|
2449
|
+
rows: this._progress.rows + resp.changes.length,
|
|
2450
|
+
pages: this._progress.pages + 1,
|
|
2451
|
+
});
|
|
2332
2452
|
// Guard against a server that reports more pages without
|
|
2333
2453
|
// advancing the cursor (a transient no-progress page): break
|
|
2334
2454
|
// rather than refetch the same page forever. The next poll /
|
|
@@ -2340,6 +2460,7 @@ export class SyncEngine {
|
|
|
2340
2460
|
hasMore = !snapshotAfter && resp.has_more && advanced;
|
|
2341
2461
|
}
|
|
2342
2462
|
if (pendingApply) await pendingApply;
|
|
2463
|
+
if (replacedRows && !snapshotCompleted) this.sweepNeeded = true;
|
|
2343
2464
|
// Clear the resync circuit breaker ONLY on a successful DELTA
|
|
2344
2465
|
// pull — one that started from a real, non-zero cursor the server
|
|
2345
2466
|
// honored. A snapshot pull from cursor=0 succeeding does NOT prove
|
|
@@ -2448,7 +2569,47 @@ export class SyncEngine {
|
|
|
2448
2569
|
// already drained + nulled the hold, so this is a no-op there — and
|
|
2449
2570
|
// the 410 recursion drained or re-owned it before we got here.
|
|
2450
2571
|
if (this.pullHold !== null) this.pullHold = null;
|
|
2572
|
+
this.setProgress({ pulling: false });
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
|
|
2576
|
+
/** Rows a reconcile fetches per entity, at most. */
|
|
2577
|
+
private static readonly RECONCILE_ROW_CAP = 20_000;
|
|
2578
|
+
/** The server's page maximum for replication reads (`?sync=1`). */
|
|
2579
|
+
private static readonly MAX_RECONCILE_PAGE_ROWS = 5_000;
|
|
2580
|
+
|
|
2581
|
+
/** Rows the first page of a pull asks for. Later pages double. */
|
|
2582
|
+
private static readonly FIRST_PULL_PAGE_ROWS = 1000;
|
|
2583
|
+
/** The server's page maximum (MAX_PAGE_ROWS in routes/sync.rs). */
|
|
2584
|
+
private static readonly MAX_PULL_PAGE_ROWS = 10_000;
|
|
2585
|
+
|
|
2586
|
+
/**
|
|
2587
|
+
* After a complete snapshot: delete local rows of `entities` that the
|
|
2588
|
+
* snapshot did not send. Rows with writes not yet on the server are kept.
|
|
2589
|
+
* The deletes are tombstoned at the snapshot's seq, so an older event
|
|
2590
|
+
* cannot bring a row back.
|
|
2591
|
+
*/
|
|
2592
|
+
private async sweepAfterSnapshot(
|
|
2593
|
+
entities: string[],
|
|
2594
|
+
seen: Map<string, Set<string>>,
|
|
2595
|
+
tombstoneSeq: number,
|
|
2596
|
+
): Promise<void> {
|
|
2597
|
+
const pendingKeys = this.mutations.pendingRowKeys();
|
|
2598
|
+
for (const entity of entities) {
|
|
2599
|
+
const ids = seen.get(entity);
|
|
2600
|
+
const removalIds: string[] = [];
|
|
2601
|
+
for (const row of this.store.list(entity)) {
|
|
2602
|
+
const id = (row as { id?: unknown }).id;
|
|
2603
|
+
if (typeof id !== "string") continue;
|
|
2604
|
+
if (ids?.has(id) || pendingKeys.has(`${entity}/${id}`)) continue;
|
|
2605
|
+
removalIds.push(id);
|
|
2606
|
+
}
|
|
2607
|
+
if (removalIds.length > 0) {
|
|
2608
|
+
await this.enqueueReconcile(entity, [], removalIds, tombstoneSeq);
|
|
2609
|
+
}
|
|
2451
2610
|
}
|
|
2611
|
+
this.markMatchedServer();
|
|
2612
|
+
this.snapshotEpoch = this.replicaEpoch;
|
|
2452
2613
|
}
|
|
2453
2614
|
|
|
2454
2615
|
/** Consecutive 410 RESYNC_REQUIRED responses since the last successful
|
|
@@ -2473,6 +2634,42 @@ export class SyncEngine {
|
|
|
2473
2634
|
* cleared on read so a subsequent reconnect-after-disconnect still
|
|
2474
2635
|
* runs reconcile normally. */
|
|
2475
2636
|
private lastPullStartedFromZero = false;
|
|
2637
|
+
/** When the replica last matched the server: a complete snapshot
|
|
2638
|
+
* (local rows it did not hold deleted) or a full reconcile. 0 = never
|
|
2639
|
+
* this run. */
|
|
2640
|
+
private matchedServerAt = 0;
|
|
2641
|
+
/** The replica epoch a complete snapshot filled. An entity with no
|
|
2642
|
+
* rows in it is empty on the server too. */
|
|
2643
|
+
private snapshotEpoch = -1;
|
|
2644
|
+
|
|
2645
|
+
/** Whether an automatic sweep (connect, visibility) should run. */
|
|
2646
|
+
/** The connection dropped (or this run started from a cached replica)
|
|
2647
|
+
* since the replica last matched the server. */
|
|
2648
|
+
private offlineSinceMatch = false;
|
|
2649
|
+
/** A snapshot replaced cached rows without saying which entities it
|
|
2650
|
+
* covers, so rows deleted on the server may still be here. */
|
|
2651
|
+
private sweepNeeded = false;
|
|
2652
|
+
|
|
2653
|
+
/**
|
|
2654
|
+
* Whether an automatic sweep (connect, visibility) should run: after a
|
|
2655
|
+
* snapshot that could not sweep its own deletes, or when the app was
|
|
2656
|
+
* offline since the replica last matched the server and the last match
|
|
2657
|
+
* is `autoReconcileIntervalMs` old. Never right after a complete
|
|
2658
|
+
* snapshot, and not merely because the app came to the foreground.
|
|
2659
|
+
*/
|
|
2660
|
+
private autoReconcileDue(): boolean {
|
|
2661
|
+
if (this.sweepNeeded) return true;
|
|
2662
|
+
if (!this.offlineSinceMatch) return false;
|
|
2663
|
+
const interval = this.config.autoReconcileIntervalMs ?? 300_000;
|
|
2664
|
+
return this.matchedServerAt === 0 || Date.now() - this.matchedServerAt >= interval;
|
|
2665
|
+
}
|
|
2666
|
+
|
|
2667
|
+
/** The replica matches the server now. */
|
|
2668
|
+
private markMatchedServer(): void {
|
|
2669
|
+
this.matchedServerAt = Date.now();
|
|
2670
|
+
this.offlineSinceMatch = false;
|
|
2671
|
+
this.sweepNeeded = false;
|
|
2672
|
+
}
|
|
2476
2673
|
|
|
2477
2674
|
/** Timestamp of the last `reconcile()` invocation. Used to debounce —
|
|
2478
2675
|
* reconcile runs on connect, WS reconnect, AND visibility-change, so
|
|
@@ -2563,13 +2760,22 @@ export class SyncEngine {
|
|
|
2563
2760
|
this.broadcastToTabs({ type: "entity-observe", entity });
|
|
2564
2761
|
return;
|
|
2565
2762
|
}
|
|
2566
|
-
if (this.isHydrated() && this.store.list(entity).length === 0) {
|
|
2763
|
+
if (this.isHydrated() && this.store.list(entity).length === 0 && this.emptyMayBeStale()) {
|
|
2567
2764
|
// Scoped reconcile bypasses the no-arg debounce and reuses the
|
|
2568
2765
|
// session-flip / cursor-drift guards in reconcileInner.
|
|
2569
2766
|
void this.reconcile([entity]);
|
|
2570
2767
|
}
|
|
2571
2768
|
}
|
|
2572
2769
|
|
|
2770
|
+
/** Whether an entity with no local rows may still have rows on the
|
|
2771
|
+
* server that no pull will deliver. Not when a snapshot is coming (the
|
|
2772
|
+
* cursor is 0 and no pull has completed since the last reset) or one
|
|
2773
|
+
* already filled this replica: a snapshot sends every visible row. */
|
|
2774
|
+
private emptyMayBeStale(): boolean {
|
|
2775
|
+
const snapshotComing = this.cursor.last_seq === 0 && !this._synced;
|
|
2776
|
+
return !snapshotComing && this.snapshotEpoch !== this.replicaEpoch;
|
|
2777
|
+
}
|
|
2778
|
+
|
|
2573
2779
|
async reconcile(entities?: string[]): Promise<void> {
|
|
2574
2780
|
const epoch = this.pendingReconcileEpoch;
|
|
2575
2781
|
const { done, skipped, startedAt } = await this.reconcileReportingSkips(entities);
|
|
@@ -2873,6 +3079,9 @@ export class SyncEngine {
|
|
|
2873
3079
|
done.push(entity);
|
|
2874
3080
|
}),
|
|
2875
3081
|
);
|
|
3082
|
+
// A full sweep that reconciled every entity matched the replica to the
|
|
3083
|
+
// server (see autoReconcileDue).
|
|
3084
|
+
if (entities === undefined && skipped.length === 0) this.markMatchedServer();
|
|
2876
3085
|
return { done, skipped, startedAt };
|
|
2877
3086
|
}
|
|
2878
3087
|
|
|
@@ -2886,20 +3095,22 @@ export class SyncEngine {
|
|
|
2886
3095
|
): Promise<{ rows: Row[]; truncated: boolean }> {
|
|
2887
3096
|
const out: Row[] = [];
|
|
2888
3097
|
let cursor: string | null = null;
|
|
2889
|
-
//
|
|
2890
|
-
//
|
|
2891
|
-
// 1000
|
|
2892
|
-
//
|
|
2893
|
-
//
|
|
2894
|
-
//
|
|
2895
|
-
|
|
3098
|
+
// At most RECONCILE_ROW_CAP rows in 20 pages. A `sync:true` entity larger than this
|
|
3099
|
+
// should switch to `sync: false` + search/by-id (see useInfiniteQuery).
|
|
3100
|
+
// Pages start at 1000 rows and double up to the server's replication
|
|
3101
|
+
// maximum: each page costs a round trip, so a 7k-row entity takes 3
|
|
3102
|
+
// requests instead of 8.
|
|
3103
|
+
// The page cap bounds requests when pages come back short (a sparse
|
|
3104
|
+
// policy can leave few visible rows per page).
|
|
3105
|
+
let limit = 1000;
|
|
3106
|
+
for (let page = 0; page < 20 && out.length < SyncEngine.RECONCILE_ROW_CAP; page++) {
|
|
2896
3107
|
// `sync=1` marks this as a REPLICATION fetch, which is what makes an
|
|
2897
3108
|
// entity's `sync` scope apply. Without it the server treats the request
|
|
2898
3109
|
// as an ordinary app read and returns unscoped rows — correct for a
|
|
2899
3110
|
// direct read, wrong here, since these rows land in the replica.
|
|
2900
3111
|
const qs: string = cursor
|
|
2901
|
-
? `?limit
|
|
2902
|
-
: `?limit
|
|
3112
|
+
? `?limit=${limit}&sync=1&after=${encodeURIComponent(cursor)}`
|
|
3113
|
+
: `?limit=${limit}&sync=1`;
|
|
2903
3114
|
const resp: {
|
|
2904
3115
|
data: Row[];
|
|
2905
3116
|
next_cursor: string | null;
|
|
@@ -2910,6 +3121,9 @@ export class SyncEngine {
|
|
|
2910
3121
|
return { rows: out, truncated: false };
|
|
2911
3122
|
}
|
|
2912
3123
|
cursor = resp.next_cursor;
|
|
3124
|
+
if (resp.data.length >= limit) {
|
|
3125
|
+
limit = Math.min(limit * 2, SyncEngine.MAX_RECONCILE_PAGE_ROWS);
|
|
3126
|
+
}
|
|
2913
3127
|
}
|
|
2914
3128
|
// Exhausted the page cap while the server still had more rows: the set
|
|
2915
3129
|
// is incomplete. Surface it so reconcile upserts-only instead of
|
|
@@ -2957,7 +3171,7 @@ export class SyncEngine {
|
|
|
2957
3171
|
const removalIds: string[] = [];
|
|
2958
3172
|
if (truncated) {
|
|
2959
3173
|
console.warn(
|
|
2960
|
-
`[reconcile] "${entity}" exceeds the ${
|
|
3174
|
+
`[reconcile] "${entity}" exceeds the ${SyncEngine.RECONCILE_ROW_CAP}-row client-replication ` +
|
|
2961
3175
|
`cap; skipping stale-row cleanup to avoid deleting un-fetched rows. ` +
|
|
2962
3176
|
`Mark large server-queried entities \`sync: false\` and read them via ` +
|
|
2963
3177
|
`search + by-id instead of replicating the whole table.`,
|
|
@@ -3032,11 +3246,23 @@ export class SyncEngine {
|
|
|
3032
3246
|
* caller's next pull cycle (or the WS `session-changed` envelope)
|
|
3033
3247
|
* will retry. Errors must not abort bootstrap.
|
|
3034
3248
|
*/
|
|
3035
|
-
private
|
|
3249
|
+
private fetchSessionBootstrap(): Promise<
|
|
3036
3250
|
{ session: ResolvedSession; token: string | null } | null
|
|
3037
3251
|
> {
|
|
3252
|
+
const token = this.currentToken();
|
|
3253
|
+
const promise = this.fetchSession(token);
|
|
3254
|
+
this.sessionFetch = { token, promise };
|
|
3255
|
+
const clear = () => {
|
|
3256
|
+
if (this.sessionFetch?.promise === promise) this.sessionFetch = null;
|
|
3257
|
+
};
|
|
3258
|
+
promise.then(clear, clear);
|
|
3259
|
+
return promise;
|
|
3260
|
+
}
|
|
3261
|
+
|
|
3262
|
+
private async fetchSession(
|
|
3263
|
+
token: string | null,
|
|
3264
|
+
): Promise<{ session: ResolvedSession; token: string | null } | null> {
|
|
3038
3265
|
try {
|
|
3039
|
-
const token = this.currentToken();
|
|
3040
3266
|
const res = await this.rawFetch("/api/auth/me");
|
|
3041
3267
|
if (!res.ok) return null;
|
|
3042
3268
|
const raw = (await res.json()) as {
|
|
@@ -3144,12 +3370,58 @@ export class SyncEngine {
|
|
|
3144
3370
|
// need to handle) errors from a session refresh.
|
|
3145
3371
|
this.sessionChain = prev.then(async () => {
|
|
3146
3372
|
// Defer the null→X / X→Y / first-resolution distinction to the
|
|
3147
|
-
// resolver, but DON'T commit the new resolved session until
|
|
3148
|
-
//
|
|
3149
|
-
//
|
|
3150
|
-
//
|
|
3373
|
+
// resolver, but DON'T commit the new resolved session until the
|
|
3374
|
+
// old identity's rows are gone — that closes the window where
|
|
3375
|
+
// useSession would report the new tenant while useQuery still has
|
|
3376
|
+
// the old tenant's rows. It does not wait for the new rows: the
|
|
3377
|
+
// re-pull can take seconds on a slow phone.
|
|
3151
3378
|
const verdict = this.session.inspectSession(next);
|
|
3152
3379
|
const prevUserId = this.session.resolved().userId;
|
|
3380
|
+
// Commit the session and act on it. Runs once: right after the wipe
|
|
3381
|
+
// when there is one, so `useSession` and `sessionResolved()` update
|
|
3382
|
+
// while the replica refills from the server (an app renders from the
|
|
3383
|
+
// session and shows `useSyncProgress` for the data), else after the
|
|
3384
|
+
// branch below.
|
|
3385
|
+
let committed = false;
|
|
3386
|
+
const commit = () => {
|
|
3387
|
+
if (committed) return;
|
|
3388
|
+
committed = true;
|
|
3389
|
+
const firstResolution = !this.session.hasObserved();
|
|
3390
|
+
this.session.commitObservation(next);
|
|
3391
|
+
// The reset above tagged the replica with the outgoing user (the
|
|
3392
|
+
// session was not committed yet). Re-tag it with the new owner.
|
|
3393
|
+
if (userFlipped && this.isMultiTabLeader) this.persistReplicaIdentity();
|
|
3394
|
+
// The session now says who the token belongs to: push writes
|
|
3395
|
+
// that were waiting for it (their owner may be this user).
|
|
3396
|
+
if (this.isMultiTabLeader) {
|
|
3397
|
+
const wasPending = this.identityPending || firstResolution;
|
|
3398
|
+
if (opts.token !== undefined) this.sessionToken = opts.token;
|
|
3399
|
+
// Writes made before any session resolved belong to this one.
|
|
3400
|
+
if (firstResolution) this.mutations.stampPendingOwner(next.userId);
|
|
3401
|
+
this.identityPending = false;
|
|
3402
|
+
this.identityRetryAttempts = 0;
|
|
3403
|
+
if ((wasPending || userFlipped) && this.mutations.pending().length > 0) {
|
|
3404
|
+
void this.push();
|
|
3405
|
+
}
|
|
3406
|
+
// Uploads follow the same owner rule: stamp the ones made before
|
|
3407
|
+
// the session was known, then send (or, for another user's,
|
|
3408
|
+
// discard) what is queued.
|
|
3409
|
+
if (wasPending || userFlipped) {
|
|
3410
|
+
const stamp = firstResolution
|
|
3411
|
+
? this.uploads.stampPendingOwner(next.userId)
|
|
3412
|
+
: Promise.resolve();
|
|
3413
|
+
void stamp.then(() => this.uploads.process());
|
|
3414
|
+
}
|
|
3415
|
+
}
|
|
3416
|
+
if (verdict.identityChanged || firstResolution) {
|
|
3417
|
+
// The first answer flips `sessionResolved()` even when the
|
|
3418
|
+
// session itself matches the placeholder (anonymous caller).
|
|
3419
|
+
this.store.notify();
|
|
3420
|
+
}
|
|
3421
|
+
if (broadcast && this.isMultiTabLeader) {
|
|
3422
|
+
this.broadcastToTabs({ type: "session", resolved: next });
|
|
3423
|
+
}
|
|
3424
|
+
};
|
|
3153
3425
|
// USER flip (sign-in, sign-out, account switch) observed on a
|
|
3154
3426
|
// session refresh. The rows, the cursor, and the live socket all
|
|
3155
3427
|
// belong to the previous identity, and a tenant verdict alone
|
|
@@ -3176,7 +3448,7 @@ export class SyncEngine {
|
|
|
3176
3448
|
this.cycleTransport();
|
|
3177
3449
|
// Queued writes stay: each carries its owner, and push drops
|
|
3178
3450
|
// the ones that belong to someone else.
|
|
3179
|
-
await this.resetAndPull({ wipeMutations: false });
|
|
3451
|
+
await this.resetAndPull({ wipeMutations: false }, commit);
|
|
3180
3452
|
}
|
|
3181
3453
|
} else if (userFlipped) {
|
|
3182
3454
|
// Follower: the leader's reset already wiped this tab.
|
|
@@ -3196,12 +3468,17 @@ export class SyncEngine {
|
|
|
3196
3468
|
// Reset and re-pull in one op-queue slot (see resetAndPull).
|
|
3197
3469
|
// The tenant's view changed → wipe the outgoing tenant's
|
|
3198
3470
|
// pending offline writes too.
|
|
3199
|
-
await this.resetAndPull({ wipeMutations: true });
|
|
3471
|
+
await this.resetAndPull({ wipeMutations: true }, commit);
|
|
3200
3472
|
}
|
|
3201
3473
|
if (this.isMultiTabLeader) {
|
|
3202
3474
|
// Only the leader pulls — followers receive subsequent
|
|
3203
3475
|
// applied broadcasts that close the catch-up window.
|
|
3204
|
-
|
|
3476
|
+
// No wipe: the replica is valid for the new tenant as it is
|
|
3477
|
+
// (membership-scoped reads), so the session commits first.
|
|
3478
|
+
if (!reset) {
|
|
3479
|
+
commit();
|
|
3480
|
+
await this.pull();
|
|
3481
|
+
}
|
|
3205
3482
|
if (skipReset) {
|
|
3206
3483
|
// No wipe happened, so rows of an org the user only just
|
|
3207
3484
|
// JOINED aren't in the replica and no change event will
|
|
@@ -3211,41 +3488,7 @@ export class SyncEngine {
|
|
|
3211
3488
|
}
|
|
3212
3489
|
}
|
|
3213
3490
|
}
|
|
3214
|
-
|
|
3215
|
-
this.session.commitObservation(next);
|
|
3216
|
-
// The reset above tagged the replica with the outgoing user (the
|
|
3217
|
-
// session was not committed yet). Re-tag it with the new owner.
|
|
3218
|
-
if (userFlipped && this.isMultiTabLeader) this.persistReplicaIdentity();
|
|
3219
|
-
// The session now says who the token belongs to: push writes
|
|
3220
|
-
// that were waiting for it (their owner may be this user).
|
|
3221
|
-
if (this.isMultiTabLeader) {
|
|
3222
|
-
const wasPending = this.identityPending || firstResolution;
|
|
3223
|
-
if (opts.token !== undefined) this.sessionToken = opts.token;
|
|
3224
|
-
// Writes made before any session resolved belong to this one.
|
|
3225
|
-
if (firstResolution) this.mutations.stampPendingOwner(next.userId);
|
|
3226
|
-
this.identityPending = false;
|
|
3227
|
-
this.identityRetryAttempts = 0;
|
|
3228
|
-
if ((wasPending || userFlipped) && this.mutations.pending().length > 0) {
|
|
3229
|
-
void this.push();
|
|
3230
|
-
}
|
|
3231
|
-
// Uploads follow the same owner rule: stamp the ones made before
|
|
3232
|
-
// the session was known, then send (or, for another user's,
|
|
3233
|
-
// discard) what is queued.
|
|
3234
|
-
if (wasPending || userFlipped) {
|
|
3235
|
-
const stamp = firstResolution
|
|
3236
|
-
? this.uploads.stampPendingOwner(next.userId)
|
|
3237
|
-
: Promise.resolve();
|
|
3238
|
-
void stamp.then(() => this.uploads.process());
|
|
3239
|
-
}
|
|
3240
|
-
}
|
|
3241
|
-
if (verdict.identityChanged || firstResolution) {
|
|
3242
|
-
// The first answer flips `sessionResolved()` even when the
|
|
3243
|
-
// session itself matches the placeholder (anonymous caller).
|
|
3244
|
-
this.store.notify();
|
|
3245
|
-
}
|
|
3246
|
-
if (broadcast && this.isMultiTabLeader) {
|
|
3247
|
-
this.broadcastToTabs({ type: "session", resolved: next });
|
|
3248
|
-
}
|
|
3491
|
+
commit();
|
|
3249
3492
|
}).catch(() => {});
|
|
3250
3493
|
return this.sessionChain;
|
|
3251
3494
|
}
|
|
@@ -3253,9 +3496,15 @@ export class SyncEngine {
|
|
|
3253
3496
|
/** Reset the replica and pull from zero in ONE op-queue slot, so no
|
|
3254
3497
|
* live frame can move the cursor between the reset and the pull's
|
|
3255
3498
|
* hold (the pull would then run as a delta and miss rows). */
|
|
3256
|
-
private resetAndPull(
|
|
3499
|
+
private resetAndPull(
|
|
3500
|
+
opts: { wipeMutations: boolean },
|
|
3501
|
+
afterReset?: () => void,
|
|
3502
|
+
): Promise<void> {
|
|
3257
3503
|
return this.opQueue.enqueue("reset-pull", async () => {
|
|
3258
3504
|
await this.resetReplicaInner(opts);
|
|
3505
|
+
// The replica is empty: the caller commits the new session here, so
|
|
3506
|
+
// the app does not wait for the whole re-pull to learn who is signed in.
|
|
3507
|
+
afterReset?.();
|
|
3259
3508
|
await this.pullInner();
|
|
3260
3509
|
}).then(() => {
|
|
3261
3510
|
if (this.isMultiTabLeader) this.markInitialSyncSettled();
|
|
@@ -3360,6 +3609,29 @@ export class SyncEngine {
|
|
|
3360
3609
|
return this.refreshResolvedSession();
|
|
3361
3610
|
}
|
|
3362
3611
|
|
|
3612
|
+
/**
|
|
3613
|
+
* Refresh the session only when it is not known for the current token:
|
|
3614
|
+
* no answer yet this run, or the token changed since the last answer.
|
|
3615
|
+
* A request already in flight for the token is joined, not repeated.
|
|
3616
|
+
* `useSession` calls this on mount, so a screen with many session
|
|
3617
|
+
* readers makes at most one `/api/auth/me` request per resolve.
|
|
3618
|
+
* After a server-side session change, use `notifySessionChanged`,
|
|
3619
|
+
* which always fetches.
|
|
3620
|
+
*/
|
|
3621
|
+
async refreshSessionIfStale(): Promise<void> {
|
|
3622
|
+
if (!this.isMultiTabLeader) return;
|
|
3623
|
+
const token = this.currentToken();
|
|
3624
|
+
if (this.session.hasObserved() && this.sessionToken === token) return;
|
|
3625
|
+
const inflight = this.sessionFetch;
|
|
3626
|
+
if (inflight && inflight.token === token) {
|
|
3627
|
+
// The request's owner (start() or a refresh) applies the answer.
|
|
3628
|
+
await inflight.promise;
|
|
3629
|
+
await this.sessionChain;
|
|
3630
|
+
return;
|
|
3631
|
+
}
|
|
3632
|
+
await this.refreshResolvedSession();
|
|
3633
|
+
}
|
|
3634
|
+
|
|
3363
3635
|
/**
|
|
3364
3636
|
* Drop a row from the local replica because the server signaled
|
|
3365
3637
|
* that the current subscriber's read policy was revoked for it.
|
|
@@ -4443,13 +4715,18 @@ export class SyncEngine {
|
|
|
4443
4715
|
return;
|
|
4444
4716
|
}
|
|
4445
4717
|
this.lastPullStartedFromZero = false;
|
|
4718
|
+
// A complete snapshot or a full reconcile in the last few
|
|
4719
|
+
// minutes already matched the replica to the server, deletions
|
|
4720
|
+
// included (see `autoReconcileIntervalMs`).
|
|
4721
|
+
if (!this.autoReconcileDue()) return;
|
|
4446
4722
|
return this.reconcile();
|
|
4447
4723
|
});
|
|
4448
4724
|
},
|
|
4449
4725
|
onDisconnected: () => {
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4726
|
+
// The transport's own reconnect loop drives the recovery; the
|
|
4727
|
+
// connection-status flip already happened inside the transport.
|
|
4728
|
+
// Record the gap: the next automatic sweep may be due.
|
|
4729
|
+
this.offlineSinceMatch = true;
|
|
4453
4730
|
},
|
|
4454
4731
|
setStatus: (s) => this.setConnectionStatus(s),
|
|
4455
4732
|
performPollTick: async () => {
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// A sign-in resolves the session as soon as /api/auth/me answers. The
|
|
2
|
+
// re-pull of the new identity's rows (seconds on a slow phone) does not
|
|
3
|
+
// hold `sessionResolved()` or `useSession` back; the app renders from the
|
|
4
|
+
// session and reads progress from `syncProgress()`.
|
|
5
|
+
|
|
6
|
+
import { afterEach, expect, test } from "bun:test";
|
|
7
|
+
import { createTestEnv, type TestEnv } from "./test-harness";
|
|
8
|
+
|
|
9
|
+
let env: TestEnv | null = null;
|
|
10
|
+
afterEach(async () => {
|
|
11
|
+
if (env) {
|
|
12
|
+
await env.dispose();
|
|
13
|
+
env = null;
|
|
14
|
+
}
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test("the session commits before the new identity's snapshot finishes", async () => {
|
|
18
|
+
let release: () => void = () => {};
|
|
19
|
+
const gate = new Promise<void>((r) => (release = r));
|
|
20
|
+
let holding = false;
|
|
21
|
+
env = await createTestEnv({
|
|
22
|
+
transport: "poll",
|
|
23
|
+
beforePull: async (auth) => {
|
|
24
|
+
if (auth.userId === "u1") {
|
|
25
|
+
holding = true;
|
|
26
|
+
await gate;
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
});
|
|
30
|
+
env.server.seed("Lead", [{ id: "a", name: "A" }]);
|
|
31
|
+
await env.engine.start();
|
|
32
|
+
expect(env.engine.resolvedSession().userId).toBeNull();
|
|
33
|
+
|
|
34
|
+
env.signIn({ userId: "u1" });
|
|
35
|
+
const refreshed = env.engine.notifySessionChanged();
|
|
36
|
+
// Wait until the re-pull is stuck at the server.
|
|
37
|
+
for (let i = 0; i < 200 && !holding; i++) await new Promise((r) => setTimeout(r, 1));
|
|
38
|
+
expect(holding).toBe(true);
|
|
39
|
+
for (let i = 0; i < 20; i++) await new Promise((r) => setTimeout(r, 0));
|
|
40
|
+
|
|
41
|
+
expect(env.engine.sessionResolved()).toBe(true);
|
|
42
|
+
expect(env.engine.resolvedSession().userId).toBe("u1");
|
|
43
|
+
// The old identity's rows are gone; the new ones have not arrived.
|
|
44
|
+
expect(env.engine.store.list("Lead")).toEqual([]);
|
|
45
|
+
expect(env.engine.syncProgress().pulling).toBe(true);
|
|
46
|
+
|
|
47
|
+
release();
|
|
48
|
+
await refreshed;
|
|
49
|
+
expect(env.engine.store.list("Lead").map((r) => r.id)).toEqual(["a"]);
|
|
50
|
+
expect(env.engine.syncProgress().pulling).toBe(false);
|
|
51
|
+
});
|