@pylonsync/sync 0.22.41 → 0.22.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +36 -23
- package/package.json +1 -1
- package/src/index.ts +160 -10
- package/src/reconcile.test.ts +349 -0
package/dist/index.d.ts
CHANGED
|
@@ -106,29 +106,6 @@ export interface SyncEngineConfig {
|
|
|
106
106
|
*/
|
|
107
107
|
multiTab?: boolean;
|
|
108
108
|
}
|
|
109
|
-
/**
|
|
110
|
-
* Generate a stable client_id. Prefers a persisted id from `storage`
|
|
111
|
-
* (so a reload keeps the same identifier) and falls back to a fresh UUID.
|
|
112
|
-
*/
|
|
113
|
-
/**
|
|
114
|
-
* Generate a Pylon-shaped row id (40-char lowercase hex).
|
|
115
|
-
*
|
|
116
|
-
* Mirrors the runtime's `generate_id` shape: 32 hex of milliseconds
|
|
117
|
-
* since epoch (extended to nanos so it lex-sorts alongside server-
|
|
118
|
-
* generated ids) + 8 hex of a per-tab counter. Lex-sortable, monotonic
|
|
119
|
-
* within a tab, statistically unique across tabs (the timestamp
|
|
120
|
-
* disambiguates almost every cross-tab collision; the counter handles
|
|
121
|
-
* the rest within a single tick).
|
|
122
|
-
*
|
|
123
|
-
* Used by `useMutation({ optimistic })` to mint client-side ids that
|
|
124
|
-
* the runtime will accept verbatim — so the optimistic ghost and the
|
|
125
|
-
* canonical row share the same `row_id` and the WS broadcast is an
|
|
126
|
-
* idempotent merge instead of a delete-then-replace flash.
|
|
127
|
-
*
|
|
128
|
-
* Apps can call this directly when they need a stable id earlier than
|
|
129
|
-
* the mutation (e.g. to reference the row from another optimistic
|
|
130
|
-
* insert in the same gesture).
|
|
131
|
-
*/
|
|
132
109
|
export declare class SyncEngine {
|
|
133
110
|
private config;
|
|
134
111
|
private cursor;
|
|
@@ -733,6 +710,42 @@ export declare class SyncEngine {
|
|
|
733
710
|
*/
|
|
734
711
|
observeEntity(entity: string): void;
|
|
735
712
|
reconcile(entities?: string[]): Promise<void>;
|
|
713
|
+
/** Entities to reconcile that have not reconciled yet: named by a server
|
|
714
|
+
* hint (`sync-reconcile`), or skipped by an earlier reconcile. */
|
|
715
|
+
private pendingReconcile;
|
|
716
|
+
/** Numbers each pending request, so a pass clears only older ones. */
|
|
717
|
+
private pendingReconcileSeq;
|
|
718
|
+
private pendingReconcileTimer;
|
|
719
|
+
private pendingReconcileRetries;
|
|
720
|
+
/** Bumped by stop() and resetReplica(): pending work queued before it
|
|
721
|
+
* does not run. */
|
|
722
|
+
private pendingReconcileEpoch;
|
|
723
|
+
private lastPendingReconcileAt;
|
|
724
|
+
/** Hints that arrive this close together share one reconcile. */
|
|
725
|
+
private static readonly PENDING_RECONCILE_DEBOUNCE_MS;
|
|
726
|
+
/** At most one hinted reconcile per this interval. */
|
|
727
|
+
private static readonly PENDING_RECONCILE_MIN_INTERVAL_MS;
|
|
728
|
+
/** Backoff for entities whose fetch raced a newer change: doubling from
|
|
729
|
+
* the base, capped. */
|
|
730
|
+
private static readonly PENDING_RECONCILE_RETRY_BASE_MS;
|
|
731
|
+
private static readonly PENDING_RECONCILE_RETRY_MAX_MS;
|
|
732
|
+
/**
|
|
733
|
+
* Keep `entities` pending until a reconcile of them completes. Used for
|
|
734
|
+
* server hints (`sync-reconcile`: a share, an unshare, a folder
|
|
735
|
+
* membership changed which rows this user may read, and no other event
|
|
736
|
+
* will tell this replica) and for entities an earlier reconcile skipped.
|
|
737
|
+
* Requests are merged, run at most once a second, and an entity whose
|
|
738
|
+
* fetch failed or raced a newer change is retried with backoff until it
|
|
739
|
+
* lands or the engine stops.
|
|
740
|
+
*/
|
|
741
|
+
private queuePendingReconcile;
|
|
742
|
+
private schedulePendingReconcile;
|
|
743
|
+
private flushPendingReconcile;
|
|
744
|
+
/** Drop pending reconciles, including queued ones (stop, replica reset). */
|
|
745
|
+
private cancelPendingReconcile;
|
|
746
|
+
/** `reconcile`, resolving with the entities it reconciled and the ones
|
|
747
|
+
* it skipped (the fetch failed, or the cursor moved during it). */
|
|
748
|
+
private reconcileReportingSkips;
|
|
736
749
|
/** Scoped reconcile batch that has not started running yet. */
|
|
737
750
|
private scopedReconcileBatch;
|
|
738
751
|
private scopedReconcileSeq;
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -230,6 +230,17 @@ export interface SyncEngineConfig {
|
|
|
230
230
|
* insert in the same gesture).
|
|
231
231
|
*/
|
|
232
232
|
|
|
233
|
+
/** What a reconcile pass did per entity. */
|
|
234
|
+
interface ReconcileOutcome {
|
|
235
|
+
/** Reconciled: the replica matches the server for these. */
|
|
236
|
+
done: string[];
|
|
237
|
+
/** Not reconciled: the fetch failed, or a newer change raced it. */
|
|
238
|
+
skipped: string[];
|
|
239
|
+
/** The pending-request sequence when the pass began fetching. Callers
|
|
240
|
+
* that share the pass clear only requests at or below it. */
|
|
241
|
+
startedAt: number;
|
|
242
|
+
}
|
|
243
|
+
|
|
233
244
|
export class SyncEngine {
|
|
234
245
|
private config: SyncEngineConfig;
|
|
235
246
|
private cursor: SyncCursor = { last_seq: 0 };
|
|
@@ -1651,6 +1662,7 @@ export class SyncEngine {
|
|
|
1651
1662
|
/** Stop the sync engine. */
|
|
1652
1663
|
stop(): void {
|
|
1653
1664
|
this.running = false;
|
|
1665
|
+
this.cancelPendingReconcile();
|
|
1654
1666
|
if (this.identityRetryTimer !== null) {
|
|
1655
1667
|
clearTimeout(this.identityRetryTimer);
|
|
1656
1668
|
this.identityRetryTimer = null;
|
|
@@ -1692,6 +1704,8 @@ export class SyncEngine {
|
|
|
1692
1704
|
* in-memory state could fix.
|
|
1693
1705
|
*/
|
|
1694
1706
|
async resetReplica(opts: { wipeMutations?: boolean } = {}): Promise<void> {
|
|
1707
|
+
// Rows fetched for the old replica must not land in the new one.
|
|
1708
|
+
this.cancelPendingReconcile();
|
|
1695
1709
|
// Public callers go through the queue so a reset can't race with
|
|
1696
1710
|
// an in-flight pull / push / reconcile. Internal callers that
|
|
1697
1711
|
// already hold the queue slot use `resetReplicaInner` directly.
|
|
@@ -2303,10 +2317,123 @@ export class SyncEngine {
|
|
|
2303
2317
|
}
|
|
2304
2318
|
|
|
2305
2319
|
async reconcile(entities?: string[]): Promise<void> {
|
|
2320
|
+
const epoch = this.pendingReconcileEpoch;
|
|
2321
|
+
const { done, skipped, startedAt } = await this.reconcileReportingSkips(entities);
|
|
2322
|
+
// stop() or resetReplica() ran meanwhile: leave pending work alone.
|
|
2323
|
+
if (epoch !== this.pendingReconcileEpoch) return;
|
|
2324
|
+
// An entity skipped because its fetch failed or raced a newer change
|
|
2325
|
+
// stays pending until a reconcile of it completes. Otherwise a row that
|
|
2326
|
+
// became readable while this replica was away (a share, a new folder)
|
|
2327
|
+
// stays missing until the next focus or reconnect.
|
|
2328
|
+
// Clear only requests this pass covers: one made after it started (a
|
|
2329
|
+
// hint during the fetch) may name rows its response does not hold.
|
|
2330
|
+
for (const e of done) {
|
|
2331
|
+
const requestedAt = this.pendingReconcile.get(e);
|
|
2332
|
+
if (requestedAt !== undefined && requestedAt <= startedAt) this.pendingReconcile.delete(e);
|
|
2333
|
+
}
|
|
2334
|
+
if (skipped.length > 0) this.queuePendingReconcile(skipped);
|
|
2335
|
+
}
|
|
2336
|
+
|
|
2337
|
+
/** Entities to reconcile that have not reconciled yet: named by a server
|
|
2338
|
+
* hint (`sync-reconcile`), or skipped by an earlier reconcile. */
|
|
2339
|
+
private pendingReconcile = new Map<string, number>();
|
|
2340
|
+
/** Numbers each pending request, so a pass clears only older ones. */
|
|
2341
|
+
private pendingReconcileSeq = 0;
|
|
2342
|
+
private pendingReconcileTimer: ReturnType<typeof setTimeout> | null = null;
|
|
2343
|
+
private pendingReconcileRetries = 0;
|
|
2344
|
+
/** Bumped by stop() and resetReplica(): pending work queued before it
|
|
2345
|
+
* does not run. */
|
|
2346
|
+
private pendingReconcileEpoch = 0;
|
|
2347
|
+
private lastPendingReconcileAt = 0;
|
|
2348
|
+
/** Hints that arrive this close together share one reconcile. */
|
|
2349
|
+
private static readonly PENDING_RECONCILE_DEBOUNCE_MS = 100;
|
|
2350
|
+
/** At most one hinted reconcile per this interval. */
|
|
2351
|
+
private static readonly PENDING_RECONCILE_MIN_INTERVAL_MS = 1_000;
|
|
2352
|
+
/** Backoff for entities whose fetch raced a newer change: doubling from
|
|
2353
|
+
* the base, capped. */
|
|
2354
|
+
private static readonly PENDING_RECONCILE_RETRY_BASE_MS = 250;
|
|
2355
|
+
private static readonly PENDING_RECONCILE_RETRY_MAX_MS = 10_000;
|
|
2356
|
+
|
|
2357
|
+
/**
|
|
2358
|
+
* Keep `entities` pending until a reconcile of them completes. Used for
|
|
2359
|
+
* server hints (`sync-reconcile`: a share, an unshare, a folder
|
|
2360
|
+
* membership changed which rows this user may read, and no other event
|
|
2361
|
+
* will tell this replica) and for entities an earlier reconcile skipped.
|
|
2362
|
+
* Requests are merged, run at most once a second, and an entity whose
|
|
2363
|
+
* fetch failed or raced a newer change is retried with backoff until it
|
|
2364
|
+
* lands or the engine stops.
|
|
2365
|
+
*/
|
|
2366
|
+
private queuePendingReconcile(entities: string[]): void {
|
|
2367
|
+
for (const e of entities) this.pendingReconcile.set(e, ++this.pendingReconcileSeq);
|
|
2368
|
+
this.schedulePendingReconcile(SyncEngine.PENDING_RECONCILE_DEBOUNCE_MS);
|
|
2369
|
+
}
|
|
2370
|
+
|
|
2371
|
+
private schedulePendingReconcile(delayMs: number): void {
|
|
2372
|
+
if (this.pendingReconcileTimer !== null || !this.running) return;
|
|
2373
|
+
const wait = Math.max(delayMs, this.lastPendingReconcileAt + SyncEngine.PENDING_RECONCILE_MIN_INTERVAL_MS - Date.now());
|
|
2374
|
+
this.pendingReconcileTimer = setTimeout(() => {
|
|
2375
|
+
this.pendingReconcileTimer = null;
|
|
2376
|
+
void this.flushPendingReconcile();
|
|
2377
|
+
}, wait);
|
|
2378
|
+
}
|
|
2379
|
+
|
|
2380
|
+
private async flushPendingReconcile(): Promise<void> {
|
|
2381
|
+
if (!this.running || this.pendingReconcile.size === 0) return;
|
|
2382
|
+
const epoch = this.pendingReconcileEpoch;
|
|
2383
|
+
const entities = [...this.pendingReconcile.keys()];
|
|
2384
|
+
this.pendingReconcile.clear();
|
|
2385
|
+
this.lastPendingReconcileAt = Date.now();
|
|
2386
|
+
let skipped: string[];
|
|
2387
|
+
try {
|
|
2388
|
+
// Queued like any reconcile, and dropped if stop() or resetReplica()
|
|
2389
|
+
// ran while it waited.
|
|
2390
|
+
const result = await this.opQueue.enqueue(`reconcile-pending:${epoch}:${Date.now()}`, async () => {
|
|
2391
|
+
if (epoch !== this.pendingReconcileEpoch || !this.running) return null;
|
|
2392
|
+
try {
|
|
2393
|
+
return await this.reconcileInner(entities);
|
|
2394
|
+
} finally {
|
|
2395
|
+
this.lastReconcileAt = Date.now();
|
|
2396
|
+
}
|
|
2397
|
+
});
|
|
2398
|
+
if (result === null) return;
|
|
2399
|
+
skipped = result.skipped;
|
|
2400
|
+
} catch {
|
|
2401
|
+
skipped = entities;
|
|
2402
|
+
}
|
|
2403
|
+
if (epoch !== this.pendingReconcileEpoch) return;
|
|
2404
|
+
for (const e of skipped) {
|
|
2405
|
+
if (!this.pendingReconcile.has(e)) this.pendingReconcile.set(e, ++this.pendingReconcileSeq);
|
|
2406
|
+
}
|
|
2407
|
+
this.pendingReconcileRetries = skipped.length > 0 ? this.pendingReconcileRetries + 1 : 0;
|
|
2408
|
+
if (this.pendingReconcile.size === 0) return;
|
|
2409
|
+
const delay =
|
|
2410
|
+
skipped.length > 0
|
|
2411
|
+
? Math.min(
|
|
2412
|
+
SyncEngine.PENDING_RECONCILE_RETRY_BASE_MS * 2 ** (this.pendingReconcileRetries - 1),
|
|
2413
|
+
SyncEngine.PENDING_RECONCILE_RETRY_MAX_MS,
|
|
2414
|
+
)
|
|
2415
|
+
: SyncEngine.PENDING_RECONCILE_DEBOUNCE_MS;
|
|
2416
|
+
this.schedulePendingReconcile(delay);
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
/** Drop pending reconciles, including queued ones (stop, replica reset). */
|
|
2420
|
+
private cancelPendingReconcile(): void {
|
|
2421
|
+
this.pendingReconcileEpoch += 1;
|
|
2422
|
+
if (this.pendingReconcileTimer !== null) {
|
|
2423
|
+
clearTimeout(this.pendingReconcileTimer);
|
|
2424
|
+
this.pendingReconcileTimer = null;
|
|
2425
|
+
}
|
|
2426
|
+
this.pendingReconcile.clear();
|
|
2427
|
+
this.pendingReconcileRetries = 0;
|
|
2428
|
+
}
|
|
2429
|
+
|
|
2430
|
+
/** `reconcile`, resolving with the entities it reconciled and the ones
|
|
2431
|
+
* it skipped (the fetch failed, or the cursor moved during it). */
|
|
2432
|
+
private async reconcileReportingSkips(entities?: string[]): Promise<ReconcileOutcome> {
|
|
2306
2433
|
const minIntervalMs = this.config.reconcileMinIntervalMs ?? 2_000;
|
|
2307
2434
|
const now = Date.now();
|
|
2308
2435
|
if (entities === undefined && now - this.lastReconcileAt < minIntervalMs) {
|
|
2309
|
-
return;
|
|
2436
|
+
return { done: [], skipped: [], startedAt: this.pendingReconcileSeq };
|
|
2310
2437
|
}
|
|
2311
2438
|
// Reconcile waits behind any in-flight pull / refresh / push so it
|
|
2312
2439
|
// can't apply rows captured under a stale session.
|
|
@@ -2315,7 +2442,7 @@ export class SyncEngine {
|
|
|
2315
2442
|
// the same window share one fetch per entity.
|
|
2316
2443
|
return this.opQueue.enqueue("reconcile", async () => {
|
|
2317
2444
|
try {
|
|
2318
|
-
await this.reconcileInner();
|
|
2445
|
+
return await this.reconcileInner();
|
|
2319
2446
|
} finally {
|
|
2320
2447
|
this.lastReconcileAt = Date.now();
|
|
2321
2448
|
}
|
|
@@ -2330,9 +2457,9 @@ export class SyncEngine {
|
|
|
2330
2457
|
for (const e of entities) open.entities.add(e);
|
|
2331
2458
|
return open.promise;
|
|
2332
2459
|
}
|
|
2333
|
-
const batch: { entities: Set<string>; promise: Promise<
|
|
2460
|
+
const batch: { entities: Set<string>; promise: Promise<ReconcileOutcome> } = {
|
|
2334
2461
|
entities: new Set(entities),
|
|
2335
|
-
promise: Promise.resolve(),
|
|
2462
|
+
promise: Promise.resolve({ done: [], skipped: [], startedAt: this.pendingReconcileSeq }),
|
|
2336
2463
|
};
|
|
2337
2464
|
this.scopedReconcileBatch = batch;
|
|
2338
2465
|
this.scopedReconcileSeq += 1;
|
|
@@ -2341,7 +2468,7 @@ export class SyncEngine {
|
|
|
2341
2468
|
async () => {
|
|
2342
2469
|
if (this.scopedReconcileBatch === batch) this.scopedReconcileBatch = null;
|
|
2343
2470
|
try {
|
|
2344
|
-
await this.reconcileInner([...batch.entities]);
|
|
2471
|
+
return await this.reconcileInner([...batch.entities]);
|
|
2345
2472
|
} finally {
|
|
2346
2473
|
this.lastReconcileAt = Date.now();
|
|
2347
2474
|
}
|
|
@@ -2353,14 +2480,15 @@ export class SyncEngine {
|
|
|
2353
2480
|
/** Scoped reconcile batch that has not started running yet. */
|
|
2354
2481
|
private scopedReconcileBatch: {
|
|
2355
2482
|
entities: Set<string>;
|
|
2356
|
-
promise: Promise<
|
|
2483
|
+
promise: Promise<ReconcileOutcome>;
|
|
2357
2484
|
} | null = null;
|
|
2358
2485
|
private scopedReconcileSeq = 0;
|
|
2359
2486
|
|
|
2360
|
-
private async reconcileInner(entities?: string[]): Promise<
|
|
2487
|
+
private async reconcileInner(entities?: string[]): Promise<ReconcileOutcome> {
|
|
2488
|
+
const startedAt = this.pendingReconcileSeq;
|
|
2361
2489
|
// Same reasoning as pullInner: the leader reconciles, broadcasts
|
|
2362
2490
|
// results, and follower replicas converge via the channel.
|
|
2363
|
-
if (!this.isMultiTabLeader) return;
|
|
2491
|
+
if (!this.isMultiTabLeader) return { done: [], skipped: [], startedAt };
|
|
2364
2492
|
// Sweep entities with local rows PLUS entities the app has observed
|
|
2365
2493
|
// via useQuery (even when empty locally). Without the observed set,
|
|
2366
2494
|
// a server row in a never-cached entity is never reconciled and
|
|
@@ -2368,7 +2496,9 @@ export class SyncEngine {
|
|
|
2368
2496
|
const names =
|
|
2369
2497
|
entities ??
|
|
2370
2498
|
[...new Set([...this.store.entityNames(), ...this.observedEntities])];
|
|
2371
|
-
if (names.length === 0) return;
|
|
2499
|
+
if (names.length === 0) return { done: [], skipped: [], startedAt };
|
|
2500
|
+
const skipped: string[] = [];
|
|
2501
|
+
const done: string[] = [];
|
|
2372
2502
|
// Tombstone seq for any local row the server doesn't return. Using
|
|
2373
2503
|
// the current cursor means future inserts (which have higher seqs)
|
|
2374
2504
|
// bypass the tombstone — re-creation server-side still propagates.
|
|
@@ -2423,6 +2553,7 @@ export class SyncEngine {
|
|
|
2423
2553
|
// Entity removed from the manifest — definitive. Drop its rows.
|
|
2424
2554
|
this.reconcile403Streak.delete(entity);
|
|
2425
2555
|
await this.dropEntity(entity, tombstoneSeq);
|
|
2556
|
+
done.push(entity);
|
|
2426
2557
|
return;
|
|
2427
2558
|
}
|
|
2428
2559
|
if (status === 403) {
|
|
@@ -2444,11 +2575,17 @@ export class SyncEngine {
|
|
|
2444
2575
|
const streak = (this.reconcile403Streak.get(entity) ?? 0) + 1;
|
|
2445
2576
|
if (streak < 2) {
|
|
2446
2577
|
this.reconcile403Streak.set(entity, streak);
|
|
2578
|
+
skipped.push(entity);
|
|
2447
2579
|
return;
|
|
2448
2580
|
}
|
|
2449
2581
|
this.reconcile403Streak.delete(entity);
|
|
2450
2582
|
await this.dropEntity(entity, tombstoneSeq);
|
|
2583
|
+
done.push(entity);
|
|
2584
|
+
return;
|
|
2451
2585
|
}
|
|
2586
|
+
// Not reconciled (offline, a 5xx): reported, so a server hint
|
|
2587
|
+
// stays pending and retries.
|
|
2588
|
+
skipped.push(entity);
|
|
2452
2589
|
return;
|
|
2453
2590
|
}
|
|
2454
2591
|
// Successful fetch — the entity is readable, so any prior 403 streak
|
|
@@ -2460,7 +2597,9 @@ export class SyncEngine {
|
|
|
2460
2597
|
// for a row our snapshot just captured. Bail out for this
|
|
2461
2598
|
// entity; reconcile() is triggered again on visibility-change
|
|
2462
2599
|
// and reconnect, and the WS event already carried the latest
|
|
2463
|
-
// state for the affected row.
|
|
2600
|
+
// state for the affected row. Reported, so a server hint can
|
|
2601
|
+
// retry (flushPendingReconcile).
|
|
2602
|
+
skipped.push(entity);
|
|
2464
2603
|
return;
|
|
2465
2604
|
}
|
|
2466
2605
|
if (this.session.signature() !== sessionBeforeFetch) {
|
|
@@ -2477,8 +2616,10 @@ export class SyncEngine {
|
|
|
2477
2616
|
tombstoneSeq,
|
|
2478
2617
|
fetchTruncated,
|
|
2479
2618
|
);
|
|
2619
|
+
done.push(entity);
|
|
2480
2620
|
}),
|
|
2481
2621
|
);
|
|
2622
|
+
return { done, skipped, startedAt };
|
|
2482
2623
|
}
|
|
2483
2624
|
|
|
2484
2625
|
/** Fetch every row for an entity. Uses cursor pagination so big tables
|
|
@@ -4105,6 +4246,15 @@ export class SyncEngine {
|
|
|
4105
4246
|
// fresh /api/auth/me read which updates the cached session AND,
|
|
4106
4247
|
// on tenant flip, resets the replica so stale rows from the
|
|
4107
4248
|
// previous tenant disappear.
|
|
4249
|
+
// A change elsewhere (a share, an unshare, a folder membership)
|
|
4250
|
+
// changed which rows of these entities this user may read, without a
|
|
4251
|
+
// change event for the rows themselves. Reconcile them.
|
|
4252
|
+
if (msg.type === "sync-reconcile" && Array.isArray(msg.entities)) {
|
|
4253
|
+
const entities = msg.entities.filter((e): e is string => typeof e === "string");
|
|
4254
|
+
if (entities.length > 0) this.queuePendingReconcile(entities);
|
|
4255
|
+
return;
|
|
4256
|
+
}
|
|
4257
|
+
|
|
4108
4258
|
if (msg.type === "session-changed") {
|
|
4109
4259
|
void this.refreshResolvedSession();
|
|
4110
4260
|
return;
|
package/src/reconcile.test.ts
CHANGED
|
@@ -668,3 +668,352 @@ describe("SyncEngine cookie auth", () => {
|
|
|
668
668
|
expect(notifications).toBe(1);
|
|
669
669
|
});
|
|
670
670
|
});
|
|
671
|
+
|
|
672
|
+
// The server sends `sync-reconcile` when a row of another entity changed
|
|
673
|
+
// which rows of these entities the user may read (a share, an unshare):
|
|
674
|
+
// no change event of the rows themselves will come.
|
|
675
|
+
describe("sync-reconcile hint", () => {
|
|
676
|
+
let restore: (() => void) | null = null;
|
|
677
|
+
|
|
678
|
+
afterEach(() => {
|
|
679
|
+
restore?.();
|
|
680
|
+
restore = null;
|
|
681
|
+
});
|
|
682
|
+
|
|
683
|
+
type Internals = {
|
|
684
|
+
dispatchInboundMessage(m: Record<string, unknown>): void;
|
|
685
|
+
running: boolean;
|
|
686
|
+
cursor: { last_seq: number };
|
|
687
|
+
pendingReconcile: Map<string, number>;
|
|
688
|
+
};
|
|
689
|
+
const inner = (engine: SyncEngine) => engine as unknown as Internals;
|
|
690
|
+
const deliver = (engine: SyncEngine, msg: Record<string, unknown>) => inner(engine).dispatchInboundMessage(msg);
|
|
691
|
+
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
692
|
+
|
|
693
|
+
function documentServer(onFetch: (n: number) => void = () => {}) {
|
|
694
|
+
let fetches = 0;
|
|
695
|
+
restore = installFetch(async (url) => {
|
|
696
|
+
if (url.includes("/api/entities/Document/cursor")) {
|
|
697
|
+
fetches += 1;
|
|
698
|
+
onFetch(fetches);
|
|
699
|
+
return {
|
|
700
|
+
status: 200,
|
|
701
|
+
body: { data: [{ id: "shared", title: "Now shared" }], next_cursor: null, has_more: false },
|
|
702
|
+
};
|
|
703
|
+
}
|
|
704
|
+
return { status: 404, body: {} };
|
|
705
|
+
});
|
|
706
|
+
return () => fetches;
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
test("reconciles the named entities: the shared row arrives, the unshared one leaves", async () => {
|
|
710
|
+
const fetches = documentServer();
|
|
711
|
+
const engine = makeEngine();
|
|
712
|
+
inner(engine).running = true;
|
|
713
|
+
seedStore(engine, "Document", [{ id: "unshared", title: "No longer shared" }]);
|
|
714
|
+
|
|
715
|
+
deliver(engine, { type: "sync-reconcile", entities: ["Document"] });
|
|
716
|
+
await sleep(250);
|
|
717
|
+
|
|
718
|
+
expect(fetches()).toBe(1);
|
|
719
|
+
expect(engine.store.list("Document").map((r) => (r as { id: string }).id)).toEqual(["shared"]);
|
|
720
|
+
engine.stop();
|
|
721
|
+
});
|
|
722
|
+
|
|
723
|
+
test("hints that arrive together share one fetch", async () => {
|
|
724
|
+
const fetches = documentServer();
|
|
725
|
+
const engine = makeEngine();
|
|
726
|
+
inner(engine).running = true;
|
|
727
|
+
for (let i = 0; i < 5; i++) deliver(engine, { type: "sync-reconcile", entities: ["Document"] });
|
|
728
|
+
await sleep(250);
|
|
729
|
+
expect(fetches()).toBe(1);
|
|
730
|
+
engine.stop();
|
|
731
|
+
});
|
|
732
|
+
|
|
733
|
+
test("keeps retrying while newer changes race the fetch, then lands", async () => {
|
|
734
|
+
const engine = makeEngine();
|
|
735
|
+
// The first two fetches race a change event: the cursor moves.
|
|
736
|
+
const fetches = documentServer((n) => {
|
|
737
|
+
if (n <= 2) inner(engine).cursor.last_seq += 1;
|
|
738
|
+
});
|
|
739
|
+
inner(engine).running = true;
|
|
740
|
+
|
|
741
|
+
deliver(engine, { type: "sync-reconcile", entities: ["Document"] });
|
|
742
|
+
await sleep(250);
|
|
743
|
+
expect(fetches()).toBe(1);
|
|
744
|
+
expect(engine.store.list("Document")).toEqual([]);
|
|
745
|
+
expect([...inner(engine).pendingReconcile.keys()]).toEqual(["Document"]);
|
|
746
|
+
|
|
747
|
+
// Retries run at most once a second.
|
|
748
|
+
await sleep(2300);
|
|
749
|
+
expect(fetches()).toBe(3);
|
|
750
|
+
expect(engine.store.list("Document").map((r) => (r as { id: string }).id)).toEqual(["shared"]);
|
|
751
|
+
expect(inner(engine).pendingReconcile.size).toBe(0);
|
|
752
|
+
engine.stop();
|
|
753
|
+
});
|
|
754
|
+
|
|
755
|
+
test("stop drops pending hints", async () => {
|
|
756
|
+
const fetches = documentServer();
|
|
757
|
+
const engine = makeEngine();
|
|
758
|
+
inner(engine).running = true;
|
|
759
|
+
deliver(engine, { type: "sync-reconcile", entities: ["Document"] });
|
|
760
|
+
engine.stop();
|
|
761
|
+
await sleep(250);
|
|
762
|
+
expect(fetches()).toBe(0);
|
|
763
|
+
expect(inner(engine).pendingReconcile.size).toBe(0);
|
|
764
|
+
});
|
|
765
|
+
|
|
766
|
+
test("ignores a hint without entity names", async () => {
|
|
767
|
+
const fetches = documentServer();
|
|
768
|
+
const engine = makeEngine();
|
|
769
|
+
inner(engine).running = true;
|
|
770
|
+
deliver(engine, { type: "sync-reconcile", entities: [42, null] });
|
|
771
|
+
deliver(engine, { type: "sync-reconcile" });
|
|
772
|
+
await sleep(250);
|
|
773
|
+
expect(fetches()).toBe(0);
|
|
774
|
+
engine.stop();
|
|
775
|
+
});
|
|
776
|
+
});
|
|
777
|
+
|
|
778
|
+
describe("sync-reconcile hint after a failed fetch", () => {
|
|
779
|
+
let restore: (() => void) | null = null;
|
|
780
|
+
afterEach(() => {
|
|
781
|
+
restore?.();
|
|
782
|
+
restore = null;
|
|
783
|
+
});
|
|
784
|
+
|
|
785
|
+
test("a 5xx keeps the hint pending and the retry lands", async () => {
|
|
786
|
+
let fetches = 0;
|
|
787
|
+
restore = installFetch(async (url) => {
|
|
788
|
+
if (url.includes("/api/entities/Document/cursor")) {
|
|
789
|
+
fetches += 1;
|
|
790
|
+
if (fetches === 1) return { status: 503, body: {} };
|
|
791
|
+
return { status: 200, body: { data: [], next_cursor: null, has_more: false } };
|
|
792
|
+
}
|
|
793
|
+
return { status: 404, body: {} };
|
|
794
|
+
});
|
|
795
|
+
const engine = makeEngine();
|
|
796
|
+
const inner = engine as unknown as {
|
|
797
|
+
running: boolean;
|
|
798
|
+
pendingReconcile: Map<string, number>;
|
|
799
|
+
dispatchInboundMessage(m: Record<string, unknown>): void;
|
|
800
|
+
};
|
|
801
|
+
inner.running = true;
|
|
802
|
+
seedStore(engine, "Document", [{ id: "unshared", title: "No longer shared" }]);
|
|
803
|
+
|
|
804
|
+
inner.dispatchInboundMessage({ type: "sync-reconcile", entities: ["Document"] });
|
|
805
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
806
|
+
expect(fetches).toBe(1);
|
|
807
|
+
expect([...inner.pendingReconcile.keys()]).toEqual(["Document"]);
|
|
808
|
+
expect(engine.store.list("Document").length).toBe(1);
|
|
809
|
+
|
|
810
|
+
await new Promise((r) => setTimeout(r, 1200));
|
|
811
|
+
expect(fetches).toBe(2);
|
|
812
|
+
expect(engine.store.list("Document")).toEqual([]);
|
|
813
|
+
engine.stop();
|
|
814
|
+
});
|
|
815
|
+
});
|
|
816
|
+
|
|
817
|
+
// Any reconcile, not only a hinted one: an entity skipped because its fetch
|
|
818
|
+
// raced a newer change is retried, so a row that became readable while the
|
|
819
|
+
// page was away (a new folder, a share) still arrives.
|
|
820
|
+
describe("a skipped reconcile retries", () => {
|
|
821
|
+
let restore: (() => void) | null = null;
|
|
822
|
+
afterEach(() => {
|
|
823
|
+
restore?.();
|
|
824
|
+
restore = null;
|
|
825
|
+
});
|
|
826
|
+
|
|
827
|
+
test("a startup reconcile that raced a change lands on the retry", async () => {
|
|
828
|
+
const engine = makeEngine();
|
|
829
|
+
const inner = engine as unknown as { running: boolean; cursor: { last_seq: number }; pendingReconcile: Set<string> };
|
|
830
|
+
let fetches = 0;
|
|
831
|
+
restore = installFetch(async (url) => {
|
|
832
|
+
if (url.includes("/api/entities/Folder/cursor")) {
|
|
833
|
+
fetches += 1;
|
|
834
|
+
if (fetches === 1) inner.cursor.last_seq += 1;
|
|
835
|
+
return { status: 200, body: { data: [{ id: "f1", name: "Team" }], next_cursor: null, has_more: false } };
|
|
836
|
+
}
|
|
837
|
+
return { status: 404, body: {} };
|
|
838
|
+
});
|
|
839
|
+
inner.running = true;
|
|
840
|
+
|
|
841
|
+
await engine.reconcile(["Folder"]);
|
|
842
|
+
expect(engine.store.list("Folder")).toEqual([]);
|
|
843
|
+
expect([...inner.pendingReconcile.keys()]).toEqual(["Folder"]);
|
|
844
|
+
|
|
845
|
+
await new Promise((r) => setTimeout(r, 1300));
|
|
846
|
+
expect(fetches).toBe(2);
|
|
847
|
+
expect(engine.store.list("Folder").map((r) => (r as { id: string }).id)).toEqual(["f1"]);
|
|
848
|
+
engine.stop();
|
|
849
|
+
});
|
|
850
|
+
});
|
|
851
|
+
|
|
852
|
+
describe("pending reconciles end when they should", () => {
|
|
853
|
+
let restore: (() => void) | null = null;
|
|
854
|
+
afterEach(() => {
|
|
855
|
+
restore?.();
|
|
856
|
+
restore = null;
|
|
857
|
+
});
|
|
858
|
+
|
|
859
|
+
type Internals = {
|
|
860
|
+
running: boolean;
|
|
861
|
+
pendingReconcile: Map<string, number>;
|
|
862
|
+
opQueue: { enqueue<T>(key: string, fn: () => Promise<T>): Promise<T> };
|
|
863
|
+
dispatchInboundMessage(m: Record<string, unknown>): void;
|
|
864
|
+
};
|
|
865
|
+
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
866
|
+
|
|
867
|
+
function server(status: () => number) {
|
|
868
|
+
let fetches = 0;
|
|
869
|
+
restore = installFetch(async (url) => {
|
|
870
|
+
if (url.includes("/api/entities/Document/cursor")) {
|
|
871
|
+
fetches += 1;
|
|
872
|
+
const s = status();
|
|
873
|
+
if (s !== 200) return { status: s, body: {} };
|
|
874
|
+
return { status: 200, body: { data: [], next_cursor: null, has_more: false } };
|
|
875
|
+
}
|
|
876
|
+
return { status: 404, body: {} };
|
|
877
|
+
});
|
|
878
|
+
return () => fetches;
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
test("a later successful reconcile clears the pending retry", async () => {
|
|
882
|
+
let failing = true;
|
|
883
|
+
const fetches = server(() => (failing ? 503 : 200));
|
|
884
|
+
const engine = makeEngine();
|
|
885
|
+
const inner = engine as unknown as Internals;
|
|
886
|
+
inner.running = true;
|
|
887
|
+
await engine.reconcile(["Document"]);
|
|
888
|
+
expect([...inner.pendingReconcile.keys()]).toEqual(["Document"]);
|
|
889
|
+
failing = false;
|
|
890
|
+
await engine.reconcile(["Document"]);
|
|
891
|
+
expect(inner.pendingReconcile.size).toBe(0);
|
|
892
|
+
await sleep(1300);
|
|
893
|
+
expect(fetches()).toBe(2);
|
|
894
|
+
engine.stop();
|
|
895
|
+
});
|
|
896
|
+
|
|
897
|
+
test("stop cancels a retry already waiting in the queue", async () => {
|
|
898
|
+
const fetches = server(() => 200);
|
|
899
|
+
const engine = makeEngine();
|
|
900
|
+
const inner = engine as unknown as Internals;
|
|
901
|
+
inner.running = true;
|
|
902
|
+
// A slow operation holds the queue while the retry's timer fires.
|
|
903
|
+
void inner.opQueue.enqueue("slow", () => sleep(400));
|
|
904
|
+
inner.dispatchInboundMessage({ type: "sync-reconcile", entities: ["Document"] });
|
|
905
|
+
await sleep(200);
|
|
906
|
+
engine.stop();
|
|
907
|
+
await sleep(400);
|
|
908
|
+
expect(fetches()).toBe(0);
|
|
909
|
+
});
|
|
910
|
+
|
|
911
|
+
test("a replica reset drops pending retries", async () => {
|
|
912
|
+
const fetches = server(() => 503);
|
|
913
|
+
const engine = makeEngine();
|
|
914
|
+
const inner = engine as unknown as Internals;
|
|
915
|
+
inner.running = true;
|
|
916
|
+
await engine.reconcile(["Document"]);
|
|
917
|
+
expect(fetches()).toBe(1);
|
|
918
|
+
await engine.resetReplica();
|
|
919
|
+
expect(inner.pendingReconcile.size).toBe(0);
|
|
920
|
+
await sleep(1300);
|
|
921
|
+
expect(fetches()).toBe(1);
|
|
922
|
+
engine.stop();
|
|
923
|
+
});
|
|
924
|
+
});
|
|
925
|
+
|
|
926
|
+
describe("pending reconciles and newer requests", () => {
|
|
927
|
+
let restore: (() => void) | null = null;
|
|
928
|
+
afterEach(() => {
|
|
929
|
+
restore?.();
|
|
930
|
+
restore = null;
|
|
931
|
+
});
|
|
932
|
+
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
933
|
+
type Internals = {
|
|
934
|
+
running: boolean;
|
|
935
|
+
pendingReconcile: Map<string, number>;
|
|
936
|
+
dispatchInboundMessage(m: Record<string, unknown>): void;
|
|
937
|
+
};
|
|
938
|
+
|
|
939
|
+
test("a hint that arrives during a fetch survives that fetch", async () => {
|
|
940
|
+
const engine = makeEngine();
|
|
941
|
+
const inner = engine as unknown as Internals;
|
|
942
|
+
let fetches = 0;
|
|
943
|
+
restore = installFetch(async (url) => {
|
|
944
|
+
if (url.includes("/api/entities/Document/cursor")) {
|
|
945
|
+
fetches += 1;
|
|
946
|
+
// The server already built this response when the share lands.
|
|
947
|
+
if (fetches === 1) inner.dispatchInboundMessage({ type: "sync-reconcile", entities: ["Document"] });
|
|
948
|
+
return { status: 200, body: { data: [], next_cursor: null, has_more: false } };
|
|
949
|
+
}
|
|
950
|
+
return { status: 404, body: {} };
|
|
951
|
+
});
|
|
952
|
+
inner.running = true;
|
|
953
|
+
await engine.reconcile(["Document"]);
|
|
954
|
+
expect([...inner.pendingReconcile.keys()]).toEqual(["Document"]);
|
|
955
|
+
await sleep(1300);
|
|
956
|
+
expect(fetches).toBe(2);
|
|
957
|
+
engine.stop();
|
|
958
|
+
});
|
|
959
|
+
|
|
960
|
+
test("a reconcile in flight during a reset does not bring retries back", async () => {
|
|
961
|
+
const engine = makeEngine();
|
|
962
|
+
const inner = engine as unknown as Internals;
|
|
963
|
+
let fetches = 0;
|
|
964
|
+
let release: () => void = () => {};
|
|
965
|
+
restore = installFetch(async (url) => {
|
|
966
|
+
if (url.includes("/api/entities/Document/cursor")) {
|
|
967
|
+
fetches += 1;
|
|
968
|
+
if (fetches === 1) await new Promise<void>((r) => (release = r));
|
|
969
|
+
return { status: 503, body: {} };
|
|
970
|
+
}
|
|
971
|
+
return { status: 404, body: {} };
|
|
972
|
+
});
|
|
973
|
+
inner.running = true;
|
|
974
|
+
const pass = engine.reconcile(["Document"]);
|
|
975
|
+
await sleep(20);
|
|
976
|
+
const reset = engine.resetReplica();
|
|
977
|
+
release();
|
|
978
|
+
await pass;
|
|
979
|
+
await reset;
|
|
980
|
+
expect(inner.pendingReconcile.size).toBe(0);
|
|
981
|
+
await sleep(1300);
|
|
982
|
+
expect(fetches).toBe(1);
|
|
983
|
+
engine.stop();
|
|
984
|
+
});
|
|
985
|
+
});
|
|
986
|
+
|
|
987
|
+
test("two callers sharing one full reconcile keep a hint that arrived between them", async () => {
|
|
988
|
+
const engine = makeEngine();
|
|
989
|
+
const inner = engine as unknown as {
|
|
990
|
+
running: boolean;
|
|
991
|
+
pendingReconcile: Map<string, number>;
|
|
992
|
+
store: { applyChange(ev: ChangeEvent): void };
|
|
993
|
+
dispatchInboundMessage(m: Record<string, unknown>): void;
|
|
994
|
+
};
|
|
995
|
+
seedStore(engine, "Document", [{ id: "d0", title: "seed" }]);
|
|
996
|
+
let fetches = 0;
|
|
997
|
+
let release: () => void = () => {};
|
|
998
|
+
const restore = installFetch(async (url) => {
|
|
999
|
+
if (url.includes("/api/entities/Document/cursor")) {
|
|
1000
|
+
fetches += 1;
|
|
1001
|
+
if (fetches === 1) await new Promise<void>((r) => (release = r));
|
|
1002
|
+
return { status: 200, body: { data: [{ id: "d0", title: "seed" }], next_cursor: null, has_more: false } };
|
|
1003
|
+
}
|
|
1004
|
+
return { status: 404, body: {} };
|
|
1005
|
+
});
|
|
1006
|
+
try {
|
|
1007
|
+
inner.running = true;
|
|
1008
|
+
const first = engine.reconcile();
|
|
1009
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
1010
|
+
inner.dispatchInboundMessage({ type: "sync-reconcile", entities: ["Document"] });
|
|
1011
|
+
const second = engine.reconcile();
|
|
1012
|
+
release();
|
|
1013
|
+
await Promise.all([first, second]);
|
|
1014
|
+
expect([...inner.pendingReconcile.keys()]).toEqual(["Document"]);
|
|
1015
|
+
} finally {
|
|
1016
|
+
engine.stop();
|
|
1017
|
+
restore();
|
|
1018
|
+
}
|
|
1019
|
+
});
|