@pylonsync/sync 0.22.70 → 0.22.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +32 -4
- package/package.json +1 -1
- package/src/index.ts +176 -104
- package/src/reconcile.test.ts +180 -23
package/dist/index.d.ts
CHANGED
|
@@ -636,6 +636,10 @@ export declare class SyncEngine {
|
|
|
636
636
|
* before the batch is ever built.
|
|
637
637
|
*/
|
|
638
638
|
private enqueueReconcile;
|
|
639
|
+
/** `enqueueReconcile`, with the rows decided when the step runs on the
|
|
640
|
+
* apply queue, after every change event queued ahead of it. `plan`
|
|
641
|
+
* returns null to apply nothing. */
|
|
642
|
+
private enqueueReconcilePlan;
|
|
639
643
|
/** Stop the sync engine. */
|
|
640
644
|
stop(): void;
|
|
641
645
|
/**
|
|
@@ -883,9 +887,13 @@ export declare class SyncEngine {
|
|
|
883
887
|
private static readonly PENDING_RECONCILE_DEBOUNCE_MS;
|
|
884
888
|
/** At most one hinted reconcile per this interval. */
|
|
885
889
|
private static readonly PENDING_RECONCILE_MIN_INTERVAL_MS;
|
|
886
|
-
/** Backoff for entities whose fetch
|
|
887
|
-
*
|
|
890
|
+
/** Backoff for entities whose fetch failed: doubling from the base,
|
|
891
|
+
* capped. */
|
|
888
892
|
private static readonly PENDING_RECONCILE_RETRY_BASE_MS;
|
|
893
|
+
/** Five minutes. A hinted reconcile must land (no other event tells this
|
|
894
|
+
* replica that a share changed), so retries never stop; they slow down.
|
|
895
|
+
* With the old cap of 10 s, a tab asked a server that kept failing every
|
|
896
|
+
* 10 s for as long as it was open. */
|
|
889
897
|
private static readonly PENDING_RECONCILE_RETRY_MAX_MS;
|
|
890
898
|
/**
|
|
891
899
|
* Keep `entities` pending until a reconcile of them completes. Used for
|
|
@@ -893,8 +901,8 @@ export declare class SyncEngine {
|
|
|
893
901
|
* membership changed which rows this user may read, and no other event
|
|
894
902
|
* will tell this replica) and for entities an earlier reconcile skipped.
|
|
895
903
|
* Requests are merged, run at most once a second, and an entity whose
|
|
896
|
-
* fetch failed
|
|
897
|
-
*
|
|
904
|
+
* fetch failed is retried with backoff until it lands or the engine
|
|
905
|
+
* stops. The backoff doubles up to PENDING_RECONCILE_RETRY_MAX_MS.
|
|
898
906
|
*/
|
|
899
907
|
private queuePendingReconcile;
|
|
900
908
|
private schedulePendingReconcile;
|
|
@@ -904,6 +912,13 @@ export declare class SyncEngine {
|
|
|
904
912
|
/** `reconcile`, resolving with the entities it reconciled and the ones
|
|
905
913
|
* it skipped (the fetch failed, or the cursor moved during it). */
|
|
906
914
|
private reconcileReportingSkips;
|
|
915
|
+
/** One per reconcile fetch in flight: the ids of the entity's rows that a
|
|
916
|
+
* change event wrote since the fetch started. The fetched rows may be
|
|
917
|
+
* older than those events, so the reconcile leaves these rows alone. */
|
|
918
|
+
private reconcileWatches;
|
|
919
|
+
/** Record a row that something newer than a running fetch just wrote or
|
|
920
|
+
* removed: a change event, a revocation. */
|
|
921
|
+
private noteReconcileTouched;
|
|
907
922
|
/** Scoped reconcile batch that has not started running yet. */
|
|
908
923
|
private scopedReconcileBatch;
|
|
909
924
|
private scopedReconcileSeq;
|
|
@@ -914,7 +929,20 @@ export declare class SyncEngine {
|
|
|
914
929
|
* with rows STILL on the server — the fetched set is then INCOMPLETE and
|
|
915
930
|
* the caller must not infer deletions from it (see applyEntityReconcile). */
|
|
916
931
|
private fetchEntityRows;
|
|
932
|
+
/**
|
|
933
|
+
* Apply one entity's fetched rows. Resolves false when `stillValid` said
|
|
934
|
+
* no and nothing was applied.
|
|
935
|
+
*
|
|
936
|
+
* The diff against the replica is computed on the apply queue, behind
|
|
937
|
+
* every change event that arrived during the fetch. At that point
|
|
938
|
+
* `touched` holds the id of each row such an event wrote. Those rows are
|
|
939
|
+
* neither upserted nor removed: the fetch may have read them before the
|
|
940
|
+
* event, and the event is the newer state.
|
|
941
|
+
*/
|
|
917
942
|
private applyEntityReconcile;
|
|
943
|
+
/** The upserts and removals that match the replica to `serverRows`. Runs
|
|
944
|
+
* on the apply queue. */
|
|
945
|
+
private planEntityReconcile;
|
|
918
946
|
private dropEntity;
|
|
919
947
|
/**
|
|
920
948
|
* Fetch `/api/auth/me` and feed the result into the SessionResolver,
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -1861,6 +1861,7 @@ export class SyncEngine {
|
|
|
1861
1861
|
if (c.seq > high) high = c.seq;
|
|
1862
1862
|
}
|
|
1863
1863
|
}
|
|
1864
|
+
for (const c of filtered) this.noteReconcileTouched(c.entity, c.row_id);
|
|
1864
1865
|
// Pick the cursor target. Explicit `targetCursor` (from pull) wins
|
|
1865
1866
|
// — pull's response carries the server's authoritative current_seq
|
|
1866
1867
|
// even when no changes landed in this window. Otherwise derive
|
|
@@ -1948,10 +1949,26 @@ export class SyncEngine {
|
|
|
1948
1949
|
removalIds: string[],
|
|
1949
1950
|
tombstoneSeq: number,
|
|
1950
1951
|
opts: { fromBroadcast?: boolean } = {},
|
|
1952
|
+
): Promise<void> {
|
|
1953
|
+
return this.enqueueReconcilePlan(entity, tombstoneSeq, () => ({ upserts, removalIds }), opts);
|
|
1954
|
+
}
|
|
1955
|
+
|
|
1956
|
+
/** `enqueueReconcile`, with the rows decided when the step runs on the
|
|
1957
|
+
* apply queue, after every change event queued ahead of it. `plan`
|
|
1958
|
+
* returns null to apply nothing. */
|
|
1959
|
+
private enqueueReconcilePlan(
|
|
1960
|
+
entity: string,
|
|
1961
|
+
tombstoneSeq: number,
|
|
1962
|
+
plan: () => { upserts: Row[]; removalIds: string[] } | null,
|
|
1963
|
+
opts: { fromBroadcast?: boolean } = {},
|
|
1951
1964
|
): Promise<void> {
|
|
1952
1965
|
const epoch = this.replicaEpoch;
|
|
1953
1966
|
return this.chainApply(async () => {
|
|
1954
1967
|
if (epoch !== this.replicaEpoch) return;
|
|
1968
|
+
const planned = plan();
|
|
1969
|
+
if (planned === null) return;
|
|
1970
|
+
const { upserts, removalIds } = planned;
|
|
1971
|
+
if (upserts.length === 0 && removalIds.length === 0) return;
|
|
1955
1972
|
const changes = this.store.applyReconcileInMemory(
|
|
1956
1973
|
entity, upserts, removalIds, tombstoneSeq,
|
|
1957
1974
|
);
|
|
@@ -2811,10 +2828,14 @@ export class SyncEngine {
|
|
|
2811
2828
|
private static readonly PENDING_RECONCILE_DEBOUNCE_MS = 100;
|
|
2812
2829
|
/** At most one hinted reconcile per this interval. */
|
|
2813
2830
|
private static readonly PENDING_RECONCILE_MIN_INTERVAL_MS = 1_000;
|
|
2814
|
-
/** Backoff for entities whose fetch
|
|
2815
|
-
*
|
|
2831
|
+
/** Backoff for entities whose fetch failed: doubling from the base,
|
|
2832
|
+
* capped. */
|
|
2816
2833
|
private static readonly PENDING_RECONCILE_RETRY_BASE_MS = 250;
|
|
2817
|
-
|
|
2834
|
+
/** Five minutes. A hinted reconcile must land (no other event tells this
|
|
2835
|
+
* replica that a share changed), so retries never stop; they slow down.
|
|
2836
|
+
* With the old cap of 10 s, a tab asked a server that kept failing every
|
|
2837
|
+
* 10 s for as long as it was open. */
|
|
2838
|
+
private static readonly PENDING_RECONCILE_RETRY_MAX_MS = 300_000;
|
|
2818
2839
|
|
|
2819
2840
|
/**
|
|
2820
2841
|
* Keep `entities` pending until a reconcile of them completes. Used for
|
|
@@ -2822,8 +2843,8 @@ export class SyncEngine {
|
|
|
2822
2843
|
* membership changed which rows this user may read, and no other event
|
|
2823
2844
|
* will tell this replica) and for entities an earlier reconcile skipped.
|
|
2824
2845
|
* Requests are merged, run at most once a second, and an entity whose
|
|
2825
|
-
* fetch failed
|
|
2826
|
-
*
|
|
2846
|
+
* fetch failed is retried with backoff until it lands or the engine
|
|
2847
|
+
* stops. The backoff doubles up to PENDING_RECONCILE_RETRY_MAX_MS.
|
|
2827
2848
|
*/
|
|
2828
2849
|
private queuePendingReconcile(entities: string[]): void {
|
|
2829
2850
|
for (const e of entities) this.pendingReconcile.set(e, ++this.pendingReconcileSeq);
|
|
@@ -2871,7 +2892,8 @@ export class SyncEngine {
|
|
|
2871
2892
|
const delay =
|
|
2872
2893
|
skipped.length > 0
|
|
2873
2894
|
? Math.min(
|
|
2874
|
-
|
|
2895
|
+
// The exponent is capped so the product stays a finite number.
|
|
2896
|
+
SyncEngine.PENDING_RECONCILE_RETRY_BASE_MS * 2 ** Math.min(this.pendingReconcileRetries - 1, 30),
|
|
2875
2897
|
SyncEngine.PENDING_RECONCILE_RETRY_MAX_MS,
|
|
2876
2898
|
)
|
|
2877
2899
|
: SyncEngine.PENDING_RECONCILE_DEBOUNCE_MS;
|
|
@@ -2939,6 +2961,19 @@ export class SyncEngine {
|
|
|
2939
2961
|
return batch.promise;
|
|
2940
2962
|
}
|
|
2941
2963
|
|
|
2964
|
+
/** One per reconcile fetch in flight: the ids of the entity's rows that a
|
|
2965
|
+
* change event wrote since the fetch started. The fetched rows may be
|
|
2966
|
+
* older than those events, so the reconcile leaves these rows alone. */
|
|
2967
|
+
private reconcileWatches = new Set<{ entity: string; touched: Set<string> }>();
|
|
2968
|
+
|
|
2969
|
+
/** Record a row that something newer than a running fetch just wrote or
|
|
2970
|
+
* removed: a change event, a revocation. */
|
|
2971
|
+
private noteReconcileTouched(entity: string, rowId: string): void {
|
|
2972
|
+
for (const watch of this.reconcileWatches) {
|
|
2973
|
+
if (watch.entity === entity) watch.touched.add(rowId);
|
|
2974
|
+
}
|
|
2975
|
+
}
|
|
2976
|
+
|
|
2942
2977
|
/** Scoped reconcile batch that has not started running yet. */
|
|
2943
2978
|
private scopedReconcileBatch: {
|
|
2944
2979
|
entities: Set<string>;
|
|
@@ -2965,22 +3000,101 @@ export class SyncEngine {
|
|
|
2965
3000
|
// the current cursor means future inserts (which have higher seqs)
|
|
2966
3001
|
// bypass the tombstone — re-creation server-side still propagates.
|
|
2967
3002
|
const tombstoneSeq = this.cursor.last_seq;
|
|
3003
|
+
const reconcileEntity = async (entity: string, touched: Set<string>): Promise<void> => {
|
|
3004
|
+
const sessionBeforeFetch = this.session.signature();
|
|
3005
|
+
const epochBeforeFetch = this.replicaEpoch;
|
|
3006
|
+
let serverRows: Row[];
|
|
3007
|
+
let fetchTruncated: boolean;
|
|
3008
|
+
try {
|
|
3009
|
+
const fetched = await this.fetchEntityRows(entity);
|
|
3010
|
+
serverRows = fetched.rows;
|
|
3011
|
+
fetchTruncated = fetched.truncated;
|
|
3012
|
+
} catch (err) {
|
|
3013
|
+
// Network errors are expected (offline, transient 5xx). Skip
|
|
3014
|
+
// this entity; the next reconcile trigger will retry.
|
|
3015
|
+
const status = (err as { status?: number })?.status;
|
|
3016
|
+
if (this.replicaEpoch !== epochBeforeFetch || this.session.signature() !== sessionBeforeFetch) {
|
|
3017
|
+
return;
|
|
3018
|
+
}
|
|
3019
|
+
if (status === 404) {
|
|
3020
|
+
// Entity removed from the manifest — definitive. Drop its rows.
|
|
3021
|
+
this.reconcile403Streak.delete(entity);
|
|
3022
|
+
await this.dropEntity(entity, tombstoneSeq);
|
|
3023
|
+
done.push(entity);
|
|
3024
|
+
return;
|
|
3025
|
+
}
|
|
3026
|
+
if (status === 403) {
|
|
3027
|
+
// A 403 can be TRANSIENT — a bearer caught mid-refresh, a tenant
|
|
3028
|
+
// flip that raced the fetch, or a momentary server-side policy
|
|
3029
|
+
// blip. Nuking every local row on the first one makes the data
|
|
3030
|
+
// flash away and reappear next reconcile (the "rows vanish on
|
|
3031
|
+
// load" bug, on the error path this time). Two gates before we
|
|
3032
|
+
// drop:
|
|
3033
|
+
// 1. If the session flipped during the fetch, the 403 reflects
|
|
3034
|
+
// the OLD context — never drop; the next reconcile under the
|
|
3035
|
+
// new session decides (mirrors the success-path guard below).
|
|
3036
|
+
if (this.session.signature() !== sessionBeforeFetch) {
|
|
3037
|
+
return;
|
|
3038
|
+
}
|
|
3039
|
+
// 2. Otherwise require TWO consecutive 403s for this entity, so
|
|
3040
|
+
// a one-off blip can't wipe the cache. A successful fetch
|
|
3041
|
+
// (below) resets the streak.
|
|
3042
|
+
const streak = (this.reconcile403Streak.get(entity) ?? 0) + 1;
|
|
3043
|
+
if (streak < 2) {
|
|
3044
|
+
this.reconcile403Streak.set(entity, streak);
|
|
3045
|
+
skipped.push(entity);
|
|
3046
|
+
return;
|
|
3047
|
+
}
|
|
3048
|
+
this.reconcile403Streak.delete(entity);
|
|
3049
|
+
await this.dropEntity(entity, tombstoneSeq);
|
|
3050
|
+
done.push(entity);
|
|
3051
|
+
return;
|
|
3052
|
+
}
|
|
3053
|
+
// Not reconciled (offline, a 5xx): reported, so a server hint
|
|
3054
|
+
// stays pending and retries.
|
|
3055
|
+
skipped.push(entity);
|
|
3056
|
+
return;
|
|
3057
|
+
}
|
|
3058
|
+
// Successful fetch — the entity is readable, so any prior 403 streak
|
|
3059
|
+
// is broken (even if the session check below skips the apply).
|
|
3060
|
+
this.reconcile403Streak.delete(entity);
|
|
3061
|
+
if (this.session.signature() !== sessionBeforeFetch) {
|
|
3062
|
+
// Session changed (token flipped, tenant switched, user
|
|
3063
|
+
// signed out → in, etc.). The rows we fetched reflect the
|
|
3064
|
+
// OLD session's policy view; applying them now would
|
|
3065
|
+
// tombstone rows visible under the NEW session. Bail and let
|
|
3066
|
+
// the session-changed envelope drive the next reconcile.
|
|
3067
|
+
return;
|
|
3068
|
+
}
|
|
3069
|
+
const applied = await this.applyEntityReconcile(
|
|
3070
|
+
entity,
|
|
3071
|
+
serverRows,
|
|
3072
|
+
tombstoneSeq,
|
|
3073
|
+
fetchTruncated,
|
|
3074
|
+
touched,
|
|
3075
|
+
// Checked when the apply runs: either can change while the apply
|
|
3076
|
+
// waits behind queued change events.
|
|
3077
|
+
// A replica reset during the fetch wiped the rows this fetch
|
|
3078
|
+
// would be compared with; the pull after the reset refills them.
|
|
3079
|
+
() => this.session.signature() === sessionBeforeFetch && this.replicaEpoch === epochBeforeFetch,
|
|
3080
|
+
);
|
|
3081
|
+
if (applied) done.push(entity);
|
|
3082
|
+
};
|
|
2968
3083
|
// Fan out the per-entity fetches in parallel. Bootstrap reconcile
|
|
2969
3084
|
// used to serialize 5 entities × ~60ms each → 300ms of dead time
|
|
2970
|
-
// on the critical path before channels render.
|
|
2971
|
-
//
|
|
2972
|
-
// each task's closure, so each entity still bails individually
|
|
2973
|
-
// if its OWN fetch raced a WS event or a session flip — parallel
|
|
2974
|
-
// fan-out doesn't weaken either guard.
|
|
3085
|
+
// on the critical path before channels render. Each entity has its
|
|
3086
|
+
// own watch and its own session check, so the fan-out weakens neither.
|
|
2975
3087
|
await Promise.all(
|
|
2976
3088
|
names.map(async (entity) => {
|
|
2977
|
-
//
|
|
2978
|
-
// detect drift mid-reconcile. Two distinct races:
|
|
3089
|
+
// Two things can change while the paginated fetch is in flight:
|
|
2979
3090
|
//
|
|
2980
|
-
// 1.
|
|
2981
|
-
//
|
|
2982
|
-
//
|
|
2983
|
-
//
|
|
3091
|
+
// 1. Change events arrive (WS, pull, another tab). A fetched row
|
|
3092
|
+
// may be older than the event that wrote it, and a row the
|
|
3093
|
+
// event inserted may be missing from the fetch. The watch
|
|
3094
|
+
// records the ids those events wrote, and the apply leaves
|
|
3095
|
+
// those rows alone (see applyEntityReconcile). The rest of the
|
|
3096
|
+
// fetch still applies, so an entity that changes all the time
|
|
3097
|
+
// (an append-only log) reconciles in one fetch.
|
|
2984
3098
|
//
|
|
2985
3099
|
// 2. Session flips: the resolved tenant/user changed while
|
|
2986
3100
|
// the fetch was in flight (e.g., the app called
|
|
@@ -2995,90 +3109,13 @@ export class SyncEngine {
|
|
|
2995
3109
|
// the session signature changed; the next reconcile
|
|
2996
3110
|
// (triggered by session-changed envelope) will re-fetch
|
|
2997
3111
|
// under the new context.
|
|
2998
|
-
const
|
|
2999
|
-
|
|
3000
|
-
const epochBeforeFetch = this.replicaEpoch;
|
|
3001
|
-
let serverRows: Row[];
|
|
3002
|
-
let fetchTruncated: boolean;
|
|
3112
|
+
const watch = { entity, touched: new Set<string>() };
|
|
3113
|
+
this.reconcileWatches.add(watch);
|
|
3003
3114
|
try {
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
} catch (err) {
|
|
3008
|
-
// Network errors are expected (offline, transient 5xx). Skip
|
|
3009
|
-
// this entity; the next reconcile trigger will retry.
|
|
3010
|
-
const status = (err as { status?: number })?.status;
|
|
3011
|
-
if (this.replicaEpoch !== epochBeforeFetch || this.session.signature() !== sessionBeforeFetch) {
|
|
3012
|
-
return;
|
|
3013
|
-
}
|
|
3014
|
-
if (status === 404) {
|
|
3015
|
-
// Entity removed from the manifest — definitive. Drop its rows.
|
|
3016
|
-
this.reconcile403Streak.delete(entity);
|
|
3017
|
-
await this.dropEntity(entity, tombstoneSeq);
|
|
3018
|
-
done.push(entity);
|
|
3019
|
-
return;
|
|
3020
|
-
}
|
|
3021
|
-
if (status === 403) {
|
|
3022
|
-
// A 403 can be TRANSIENT — a bearer caught mid-refresh, a tenant
|
|
3023
|
-
// flip that raced the fetch, or a momentary server-side policy
|
|
3024
|
-
// blip. Nuking every local row on the first one makes the data
|
|
3025
|
-
// flash away and reappear next reconcile (the "rows vanish on
|
|
3026
|
-
// load" bug, on the error path this time). Two gates before we
|
|
3027
|
-
// drop:
|
|
3028
|
-
// 1. If the session flipped during the fetch, the 403 reflects
|
|
3029
|
-
// the OLD context — never drop; the next reconcile under the
|
|
3030
|
-
// new session decides (mirrors the success-path guard below).
|
|
3031
|
-
if (this.session.signature() !== sessionBeforeFetch) {
|
|
3032
|
-
return;
|
|
3033
|
-
}
|
|
3034
|
-
// 2. Otherwise require TWO consecutive 403s for this entity, so
|
|
3035
|
-
// a one-off blip can't wipe the cache. A successful fetch
|
|
3036
|
-
// (below) resets the streak.
|
|
3037
|
-
const streak = (this.reconcile403Streak.get(entity) ?? 0) + 1;
|
|
3038
|
-
if (streak < 2) {
|
|
3039
|
-
this.reconcile403Streak.set(entity, streak);
|
|
3040
|
-
skipped.push(entity);
|
|
3041
|
-
return;
|
|
3042
|
-
}
|
|
3043
|
-
this.reconcile403Streak.delete(entity);
|
|
3044
|
-
await this.dropEntity(entity, tombstoneSeq);
|
|
3045
|
-
done.push(entity);
|
|
3046
|
-
return;
|
|
3047
|
-
}
|
|
3048
|
-
// Not reconciled (offline, a 5xx): reported, so a server hint
|
|
3049
|
-
// stays pending and retries.
|
|
3050
|
-
skipped.push(entity);
|
|
3051
|
-
return;
|
|
3052
|
-
}
|
|
3053
|
-
// Successful fetch — the entity is readable, so any prior 403 streak
|
|
3054
|
-
// is broken (even if a drift guard below makes us skip the apply).
|
|
3055
|
-
this.reconcile403Streak.delete(entity);
|
|
3056
|
-
if (this.cursor.last_seq !== cursorBeforeFetch) {
|
|
3057
|
-
// Cursor moved during fetch — at least one WS event for this
|
|
3058
|
-
// (or another) entity landed and might have a fresher value
|
|
3059
|
-
// for a row our snapshot just captured. Bail out for this
|
|
3060
|
-
// entity; reconcile() is triggered again on visibility-change
|
|
3061
|
-
// and reconnect, and the WS event already carried the latest
|
|
3062
|
-
// state for the affected row. Reported, so a server hint can
|
|
3063
|
-
// retry (flushPendingReconcile).
|
|
3064
|
-
skipped.push(entity);
|
|
3065
|
-
return;
|
|
3066
|
-
}
|
|
3067
|
-
if (this.session.signature() !== sessionBeforeFetch) {
|
|
3068
|
-
// Session changed (token flipped, tenant switched, user
|
|
3069
|
-
// signed out → in, etc.). The rows we fetched reflect the
|
|
3070
|
-
// OLD session's policy view; applying them now would
|
|
3071
|
-
// tombstone rows visible under the NEW session. Bail and let
|
|
3072
|
-
// the session-changed envelope drive the next reconcile.
|
|
3073
|
-
return;
|
|
3115
|
+
await reconcileEntity(entity, watch.touched);
|
|
3116
|
+
} finally {
|
|
3117
|
+
this.reconcileWatches.delete(watch);
|
|
3074
3118
|
}
|
|
3075
|
-
await this.applyEntityReconcile(
|
|
3076
|
-
entity,
|
|
3077
|
-
serverRows,
|
|
3078
|
-
tombstoneSeq,
|
|
3079
|
-
fetchTruncated,
|
|
3080
|
-
);
|
|
3081
|
-
done.push(entity);
|
|
3082
3119
|
}),
|
|
3083
3120
|
);
|
|
3084
3121
|
// A full sweep that reconciled every entity matched the replica to the
|
|
@@ -3133,12 +3170,43 @@ export class SyncEngine {
|
|
|
3133
3170
|
return { rows: out, truncated: true };
|
|
3134
3171
|
}
|
|
3135
3172
|
|
|
3173
|
+
/**
|
|
3174
|
+
* Apply one entity's fetched rows. Resolves false when `stillValid` said
|
|
3175
|
+
* no and nothing was applied.
|
|
3176
|
+
*
|
|
3177
|
+
* The diff against the replica is computed on the apply queue, behind
|
|
3178
|
+
* every change event that arrived during the fetch. At that point
|
|
3179
|
+
* `touched` holds the id of each row such an event wrote. Those rows are
|
|
3180
|
+
* neither upserted nor removed: the fetch may have read them before the
|
|
3181
|
+
* event, and the event is the newer state.
|
|
3182
|
+
*/
|
|
3136
3183
|
private async applyEntityReconcile(
|
|
3137
3184
|
entity: string,
|
|
3138
3185
|
serverRows: Row[],
|
|
3139
3186
|
tombstoneSeq: number,
|
|
3140
3187
|
truncated: boolean,
|
|
3141
|
-
|
|
3188
|
+
touched: ReadonlySet<string>,
|
|
3189
|
+
stillValid: () => boolean,
|
|
3190
|
+
): Promise<boolean> {
|
|
3191
|
+
let applied = true;
|
|
3192
|
+
await this.enqueueReconcilePlan(entity, tombstoneSeq, () => {
|
|
3193
|
+
if (!stillValid()) {
|
|
3194
|
+
applied = false;
|
|
3195
|
+
return null;
|
|
3196
|
+
}
|
|
3197
|
+
return this.planEntityReconcile(entity, serverRows, truncated, touched);
|
|
3198
|
+
});
|
|
3199
|
+
return applied;
|
|
3200
|
+
}
|
|
3201
|
+
|
|
3202
|
+
/** The upserts and removals that match the replica to `serverRows`. Runs
|
|
3203
|
+
* on the apply queue. */
|
|
3204
|
+
private planEntityReconcile(
|
|
3205
|
+
entity: string,
|
|
3206
|
+
serverRows: Row[],
|
|
3207
|
+
truncated: boolean,
|
|
3208
|
+
touched: ReadonlySet<string>,
|
|
3209
|
+
): { upserts: Row[]; removalIds: string[] } {
|
|
3142
3210
|
// Invariant: rows with in-flight or failed mutations are
|
|
3143
3211
|
// off-limits to reconcile. Neither the upsert branch nor the
|
|
3144
3212
|
// tombstone branch may touch them. A hydrated offline mutation
|
|
@@ -3154,6 +3222,7 @@ export class SyncEngine {
|
|
|
3154
3222
|
if (typeof id !== "string" || id.length === 0) continue;
|
|
3155
3223
|
serverIds.add(id);
|
|
3156
3224
|
if (pendingKeys.has(`${entity}/${id}`)) continue;
|
|
3225
|
+
if (touched.has(id)) continue;
|
|
3157
3226
|
const local = this.store.get(entity, id);
|
|
3158
3227
|
if (!local || rowsDiffer(local, row)) {
|
|
3159
3228
|
upserts.push(row);
|
|
@@ -3183,13 +3252,12 @@ export class SyncEngine {
|
|
|
3183
3252
|
const id = (local as { id?: unknown }).id;
|
|
3184
3253
|
if (typeof id !== "string") continue;
|
|
3185
3254
|
if (pendingKeys.has(`${entity}/${id}`)) continue;
|
|
3255
|
+
// Inserted by an event after the fetch read past its position.
|
|
3256
|
+
if (touched.has(id)) continue;
|
|
3186
3257
|
if (!serverIds.has(id)) removalIds.push(id);
|
|
3187
3258
|
}
|
|
3188
3259
|
}
|
|
3189
|
-
|
|
3190
|
-
// Route through the apply queue so a reconcile batch can't
|
|
3191
|
-
// interleave with a fresher WS/pull change event mid-apply.
|
|
3192
|
-
await this.enqueueReconcile(entity, upserts, removalIds, tombstoneSeq);
|
|
3260
|
+
return { upserts, removalIds };
|
|
3193
3261
|
}
|
|
3194
3262
|
|
|
3195
3263
|
private async dropEntity(
|
|
@@ -3658,6 +3726,10 @@ export class SyncEngine {
|
|
|
3658
3726
|
rowId: string,
|
|
3659
3727
|
tombstoneSeq: number,
|
|
3660
3728
|
): void {
|
|
3729
|
+
// A reconcile fetch in flight may have read the row before the server
|
|
3730
|
+
// revoked it. Without this it would put the row back, and nothing later
|
|
3731
|
+
// would remove it again.
|
|
3732
|
+
this.noteReconcileTouched(entity, rowId);
|
|
3661
3733
|
const removed = this.store.revokeRow(entity, rowId, tombstoneSeq);
|
|
3662
3734
|
if (removed) {
|
|
3663
3735
|
if (this.persistence) {
|
package/src/reconcile.test.ts
CHANGED
|
@@ -730,25 +730,28 @@ describe("sync-reconcile hint", () => {
|
|
|
730
730
|
engine.stop();
|
|
731
731
|
});
|
|
732
732
|
|
|
733
|
-
test("
|
|
733
|
+
test("lands in one fetch when newer changes arrive during the fetch", async () => {
|
|
734
734
|
const engine = makeEngine();
|
|
735
|
-
//
|
|
736
|
-
|
|
737
|
-
|
|
735
|
+
// Every fetch races a change event, as on an entity that is written
|
|
736
|
+
// all the time. The reconcile used to discard each fetch and retry
|
|
737
|
+
// with no end.
|
|
738
|
+
let seq = 0;
|
|
739
|
+
const fetches = documentServer(() => {
|
|
740
|
+
seq += 1;
|
|
741
|
+
void (inner(engine) as unknown as { enqueueApply(c: ChangeEvent[]): Promise<void> }).enqueueApply([
|
|
742
|
+
{ seq, entity: "Document", row_id: `live-${seq}`, kind: "insert", data: { id: `live-${seq}` }, timestamp: "" },
|
|
743
|
+
]);
|
|
738
744
|
});
|
|
739
745
|
inner(engine).running = true;
|
|
740
746
|
|
|
741
747
|
deliver(engine, { type: "sync-reconcile", entities: ["Document"] });
|
|
742
748
|
await sleep(250);
|
|
743
749
|
expect(fetches()).toBe(1);
|
|
744
|
-
expect(engine.store.list("Document")).toEqual([]);
|
|
745
|
-
expect(
|
|
750
|
+
expect(engine.store.list("Document").map((r) => (r as { id: string }).id).sort()).toEqual(["live-1", "shared"]);
|
|
751
|
+
expect(inner(engine).pendingReconcile.size).toBe(0);
|
|
746
752
|
|
|
747
|
-
// Retries run at most once a second.
|
|
748
753
|
await sleep(2300);
|
|
749
|
-
expect(fetches()).toBe(
|
|
750
|
-
expect(engine.store.list("Document").map((r) => (r as { id: string }).id)).toEqual(["shared"]);
|
|
751
|
-
expect(inner(engine).pendingReconcile.size).toBe(0);
|
|
754
|
+
expect(fetches()).toBe(1);
|
|
752
755
|
engine.stop();
|
|
753
756
|
});
|
|
754
757
|
|
|
@@ -814,37 +817,153 @@ describe("sync-reconcile hint after a failed fetch", () => {
|
|
|
814
817
|
});
|
|
815
818
|
});
|
|
816
819
|
|
|
817
|
-
//
|
|
818
|
-
//
|
|
819
|
-
//
|
|
820
|
-
describe("a
|
|
820
|
+
// Change events that arrive while a reconcile fetch is in flight. The fetch
|
|
821
|
+
// still applies. Only the rows those events wrote are left alone: the fetch
|
|
822
|
+
// may have read them before the event.
|
|
823
|
+
describe("a reconcile whose fetch raced change events", () => {
|
|
821
824
|
let restore: (() => void) | null = null;
|
|
822
825
|
afterEach(() => {
|
|
823
826
|
restore?.();
|
|
824
827
|
restore = null;
|
|
825
828
|
});
|
|
826
829
|
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
+
type Internals = {
|
|
831
|
+
running: boolean;
|
|
832
|
+
pendingReconcile: Map<string, number>;
|
|
833
|
+
enqueueApply(changes: ChangeEvent[]): Promise<void>;
|
|
834
|
+
};
|
|
835
|
+
|
|
836
|
+
/** A server that answers `rows` for Folder, after `during` ran. */
|
|
837
|
+
function folderServer(rows: Row[], during: () => void) {
|
|
830
838
|
let fetches = 0;
|
|
831
839
|
restore = installFetch(async (url) => {
|
|
832
840
|
if (url.includes("/api/entities/Folder/cursor")) {
|
|
833
841
|
fetches += 1;
|
|
834
|
-
|
|
835
|
-
return { status: 200, body: { data:
|
|
842
|
+
during();
|
|
843
|
+
return { status: 200, body: { data: rows, next_cursor: null, has_more: false } };
|
|
836
844
|
}
|
|
837
845
|
return { status: 404, body: {} };
|
|
838
846
|
});
|
|
847
|
+
return () => fetches;
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
const event = (seq: number, kind: ChangeEvent["kind"], id: string, data?: Row): ChangeEvent => ({
|
|
851
|
+
seq,
|
|
852
|
+
entity: "Folder",
|
|
853
|
+
row_id: id,
|
|
854
|
+
kind,
|
|
855
|
+
data,
|
|
856
|
+
timestamp: "",
|
|
857
|
+
});
|
|
858
|
+
const names = (engine: SyncEngine) =>
|
|
859
|
+
Object.fromEntries(engine.store.list("Folder").map((r) => [(r as { id: string }).id, (r as { name?: string }).name]));
|
|
860
|
+
|
|
861
|
+
test("applies the fetch at once, with no retry", async () => {
|
|
862
|
+
const engine = makeEngine();
|
|
863
|
+
const inner = engine as unknown as Internals;
|
|
864
|
+
const fetches = folderServer([{ id: "f1", name: "Team" }], () => {
|
|
865
|
+
void inner.enqueueApply([event(1, "insert", "other", { id: "other", name: "Other" })]);
|
|
866
|
+
});
|
|
839
867
|
inner.running = true;
|
|
840
868
|
|
|
841
869
|
await engine.reconcile(["Folder"]);
|
|
842
|
-
expect(engine
|
|
843
|
-
expect(
|
|
870
|
+
expect(names(engine)).toEqual({ f1: "Team", other: "Other" });
|
|
871
|
+
expect(inner.pendingReconcile.size).toBe(0);
|
|
844
872
|
|
|
845
873
|
await new Promise((r) => setTimeout(r, 1300));
|
|
846
|
-
expect(fetches).toBe(
|
|
847
|
-
|
|
874
|
+
expect(fetches()).toBe(1);
|
|
875
|
+
engine.stop();
|
|
876
|
+
});
|
|
877
|
+
|
|
878
|
+
test("a row an event updated during the fetch keeps the event's value", async () => {
|
|
879
|
+
const engine = makeEngine();
|
|
880
|
+
const inner = engine as unknown as Internals;
|
|
881
|
+
seedStore(engine, "Folder", [{ id: "f1", name: "Old" }]);
|
|
882
|
+
// The server read f1 before the update, so the fetch carries the old name.
|
|
883
|
+
folderServer([{ id: "f1", name: "Old" }, { id: "f2", name: "Second" }], () => {
|
|
884
|
+
void inner.enqueueApply([event(1, "update", "f1", { id: "f1", name: "New" })]);
|
|
885
|
+
});
|
|
886
|
+
inner.running = true;
|
|
887
|
+
|
|
888
|
+
await engine.reconcile(["Folder"]);
|
|
889
|
+
expect(names(engine)).toEqual({ f1: "New", f2: "Second" });
|
|
890
|
+
engine.stop();
|
|
891
|
+
});
|
|
892
|
+
|
|
893
|
+
test("a row an event inserted during the fetch is not removed, and a stale row is", async () => {
|
|
894
|
+
const engine = makeEngine();
|
|
895
|
+
const inner = engine as unknown as Internals;
|
|
896
|
+
seedStore(engine, "Folder", [{ id: "stale", name: "Deleted on the server" }]);
|
|
897
|
+
// The insert landed after the server read past its position.
|
|
898
|
+
folderServer([], () => {
|
|
899
|
+
void inner.enqueueApply([event(1, "insert", "fresh", { id: "fresh", name: "Fresh" })]);
|
|
900
|
+
});
|
|
901
|
+
inner.running = true;
|
|
902
|
+
|
|
903
|
+
await engine.reconcile(["Folder"]);
|
|
904
|
+
expect(names(engine)).toEqual({ fresh: "Fresh" });
|
|
905
|
+
engine.stop();
|
|
906
|
+
});
|
|
907
|
+
|
|
908
|
+
test("a row an event deleted during the fetch is not brought back", async () => {
|
|
909
|
+
const engine = makeEngine();
|
|
910
|
+
const inner = engine as unknown as Internals;
|
|
911
|
+
seedStore(engine, "Folder", [{ id: "f1", name: "Team" }]);
|
|
912
|
+
// The server read f1 before the delete.
|
|
913
|
+
folderServer([{ id: "f1", name: "Team" }], () => {
|
|
914
|
+
void inner.enqueueApply([event(1, "delete", "f1")]);
|
|
915
|
+
});
|
|
916
|
+
inner.running = true;
|
|
917
|
+
|
|
918
|
+
await engine.reconcile(["Folder"]);
|
|
919
|
+
expect(names(engine)).toEqual({});
|
|
920
|
+
engine.stop();
|
|
921
|
+
});
|
|
922
|
+
|
|
923
|
+
test("a row revoked during the fetch is not put back", async () => {
|
|
924
|
+
const engine = makeEngine();
|
|
925
|
+
const inner = engine as unknown as Internals & {
|
|
926
|
+
handleRowRevocation(entity: string, rowId: string, tombstoneSeq: number): void;
|
|
927
|
+
};
|
|
928
|
+
seedStore(engine, "Folder", [{ id: "f1", name: "Team" }]);
|
|
929
|
+
// The server read f1 before it revoked it. A later event for another
|
|
930
|
+
// row moves the cursor past the revocation.
|
|
931
|
+
folderServer([{ id: "f1", name: "Team" }, { id: "f2", name: "Second" }], () => {
|
|
932
|
+
inner.handleRowRevocation("Folder", "f1", 1);
|
|
933
|
+
void inner.enqueueApply([event(2, "insert", "f3", { id: "f3", name: "Third" })]);
|
|
934
|
+
});
|
|
935
|
+
inner.running = true;
|
|
936
|
+
|
|
937
|
+
await engine.reconcile(["Folder"]);
|
|
938
|
+
expect(names(engine)).toEqual({ f2: "Second", f3: "Third" });
|
|
939
|
+
engine.stop();
|
|
940
|
+
});
|
|
941
|
+
|
|
942
|
+
test("a fetch that a replica reset interrupted is not applied", async () => {
|
|
943
|
+
const engine = makeEngine();
|
|
944
|
+
const inner = engine as unknown as Internals & { replicaEpoch: number };
|
|
945
|
+
// The reset wipes the replica; the pull that follows it refills it.
|
|
946
|
+
folderServer([{ id: "f1", name: "From before the reset" }], () => {
|
|
947
|
+
inner.replicaEpoch += 1;
|
|
948
|
+
});
|
|
949
|
+
inner.running = true;
|
|
950
|
+
|
|
951
|
+
await engine.reconcile(["Folder"]);
|
|
952
|
+
expect(names(engine)).toEqual({});
|
|
953
|
+
engine.stop();
|
|
954
|
+
});
|
|
955
|
+
|
|
956
|
+
test("an event for another entity does not protect this entity's rows", async () => {
|
|
957
|
+
const engine = makeEngine();
|
|
958
|
+
const inner = engine as unknown as Internals;
|
|
959
|
+
seedStore(engine, "Folder", [{ id: "stale", name: "Deleted on the server" }]);
|
|
960
|
+
folderServer([{ id: "f1", name: "Team" }], () => {
|
|
961
|
+
void inner.enqueueApply([{ ...event(1, "insert", "stale", { id: "stale" }), entity: "Document" }]);
|
|
962
|
+
});
|
|
963
|
+
inner.running = true;
|
|
964
|
+
|
|
965
|
+
await engine.reconcile(["Folder"]);
|
|
966
|
+
expect(names(engine)).toEqual({ f1: "Team" });
|
|
848
967
|
engine.stop();
|
|
849
968
|
});
|
|
850
969
|
});
|
|
@@ -894,6 +1013,44 @@ describe("pending reconciles end when they should", () => {
|
|
|
894
1013
|
engine.stop();
|
|
895
1014
|
});
|
|
896
1015
|
|
|
1016
|
+
test("retries of a failing fetch slow down to the cap and still land", async () => {
|
|
1017
|
+
const statics = SyncEngine as unknown as {
|
|
1018
|
+
PENDING_RECONCILE_RETRY_BASE_MS: number;
|
|
1019
|
+
PENDING_RECONCILE_RETRY_MAX_MS: number;
|
|
1020
|
+
PENDING_RECONCILE_MIN_INTERVAL_MS: number;
|
|
1021
|
+
};
|
|
1022
|
+
const saved = { ...statics };
|
|
1023
|
+
statics.PENDING_RECONCILE_RETRY_BASE_MS = 10;
|
|
1024
|
+
statics.PENDING_RECONCILE_RETRY_MAX_MS = 200;
|
|
1025
|
+
statics.PENDING_RECONCILE_MIN_INTERVAL_MS = 1;
|
|
1026
|
+
try {
|
|
1027
|
+
let failing = true;
|
|
1028
|
+
const fetches = server(() => (failing ? 503 : 200));
|
|
1029
|
+
const engine = makeEngine();
|
|
1030
|
+
const inner = engine as unknown as Internals;
|
|
1031
|
+
inner.running = true;
|
|
1032
|
+
await engine.reconcile(["Document"]);
|
|
1033
|
+
// Delays: 100 (debounce), 20, 40, 80, 160, then 200 each. A fixed
|
|
1034
|
+
// 10 ms interval would have made about 90 requests by now.
|
|
1035
|
+
await sleep(900);
|
|
1036
|
+
const whileFailing = fetches();
|
|
1037
|
+
expect(whileFailing).toBeGreaterThanOrEqual(6);
|
|
1038
|
+
expect(whileFailing).toBeLessThanOrEqual(9);
|
|
1039
|
+
expect([...inner.pendingReconcile.keys()]).toEqual(["Document"]);
|
|
1040
|
+
|
|
1041
|
+
// Nothing else triggers a reconcile. The timer alone lands it.
|
|
1042
|
+
failing = false;
|
|
1043
|
+
await sleep(450);
|
|
1044
|
+
expect(inner.pendingReconcile.size).toBe(0);
|
|
1045
|
+
expect(fetches()).toBeLessThanOrEqual(whileFailing + 2);
|
|
1046
|
+
engine.stop();
|
|
1047
|
+
} finally {
|
|
1048
|
+
statics.PENDING_RECONCILE_RETRY_BASE_MS = saved.PENDING_RECONCILE_RETRY_BASE_MS;
|
|
1049
|
+
statics.PENDING_RECONCILE_RETRY_MAX_MS = saved.PENDING_RECONCILE_RETRY_MAX_MS;
|
|
1050
|
+
statics.PENDING_RECONCILE_MIN_INTERVAL_MS = saved.PENDING_RECONCILE_MIN_INTERVAL_MS;
|
|
1051
|
+
}
|
|
1052
|
+
});
|
|
1053
|
+
|
|
897
1054
|
test("stop cancels a retry already waiting in the queue", async () => {
|
|
898
1055
|
const fetches = server(() => 200);
|
|
899
1056
|
const engine = makeEngine();
|