@pylonsync/sync 0.22.70 → 0.22.72

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -636,6 +636,10 @@ export declare class SyncEngine {
636
636
  * before the batch is ever built.
637
637
  */
638
638
  private enqueueReconcile;
639
+ /** `enqueueReconcile`, with the rows decided when the step runs on the
640
+ * apply queue, after every change event queued ahead of it. `plan`
641
+ * returns null to apply nothing. */
642
+ private enqueueReconcilePlan;
639
643
  /** Stop the sync engine. */
640
644
  stop(): void;
641
645
  /**
@@ -883,9 +887,13 @@ export declare class SyncEngine {
883
887
  private static readonly PENDING_RECONCILE_DEBOUNCE_MS;
884
888
  /** At most one hinted reconcile per this interval. */
885
889
  private static readonly PENDING_RECONCILE_MIN_INTERVAL_MS;
886
- /** Backoff for entities whose fetch raced a newer change: doubling from
887
- * the base, capped. */
890
+ /** Backoff for entities whose fetch failed: doubling from the base,
891
+ * capped. */
888
892
  private static readonly PENDING_RECONCILE_RETRY_BASE_MS;
893
+ /** Five minutes. A hinted reconcile must land (no other event tells this
894
+ * replica that a share changed), so retries never stop; they slow down.
895
+ * With the old cap of 10 s, a tab asked a server that kept failing every
896
+ * 10 s for as long as it was open. */
889
897
  private static readonly PENDING_RECONCILE_RETRY_MAX_MS;
890
898
  /**
891
899
  * Keep `entities` pending until a reconcile of them completes. Used for
@@ -893,8 +901,8 @@ export declare class SyncEngine {
893
901
  * membership changed which rows this user may read, and no other event
894
902
  * will tell this replica) and for entities an earlier reconcile skipped.
895
903
  * Requests are merged, run at most once a second, and an entity whose
896
- * fetch failed or raced a newer change is retried with backoff until it
897
- * lands or the engine stops.
904
+ * fetch failed is retried with backoff until it lands or the engine
905
+ * stops. The backoff doubles up to PENDING_RECONCILE_RETRY_MAX_MS.
898
906
  */
899
907
  private queuePendingReconcile;
900
908
  private schedulePendingReconcile;
@@ -904,6 +912,13 @@ export declare class SyncEngine {
904
912
  /** `reconcile`, resolving with the entities it reconciled and the ones
905
913
  * it skipped (the fetch failed, or the cursor moved during it). */
906
914
  private reconcileReportingSkips;
915
+ /** One per reconcile fetch in flight: the ids of the entity's rows that a
916
+ * change event wrote since the fetch started. The fetched rows may be
917
+ * older than those events, so the reconcile leaves these rows alone. */
918
+ private reconcileWatches;
919
+ /** Record a row that something newer than a running fetch just wrote or
920
+ * removed: a change event, a revocation. */
921
+ private noteReconcileTouched;
907
922
  /** Scoped reconcile batch that has not started running yet. */
908
923
  private scopedReconcileBatch;
909
924
  private scopedReconcileSeq;
@@ -914,7 +929,20 @@ export declare class SyncEngine {
914
929
  * with rows STILL on the server — the fetched set is then INCOMPLETE and
915
930
  * the caller must not infer deletions from it (see applyEntityReconcile). */
916
931
  private fetchEntityRows;
932
+ /**
933
+ * Apply one entity's fetched rows. Resolves false when `stillValid` said
934
+ * no and nothing was applied.
935
+ *
936
+ * The diff against the replica is computed on the apply queue, behind
937
+ * every change event that arrived during the fetch. At that point
938
+ * `touched` holds the id of each row such an event wrote. Those rows are
939
+ * neither upserted nor removed: the fetch may have read them before the
940
+ * event, and the event is the newer state.
941
+ */
917
942
  private applyEntityReconcile;
943
+ /** The upserts and removals that match the replica to `serverRows`. Runs
944
+ * on the apply queue. */
945
+ private planEntityReconcile;
918
946
  private dropEntity;
919
947
  /**
920
948
  * Fetch `/api/auth/me` and feed the result into the SessionResolver,
package/package.json CHANGED
@@ -3,7 +3,7 @@
3
3
  "publishConfig": {
4
4
  "access": "public"
5
5
  },
6
- "version": "0.22.70",
6
+ "version": "0.22.72",
7
7
  "type": "module",
8
8
  "main": "./src/index.ts",
9
9
  "types": "./dist/index.d.ts",
package/src/index.ts CHANGED
@@ -1861,6 +1861,7 @@ export class SyncEngine {
1861
1861
  if (c.seq > high) high = c.seq;
1862
1862
  }
1863
1863
  }
1864
+ for (const c of filtered) this.noteReconcileTouched(c.entity, c.row_id);
1864
1865
  // Pick the cursor target. Explicit `targetCursor` (from pull) wins
1865
1866
  // — pull's response carries the server's authoritative current_seq
1866
1867
  // even when no changes landed in this window. Otherwise derive
@@ -1948,10 +1949,26 @@ export class SyncEngine {
1948
1949
  removalIds: string[],
1949
1950
  tombstoneSeq: number,
1950
1951
  opts: { fromBroadcast?: boolean } = {},
1952
+ ): Promise<void> {
1953
+ return this.enqueueReconcilePlan(entity, tombstoneSeq, () => ({ upserts, removalIds }), opts);
1954
+ }
1955
+
1956
+ /** `enqueueReconcile`, with the rows decided when the step runs on the
1957
+ * apply queue, after every change event queued ahead of it. `plan`
1958
+ * returns null to apply nothing. */
1959
+ private enqueueReconcilePlan(
1960
+ entity: string,
1961
+ tombstoneSeq: number,
1962
+ plan: () => { upserts: Row[]; removalIds: string[] } | null,
1963
+ opts: { fromBroadcast?: boolean } = {},
1951
1964
  ): Promise<void> {
1952
1965
  const epoch = this.replicaEpoch;
1953
1966
  return this.chainApply(async () => {
1954
1967
  if (epoch !== this.replicaEpoch) return;
1968
+ const planned = plan();
1969
+ if (planned === null) return;
1970
+ const { upserts, removalIds } = planned;
1971
+ if (upserts.length === 0 && removalIds.length === 0) return;
1955
1972
  const changes = this.store.applyReconcileInMemory(
1956
1973
  entity, upserts, removalIds, tombstoneSeq,
1957
1974
  );
@@ -2811,10 +2828,14 @@ export class SyncEngine {
2811
2828
  private static readonly PENDING_RECONCILE_DEBOUNCE_MS = 100;
2812
2829
  /** At most one hinted reconcile per this interval. */
2813
2830
  private static readonly PENDING_RECONCILE_MIN_INTERVAL_MS = 1_000;
2814
- /** Backoff for entities whose fetch raced a newer change: doubling from
2815
- * the base, capped. */
2831
+ /** Backoff for entities whose fetch failed: doubling from the base,
2832
+ * capped. */
2816
2833
  private static readonly PENDING_RECONCILE_RETRY_BASE_MS = 250;
2817
- private static readonly PENDING_RECONCILE_RETRY_MAX_MS = 10_000;
2834
+ /** Five minutes. A hinted reconcile must land (no other event tells this
2835
+ * replica that a share changed), so retries never stop; they slow down.
2836
+ * With the old cap of 10 s, a tab asked a server that kept failing every
2837
+ * 10 s for as long as it was open. */
2838
+ private static readonly PENDING_RECONCILE_RETRY_MAX_MS = 300_000;
2818
2839
 
2819
2840
  /**
2820
2841
  * Keep `entities` pending until a reconcile of them completes. Used for
@@ -2822,8 +2843,8 @@ export class SyncEngine {
2822
2843
  * membership changed which rows this user may read, and no other event
2823
2844
  * will tell this replica) and for entities an earlier reconcile skipped.
2824
2845
  * Requests are merged, run at most once a second, and an entity whose
2825
- * fetch failed or raced a newer change is retried with backoff until it
2826
- * lands or the engine stops.
2846
+ * fetch failed is retried with backoff until it lands or the engine
2847
+ * stops. The backoff doubles up to PENDING_RECONCILE_RETRY_MAX_MS.
2827
2848
  */
2828
2849
  private queuePendingReconcile(entities: string[]): void {
2829
2850
  for (const e of entities) this.pendingReconcile.set(e, ++this.pendingReconcileSeq);
@@ -2871,7 +2892,8 @@ export class SyncEngine {
2871
2892
  const delay =
2872
2893
  skipped.length > 0
2873
2894
  ? Math.min(
2874
- SyncEngine.PENDING_RECONCILE_RETRY_BASE_MS * 2 ** (this.pendingReconcileRetries - 1),
2895
+ // The exponent is capped so the product stays a finite number.
2896
+ SyncEngine.PENDING_RECONCILE_RETRY_BASE_MS * 2 ** Math.min(this.pendingReconcileRetries - 1, 30),
2875
2897
  SyncEngine.PENDING_RECONCILE_RETRY_MAX_MS,
2876
2898
  )
2877
2899
  : SyncEngine.PENDING_RECONCILE_DEBOUNCE_MS;
@@ -2939,6 +2961,19 @@ export class SyncEngine {
2939
2961
  return batch.promise;
2940
2962
  }
2941
2963
 
2964
+ /** One per reconcile fetch in flight: the ids of the entity's rows that a
2965
+ * change event wrote since the fetch started. The fetched rows may be
2966
+ * older than those events, so the reconcile leaves these rows alone. */
2967
+ private reconcileWatches = new Set<{ entity: string; touched: Set<string> }>();
2968
+
2969
+ /** Record a row that something newer than a running fetch just wrote or
2970
+ * removed: a change event, a revocation. */
2971
+ private noteReconcileTouched(entity: string, rowId: string): void {
2972
+ for (const watch of this.reconcileWatches) {
2973
+ if (watch.entity === entity) watch.touched.add(rowId);
2974
+ }
2975
+ }
2976
+
2942
2977
  /** Scoped reconcile batch that has not started running yet. */
2943
2978
  private scopedReconcileBatch: {
2944
2979
  entities: Set<string>;
@@ -2965,22 +3000,101 @@ export class SyncEngine {
2965
3000
  // the current cursor means future inserts (which have higher seqs)
2966
3001
  // bypass the tombstone — re-creation server-side still propagates.
2967
3002
  const tombstoneSeq = this.cursor.last_seq;
3003
+ const reconcileEntity = async (entity: string, touched: Set<string>): Promise<void> => {
3004
+ const sessionBeforeFetch = this.session.signature();
3005
+ const epochBeforeFetch = this.replicaEpoch;
3006
+ let serverRows: Row[];
3007
+ let fetchTruncated: boolean;
3008
+ try {
3009
+ const fetched = await this.fetchEntityRows(entity);
3010
+ serverRows = fetched.rows;
3011
+ fetchTruncated = fetched.truncated;
3012
+ } catch (err) {
3013
+ // Network errors are expected (offline, transient 5xx). Skip
3014
+ // this entity; the next reconcile trigger will retry.
3015
+ const status = (err as { status?: number })?.status;
3016
+ if (this.replicaEpoch !== epochBeforeFetch || this.session.signature() !== sessionBeforeFetch) {
3017
+ return;
3018
+ }
3019
+ if (status === 404) {
3020
+ // Entity removed from the manifest — definitive. Drop its rows.
3021
+ this.reconcile403Streak.delete(entity);
3022
+ await this.dropEntity(entity, tombstoneSeq);
3023
+ done.push(entity);
3024
+ return;
3025
+ }
3026
+ if (status === 403) {
3027
+ // A 403 can be TRANSIENT — a bearer caught mid-refresh, a tenant
3028
+ // flip that raced the fetch, or a momentary server-side policy
3029
+ // blip. Nuking every local row on the first one makes the data
3030
+ // flash away and reappear next reconcile (the "rows vanish on
3031
+ // load" bug, on the error path this time). Two gates before we
3032
+ // drop:
3033
+ // 1. If the session flipped during the fetch, the 403 reflects
3034
+ // the OLD context — never drop; the next reconcile under the
3035
+ // new session decides (mirrors the success-path guard below).
3036
+ if (this.session.signature() !== sessionBeforeFetch) {
3037
+ return;
3038
+ }
3039
+ // 2. Otherwise require TWO consecutive 403s for this entity, so
3040
+ // a one-off blip can't wipe the cache. A successful fetch
3041
+ // (below) resets the streak.
3042
+ const streak = (this.reconcile403Streak.get(entity) ?? 0) + 1;
3043
+ if (streak < 2) {
3044
+ this.reconcile403Streak.set(entity, streak);
3045
+ skipped.push(entity);
3046
+ return;
3047
+ }
3048
+ this.reconcile403Streak.delete(entity);
3049
+ await this.dropEntity(entity, tombstoneSeq);
3050
+ done.push(entity);
3051
+ return;
3052
+ }
3053
+ // Not reconciled (offline, a 5xx): reported, so a server hint
3054
+ // stays pending and retries.
3055
+ skipped.push(entity);
3056
+ return;
3057
+ }
3058
+ // Successful fetch — the entity is readable, so any prior 403 streak
3059
+ // is broken (even if the session check below skips the apply).
3060
+ this.reconcile403Streak.delete(entity);
3061
+ if (this.session.signature() !== sessionBeforeFetch) {
3062
+ // Session changed (token flipped, tenant switched, user
3063
+ // signed out → in, etc.). The rows we fetched reflect the
3064
+ // OLD session's policy view; applying them now would
3065
+ // tombstone rows visible under the NEW session. Bail and let
3066
+ // the session-changed envelope drive the next reconcile.
3067
+ return;
3068
+ }
3069
+ const applied = await this.applyEntityReconcile(
3070
+ entity,
3071
+ serverRows,
3072
+ tombstoneSeq,
3073
+ fetchTruncated,
3074
+ touched,
3075
+ // Checked when the apply runs: either can change while the apply
3076
+ // waits behind queued change events.
3077
+ // A replica reset during the fetch wiped the rows this fetch
3078
+ // would be compared with; the pull after the reset refills them.
3079
+ () => this.session.signature() === sessionBeforeFetch && this.replicaEpoch === epochBeforeFetch,
3080
+ );
3081
+ if (applied) done.push(entity);
3082
+ };
2968
3083
  // Fan out the per-entity fetches in parallel. Bootstrap reconcile
2969
3084
  // used to serialize 5 entities × ~60ms each → 300ms of dead time
2970
- // on the critical path before channels render. The per-entity
2971
- // drift checks (cursor + session signature) are captured inside
2972
- // each task's closure, so each entity still bails individually
2973
- // if its OWN fetch raced a WS event or a session flip — parallel
2974
- // fan-out doesn't weaken either guard.
3085
+ // on the critical path before channels render. Each entity has its
3086
+ // own watch and its own session check, so the fan-out weakens neither.
2975
3087
  await Promise.all(
2976
3088
  names.map(async (entity) => {
2977
- // Capture cursor + resolved session BEFORE the fetch so we can
2978
- // detect drift mid-reconcile. Two distinct races:
3089
+ // Two things can change while the paginated fetch is in flight:
2979
3090
  //
2980
- // 1. Cursor moves: a WS event for this (or another) entity
2981
- // landed while the page-paginated fetch was in flight. Our
2982
- // snapshot is stale; applying it would clobber the fresher
2983
- // WS-delivered row.
3091
+ // 1. Change events arrive (WS, pull, another tab). A fetched row
3092
+ // may be older than the event that wrote it, and a row the
3093
+ // event inserted may be missing from the fetch. The watch
3094
+ // records the ids those events wrote, and the apply leaves
3095
+ // those rows alone (see applyEntityReconcile). The rest of the
3096
+ // fetch still applies, so an entity that changes all the time
3097
+ // (an append-only log) reconciles in one fetch.
2984
3098
  //
2985
3099
  // 2. Session flips: the resolved tenant/user changed while
2986
3100
  // the fetch was in flight (e.g., the app called
@@ -2995,90 +3109,13 @@ export class SyncEngine {
2995
3109
  // the session signature changed; the next reconcile
2996
3110
  // (triggered by session-changed envelope) will re-fetch
2997
3111
  // under the new context.
2998
- const cursorBeforeFetch = this.cursor.last_seq;
2999
- const sessionBeforeFetch = this.session.signature();
3000
- const epochBeforeFetch = this.replicaEpoch;
3001
- let serverRows: Row[];
3002
- let fetchTruncated: boolean;
3112
+ const watch = { entity, touched: new Set<string>() };
3113
+ this.reconcileWatches.add(watch);
3003
3114
  try {
3004
- const fetched = await this.fetchEntityRows(entity);
3005
- serverRows = fetched.rows;
3006
- fetchTruncated = fetched.truncated;
3007
- } catch (err) {
3008
- // Network errors are expected (offline, transient 5xx). Skip
3009
- // this entity; the next reconcile trigger will retry.
3010
- const status = (err as { status?: number })?.status;
3011
- if (this.replicaEpoch !== epochBeforeFetch || this.session.signature() !== sessionBeforeFetch) {
3012
- return;
3013
- }
3014
- if (status === 404) {
3015
- // Entity removed from the manifest — definitive. Drop its rows.
3016
- this.reconcile403Streak.delete(entity);
3017
- await this.dropEntity(entity, tombstoneSeq);
3018
- done.push(entity);
3019
- return;
3020
- }
3021
- if (status === 403) {
3022
- // A 403 can be TRANSIENT — a bearer caught mid-refresh, a tenant
3023
- // flip that raced the fetch, or a momentary server-side policy
3024
- // blip. Nuking every local row on the first one makes the data
3025
- // flash away and reappear next reconcile (the "rows vanish on
3026
- // load" bug, on the error path this time). Two gates before we
3027
- // drop:
3028
- // 1. If the session flipped during the fetch, the 403 reflects
3029
- // the OLD context — never drop; the next reconcile under the
3030
- // new session decides (mirrors the success-path guard below).
3031
- if (this.session.signature() !== sessionBeforeFetch) {
3032
- return;
3033
- }
3034
- // 2. Otherwise require TWO consecutive 403s for this entity, so
3035
- // a one-off blip can't wipe the cache. A successful fetch
3036
- // (below) resets the streak.
3037
- const streak = (this.reconcile403Streak.get(entity) ?? 0) + 1;
3038
- if (streak < 2) {
3039
- this.reconcile403Streak.set(entity, streak);
3040
- skipped.push(entity);
3041
- return;
3042
- }
3043
- this.reconcile403Streak.delete(entity);
3044
- await this.dropEntity(entity, tombstoneSeq);
3045
- done.push(entity);
3046
- return;
3047
- }
3048
- // Not reconciled (offline, a 5xx): reported, so a server hint
3049
- // stays pending and retries.
3050
- skipped.push(entity);
3051
- return;
3052
- }
3053
- // Successful fetch — the entity is readable, so any prior 403 streak
3054
- // is broken (even if a drift guard below makes us skip the apply).
3055
- this.reconcile403Streak.delete(entity);
3056
- if (this.cursor.last_seq !== cursorBeforeFetch) {
3057
- // Cursor moved during fetch — at least one WS event for this
3058
- // (or another) entity landed and might have a fresher value
3059
- // for a row our snapshot just captured. Bail out for this
3060
- // entity; reconcile() is triggered again on visibility-change
3061
- // and reconnect, and the WS event already carried the latest
3062
- // state for the affected row. Reported, so a server hint can
3063
- // retry (flushPendingReconcile).
3064
- skipped.push(entity);
3065
- return;
3066
- }
3067
- if (this.session.signature() !== sessionBeforeFetch) {
3068
- // Session changed (token flipped, tenant switched, user
3069
- // signed out → in, etc.). The rows we fetched reflect the
3070
- // OLD session's policy view; applying them now would
3071
- // tombstone rows visible under the NEW session. Bail and let
3072
- // the session-changed envelope drive the next reconcile.
3073
- return;
3115
+ await reconcileEntity(entity, watch.touched);
3116
+ } finally {
3117
+ this.reconcileWatches.delete(watch);
3074
3118
  }
3075
- await this.applyEntityReconcile(
3076
- entity,
3077
- serverRows,
3078
- tombstoneSeq,
3079
- fetchTruncated,
3080
- );
3081
- done.push(entity);
3082
3119
  }),
3083
3120
  );
3084
3121
  // A full sweep that reconciled every entity matched the replica to the
@@ -3133,12 +3170,43 @@ export class SyncEngine {
3133
3170
  return { rows: out, truncated: true };
3134
3171
  }
3135
3172
 
3173
+ /**
3174
+ * Apply one entity's fetched rows. Resolves false when `stillValid` said
3175
+ * no and nothing was applied.
3176
+ *
3177
+ * The diff against the replica is computed on the apply queue, behind
3178
+ * every change event that arrived during the fetch. At that point
3179
+ * `touched` holds the id of each row such an event wrote. Those rows are
3180
+ * neither upserted nor removed: the fetch may have read them before the
3181
+ * event, and the event is the newer state.
3182
+ */
3136
3183
  private async applyEntityReconcile(
3137
3184
  entity: string,
3138
3185
  serverRows: Row[],
3139
3186
  tombstoneSeq: number,
3140
3187
  truncated: boolean,
3141
- ): Promise<void> {
3188
+ touched: ReadonlySet<string>,
3189
+ stillValid: () => boolean,
3190
+ ): Promise<boolean> {
3191
+ let applied = true;
3192
+ await this.enqueueReconcilePlan(entity, tombstoneSeq, () => {
3193
+ if (!stillValid()) {
3194
+ applied = false;
3195
+ return null;
3196
+ }
3197
+ return this.planEntityReconcile(entity, serverRows, truncated, touched);
3198
+ });
3199
+ return applied;
3200
+ }
3201
+
3202
+ /** The upserts and removals that match the replica to `serverRows`. Runs
3203
+ * on the apply queue. */
3204
+ private planEntityReconcile(
3205
+ entity: string,
3206
+ serverRows: Row[],
3207
+ truncated: boolean,
3208
+ touched: ReadonlySet<string>,
3209
+ ): { upserts: Row[]; removalIds: string[] } {
3142
3210
  // Invariant: rows with in-flight or failed mutations are
3143
3211
  // off-limits to reconcile. Neither the upsert branch nor the
3144
3212
  // tombstone branch may touch them. A hydrated offline mutation
@@ -3154,6 +3222,7 @@ export class SyncEngine {
3154
3222
  if (typeof id !== "string" || id.length === 0) continue;
3155
3223
  serverIds.add(id);
3156
3224
  if (pendingKeys.has(`${entity}/${id}`)) continue;
3225
+ if (touched.has(id)) continue;
3157
3226
  const local = this.store.get(entity, id);
3158
3227
  if (!local || rowsDiffer(local, row)) {
3159
3228
  upserts.push(row);
@@ -3183,13 +3252,12 @@ export class SyncEngine {
3183
3252
  const id = (local as { id?: unknown }).id;
3184
3253
  if (typeof id !== "string") continue;
3185
3254
  if (pendingKeys.has(`${entity}/${id}`)) continue;
3255
+ // Inserted by an event after the fetch read past its position.
3256
+ if (touched.has(id)) continue;
3186
3257
  if (!serverIds.has(id)) removalIds.push(id);
3187
3258
  }
3188
3259
  }
3189
- if (upserts.length === 0 && removalIds.length === 0) return;
3190
- // Route through the apply queue so a reconcile batch can't
3191
- // interleave with a fresher WS/pull change event mid-apply.
3192
- await this.enqueueReconcile(entity, upserts, removalIds, tombstoneSeq);
3260
+ return { upserts, removalIds };
3193
3261
  }
3194
3262
 
3195
3263
  private async dropEntity(
@@ -3658,6 +3726,10 @@ export class SyncEngine {
3658
3726
  rowId: string,
3659
3727
  tombstoneSeq: number,
3660
3728
  ): void {
3729
+ // A reconcile fetch in flight may have read the row before the server
3730
+ // revoked it. Without this it would put the row back, and nothing later
3731
+ // would remove it again.
3732
+ this.noteReconcileTouched(entity, rowId);
3661
3733
  const removed = this.store.revokeRow(entity, rowId, tombstoneSeq);
3662
3734
  if (removed) {
3663
3735
  if (this.persistence) {
@@ -730,25 +730,28 @@ describe("sync-reconcile hint", () => {
730
730
  engine.stop();
731
731
  });
732
732
 
733
- test("keeps retrying while newer changes race the fetch, then lands", async () => {
733
+ test("lands in one fetch when newer changes arrive during the fetch", async () => {
734
734
  const engine = makeEngine();
735
- // The first two fetches race a change event: the cursor moves.
736
- const fetches = documentServer((n) => {
737
- if (n <= 2) inner(engine).cursor.last_seq += 1;
735
+ // Every fetch races a change event, as on an entity that is written
736
+ // all the time. The reconcile used to discard each fetch and retry
737
+ // with no end.
738
+ let seq = 0;
739
+ const fetches = documentServer(() => {
740
+ seq += 1;
741
+ void (inner(engine) as unknown as { enqueueApply(c: ChangeEvent[]): Promise<void> }).enqueueApply([
742
+ { seq, entity: "Document", row_id: `live-${seq}`, kind: "insert", data: { id: `live-${seq}` }, timestamp: "" },
743
+ ]);
738
744
  });
739
745
  inner(engine).running = true;
740
746
 
741
747
  deliver(engine, { type: "sync-reconcile", entities: ["Document"] });
742
748
  await sleep(250);
743
749
  expect(fetches()).toBe(1);
744
- expect(engine.store.list("Document")).toEqual([]);
745
- expect([...inner(engine).pendingReconcile.keys()]).toEqual(["Document"]);
750
+ expect(engine.store.list("Document").map((r) => (r as { id: string }).id).sort()).toEqual(["live-1", "shared"]);
751
+ expect(inner(engine).pendingReconcile.size).toBe(0);
746
752
 
747
- // Retries run at most once a second.
748
753
  await sleep(2300);
749
- expect(fetches()).toBe(3);
750
- expect(engine.store.list("Document").map((r) => (r as { id: string }).id)).toEqual(["shared"]);
751
- expect(inner(engine).pendingReconcile.size).toBe(0);
754
+ expect(fetches()).toBe(1);
752
755
  engine.stop();
753
756
  });
754
757
 
@@ -814,37 +817,153 @@ describe("sync-reconcile hint after a failed fetch", () => {
814
817
  });
815
818
  });
816
819
 
817
- // Any reconcile, not only a hinted one: an entity skipped because its fetch
818
- // raced a newer change is retried, so a row that became readable while the
819
- // page was away (a new folder, a share) still arrives.
820
- describe("a skipped reconcile retries", () => {
820
+ // Change events that arrive while a reconcile fetch is in flight. The fetch
821
+ // still applies. Only the rows those events wrote are left alone: the fetch
822
+ // may have read them before the event.
823
+ describe("a reconcile whose fetch raced change events", () => {
821
824
  let restore: (() => void) | null = null;
822
825
  afterEach(() => {
823
826
  restore?.();
824
827
  restore = null;
825
828
  });
826
829
 
827
- test("a startup reconcile that raced a change lands on the retry", async () => {
828
- const engine = makeEngine();
829
- const inner = engine as unknown as { running: boolean; cursor: { last_seq: number }; pendingReconcile: Set<string> };
830
+ type Internals = {
831
+ running: boolean;
832
+ pendingReconcile: Map<string, number>;
833
+ enqueueApply(changes: ChangeEvent[]): Promise<void>;
834
+ };
835
+
836
+ /** A server that answers `rows` for Folder, after `during` ran. */
837
+ function folderServer(rows: Row[], during: () => void) {
830
838
  let fetches = 0;
831
839
  restore = installFetch(async (url) => {
832
840
  if (url.includes("/api/entities/Folder/cursor")) {
833
841
  fetches += 1;
834
- if (fetches === 1) inner.cursor.last_seq += 1;
835
- return { status: 200, body: { data: [{ id: "f1", name: "Team" }], next_cursor: null, has_more: false } };
842
+ during();
843
+ return { status: 200, body: { data: rows, next_cursor: null, has_more: false } };
836
844
  }
837
845
  return { status: 404, body: {} };
838
846
  });
847
+ return () => fetches;
848
+ }
849
+
850
+ const event = (seq: number, kind: ChangeEvent["kind"], id: string, data?: Row): ChangeEvent => ({
851
+ seq,
852
+ entity: "Folder",
853
+ row_id: id,
854
+ kind,
855
+ data,
856
+ timestamp: "",
857
+ });
858
+ const names = (engine: SyncEngine) =>
859
+ Object.fromEntries(engine.store.list("Folder").map((r) => [(r as { id: string }).id, (r as { name?: string }).name]));
860
+
861
+ test("applies the fetch at once, with no retry", async () => {
862
+ const engine = makeEngine();
863
+ const inner = engine as unknown as Internals;
864
+ const fetches = folderServer([{ id: "f1", name: "Team" }], () => {
865
+ void inner.enqueueApply([event(1, "insert", "other", { id: "other", name: "Other" })]);
866
+ });
839
867
  inner.running = true;
840
868
 
841
869
  await engine.reconcile(["Folder"]);
842
- expect(engine.store.list("Folder")).toEqual([]);
843
- expect([...inner.pendingReconcile.keys()]).toEqual(["Folder"]);
870
+ expect(names(engine)).toEqual({ f1: "Team", other: "Other" });
871
+ expect(inner.pendingReconcile.size).toBe(0);
844
872
 
845
873
  await new Promise((r) => setTimeout(r, 1300));
846
- expect(fetches).toBe(2);
847
- expect(engine.store.list("Folder").map((r) => (r as { id: string }).id)).toEqual(["f1"]);
874
+ expect(fetches()).toBe(1);
875
+ engine.stop();
876
+ });
877
+
878
+ test("a row an event updated during the fetch keeps the event's value", async () => {
879
+ const engine = makeEngine();
880
+ const inner = engine as unknown as Internals;
881
+ seedStore(engine, "Folder", [{ id: "f1", name: "Old" }]);
882
+ // The server read f1 before the update, so the fetch carries the old name.
883
+ folderServer([{ id: "f1", name: "Old" }, { id: "f2", name: "Second" }], () => {
884
+ void inner.enqueueApply([event(1, "update", "f1", { id: "f1", name: "New" })]);
885
+ });
886
+ inner.running = true;
887
+
888
+ await engine.reconcile(["Folder"]);
889
+ expect(names(engine)).toEqual({ f1: "New", f2: "Second" });
890
+ engine.stop();
891
+ });
892
+
893
+ test("a row an event inserted during the fetch is not removed, and a stale row is", async () => {
894
+ const engine = makeEngine();
895
+ const inner = engine as unknown as Internals;
896
+ seedStore(engine, "Folder", [{ id: "stale", name: "Deleted on the server" }]);
897
+ // The insert landed after the server read past its position.
898
+ folderServer([], () => {
899
+ void inner.enqueueApply([event(1, "insert", "fresh", { id: "fresh", name: "Fresh" })]);
900
+ });
901
+ inner.running = true;
902
+
903
+ await engine.reconcile(["Folder"]);
904
+ expect(names(engine)).toEqual({ fresh: "Fresh" });
905
+ engine.stop();
906
+ });
907
+
908
+ test("a row an event deleted during the fetch is not brought back", async () => {
909
+ const engine = makeEngine();
910
+ const inner = engine as unknown as Internals;
911
+ seedStore(engine, "Folder", [{ id: "f1", name: "Team" }]);
912
+ // The server read f1 before the delete.
913
+ folderServer([{ id: "f1", name: "Team" }], () => {
914
+ void inner.enqueueApply([event(1, "delete", "f1")]);
915
+ });
916
+ inner.running = true;
917
+
918
+ await engine.reconcile(["Folder"]);
919
+ expect(names(engine)).toEqual({});
920
+ engine.stop();
921
+ });
922
+
923
+ test("a row revoked during the fetch is not put back", async () => {
924
+ const engine = makeEngine();
925
+ const inner = engine as unknown as Internals & {
926
+ handleRowRevocation(entity: string, rowId: string, tombstoneSeq: number): void;
927
+ };
928
+ seedStore(engine, "Folder", [{ id: "f1", name: "Team" }]);
929
+ // The server read f1 before it revoked it. A later event for another
930
+ // row moves the cursor past the revocation.
931
+ folderServer([{ id: "f1", name: "Team" }, { id: "f2", name: "Second" }], () => {
932
+ inner.handleRowRevocation("Folder", "f1", 1);
933
+ void inner.enqueueApply([event(2, "insert", "f3", { id: "f3", name: "Third" })]);
934
+ });
935
+ inner.running = true;
936
+
937
+ await engine.reconcile(["Folder"]);
938
+ expect(names(engine)).toEqual({ f2: "Second", f3: "Third" });
939
+ engine.stop();
940
+ });
941
+
942
+ test("a fetch that a replica reset interrupted is not applied", async () => {
943
+ const engine = makeEngine();
944
+ const inner = engine as unknown as Internals & { replicaEpoch: number };
945
+ // The reset wipes the replica; the pull that follows it refills it.
946
+ folderServer([{ id: "f1", name: "From before the reset" }], () => {
947
+ inner.replicaEpoch += 1;
948
+ });
949
+ inner.running = true;
950
+
951
+ await engine.reconcile(["Folder"]);
952
+ expect(names(engine)).toEqual({});
953
+ engine.stop();
954
+ });
955
+
956
+ test("an event for another entity does not protect this entity's rows", async () => {
957
+ const engine = makeEngine();
958
+ const inner = engine as unknown as Internals;
959
+ seedStore(engine, "Folder", [{ id: "stale", name: "Deleted on the server" }]);
960
+ folderServer([{ id: "f1", name: "Team" }], () => {
961
+ void inner.enqueueApply([{ ...event(1, "insert", "stale", { id: "stale" }), entity: "Document" }]);
962
+ });
963
+ inner.running = true;
964
+
965
+ await engine.reconcile(["Folder"]);
966
+ expect(names(engine)).toEqual({ f1: "Team" });
848
967
  engine.stop();
849
968
  });
850
969
  });
@@ -894,6 +1013,44 @@ describe("pending reconciles end when they should", () => {
894
1013
  engine.stop();
895
1014
  });
896
1015
 
1016
+ test("retries of a failing fetch slow down to the cap and still land", async () => {
1017
+ const statics = SyncEngine as unknown as {
1018
+ PENDING_RECONCILE_RETRY_BASE_MS: number;
1019
+ PENDING_RECONCILE_RETRY_MAX_MS: number;
1020
+ PENDING_RECONCILE_MIN_INTERVAL_MS: number;
1021
+ };
1022
+ const saved = { ...statics };
1023
+ statics.PENDING_RECONCILE_RETRY_BASE_MS = 10;
1024
+ statics.PENDING_RECONCILE_RETRY_MAX_MS = 200;
1025
+ statics.PENDING_RECONCILE_MIN_INTERVAL_MS = 1;
1026
+ try {
1027
+ let failing = true;
1028
+ const fetches = server(() => (failing ? 503 : 200));
1029
+ const engine = makeEngine();
1030
+ const inner = engine as unknown as Internals;
1031
+ inner.running = true;
1032
+ await engine.reconcile(["Document"]);
1033
+ // Delays: 100 (debounce), 20, 40, 80, 160, then 200 each. A fixed
1034
+ // 10 ms interval would have made about 90 requests by now.
1035
+ await sleep(900);
1036
+ const whileFailing = fetches();
1037
+ expect(whileFailing).toBeGreaterThanOrEqual(6);
1038
+ expect(whileFailing).toBeLessThanOrEqual(9);
1039
+ expect([...inner.pendingReconcile.keys()]).toEqual(["Document"]);
1040
+
1041
+ // Nothing else triggers a reconcile. The timer alone lands it.
1042
+ failing = false;
1043
+ await sleep(450);
1044
+ expect(inner.pendingReconcile.size).toBe(0);
1045
+ expect(fetches()).toBeLessThanOrEqual(whileFailing + 2);
1046
+ engine.stop();
1047
+ } finally {
1048
+ statics.PENDING_RECONCILE_RETRY_BASE_MS = saved.PENDING_RECONCILE_RETRY_BASE_MS;
1049
+ statics.PENDING_RECONCILE_RETRY_MAX_MS = saved.PENDING_RECONCILE_RETRY_MAX_MS;
1050
+ statics.PENDING_RECONCILE_MIN_INTERVAL_MS = saved.PENDING_RECONCILE_MIN_INTERVAL_MS;
1051
+ }
1052
+ });
1053
+
897
1054
  test("stop cancels a retry already waiting in the queue", async () => {
898
1055
  const fetches = server(() => 200);
899
1056
  const engine = makeEngine();