@pylonsync/sync 0.3.227 → 0.3.229
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/idb-warm-load.test.ts +144 -0
- package/src/index.ts +307 -63
- package/src/local-store.ts +57 -5
- package/src/multi-tab-orchestrator.ts +31 -5
- package/src/mutation-queue.ts +32 -3
- package/src/persistence.ts +69 -30
- package/src/round6-codex.test.ts +157 -0
- package/src/scenarios.test.ts +184 -0
- package/src/test-harness/server.ts +36 -0
- package/src/test-harness/transport.ts +16 -0
|
@@ -69,8 +69,11 @@ export interface MultiTabOrchestratorHooks {
|
|
|
69
69
|
removalIds: string[],
|
|
70
70
|
tombstoneSeq: number,
|
|
71
71
|
): void;
|
|
72
|
-
/** Replica reset broadcast
|
|
73
|
-
|
|
72
|
+
/** Replica reset broadcast from the leader. `wipeMutations` is true
|
|
73
|
+
* when the reset was an identity flip (drop the outgoing identity's
|
|
74
|
+
* pending offline writes), false for a same-user 410 RESYNC (keep
|
|
75
|
+
* them — they survive the snapshot refresh). */
|
|
76
|
+
onResetReceived(wipeMutations: boolean): void;
|
|
74
77
|
/** Resolved session update from the leader. Engine funnels through
|
|
75
78
|
* its session chain so concurrent triggers commit in order. */
|
|
76
79
|
onSessionReceived(resolved: ResolvedSession): void;
|
|
@@ -112,6 +115,16 @@ export interface MultiTabOrchestratorHooks {
|
|
|
112
115
|
/** New leader → followers: re-forward your locally-wanted room subs.
|
|
113
116
|
* Triggered alongside CRDT/reactive replay after a leader change. */
|
|
114
117
|
onReplayRoomSubs?(): void;
|
|
118
|
+
|
|
119
|
+
/** Follower → leader: a follower's `useQuery` observed an entity.
|
|
120
|
+
* Leader-only. The leader adds it to its reconcile sweep and fetches
|
|
121
|
+
* it so a server row the follower never cached is discovered and
|
|
122
|
+
* broadcast back as a `reconciled` batch. Without this, a follower's
|
|
123
|
+
* view on a never-cached entity stays empty forever. */
|
|
124
|
+
onEntityObserve?(entity: string, fromTabId: string): void;
|
|
125
|
+
/** New leader → followers: re-declare your observed entities so the
|
|
126
|
+
* new leader's reconcile sweep covers them. */
|
|
127
|
+
onReplayObservedEntities?(): void;
|
|
115
128
|
}
|
|
116
129
|
|
|
117
130
|
export interface MultiTabOrchestratorConfig {
|
|
@@ -247,8 +260,8 @@ export class MultiTabOrchestrator {
|
|
|
247
260
|
});
|
|
248
261
|
}
|
|
249
262
|
|
|
250
|
-
broadcastReset(): void {
|
|
251
|
-
this.broadcastRaw({ type: "reset" });
|
|
263
|
+
broadcastReset(wipeMutations: boolean): void {
|
|
264
|
+
this.broadcastRaw({ type: "reset", wipeMutations });
|
|
252
265
|
}
|
|
253
266
|
|
|
254
267
|
broadcastSession(resolved: ResolvedSession): void {
|
|
@@ -328,7 +341,7 @@ export class MultiTabOrchestrator {
|
|
|
328
341
|
break;
|
|
329
342
|
}
|
|
330
343
|
case "reset": {
|
|
331
|
-
this.hooks.onResetReceived();
|
|
344
|
+
this.hooks.onResetReceived(msg.wipeMutations === true);
|
|
332
345
|
break;
|
|
333
346
|
}
|
|
334
347
|
case "session": {
|
|
@@ -388,6 +401,19 @@ export class MultiTabOrchestrator {
|
|
|
388
401
|
// so the new leader can rebuild its forwarder sets and resend
|
|
389
402
|
// `room-subscribe` on the WS.
|
|
390
403
|
this.hooks.onReplayRoomSubs?.();
|
|
404
|
+
// ...and re-declare observed entities so the new leader's
|
|
405
|
+
// reconcile sweep covers them.
|
|
406
|
+
this.hooks.onReplayObservedEntities?.();
|
|
407
|
+
break;
|
|
408
|
+
}
|
|
409
|
+
case "entity-observe": {
|
|
410
|
+
// Follower → leader. Leader-only — a follower receiving this
|
|
411
|
+
// (own broadcast echo, or stale leader transition) ignores.
|
|
412
|
+
if (!this._isLeader) return;
|
|
413
|
+
const entity = msg.entity as string | undefined;
|
|
414
|
+
if (typeof entity === "string") {
|
|
415
|
+
this.hooks.onEntityObserve?.(entity, fromTabId);
|
|
416
|
+
}
|
|
391
417
|
break;
|
|
392
418
|
}
|
|
393
419
|
case "room-sub-register": {
|
package/src/mutation-queue.ts
CHANGED
|
@@ -10,13 +10,19 @@
|
|
|
10
10
|
// instead of re-applying.
|
|
11
11
|
// ---------------------------------------------------------------------------
|
|
12
12
|
|
|
13
|
-
import type { ClientChange } from "./types";
|
|
13
|
+
import type { ClientChange, Row } from "./types";
|
|
14
14
|
|
|
15
15
|
export interface PendingMutation {
|
|
16
16
|
id: string;
|
|
17
17
|
change: ClientChange;
|
|
18
18
|
status: "pending" | "applied" | "failed";
|
|
19
19
|
error?: string;
|
|
20
|
+
/** Pre-mutation snapshot of the affected row, captured at optimistic-
|
|
21
|
+
* apply time for `update`/`delete`. On a server rejection,
|
|
22
|
+
* `failPushedMutation` restores this so the local replica reverts to
|
|
23
|
+
* the value the server actually holds. `null` = the row didn't exist
|
|
24
|
+
* before the mutation; `undefined` = not captured (inserts). */
|
|
25
|
+
prevRow?: Row | null;
|
|
20
26
|
}
|
|
21
27
|
|
|
22
28
|
/**
|
|
@@ -80,14 +86,14 @@ export class MutationQueue {
|
|
|
80
86
|
* entry with the same op_id is already queued — a follower
|
|
81
87
|
* retrying its forward of the same op shouldn't double-queue on
|
|
82
88
|
* the leader. */
|
|
83
|
-
add(change: ClientChange): string {
|
|
89
|
+
add(change: ClientChange, prevRow?: Row | null): string {
|
|
84
90
|
const id =
|
|
85
91
|
typeof change.op_id === "string" && change.op_id.length > 0
|
|
86
92
|
? change.op_id
|
|
87
93
|
: `mut_${Date.now()}_${Math.random().toString(36).slice(2)}`;
|
|
88
94
|
if (this.queue.some((m) => m.id === id)) return id;
|
|
89
95
|
const changeWithOp: ClientChange = { ...change, op_id: id };
|
|
90
|
-
this.queue.push({ id, change: changeWithOp, status: "pending" });
|
|
96
|
+
this.queue.push({ id, change: changeWithOp, status: "pending", prevRow });
|
|
91
97
|
this.flush();
|
|
92
98
|
return id;
|
|
93
99
|
}
|
|
@@ -96,6 +102,13 @@ export class MutationQueue {
|
|
|
96
102
|
return this.queue.filter((m) => m.status === "pending");
|
|
97
103
|
}
|
|
98
104
|
|
|
105
|
+
/** Look up a queued mutation by op id (any status). Used by the
|
|
106
|
+
* follower's mutations-failed handler to reach the captured
|
|
107
|
+
* `prevRow` for rollback. */
|
|
108
|
+
get(id: string): PendingMutation | undefined {
|
|
109
|
+
return this.queue.find((m) => m.id === id);
|
|
110
|
+
}
|
|
111
|
+
|
|
99
112
|
/**
|
|
100
113
|
* Set of `${entity}/${row_id}` keys for every mutation currently
|
|
101
114
|
* in Pending or Failed state. Used by reconcile() to skip rows
|
|
@@ -152,6 +165,22 @@ export class MutationQueue {
|
|
|
152
165
|
this.flush();
|
|
153
166
|
}
|
|
154
167
|
|
|
168
|
+
/**
|
|
169
|
+
* Drop EVERY mutation — pending, failed, and applied — then flush the
|
|
170
|
+
* empty queue to disk. Used by the engine's identity-flip reset
|
|
171
|
+
* (`resetReplica({ wipeMutations: true })`): the queued offline writes
|
|
172
|
+
* belong to the OUTGOING identity, and replaying them under the new
|
|
173
|
+
* session would attribute one user's writes to another (or get
|
|
174
|
+
* policy-rejected on the server). Distinct from `clear()`, which only
|
|
175
|
+
* prunes already-applied entries and deliberately PRESERVES pending +
|
|
176
|
+
* failed writes — the 410-RESYNC same-user path relies on that so
|
|
177
|
+
* offline writes survive a snapshot refresh.
|
|
178
|
+
*/
|
|
179
|
+
clearAll(): void {
|
|
180
|
+
this.queue = [];
|
|
181
|
+
this.flush();
|
|
182
|
+
}
|
|
183
|
+
|
|
155
184
|
/** Fire-and-forget persistence write. */
|
|
156
185
|
private flush(): void {
|
|
157
186
|
if (!this.persistence) return;
|
package/src/persistence.ts
CHANGED
|
@@ -86,15 +86,47 @@ export class IndexedDBPersistence {
|
|
|
86
86
|
});
|
|
87
87
|
}
|
|
88
88
|
|
|
89
|
-
/**
|
|
90
|
-
|
|
91
|
-
|
|
89
|
+
/** Resolve when a write transaction settles, reporting durability:
|
|
90
|
+
* `true` on COMMIT, `false` if it aborts/errors — e.g.
|
|
91
|
+
* QuotaExceededError on a storage-pressured device. NEVER rejects or
|
|
92
|
+
* hangs: registering only `oncomplete` (the original shape) meant an
|
|
93
|
+
* aborted tx never settled, and the engine awaits the persist before
|
|
94
|
+
* advancing the cursor in `enqueueApply` — so ONE hung write wedged
|
|
95
|
+
* the whole applyQueue and live sync silently died for the session.
|
|
96
|
+
*
|
|
97
|
+
* But "resolve unconditionally" (the first fix) was also wrong: on an
|
|
98
|
+
* abort the row never reached disk, yet `enqueueApply` still advanced
|
|
99
|
+
* the PERSISTED cursor past it — so on restart the warm-load skipped
|
|
100
|
+
* those rows forever (cursor ahead of replica). The boolean lets the
|
|
101
|
+
* caller hold the on-disk cursor back when a row write fails, so the
|
|
102
|
+
* next cold start re-pulls the gap. Best-effort: log once, keep the
|
|
103
|
+
* in-memory replica authoritative, but never persist a cursor ahead of
|
|
104
|
+
* what's actually durable. Mirrors the read paths (`getRow`/
|
|
105
|
+
* `loadCursor`) which already degrade via `onerror`. */
|
|
106
|
+
private commit(tx: IDBTransaction, op: string): Promise<boolean> {
|
|
107
|
+
return new Promise((resolve) => {
|
|
108
|
+
tx.oncomplete = () => resolve(true);
|
|
109
|
+
tx.onerror = () => {
|
|
110
|
+
// eslint-disable-next-line no-console
|
|
111
|
+
console.warn(`[pylon-sync] IndexedDB ${op} failed; degrading to memory`, tx.error);
|
|
112
|
+
resolve(false);
|
|
113
|
+
};
|
|
114
|
+
tx.onabort = () => {
|
|
115
|
+
// eslint-disable-next-line no-console
|
|
116
|
+
console.warn(`[pylon-sync] IndexedDB ${op} aborted; degrading to memory`, tx.error);
|
|
117
|
+
resolve(false);
|
|
118
|
+
};
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Save a row to IndexedDB. Resolves `true` when the write is durable,
|
|
123
|
+
* `false` if it degraded (no DB / aborted) — see `commit`. */
|
|
124
|
+
async saveRow(entity: string, id: string, data: Row): Promise<boolean> {
|
|
125
|
+
if (!this.db) return false;
|
|
92
126
|
const tx = this.db.transaction(STORE_NAME, "readwrite");
|
|
93
127
|
const store = tx.objectStore(STORE_NAME);
|
|
94
128
|
store.put({ _key: `${entity}:${id}`, entity, id, data });
|
|
95
|
-
return
|
|
96
|
-
tx.oncomplete = () => resolve();
|
|
97
|
-
});
|
|
129
|
+
return this.commit(tx, "saveRow");
|
|
98
130
|
}
|
|
99
131
|
|
|
100
132
|
/** Fetch a row from IndexedDB by key. Used by `persistChange` on update
|
|
@@ -113,15 +145,14 @@ export class IndexedDBPersistence {
|
|
|
113
145
|
});
|
|
114
146
|
}
|
|
115
147
|
|
|
116
|
-
/** Delete a row from IndexedDB.
|
|
117
|
-
|
|
118
|
-
|
|
148
|
+
/** Delete a row from IndexedDB. Resolves `true` when durable, `false`
|
|
149
|
+
* if it degraded — see `commit`. */
|
|
150
|
+
async deleteRow(entity: string, id: string): Promise<boolean> {
|
|
151
|
+
if (!this.db) return false;
|
|
119
152
|
const tx = this.db.transaction(STORE_NAME, "readwrite");
|
|
120
153
|
const store = tx.objectStore(STORE_NAME);
|
|
121
154
|
store.delete(`${entity}:${id}`);
|
|
122
|
-
return
|
|
123
|
-
tx.oncomplete = () => resolve();
|
|
124
|
-
});
|
|
155
|
+
return this.commit(tx, "deleteRow");
|
|
125
156
|
}
|
|
126
157
|
|
|
127
158
|
/** Load all rows for an entity from IndexedDB. */
|
|
@@ -217,15 +248,14 @@ export class IndexedDBPersistence {
|
|
|
217
248
|
});
|
|
218
249
|
}
|
|
219
250
|
|
|
220
|
-
/** Save the sync cursor.
|
|
221
|
-
|
|
222
|
-
|
|
251
|
+
/** Save the sync cursor. Resolves `true` when durable, `false` if it
|
|
252
|
+
* degraded — see `commit`. */
|
|
253
|
+
async saveCursor(cursor: SyncCursor): Promise<boolean> {
|
|
254
|
+
if (!this.db) return false;
|
|
223
255
|
const tx = this.db.transaction(CURSOR_STORE, "readwrite");
|
|
224
256
|
const store = tx.objectStore(CURSOR_STORE);
|
|
225
257
|
store.put({ key: "cursor", ...cursor });
|
|
226
|
-
return
|
|
227
|
-
tx.oncomplete = () => resolve();
|
|
228
|
-
});
|
|
258
|
+
return this.commit(tx, "saveCursor");
|
|
229
259
|
}
|
|
230
260
|
|
|
231
261
|
/** Load the sync cursor. */
|
|
@@ -247,25 +277,34 @@ export class IndexedDBPersistence {
|
|
|
247
277
|
});
|
|
248
278
|
}
|
|
249
279
|
|
|
250
|
-
/** Clear
|
|
251
|
-
|
|
252
|
-
|
|
280
|
+
/** Clear stored rows + cursor. Deliberately does NOT touch the
|
|
281
|
+
* durable mutation queue: `resetReplica` calls this on a 410 RESYNC
|
|
282
|
+
* (same user, needs a fresh snapshot) where pending offline writes
|
|
283
|
+
* MUST survive. On an IDENTITY flip the old identity's pending
|
|
284
|
+
* mutations must instead be discarded — that wipe runs through
|
|
285
|
+
* `MutationQueue.clearAll()` (which persists an empty `MUTATIONS_STORE`
|
|
286
|
+
* via its own backend), gated on `resetReplica({ wipeMutations })` at
|
|
287
|
+
* the token/tenant-flip call sites. Splitting the two stores keeps each
|
|
288
|
+
* reset path honest: 410 keeps writes, identity flip drops them. */
|
|
289
|
+
async clear(): Promise<boolean> {
|
|
290
|
+
if (!this.db) return false;
|
|
253
291
|
const tx = this.db.transaction([STORE_NAME, CURSOR_STORE], "readwrite");
|
|
254
292
|
tx.objectStore(STORE_NAME).clear();
|
|
255
293
|
tx.objectStore(CURSOR_STORE).clear();
|
|
256
|
-
return
|
|
257
|
-
tx.oncomplete = () => resolve();
|
|
258
|
-
});
|
|
294
|
+
return this.commit(tx, "clear");
|
|
259
295
|
}
|
|
260
296
|
}
|
|
261
297
|
|
|
262
298
|
/**
|
|
263
|
-
* Apply a change event to IndexedDB persistence.
|
|
299
|
+
* Apply a change event to IndexedDB persistence. Returns `true` when the
|
|
300
|
+
* write reached disk durably, `false` when it degraded (so `enqueueApply`
|
|
301
|
+
* can hold the persisted cursor back rather than skip the row on restart).
|
|
302
|
+
* A change with no `data` is a no-op and counts as durable.
|
|
264
303
|
*/
|
|
265
304
|
export async function persistChange(
|
|
266
305
|
persistence: IndexedDBPersistence,
|
|
267
306
|
change: ChangeEvent
|
|
268
|
-
): Promise<
|
|
307
|
+
): Promise<boolean> {
|
|
269
308
|
switch (change.kind) {
|
|
270
309
|
case "insert":
|
|
271
310
|
case "update":
|
|
@@ -274,13 +313,13 @@ export async function persistChange(
|
|
|
274
313
|
// change's `data` with the post-merge row from memory — so even
|
|
275
314
|
// on update the full row lands here. Overwriting is correct;
|
|
276
315
|
// pre-merge patches would have dropped unpatched columns.
|
|
277
|
-
|
|
316
|
+
return persistence.saveRow(change.entity, change.row_id, change.data);
|
|
278
317
|
}
|
|
279
|
-
|
|
318
|
+
return true;
|
|
280
319
|
case "delete":
|
|
281
|
-
|
|
282
|
-
break;
|
|
320
|
+
return persistence.deleteRow(change.entity, change.row_id);
|
|
283
321
|
}
|
|
322
|
+
return true;
|
|
284
323
|
}
|
|
285
324
|
|
|
286
325
|
// ---------------------------------------------------------------------------
|
package/src/round6-codex.test.ts
CHANGED
|
@@ -326,3 +326,160 @@ describe("codex round-6: peer leaving scrubs forwarded subs", () => {
|
|
|
326
326
|
expect(engine.serverSubs.has("sub-1")).toBe(false);
|
|
327
327
|
});
|
|
328
328
|
});
|
|
329
|
+
|
|
330
|
+
describe("sync hardening: multi-tab follower coverage", () => {
|
|
331
|
+
let env: TestEnv | null = null;
|
|
332
|
+
afterEach(async () => {
|
|
333
|
+
if (env) await env.dispose();
|
|
334
|
+
env = null;
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
// FOLLOWER GHOST ROLLBACK. The leader broadcasts `mutations-failed`;
|
|
338
|
+
// the follower must roll back its OWN optimistic ghost, not just mark
|
|
339
|
+
// the queue entry failed. Pre-fix onMutationsFailed called markFailed
|
|
340
|
+
// directly, so the ghost row stuck around in the tab the user sees.
|
|
341
|
+
test("a mutations-failed for an optimistic insert removes the follower's ghost", async () => {
|
|
342
|
+
env = createTestEnv();
|
|
343
|
+
env.signIn({ userId: "u1" });
|
|
344
|
+
await env.start();
|
|
345
|
+
const engine = env.engine;
|
|
346
|
+
|
|
347
|
+
// Optimistic ghost in the store + a matching queued mutation (as if
|
|
348
|
+
// this follower had forwarded the insert to the leader).
|
|
349
|
+
engine.store.optimisticInsertWithId("Todo", "t1", { id: "t1", text: "ghost" });
|
|
350
|
+
engine.mutations.add({
|
|
351
|
+
op_id: "op-1",
|
|
352
|
+
entity: "Todo",
|
|
353
|
+
row_id: "t1",
|
|
354
|
+
kind: "insert",
|
|
355
|
+
data: { id: "t1", text: "ghost" },
|
|
356
|
+
});
|
|
357
|
+
expect(engine.store.get("Todo", "t1")).not.toBeNull();
|
|
358
|
+
|
|
359
|
+
(engine as unknown as {
|
|
360
|
+
orchestrator: { handleMessage(msg: unknown, from: string): void };
|
|
361
|
+
}).orchestrator.handleMessage(
|
|
362
|
+
{ type: "mutations-failed", ops: [{ opId: "op-1", error: "rejected" }] },
|
|
363
|
+
"leader-tab",
|
|
364
|
+
);
|
|
365
|
+
|
|
366
|
+
// Ghost gone (rolled back), mutation marked failed.
|
|
367
|
+
expect(engine.store.get("Todo", "t1")).toBeNull();
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
// ENTITY-OBSERVE FORWARDING. A leader that receives a forwarded
|
|
371
|
+
// `entity-observe` from a follower must add the entity to its reconcile
|
|
372
|
+
// sweep and fetch it — so a server row the follower never cached is
|
|
373
|
+
// discovered + broadcast back. Without this a follower's useQuery on a
|
|
374
|
+
// never-cached entity renders empty forever.
|
|
375
|
+
test("a leader fetches an entity a follower forwarded via entity-observe", async () => {
|
|
376
|
+
env = createTestEnv({ transport: "poll" });
|
|
377
|
+
env.signIn({ userId: "u1" });
|
|
378
|
+
await env.start(); // solo → this engine is the leader
|
|
379
|
+
const engine = env.engine;
|
|
380
|
+
|
|
381
|
+
// A server row exists for an entity this engine never cached.
|
|
382
|
+
env.server.insert("Domain", { id: "d1", host: "x.com" });
|
|
383
|
+
expect(engine.store.list("Domain")).toHaveLength(0);
|
|
384
|
+
|
|
385
|
+
(engine as unknown as {
|
|
386
|
+
orchestrator: { handleMessage(msg: unknown, from: string): void };
|
|
387
|
+
}).orchestrator.handleMessage(
|
|
388
|
+
{ type: "entity-observe", entity: "Domain" },
|
|
389
|
+
"follower-tab",
|
|
390
|
+
);
|
|
391
|
+
await env.flush();
|
|
392
|
+
|
|
393
|
+
expect(engine.store.get("Domain", "d1")).not.toBeNull();
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
// FORWARDED-MUTATION ROLLBACK (pins onMutationsForwarded prevRow
|
|
397
|
+
// threading). When a follower forwards an update/delete and the leader's
|
|
398
|
+
// push of it is PERMANENTLY rejected, the leader must restore the
|
|
399
|
+
// canonical row — NOT delete it. Pre-fix the leader queued the forwarded
|
|
400
|
+
// op via mutations.add(op.change) WITHOUT prevRow, so failPushedMutation
|
|
401
|
+
// ran restoreRow(undefined ?? null) → restoreRow(null) → DELETED the
|
|
402
|
+
// leader's still-valid canonical row. Data loss on every rejected
|
|
403
|
+
// forwarded edit.
|
|
404
|
+
test("a rejected forwarded UPDATE keeps the leader's canonical row", async () => {
|
|
405
|
+
env = createTestEnv({ transport: "poll" });
|
|
406
|
+
env.signIn({ userId: "u1" });
|
|
407
|
+
env.server.seed("Note", [{ id: "n1", title: "canonical" }]);
|
|
408
|
+
await env.start(); // solo → leader, holds the canonical row
|
|
409
|
+
await env.flush();
|
|
410
|
+
expect(env.engine.store.get("Note", "n1")).not.toBeNull();
|
|
411
|
+
|
|
412
|
+
// A follower optimistically edited n1, captured the prior value as
|
|
413
|
+
// prevRow, and forwarded the update. The leader's push will 403.
|
|
414
|
+
env.server.primeNextPushOutcome({ kind: "status", status: 403 });
|
|
415
|
+
(env.engine as unknown as {
|
|
416
|
+
orchestrator: { handleMessage(msg: unknown, from: string): void };
|
|
417
|
+
}).orchestrator.handleMessage(
|
|
418
|
+
{
|
|
419
|
+
type: "mutations",
|
|
420
|
+
ops: [
|
|
421
|
+
{
|
|
422
|
+
id: "op-u",
|
|
423
|
+
change: {
|
|
424
|
+
op_id: "op-u",
|
|
425
|
+
entity: "Note",
|
|
426
|
+
row_id: "n1",
|
|
427
|
+
kind: "update",
|
|
428
|
+
data: { title: "edited" },
|
|
429
|
+
},
|
|
430
|
+
status: "pending",
|
|
431
|
+
prevRow: { id: "n1", title: "canonical" },
|
|
432
|
+
},
|
|
433
|
+
],
|
|
434
|
+
},
|
|
435
|
+
"follower-tab",
|
|
436
|
+
);
|
|
437
|
+
// Wait behind the forwarded push (opQueue serializes), then drain.
|
|
438
|
+
await env.engine.push();
|
|
439
|
+
await env.flush();
|
|
440
|
+
|
|
441
|
+
const row = env.engine.store.get("Note", "n1") as { title?: string } | null;
|
|
442
|
+
expect(row).not.toBeNull();
|
|
443
|
+
expect(row?.title).toBe("canonical");
|
|
444
|
+
});
|
|
445
|
+
|
|
446
|
+
test("a rejected forwarded DELETE keeps the leader's canonical row", async () => {
|
|
447
|
+
env = createTestEnv({ transport: "poll" });
|
|
448
|
+
env.signIn({ userId: "u1" });
|
|
449
|
+
env.server.seed("Note", [{ id: "n1", title: "keep" }]);
|
|
450
|
+
await env.start();
|
|
451
|
+
await env.flush();
|
|
452
|
+
expect(env.engine.store.get("Note", "n1")).not.toBeNull();
|
|
453
|
+
|
|
454
|
+
env.server.primeNextPushOutcome({ kind: "status", status: 403 });
|
|
455
|
+
(env.engine as unknown as {
|
|
456
|
+
orchestrator: { handleMessage(msg: unknown, from: string): void };
|
|
457
|
+
}).orchestrator.handleMessage(
|
|
458
|
+
{
|
|
459
|
+
type: "mutations",
|
|
460
|
+
ops: [
|
|
461
|
+
{
|
|
462
|
+
id: "op-d",
|
|
463
|
+
change: {
|
|
464
|
+
op_id: "op-d",
|
|
465
|
+
entity: "Note",
|
|
466
|
+
row_id: "n1",
|
|
467
|
+
kind: "delete",
|
|
468
|
+
},
|
|
469
|
+
status: "pending",
|
|
470
|
+
prevRow: { id: "n1", title: "keep" },
|
|
471
|
+
},
|
|
472
|
+
],
|
|
473
|
+
},
|
|
474
|
+
"follower-tab",
|
|
475
|
+
);
|
|
476
|
+
await env.engine.push();
|
|
477
|
+
await env.flush();
|
|
478
|
+
|
|
479
|
+
// Pre-fix the leader deleted its canonical row on the rejected
|
|
480
|
+
// forwarded delete; with prevRow threaded it survives.
|
|
481
|
+
const row = env.engine.store.get("Note", "n1") as { title?: string } | null;
|
|
482
|
+
expect(row).not.toBeNull();
|
|
483
|
+
expect(row?.title).toBe("keep");
|
|
484
|
+
});
|
|
485
|
+
});
|
package/src/scenarios.test.ts
CHANGED
|
@@ -487,6 +487,190 @@ describe("sync scenarios", () => {
|
|
|
487
487
|
expect(env.server.snapshotPullCount - before).toBe(1);
|
|
488
488
|
});
|
|
489
489
|
|
|
490
|
+
// EMPTY-ENTITY RECONCILE GAP (pins observeEntity). A server row in an
|
|
491
|
+
// entity the local replica has NEVER cached stays invisible: useQuery
|
|
492
|
+
// reads the empty local store, a delta pull can't recover a row
|
|
493
|
+
// created before the cursor, and the no-arg reconcile sweeps only
|
|
494
|
+
// entities with ≥1 local row (store.entityNames()) — so it skips the
|
|
495
|
+
// empty entity entirely. The user hit this as "I attached the domain
|
|
496
|
+
// but the Domains list is empty"; clearing IndexedDB (cursor→0→
|
|
497
|
+
// snapshot) was the only recovery. observeEntity (called by useQuery
|
|
498
|
+
// on mount) adds the entity to the sweep + fires a one-shot fetch.
|
|
499
|
+
test("observing an entity recovers a server row the local cache never had", async () => {
|
|
500
|
+
env = createTestEnv({ transport: "poll" });
|
|
501
|
+
env.signIn({ userId: "u1" });
|
|
502
|
+
// Client has rows for Note, but none for Domain.
|
|
503
|
+
env.server.seed("Note", [{ id: "n1", title: "x" }]);
|
|
504
|
+
await env.start();
|
|
505
|
+
await env.flush();
|
|
506
|
+
expect(env.engine.store.list("Domain")).toHaveLength(0);
|
|
507
|
+
|
|
508
|
+
// A Domain row exists server-side but was never delivered to this
|
|
509
|
+
// client (created on another surface / a missed insert). Inserted
|
|
510
|
+
// after start so the initial snapshot didn't include it; poll
|
|
511
|
+
// transport means no auto-delivery.
|
|
512
|
+
env.server.insert("Domain", { id: "d1", host: "chat.example.com" });
|
|
513
|
+
|
|
514
|
+
// A no-arg reconcile sweeps only entities with local rows (Note),
|
|
515
|
+
// so it never touches Domain — the row stays invisible. The bug.
|
|
516
|
+
await env.engine.reconcile(["Note"]);
|
|
517
|
+
await env.flush();
|
|
518
|
+
expect(env.engine.store.get("Domain", "d1")).toBeNull();
|
|
519
|
+
|
|
520
|
+
// observeEntity (what useQuery now calls on mount) adds Domain to
|
|
521
|
+
// the sweep and fires a one-shot scoped reconcile — the row appears
|
|
522
|
+
// without a cache clear.
|
|
523
|
+
env.engine.observeEntity("Domain");
|
|
524
|
+
await env.flush();
|
|
525
|
+
expect(env.engine.store.get("Domain", "d1")).not.toBeNull();
|
|
526
|
+
});
|
|
527
|
+
|
|
528
|
+
// TAIL-PULL DEADLOCK (pins index.ts:1310). When a delta pull reports
|
|
529
|
+
// has_more (the catch-up exceeds the server's per-pull limit), pullInner
|
|
530
|
+
// drains the tail by recursing. Pre-fix it called the PUBLIC pull(),
|
|
531
|
+
// which re-enters the op queue under key "pull" and coalesces onto the
|
|
532
|
+
// promise it's currently running inside → permanent self-deadlock that
|
|
533
|
+
// bricks the whole pull path. Post-fix it recurses into pullInner().
|
|
534
|
+
test("delta pull with has_more drains the tail without self-deadlocking", async () => {
|
|
535
|
+
env = createTestEnv({ transport: "poll" });
|
|
536
|
+
env.signIn({ userId: "u1" });
|
|
537
|
+
env.server.seed("Note", [{ id: "n1", title: "a" }]);
|
|
538
|
+
await env.start();
|
|
539
|
+
await env.flush();
|
|
540
|
+
|
|
541
|
+
// A later change exists; the next delta pull reports has_more once,
|
|
542
|
+
// forcing the tail recursion.
|
|
543
|
+
env.server.insert("Note", { id: "n2", title: "b" });
|
|
544
|
+
env.server.primeNextPullHasMore();
|
|
545
|
+
|
|
546
|
+
// Must complete, not hang. Pre-fix the pull never resolves and the
|
|
547
|
+
// race rejects with "pull deadlocked".
|
|
548
|
+
const e = env;
|
|
549
|
+
await Promise.race([
|
|
550
|
+
e.engine.pull().then(() => e.flush()),
|
|
551
|
+
new Promise((_, reject) =>
|
|
552
|
+
setTimeout(() => reject(new Error("pull deadlocked")), 3000),
|
|
553
|
+
),
|
|
554
|
+
]);
|
|
555
|
+
expect(env.engine.store.get("Note", "n2")).not.toBeNull();
|
|
556
|
+
});
|
|
557
|
+
|
|
558
|
+
// OFFLINE WRITES (pins the transient/permanent split in pushInner).
|
|
559
|
+
// A push that fails with a NETWORK error (offline — fetch rejects, no
|
|
560
|
+
// HTTP status) must keep the mutation `pending` and the optimistic
|
|
561
|
+
// ghost intact, then re-send on reconnect. Pre-fix every transport
|
|
562
|
+
// failure marked the mutation `failed` + rolled back the ghost, so an
|
|
563
|
+
// offline insert vanished and was never re-sent.
|
|
564
|
+
test("offline push keeps the mutation pending and the ghost intact (not failed)", async () => {
|
|
565
|
+
env = createTestEnv({ transport: "poll" });
|
|
566
|
+
env.signIn({ userId: "u1" });
|
|
567
|
+
await env.start();
|
|
568
|
+
await env.flush();
|
|
569
|
+
|
|
570
|
+
env.server.primeNextPushOutcome({ kind: "network" });
|
|
571
|
+
const id = await env.engine.insert("Note", { title: "offline" });
|
|
572
|
+
await env.flush();
|
|
573
|
+
|
|
574
|
+
// Offline (network throw, no status): the ghost survives and the
|
|
575
|
+
// mutation stays PENDING — not `failed`. Pre-fix it was marked failed
|
|
576
|
+
// and the ghost rolled back, so the offline write vanished and (being
|
|
577
|
+
// `failed`, not `pending`) was never re-sent. Pending is exactly what
|
|
578
|
+
// makes the standard reconnect push re-ship it. (insert() returns the
|
|
579
|
+
// ROW id; match the mutation by row_id, not its op_id.)
|
|
580
|
+
expect(env.engine.store.get("Note", id)).not.toBeNull();
|
|
581
|
+
const mine = (env.engine.mutations as unknown as {
|
|
582
|
+
queue: { change: { row_id: string }; status: string }[];
|
|
583
|
+
}).queue.filter((m) => m.change.row_id === id);
|
|
584
|
+
expect(mine).toHaveLength(1);
|
|
585
|
+
expect(mine[0]!.status).toBe("pending");
|
|
586
|
+
});
|
|
587
|
+
|
|
588
|
+
// FAILED UPDATE ROLLBACK (pins failPushedMutation update path). A
|
|
589
|
+
// PERMANENT rejection (403) of an update must restore the row to its
|
|
590
|
+
// pre-update value. Pre-fix only inserts rolled back; a rejected update
|
|
591
|
+
// left the local row permanently wrong.
|
|
592
|
+
test("a permanently-rejected update restores the prior value", async () => {
|
|
593
|
+
env = createTestEnv({ transport: "poll" });
|
|
594
|
+
env.signIn({ userId: "u1" });
|
|
595
|
+
env.server.seed("Note", [{ id: "n1", title: "original" }]);
|
|
596
|
+
await env.start();
|
|
597
|
+
await env.flush();
|
|
598
|
+
|
|
599
|
+
env.server.primeNextPushOutcome({ kind: "status", status: 403 });
|
|
600
|
+
await env.engine.update("Note", "n1", { title: "edited" });
|
|
601
|
+
await env.flush();
|
|
602
|
+
|
|
603
|
+
const row = env.engine.store.get("Note", "n1") as { title?: string } | null;
|
|
604
|
+
expect(row?.title).toBe("original");
|
|
605
|
+
});
|
|
606
|
+
|
|
607
|
+
// FAILED DELETE ROLLBACK (pins failPushedMutation delete path). A
|
|
608
|
+
// rejected delete must bring the row back AND clear the optimistic
|
|
609
|
+
// tombstone, so a later server insert of the same id isn't fenced out.
|
|
610
|
+
// Pre-fix the row vanished and the tombstone blocked re-creation
|
|
611
|
+
// forever.
|
|
612
|
+
test("a permanently-rejected delete restores the row and un-fences it", async () => {
|
|
613
|
+
env = createTestEnv({ transport: "poll" });
|
|
614
|
+
env.signIn({ userId: "u1" });
|
|
615
|
+
env.server.seed("Note", [{ id: "n1", title: "keep" }]);
|
|
616
|
+
await env.start();
|
|
617
|
+
await env.flush();
|
|
618
|
+
|
|
619
|
+
env.server.primeNextPushOutcome({ kind: "status", status: 403 });
|
|
620
|
+
await env.engine.delete("Note", "n1");
|
|
621
|
+
await env.flush();
|
|
622
|
+
expect(env.engine.store.get("Note", "n1")).not.toBeNull();
|
|
623
|
+
|
|
624
|
+
// Un-fenced: a server insert of the same id now lands.
|
|
625
|
+
env.server.insert("Note", { id: "n1", title: "server-updated" });
|
|
626
|
+
await env.engine.pull();
|
|
627
|
+
await env.flush();
|
|
628
|
+
const row = env.engine.store.get("Note", "n1") as { title?: string } | null;
|
|
629
|
+
expect(row?.title).toBe("server-updated");
|
|
630
|
+
});
|
|
631
|
+
|
|
632
|
+
// CROSS-IDENTITY MUTATION WIPE (pins resetReplica wipeMutations). An
|
|
633
|
+
// identity flip (token / tenant change) must DROP the outgoing
|
|
634
|
+
// identity's pending offline writes — replaying them under the new
|
|
635
|
+
// session would attribute one user's writes to another. A same-user 410
|
|
636
|
+
// RESYNC must KEEP them (they survive the snapshot refresh).
|
|
637
|
+
test("an identity-flip reset wipes the outgoing identity's pending writes", async () => {
|
|
638
|
+
env = createTestEnv({ transport: "poll" });
|
|
639
|
+
env.signIn({ userId: "u1" });
|
|
640
|
+
await env.start();
|
|
641
|
+
await env.flush();
|
|
642
|
+
|
|
643
|
+
// Queue an offline write — network failure keeps it pending.
|
|
644
|
+
env.server.primeNextPushOutcome({ kind: "network" });
|
|
645
|
+
await env.engine.insert("Note", { title: "draft" });
|
|
646
|
+
await env.flush();
|
|
647
|
+
const queue = (env.engine.mutations as unknown as { queue: unknown[] }).queue;
|
|
648
|
+
expect(queue.length).toBeGreaterThan(0);
|
|
649
|
+
|
|
650
|
+
// Identity flip → wipe.
|
|
651
|
+
await env.engine.resetReplica({ wipeMutations: true });
|
|
652
|
+
expect((env.engine.mutations as unknown as { queue: unknown[] }).queue).toHaveLength(0);
|
|
653
|
+
});
|
|
654
|
+
|
|
655
|
+
test("a same-user 410 resync PRESERVES pending offline writes", async () => {
|
|
656
|
+
env = createTestEnv({ transport: "poll" });
|
|
657
|
+
env.signIn({ userId: "u1" });
|
|
658
|
+
await env.start();
|
|
659
|
+
await env.flush();
|
|
660
|
+
|
|
661
|
+
env.server.primeNextPushOutcome({ kind: "network" });
|
|
662
|
+
await env.engine.insert("Note", { title: "draft" });
|
|
663
|
+
await env.flush();
|
|
664
|
+
const before = (env.engine.mutations as unknown as { queue: unknown[] }).queue.length;
|
|
665
|
+
expect(before).toBeGreaterThan(0);
|
|
666
|
+
|
|
667
|
+
// 410 RESYNC path is the default (wipeMutations omitted) — KEEP writes.
|
|
668
|
+
await env.engine.resetReplica();
|
|
669
|
+
expect(
|
|
670
|
+
(env.engine.mutations as unknown as { queue: unknown[] }).queue.length,
|
|
671
|
+
).toBe(before);
|
|
672
|
+
});
|
|
673
|
+
|
|
490
674
|
// Row-revoked envelope: server pushes `row-revoked` to a
|
|
491
675
|
// subscriber whose read policy was revoked for a specific row.
|
|
492
676
|
// The engine must drop the row from the local replica and
|