@pylonsync/sync 0.3.228 → 0.3.230

package/src/local-store.ts

@@ -199,12 +199,20 @@ export class LocalStore {
    * between the memory apply and the eventual disk write can persist
    * a cursor that's ahead of the replica, skipping those rows
    * forever on restart.
+   *
+   * Returns `true` when every persist write reached disk durably,
+   * `false` when at least one degraded (quota / abort). The engine
+   * uses the result to hold the PERSISTED cursor back: a row that
+   * didn't reach disk must not be skipped by an advanced on-disk
+   * cursor on the next cold start. The in-memory replica always
+   * reflects the change regardless.
    */
-  async applyChangesAsync(changes: ChangeEvent[]): Promise<void> {
+  async applyChangesAsync(changes: ChangeEvent[]): Promise<boolean> {
     for (const change of changes) {
       this.applyChange(change);
     }
     this.notify();
+    let allDurable = true;
     if (this._persistFn) {
       // Sequential await — concurrent IDB writes can resolve out of
       // order, racing an update behind its own delete on disk. The
@@ -213,10 +221,12 @@ export class LocalStore {
       for (const change of changes) {
         const result = this._persistFn(this.hydrateFromMemory(change));
         if (result instanceof Promise) {
-          await result;
+          const durable = await result;
+          if (durable === false) allDurable = false;
         }
       }
     }
+    return allDurable;
   }
 
   /**
@@ -234,9 +244,11 @@ export class LocalStore {
   }
 
   /** Persistence callback for auto-saving changes. Returns
-   *  `Promise<void>` so callers can await. Void-returning callbacks
-   *  are accepted for backwards compatibility (just not awaitable). */
-  _persistFn: ((change: ChangeEvent) => void | Promise<void>) | null = null;
+   *  `Promise<boolean>` (true = durable, false = degraded) so
+   *  `applyChangesAsync` can gate the on-disk cursor on durability.
+   *  Void-returning callbacks are accepted for backwards compatibility
+   *  (treated as durable / fire-and-forget). */
+  _persistFn: ((change: ChangeEvent) => void | Promise<boolean>) | null = null;
 
   /** Subscribe to store changes. Returns unsubscribe function. */
   subscribe(listener: () => void): () => void {
@@ -307,6 +319,46 @@ export class LocalStore {
     }
   }
 
+  /**
+   * Undo a rejected optimistic update/delete by restoring the row to its
+   * captured pre-mutation value and clearing any optimistic tombstone
+   * for it. `failPushedMutation` calls this when the server rejects an
+   * update (restore the prior field values) or a delete (bring the row
+   * back AND un-fence it so the row — and any future server insert of
+   * the id — isn't blocked by the lingering optimistic tombstone).
+   *
+   * `prev === null` means the row didn't exist before the mutation
+   * (e.g. an update on a row that was itself an un-acked insert) — in
+   * that case we just remove it + clear the fence.
+   *
+   * A REAL (server-issued) tombstone wins over the restore: if an
+   * authoritative delete/revocation for this id landed on the applyQueue
+   * while the rejected push was in flight (the opQueue and applyQueue run
+   * independently), resurrecting `prev` here would briefly un-delete a row
+   * the server says is gone — healed only at the next reconcile. So when a
+   * server tombstone is present we drop the row and let the canonical
+   * state stand; the failed mutation's own optimistic fence is cleared
+   * regardless so a later legitimate re-create of the id isn't blocked.
+   */
+  restoreRow(entity: string, id: string, prev: Row | null): void {
+    // The failed mutation's own optimistic fence always clears.
+    this.optimisticTombstones.get(entity)?.delete(id);
+    if (this.tombstones.get(entity)?.has(id)) {
+      // Server authoritatively removed this row mid-flight — its
+      // deletion outranks our local rollback.
+      this.tables.get(entity)?.delete(id);
+      this.notify();
+      return;
+    }
+    if (prev) {
+      if (!this.tables.has(entity)) this.tables.set(entity, new Map());
+      this.tables.get(entity)!.set(id, prev);
+    } else {
+      this.tables.get(entity)?.delete(id);
+    }
+    this.notify();
+  }
+
   /** Apply an optimistic delete. Block any incoming insert/update
    *  for this id until the server's authoritative delete arrives. */
   optimisticDelete(entity: string, id: string): void {

package/src/multi-tab-orchestrator.ts

@@ -69,8 +69,11 @@ export interface MultiTabOrchestratorHooks {
     removalIds: string[],
     tombstoneSeq: number,
   ): void;
-  /** Replica reset broadcast — identity flip happened on the leader. */
-  onResetReceived(): void;
+  /** Replica reset broadcast from the leader. `wipeMutations` is true
+   *  when the reset was an identity flip (drop the outgoing identity's
+   *  pending offline writes), false for a same-user 410 RESYNC (keep
+   *  them — they survive the snapshot refresh). */
+  onResetReceived(wipeMutations: boolean): void;
   /** Resolved session update from the leader. Engine funnels through
    *  its session chain so concurrent triggers commit in order. */
   onSessionReceived(resolved: ResolvedSession): void;
@@ -112,6 +115,16 @@ export interface MultiTabOrchestratorHooks {
   /** New leader → followers: re-forward your locally-wanted room subs.
    *  Triggered alongside CRDT/reactive replay after a leader change. */
   onReplayRoomSubs?(): void;
+
+  /** Follower → leader: a follower's `useQuery` observed an entity.
+   *  Leader-only. The leader adds it to its reconcile sweep and fetches
+   *  it so a server row the follower never cached is discovered and
+   *  broadcast back as a `reconciled` batch. Without this, a follower's
+   *  view on a never-cached entity stays empty forever. */
+  onEntityObserve?(entity: string, fromTabId: string): void;
+  /** New leader → followers: re-declare your observed entities so the
+   *  new leader's reconcile sweep covers them. */
+  onReplayObservedEntities?(): void;
 }
 
 export interface MultiTabOrchestratorConfig {
@@ -247,8 +260,8 @@ export class MultiTabOrchestrator {
     });
   }
 
-  broadcastReset(): void {
-    this.broadcastRaw({ type: "reset" });
+  broadcastReset(wipeMutations: boolean): void {
+    this.broadcastRaw({ type: "reset", wipeMutations });
   }
 
   broadcastSession(resolved: ResolvedSession): void {
@@ -328,7 +341,7 @@ export class MultiTabOrchestrator {
         break;
       }
       case "reset": {
-        this.hooks.onResetReceived();
+        this.hooks.onResetReceived(msg.wipeMutations === true);
         break;
       }
       case "session": {
@@ -388,6 +401,19 @@ export class MultiTabOrchestrator {
         // so the new leader can rebuild its forwarder sets and resend
         // `room-subscribe` on the WS.
         this.hooks.onReplayRoomSubs?.();
+        // ...and re-declare observed entities so the new leader's
+        // reconcile sweep covers them.
+        this.hooks.onReplayObservedEntities?.();
+        break;
+      }
+      case "entity-observe": {
+        // Follower → leader. Leader-only — a follower receiving this
+        // (own broadcast echo, or stale leader transition) ignores.
+        if (!this._isLeader) return;
+        const entity = msg.entity as string | undefined;
+        if (typeof entity === "string") {
+          this.hooks.onEntityObserve?.(entity, fromTabId);
+        }
         break;
       }
       case "room-sub-register": {

package/src/mutation-queue.ts

@@ -10,13 +10,19 @@
 // instead of re-applying.
 // ---------------------------------------------------------------------------
 
-import type { ClientChange } from "./types";
+import type { ClientChange, Row } from "./types";
 
 export interface PendingMutation {
   id: string;
   change: ClientChange;
   status: "pending" | "applied" | "failed";
   error?: string;
+  /** Pre-mutation snapshot of the affected row, captured at optimistic-
+   *  apply time for `update`/`delete`. On a server rejection,
+   *  `failPushedMutation` restores this so the local replica reverts to
+   *  the value the server actually holds. `null` = the row didn't exist
+   *  before the mutation; `undefined` = not captured (inserts). */
+  prevRow?: Row | null;
 }
 
 /**
@@ -80,14 +86,14 @@ export class MutationQueue {
    *  entry with the same op_id is already queued — a follower
    *  retrying its forward of the same op shouldn't double-queue on
    *  the leader. */
-  add(change: ClientChange): string {
+  add(change: ClientChange, prevRow?: Row | null): string {
     const id =
       typeof change.op_id === "string" && change.op_id.length > 0
         ? change.op_id
         : `mut_${Date.now()}_${Math.random().toString(36).slice(2)}`;
     if (this.queue.some((m) => m.id === id)) return id;
     const changeWithOp: ClientChange = { ...change, op_id: id };
-    this.queue.push({ id, change: changeWithOp, status: "pending" });
+    this.queue.push({ id, change: changeWithOp, status: "pending", prevRow });
     this.flush();
     return id;
   }
@@ -96,6 +102,13 @@ export class MutationQueue {
     return this.queue.filter((m) => m.status === "pending");
   }
 
+  /** Look up a queued mutation by op id (any status). Used by the
+   *  follower's mutations-failed handler to reach the captured
+   *  `prevRow` for rollback. */
+  get(id: string): PendingMutation | undefined {
+    return this.queue.find((m) => m.id === id);
+  }
+
   /**
    * Set of `${entity}/${row_id}` keys for every mutation currently
    * in Pending or Failed state. Used by reconcile() to skip rows
@@ -152,6 +165,22 @@ export class MutationQueue {
     this.flush();
   }
 
+  /**
+   * Drop EVERY mutation — pending, failed, and applied — then flush the
+   * empty queue to disk. Used by the engine's identity-flip reset
+   * (`resetReplica({ wipeMutations: true })`): the queued offline writes
+   * belong to the OUTGOING identity, and replaying them under the new
+   * session would attribute one user's writes to another (or get
+   * policy-rejected on the server). Distinct from `clear()`, which only
+   * prunes already-applied entries and deliberately PRESERVES pending +
+   * failed writes — the 410-RESYNC same-user path relies on that so
+   * offline writes survive a snapshot refresh.
+   */
+  clearAll(): void {
+    this.queue = [];
+    this.flush();
+  }
+
   /** Fire-and-forget persistence write. */
   private flush(): void {
     if (!this.persistence) return;

package/src/persistence.ts

@@ -86,15 +86,47 @@ export class IndexedDBPersistence {
     });
   }
 
-  /** Save a row to IndexedDB. */
-  async saveRow(entity: string, id: string, data: Row): Promise<void> {
-    if (!this.db) return;
+  /** Resolve when a write transaction settles, reporting durability:
+   *  `true` on COMMIT, `false` if it aborts/errors — e.g.
+   *  QuotaExceededError on a storage-pressured device. NEVER rejects or
+   *  hangs: registering only `oncomplete` (the original shape) meant an
+   *  aborted tx never settled, and the engine awaits the persist before
+   *  advancing the cursor in `enqueueApply` — so ONE hung write wedged
+   *  the whole applyQueue and live sync silently died for the session.
+   *
+   *  But "resolve unconditionally" (the first fix) was also wrong: on an
+   *  abort the row never reached disk, yet `enqueueApply` still advanced
+   *  the PERSISTED cursor past it — so on restart the warm-load skipped
+   *  those rows forever (cursor ahead of replica). The boolean lets the
+   *  caller hold the on-disk cursor back when a row write fails, so the
+   *  next cold start re-pulls the gap. Best-effort: log once, keep the
+   *  in-memory replica authoritative, but never persist a cursor ahead of
+   *  what's actually durable. Mirrors the read paths (`getRow`/
+   *  `loadCursor`) which already degrade via `onerror`. */
+  private commit(tx: IDBTransaction, op: string): Promise<boolean> {
+    return new Promise((resolve) => {
+      tx.oncomplete = () => resolve(true);
+      tx.onerror = () => {
+        // eslint-disable-next-line no-console
+        console.warn(`[pylon-sync] IndexedDB ${op} failed; degrading to memory`, tx.error);
+        resolve(false);
+      };
+      tx.onabort = () => {
+        // eslint-disable-next-line no-console
+        console.warn(`[pylon-sync] IndexedDB ${op} aborted; degrading to memory`, tx.error);
+        resolve(false);
+      };
+    });
+  }
+
+  /** Save a row to IndexedDB. Resolves `true` when the write is durable,
+   *  `false` if it degraded (no DB / aborted) — see `commit`. */
+  async saveRow(entity: string, id: string, data: Row): Promise<boolean> {
+    if (!this.db) return false;
     const tx = this.db.transaction(STORE_NAME, "readwrite");
     const store = tx.objectStore(STORE_NAME);
     store.put({ _key: `${entity}:${id}`, entity, id, data });
-    return new Promise((resolve) => {
-      tx.oncomplete = () => resolve();
-    });
+    return this.commit(tx, "saveRow");
   }
 
   /** Fetch a row from IndexedDB by key. Used by `persistChange` on update
@@ -113,15 +145,14 @@ export class IndexedDBPersistence {
     });
   }
 
-  /** Delete a row from IndexedDB. */
-  async deleteRow(entity: string, id: string): Promise<void> {
-    if (!this.db) return;
+  /** Delete a row from IndexedDB. Resolves `true` when durable, `false`
+   *  if it degraded — see `commit`. */
+  async deleteRow(entity: string, id: string): Promise<boolean> {
+    if (!this.db) return false;
     const tx = this.db.transaction(STORE_NAME, "readwrite");
     const store = tx.objectStore(STORE_NAME);
     store.delete(`${entity}:${id}`);
-    return new Promise((resolve) => {
-      tx.oncomplete = () => resolve();
-    });
+    return this.commit(tx, "deleteRow");
   }
 
   /** Load all rows for an entity from IndexedDB. */
@@ -217,15 +248,14 @@ export class IndexedDBPersistence {
     });
   }
 
-  /** Save the sync cursor. */
-  async saveCursor(cursor: SyncCursor): Promise<void> {
-    if (!this.db) return;
+  /** Save the sync cursor. Resolves `true` when durable, `false` if it
+   *  degraded — see `commit`. */
+  async saveCursor(cursor: SyncCursor): Promise<boolean> {
+    if (!this.db) return false;
     const tx = this.db.transaction(CURSOR_STORE, "readwrite");
     const store = tx.objectStore(CURSOR_STORE);
     store.put({ key: "cursor", ...cursor });
-    return new Promise((resolve) => {
-      tx.oncomplete = () => resolve();
-    });
+    return this.commit(tx, "saveCursor");
   }
 
   /** Load the sync cursor. */
@@ -247,25 +277,34 @@ export class IndexedDBPersistence {
     });
   }
 
-  /** Clear all stored data. */
-  async clear(): Promise<void> {
-    if (!this.db) return;
+  /** Clear stored rows + cursor. Deliberately does NOT touch the
+   *  durable mutation queue: `resetReplica` calls this on a 410 RESYNC
+   *  (same user, needs a fresh snapshot) where pending offline writes
+   *  MUST survive. On an IDENTITY flip the old identity's pending
+   *  mutations must instead be discarded — that wipe runs through
+   *  `MutationQueue.clearAll()` (which persists an empty `MUTATIONS_STORE`
+   *  via its own backend), gated on `resetReplica({ wipeMutations })` at
+   *  the token/tenant-flip call sites. Splitting the two stores keeps each
+   *  reset path honest: 410 keeps writes, identity flip drops them. */
+  async clear(): Promise<boolean> {
+    if (!this.db) return false;
     const tx = this.db.transaction([STORE_NAME, CURSOR_STORE], "readwrite");
     tx.objectStore(STORE_NAME).clear();
     tx.objectStore(CURSOR_STORE).clear();
-    return new Promise((resolve) => {
-      tx.oncomplete = () => resolve();
-    });
+    return this.commit(tx, "clear");
   }
 }
 
 /**
- * Apply a change event to IndexedDB persistence.
+ * Apply a change event to IndexedDB persistence. Returns `true` when the
+ * write reached disk durably, `false` when it degraded (so `enqueueApply`
+ * can hold the persisted cursor back rather than skip the row on restart).
+ * A change with no `data` is a no-op and counts as durable.
  */
 export async function persistChange(
   persistence: IndexedDBPersistence,
   change: ChangeEvent
-): Promise<void> {
+): Promise<boolean> {
   switch (change.kind) {
     case "insert":
     case "update":
@@ -274,13 +313,13 @@ export async function persistChange(
         // change's `data` with the post-merge row from memory — so even
         // on update the full row lands here. Overwriting is correct;
         // pre-merge patches would have dropped unpatched columns.
-        await persistence.saveRow(change.entity, change.row_id, change.data);
+        return persistence.saveRow(change.entity, change.row_id, change.data);
       }
-      break;
+      return true;
     case "delete":
-      await persistence.deleteRow(change.entity, change.row_id);
-      break;
+      return persistence.deleteRow(change.entity, change.row_id);
   }
+  return true;
 }
 
 // ---------------------------------------------------------------------------

package/src/round6-codex.test.ts

@@ -326,3 +326,160 @@ describe("codex round-6: peer leaving scrubs forwarded subs", () => {
     expect(engine.serverSubs.has("sub-1")).toBe(false);
   });
 });
+
+describe("sync hardening: multi-tab follower coverage", () => {
+  let env: TestEnv | null = null;
+  afterEach(async () => {
+    if (env) await env.dispose();
+    env = null;
+  });
+
+  // FOLLOWER GHOST ROLLBACK. The leader broadcasts `mutations-failed`;
+  // the follower must roll back its OWN optimistic ghost, not just mark
+  // the queue entry failed. Pre-fix onMutationsFailed called markFailed
+  // directly, so the ghost row stuck around in the tab the user sees.
+  test("a mutations-failed for an optimistic insert removes the follower's ghost", async () => {
+    env = createTestEnv();
+    env.signIn({ userId: "u1" });
+    await env.start();
+    const engine = env.engine;
+
+    // Optimistic ghost in the store + a matching queued mutation (as if
+    // this follower had forwarded the insert to the leader).
+    engine.store.optimisticInsertWithId("Todo", "t1", { id: "t1", text: "ghost" });
+    engine.mutations.add({
+      op_id: "op-1",
+      entity: "Todo",
+      row_id: "t1",
+      kind: "insert",
+      data: { id: "t1", text: "ghost" },
+    });
+    expect(engine.store.get("Todo", "t1")).not.toBeNull();
+
+    (engine as unknown as {
+      orchestrator: { handleMessage(msg: unknown, from: string): void };
+    }).orchestrator.handleMessage(
+      { type: "mutations-failed", ops: [{ opId: "op-1", error: "rejected" }] },
+      "leader-tab",
+    );
+
+    // Ghost gone (rolled back), mutation marked failed.
+    expect(engine.store.get("Todo", "t1")).toBeNull();
+  });
+
+  // ENTITY-OBSERVE FORWARDING. A leader that receives a forwarded
+  // `entity-observe` from a follower must add the entity to its reconcile
+  // sweep and fetch it — so a server row the follower never cached is
+  // discovered + broadcast back. Without this a follower's useQuery on a
+  // never-cached entity renders empty forever.
+  test("a leader fetches an entity a follower forwarded via entity-observe", async () => {
+    env = createTestEnv({ transport: "poll" });
+    env.signIn({ userId: "u1" });
+    await env.start(); // solo → this engine is the leader
+    const engine = env.engine;
+
+    // A server row exists for an entity this engine never cached.
+    env.server.insert("Domain", { id: "d1", host: "x.com" });
+    expect(engine.store.list("Domain")).toHaveLength(0);
+
+    (engine as unknown as {
+      orchestrator: { handleMessage(msg: unknown, from: string): void };
+    }).orchestrator.handleMessage(
+      { type: "entity-observe", entity: "Domain" },
+      "follower-tab",
+    );
+    await env.flush();
+
+    expect(engine.store.get("Domain", "d1")).not.toBeNull();
+  });
+
+  // FORWARDED-MUTATION ROLLBACK (pins onMutationsForwarded prevRow
+  // threading). When a follower forwards an update/delete and the leader's
+  // push of it is PERMANENTLY rejected, the leader must restore the
+  // canonical row — NOT delete it. Pre-fix the leader queued the forwarded
+  // op via mutations.add(op.change) WITHOUT prevRow, so failPushedMutation
+  // ran restoreRow(undefined ?? null) → restoreRow(null) → DELETED the
+  // leader's still-valid canonical row. Data loss on every rejected
+  // forwarded edit.
+  test("a rejected forwarded UPDATE keeps the leader's canonical row", async () => {
+    env = createTestEnv({ transport: "poll" });
+    env.signIn({ userId: "u1" });
+    env.server.seed("Note", [{ id: "n1", title: "canonical" }]);
+    await env.start(); // solo → leader, holds the canonical row
+    await env.flush();
+    expect(env.engine.store.get("Note", "n1")).not.toBeNull();
+
+    // A follower optimistically edited n1, captured the prior value as
+    // prevRow, and forwarded the update. The leader's push will 403.
+    env.server.primeNextPushOutcome({ kind: "status", status: 403 });
+    (env.engine as unknown as {
+      orchestrator: { handleMessage(msg: unknown, from: string): void };
+    }).orchestrator.handleMessage(
+      {
+        type: "mutations",
+        ops: [
+          {
+            id: "op-u",
+            change: {
+              op_id: "op-u",
+              entity: "Note",
+              row_id: "n1",
+              kind: "update",
+              data: { title: "edited" },
+            },
+            status: "pending",
+            prevRow: { id: "n1", title: "canonical" },
+          },
+        ],
+      },
+      "follower-tab",
+    );
+    // Wait behind the forwarded push (opQueue serializes), then drain.
+    await env.engine.push();
+    await env.flush();
+
+    const row = env.engine.store.get("Note", "n1") as { title?: string } | null;
+    expect(row).not.toBeNull();
+    expect(row?.title).toBe("canonical");
+  });
+
+  test("a rejected forwarded DELETE keeps the leader's canonical row", async () => {
+    env = createTestEnv({ transport: "poll" });
+    env.signIn({ userId: "u1" });
+    env.server.seed("Note", [{ id: "n1", title: "keep" }]);
+    await env.start();
+    await env.flush();
+    expect(env.engine.store.get("Note", "n1")).not.toBeNull();
+
+    env.server.primeNextPushOutcome({ kind: "status", status: 403 });
+    (env.engine as unknown as {
+      orchestrator: { handleMessage(msg: unknown, from: string): void };
+    }).orchestrator.handleMessage(
+      {
+        type: "mutations",
+        ops: [
+          {
+            id: "op-d",
+            change: {
+              op_id: "op-d",
+              entity: "Note",
+              row_id: "n1",
+              kind: "delete",
+            },
+            status: "pending",
+            prevRow: { id: "n1", title: "keep" },
+          },
+        ],
+      },
+      "follower-tab",
+    );
+    await env.engine.push();
+    await env.flush();
+
+    // Pre-fix the leader deleted its canonical row on the rejected
+    // forwarded delete; with prevRow threaded it survives.
+    const row = env.engine.store.get("Note", "n1") as { title?: string } | null;
+    expect(row).not.toBeNull();
+    expect(row?.title).toBe("keep");
+  });
+});