@pylonsync/sync 0.3.179 → 0.3.181

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.3.179",
+  "version": "0.3.181",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/index.ts

@@ -114,6 +114,19 @@ export class LocalStore {
    * dominate anything a concurrent pull could replay.
    */
   private tombstones: Map<string, Map<string, number>> = new Map();
+  /**
+   * Pending optimistic deletes — `(entity, row_id)` pairs the local
+   * client has dropped but the server hasn't yet confirmed. Stored
+   * separately from `tombstones` because the optimistic "block any
+   * incoming insert/update for this row" guard runs at infinite
+   * seq, but the real server delete seq (typically 4–6 digits)
+   * would never max-merge past `Number.MAX_SAFE_INTEGER`. The old
+   * design left MAX_SAFE_INTEGER permanently in `tombstones` for
+   * any optimistically-deleted id, so a future server-issued insert
+   * with seq=N could never pass the `seq < tombstoneSeq` check —
+   * the row id was blocked for the lifetime of the replica.
+   */
+  private optimisticTombstones: Map<string, Set<string>> = new Map();
   private listeners: Set<() => void> = new Set();
 
   /** Get all rows for an entity. */
@@ -163,6 +176,9 @@ export class LocalStore {
 
   /** Check if `(entity, id)` has a tombstone. */
   private isTombstoned(entity: string, id: string, at_seq?: number): boolean {
+    // Pending optimistic delete — block everything until the server's
+    // real delete arrives and supersedes us.
+    if (this.optimisticTombstones.get(entity)?.has(id)) return true;
     const tombSeq = this.tombstones.get(entity)?.get(id);
     if (tombSeq === undefined) return false;
     // If the caller didn't tell us when their change happened, treat as
@@ -172,6 +188,11 @@ export class LocalStore {
   }
 
   private recordTombstone(entity: string, id: string, seq: number): void {
+    // A real (server-issued) tombstone supersedes any pending optimistic
+    // entry for this id. Without this drop, the optimistic
+    // MAX_SAFE_INTEGER entry would persist forever and block future
+    // re-creations of the same id (the case codex flagged P1).
+    this.optimisticTombstones.get(entity)?.delete(id);
     if (!this.tombstones.has(entity)) {
       this.tombstones.set(entity, new Map());
     }
@@ -264,8 +285,20 @@ export class LocalStore {
     }
     this.notify();
     if (this._persistFn) {
-      const results = changes.map((c) => this._persistFn!(this.hydrateFromMemory(c)));
-      await Promise.all(results.map((r) => (r instanceof Promise ? r : Promise.resolve())));
+      // Persist sequentially in arrival order — `Promise.all` would
+      // fire every IndexedDB write concurrently and the IDB scheduler
+      // can resolve them out of order. An `update → delete` pair on
+      // the same row would race the delete behind the update on disk,
+      // leaving a stale row in the persisted replica while the cursor
+      // advanced past the delete. Sequencing here matches the in-memory
+      // apply order, which itself is sequenced by the engine's
+      // `applyQueue`.
+      for (const change of changes) {
+        const result = this._persistFn(this.hydrateFromMemory(change));
+        if (result instanceof Promise) {
+          await result;
+        }
+      }
     }
   }
 
@@ -364,11 +397,17 @@ export class LocalStore {
   /** Apply an optimistic delete. */
   optimisticDelete(entity: string, id: string): void {
     this.tables.get(entity)?.delete(id);
-    // Client-side deletes dominate any concurrent server replay until the
-    // server confirms; use MAX_SAFE_INTEGER as the tombstone seq. When the
-    // server's real delete event arrives it will refresh the tombstone with
-    // the authoritative seq (via `recordTombstone`'s max-of).
-    this.recordTombstone(entity, id, Number.MAX_SAFE_INTEGER);
+    // Optimistic delete: block any incoming insert/update for this id
+    // until the server's authoritative delete arrives. Tracked in
+    // `optimisticTombstones` rather than `tombstones` so the real
+    // server seq can supersede it cleanly — the previous design wrote
+    // MAX_SAFE_INTEGER into `tombstones` and `recordTombstone`'s
+    // max-merge would never replace it with the smaller real seq,
+    // leaving the id permanently quarantined.
+    if (!this.optimisticTombstones.has(entity)) {
+      this.optimisticTombstones.set(entity, new Set());
+    }
+    this.optimisticTombstones.get(entity)!.add(id);
     this.notify();
   }
 
@@ -381,6 +420,7 @@ export class LocalStore {
   clearAll(): void {
     this.tables.clear();
     this.tombstones.clear();
+    this.optimisticTombstones.clear();
     this.notify();
   }
 }
@@ -991,16 +1031,40 @@ export class SyncEngine {
    */
   private enqueueApply(
     changes: ChangeEvent[],
-    targetCursor?: SyncCursor,
+    options:
+      | SyncCursor
+      | { targetCursor?: SyncCursor; skipSeqGuard?: boolean; advanceCursor?: boolean } = {},
   ): Promise<void> {
+    // Back-compat: callers that pass a SyncCursor positional arg get
+    // the same semantics as before. New callers can pass an options
+    // object — `skipSeqGuard` lets reconcile bypass the seq filter
+    // (its synthetic events fabricate seqs that don't fit the natural
+    // monotonic order), and `advanceCursor: false` keeps the cursor
+    // pinned where it was so reconcile doesn't fake-advance past the
+    // server's real position.
+    const opts: { targetCursor?: SyncCursor; skipSeqGuard?: boolean; advanceCursor?: boolean } =
+      options && typeof options === "object" && "last_seq" in options
+        ? { targetCursor: options as SyncCursor }
+        : (options as {
+            targetCursor?: SyncCursor;
+            skipSeqGuard?: boolean;
+            advanceCursor?: boolean;
+          });
+    const skipSeqGuard = opts.skipSeqGuard ?? false;
+    const advanceCursor = opts.advanceCursor ?? true;
+    const targetCursor = opts.targetCursor;
+
     const prev = this.applyQueue;
     const next = prev.then(async () => {
-      const filtered = changes.filter(
-        (c) => typeof c.seq === "number" && c.seq > this.cursor.last_seq,
-      );
+      const filtered = skipSeqGuard
+        ? changes
+        : changes.filter(
+            (c) => typeof c.seq === "number" && c.seq > this.cursor.last_seq,
+          );
       if (filtered.length > 0) {
         await this.store.applyChangesAsync(filtered);
       }
+      if (!advanceCursor) return;
       // Pick the cursor target. Explicit `targetCursor` (from pull) wins
       // — pull's response carries the server's authoritative current_seq
       // even when no changes landed in this window. Otherwise derive
@@ -1096,18 +1160,22 @@ export class SyncEngine {
         this.reconnectAttempts = 0;
         this.wsStableTimer = null;
       }, 5_000);
-      // Client-side keepalive ping. The server's per-client reader
-      // thread blocks on a synchronous WS read for as long as no client
-      // message arrives. On the HTTP-multiplexed `/api/sync/ws` path
-      // tiny_http doesn't expose stream-level read timeouts, so the
-      // reader's mutex hold is bounded only by client activity. Without
-      // these pings the broadcaster contends for the same mutex and
-      // wedges — Insert events never reach the tab. A 1s cadence makes
-      // worst-case broadcast latency ~1s even when the user is idle.
-      // Browsers don't expose WebSocket-level PING frames; a JSON
-      // payload with type:"ping" achieves the same effect (the server
-      // reads, looks up an unknown "ping" type, loops back releasing
-      // the mutex; broadcaster grabs it during the gap).
+      // Client-side keepalive ping at 200ms. The server's per-client
+      // reader thread blocks synchronously on `ws.read()` and holds the
+      // per-client mutex for the duration of the call. On the dedicated
+      // `:port+1` listener that block is bounded by a 200ms TCP read
+      // timeout (`stream.set_read_timeout`), so the broadcaster gets a
+      // window every 200ms. The HTTP-multiplexed `/api/sync/ws` path
+      // goes through tiny_http's `CustomStream`, which doesn't expose
+      // the underlying TcpStream, so we can't set a timeout there.
+      // These periodic JSON pings give the broadcaster the same 200ms
+      // window by causing the read to return (the server treats
+      // unknown `type` values as no-ops; the side effect is releasing
+      // the mutex between iterations). Browsers don't expose WS-level
+      // PING frames at the application layer, so we send a Text frame.
+      // Tradeoff: ~5 inbound frames/sec/client of background traffic
+      // for sub-second broadcast latency — worth it for the demo
+      // experience and idle tabs alike.
       if (this.pingTimer) clearInterval(this.pingTimer);
       this.pingTimer = setInterval(() => {
         if (this.ws?.readyState !== WebSocket.OPEN) return;
@@ -1116,7 +1184,7 @@ export class SyncEngine {
         } catch {
           // ignore — onclose will trigger reconnect
         }
-      }, 1_000);
+      }, 200);
       // Re-send any active CRDT subscriptions across the new socket.
       // The server purged them on disconnect (`unsubscribe_all`), so
       // without this resync a tab that was subscribed before a network
@@ -1734,30 +1802,47 @@ export class SyncEngine {
       }
     }
     if (changes.length > 0) {
-      await this.store.applyChangesAsync(changes);
+      // Reconcile applies route through the same serialized queue as
+      // WS/pull so a stale reconcile response can't interleave with a
+      // newer WS/pull update mid-batch. The synthetic seqs reconcile
+      // fabricates (tombstoneSeq + 1) collide with WS-issued seqs so
+      // we skip the monotonic guard and don't advance the cursor —
+      // the cursor still reflects the server's last_seq, not our
+      // fabricated reconcile seqs.
+      await this.enqueueApply(changes, {
+        skipSeqGuard: true,
+        advanceCursor: false,
+      });
     }
     // Removals: every local row whose id isn't in the server set is
     // stale. Tombstone with the current cursor so future legitimate
-    // re-creations still flow through.
+    // re-creations still flow through. Synthesize Delete events for
+    // each removal and route through the apply queue so the order
+    // relative to WS/pull updates is preserved — a removal here is
+    // really "server says this row is gone as of tombstoneSeq", which
+    // matters if a later WS update reinstates it on the same row id.
     const locals = this.store.list(entity);
-    let removed = false;
+    const removalChanges: ChangeEvent[] = [];
     for (const local of locals) {
       const id = (local as { id?: unknown }).id;
       if (typeof id !== "string") continue;
       if (!serverIds.has(id)) {
-        if (this.store.reconcileRemove(entity, id, tombstoneSeq)) {
-          removed = true;
-          if (this.persistence) {
-            try {
-              await this.persistence.deleteRow(entity, id);
-            } catch {
-              /* best-effort */
-            }
-          }
-        }
+        removalChanges.push({
+          seq: tombstoneSeq,
+          entity,
+          row_id: id,
+          kind: "delete",
+          data: undefined,
+          timestamp: "",
+        });
       }
     }
-    if (removed) this.store.notify();
+    if (removalChanges.length > 0) {
+      await this.enqueueApply(removalChanges, {
+        skipSeqGuard: true,
+        advanceCursor: false,
+      });
+    }
   }
 
   private async dropEntity(