@pylonsync/sync 0.3.310 → 0.3.312

package/dist/index.d.ts

@@ -287,6 +287,18 @@ export declare class SyncEngine {
      * is the dedup key.
      */
     private binaryHandlers;
+    /**
+     * Latest server CRDT SNAPSHOT frame per `entity|rowId`, kept so a
+     * follower tab that registers interest in an ALREADY-subscribed row
+     * gets state immediately. The server sends its catch-up snapshot only
+     * when a fresh `crdt-subscribe` goes over the WS — when the leader
+     * (or another follower) already holds the subscription, no wire
+     * traffic happens and, pre-cache, the new tab's LoroDoc stayed empty
+     * until the next live edit. Server broadcasts are always FULL merged
+     * snapshots (CRDT_FRAME_SNAPSHOT), so one frame per row is complete
+     * state. Bounded FIFO — see LAST_CRDT_FRAMES_MAX.
+     */
+    private lastCrdtFrames;
     /**
      * Server-side ephemeral subscriptions (CRDT row subs, reactive query
      * subs, future kinds). Owns the WS replay bookkeeping — each kind
@@ -951,6 +963,9 @@ export declare class SyncEngine {
      *  follower tab forwarded a CRDT sub, mirror over the multi-tab
      *  channel so followers see Loro updates too. */
     private dispatchBinaryFrame;
+    /** Drop the cached snapshot for a revoked row so it can't be
+     *  replayed to a late-joining tab after policy said no. */
+    private evictCrdtFrame;
     private request;
 }
 /** Data shape for hydrating the client from server-rendered content. */
@@ -969,6 +984,30 @@ export interface HydrationData {
 export declare function getServerData(baseUrl: string, entities: string[], options?: {
     token?: string;
 }): Promise<HydrationData>;
+/**
+ * Stable equality check for reconciler diffs. Keys are sorted so
+ * `{a:1,b:2}` and `{b:2,a:1}` compare equal — without that, every
+ * reconcile pass would think every row had changed (insertion order
+ * varies by mutation path on the server). Recursive on objects only;
+ * arrays and primitives use their natural shape.
+ */
+/** Cap on cached per-row CRDT snapshots (see lastCrdtFrames). 64 rows of
+ *  collaborative state is far beyond what one browser session edits at
+ *  once; FIFO eviction keeps a long-lived leader tab bounded. */
+export declare const LAST_CRDT_FRAMES_MAX = 64;
+/**
+ * Header-only peek at a binary CRDT frame: returns `"entity|rowId"` for
+ * a SNAPSHOT frame, null for anything else (updates, foreign binary,
+ * truncated bytes). Mirrors the layout of
+ * `crates/router::encode_crdt_frame` / `@pylonsync/loro`'s wire.ts:
+ *
+ *   [type: u8] [entity_len: u16 BE] [entity utf8]
+ *   [row_id_len: u16 BE] [row_id utf8] [payload]
+ *
+ * The engine still treats the PAYLOAD as opaque — this reads only the
+ * routing header so the follower-catch-up cache can be keyed per row.
+ */
+export declare function crdtFrameKey(bytes: Uint8Array): string | null;
 /**
  * Create a sync engine connected to the pylon backend.
  *

package/dist/subscription-coordinator.d.ts

@@ -5,6 +5,13 @@ import type { ReactiveMessage } from "./types";
 export interface SubscriptionCoordinatorContext {
     isLeader(): boolean;
     broadcastToTabs(payload: unknown): void;
+    /** Leader-side replay of the cached CRDT snapshot for a row (see
+     *  SyncEngine.lastCrdtFrames). Called when a follower registers
+     *  interest in a row whose WS subscription is ALREADY alive — the
+     *  server only ships its catch-up snapshot on a fresh
+     *  `crdt-subscribe`, so without the replay the new tab's doc stays
+     *  empty until the next live edit. */
+    replayCrdtFrame?(entity: string, rowId: string): void;
 }
 export declare class SubscriptionCoordinator {
     private readonly serverSubs;

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.3.310",
+  "version": "0.3.312",
   "type": "module",
   "main": "./src/index.ts",
   "types": "./dist/index.d.ts",

package/src/crdt-follower-catchup.test.ts

@@ -0,0 +1,107 @@
+// Follower catch-up for CRDT rows. The server ships its catch-up
+// snapshot only on a FRESH `crdt-subscribe`; when a follower tab
+// registers interest in a row the leader (or another follower) already
+// subscribed, no wire traffic happens — pre-fix the new tab's LoroDoc
+// stayed empty until the next live edit ("open the doc in a second tab
+// and the content doesn't render"). The leader now caches the latest
+// snapshot frame per row and replays it over the tab channel.
+import { describe, expect, test } from "bun:test";
+import { SubscriptionCoordinator, crdtKey } from "./subscription-coordinator";
+import { ServerSubscriptions } from "./server-subscriptions";
+import { crdtFrameKey } from "./index";
+
+function encodeSnapshotFrame(
+  type: number,
+  entity: string,
+  rowId: string,
+  payload: Uint8Array,
+): Uint8Array {
+  const enc = new TextEncoder();
+  const e = enc.encode(entity);
+  const r = enc.encode(rowId);
+  const out = new Uint8Array(1 + 2 + e.length + 2 + r.length + payload.length);
+  const view = new DataView(out.buffer);
+  out[0] = type;
+  view.setUint16(1, e.length, false);
+  out.set(e, 3);
+  view.setUint16(3 + e.length, r.length, false);
+  out.set(r, 5 + e.length);
+  out.set(payload, 5 + e.length + r.length);
+  return out;
+}
+
+describe("crdtFrameKey", () => {
+  test("keys a snapshot frame by entity|rowId", () => {
+    const frame = encodeSnapshotFrame(0x10, "Doc", "row1", new Uint8Array(3));
+    expect(crdtFrameKey(frame)).toBe("Doc|row1");
+  });
+
+  test("ignores update frames and garbage", () => {
+    const update = encodeSnapshotFrame(0x11, "Doc", "row1", new Uint8Array(3));
+    expect(crdtFrameKey(update)).toBeNull();
+    expect(crdtFrameKey(new Uint8Array([0x10, 0, 9]))).toBeNull();
+    expect(crdtFrameKey(new Uint8Array(0))).toBeNull();
+  });
+});
+
+describe("forwarded register replay", () => {
+  function harness() {
+    const sent: unknown[] = [];
+    const replayed: Array<[string, string]> = [];
+    const serverSubs = new ServerSubscriptions((msg) => {
+      sent.push(msg);
+      return true;
+    });
+    const coordinator = new SubscriptionCoordinator(serverSubs, {
+      isLeader: () => true,
+      broadcastToTabs: () => {},
+      replayCrdtFrame: (entity, rowId) => {
+        replayed.push([entity, rowId]);
+      },
+    });
+    return { coordinator, sent, replayed };
+  }
+
+  test("already-alive subscription replays the cached snapshot to followers", () => {
+    const { coordinator, replayed } = harness();
+    // Leader's own component holds the row → WS sub is alive.
+    coordinator.subscribeCrdt("Doc", "row1");
+    // A second tab opens the same doc and forwards its interest. No new
+    // crdt-subscribe goes out (the sub exists), so the ONLY way this
+    // tab gets state is the replay.
+    coordinator.handleForwardedRegister(
+      { kind: "crdt", key: crdtKey("Doc", "row1"), entity: "Doc", rowId: "row1" },
+      "tab-2",
+    );
+    expect(replayed).toEqual([["Doc", "row1"]]);
+  });
+
+  test("fresh subscription does NOT replay — the server catch-up covers it", () => {
+    const { coordinator, replayed, sent } = harness();
+    coordinator.handleForwardedRegister(
+      { kind: "crdt", key: crdtKey("Doc", "row9"), entity: "Doc", rowId: "row9" },
+      "tab-2",
+    );
+    // First interest in the row → real crdt-subscribe goes out; the
+    // server's own catch-up snapshot will be fanned to tabs.
+    expect(replayed).toEqual([]);
+    expect(
+      sent.some(
+        (m) => (m as { type?: string }).type === "crdt-subscribe",
+      ),
+    ).toBe(true);
+  });
+
+  test("second follower for the same row also gets a replay", () => {
+    const { coordinator, replayed } = harness();
+    coordinator.handleForwardedRegister(
+      { kind: "crdt", key: crdtKey("Doc", "row1"), entity: "Doc", rowId: "row1" },
+      "tab-2",
+    );
+    coordinator.handleForwardedRegister(
+      { kind: "crdt", key: crdtKey("Doc", "row1"), entity: "Doc", rowId: "row1" },
+      "tab-3",
+    );
+    expect(replayed).toEqual([["Doc", "row1"]]);
+  });
+});

package/src/index.ts

@@ -417,6 +417,19 @@ export class SyncEngine {
    */
   private binaryHandlers: Set<(bytes: Uint8Array) => void> = new Set();
 
+  /**
+   * Latest server CRDT SNAPSHOT frame per `entity|rowId`, kept so a
+   * follower tab that registers interest in an ALREADY-subscribed row
+   * gets state immediately. The server sends its catch-up snapshot only
+   * when a fresh `crdt-subscribe` goes over the WS — when the leader
+   * (or another follower) already holds the subscription, no wire
+   * traffic happens and, pre-cache, the new tab's LoroDoc stayed empty
+   * until the next live edit. Server broadcasts are always FULL merged
+   * snapshots (CRDT_FRAME_SNAPSHOT), so one frame per row is complete
+   * state. Bounded FIFO — see LAST_CRDT_FRAMES_MAX.
+   */
+  private lastCrdtFrames: Map<string, Uint8Array> = new Map();
+
   /**
    * Server-side ephemeral subscriptions (CRDT row subs, reactive query
    * subs, future kinds). Owns the WS replay bookkeeping — each kind
@@ -534,6 +547,14 @@ export class SyncEngine {
     this.subscriptions = new SubscriptionCoordinator(this.serverSubs, {
       isLeader: () => this.isMultiTabLeader,
       broadcastToTabs: (payload) => this.broadcastToTabs(payload),
+      // Follower registered interest in a row whose WS subscription is
+      // already alive — no server catch-up will come, so replay the
+      // cached snapshot over the tab channel. Loro imports are
+      // idempotent, so tabs that already have the state are unaffected.
+      replayCrdtFrame: (entity, rowId) => {
+        const cached = this.lastCrdtFrames.get(`${entity}|${rowId}`);
+        if (cached) this.broadcastToTabs({ type: "binary", bytes: cached });
+      },
     });
     this.rooms = new RoomSubscriptions((msg) => {
       // Leader: send over the WS. Followers don't open a transport;
@@ -751,16 +772,38 @@ export class SyncEngine {
 
     if (!this.isMultiTabLeader) {
       // Follower path: rely on the leader's broadcasts for session +
-      // applied changes. Nothing else to do here — the broker is
-      // wired to forward inbound messages into the engine. The
-      // sessionPromise we kicked off above resolves into the void;
-      // the leader's broadcast will deliver the authoritative view.
-      // Swallow any pending error so it doesn't surface as an
-      // unhandled rejection.
+      // applied changes. The broker is wired to forward inbound
+      // messages into the engine. The sessionPromise we kicked off
+      // above resolves into the void; the leader's broadcast will
+      // deliver the authoritative view. Swallow any pending error so
+      // it doesn't surface as an unhandled rejection.
       void sessionPromise.then(
         () => {},
         () => {},
       );
+      // Replay every subscription made BEFORE the election settled.
+      // Those calls took the follower branch and broadcast to a
+      // not-yet-constructed orchestrator — a silent no-op — so the
+      // leader never learned this tab wants anything. The leader-side
+      // mirror of this is seedServerSubsFromLocalInterest below; the
+      // follower side was missing, which left a second tab opening a
+      // CRDT doc the leader already held rendering EMPTY forever (no
+      // crdt-subscribe ever reached the leader, so neither the server
+      // catch-up nor the cached-snapshot replay fired). Rooms,
+      // observed entities, and hydrated offline mutations strand the
+      // same way — replay the full bundle, mirroring what
+      // request-sub-replay does after a leader flip.
+      this.subscriptions.replayForwardedSubs();
+      for (const roomId of this.rooms.roomIds()) {
+        this.broadcastToTabs({ type: "room-sub-register", room: roomId });
+      }
+      for (const entity of this.observedEntities) {
+        this.broadcastToTabs({ type: "entity-observe", entity });
+      }
+      const pendingOps = this.mutations.pending();
+      if (pendingOps.length > 0) {
+        this.broadcastToTabs({ type: "mutations", ops: pendingOps });
+      }
       return;
     }
 
@@ -1169,6 +1212,28 @@ export class SyncEngine {
       this.transport.stop();
       this.transport = null;
     }
+    // Re-forward everything this tab wanted while it still believed it
+    // was the leader. Before the election settles every tab defaults to
+    // leader, so components that mounted in that window registered
+    // their subscriptions against OUR transport — which we just
+    // stopped. Without the re-forward the real leader never learns
+    // about them: a second tab opening a CRDT doc the leader already
+    // holds rendered EMPTY forever (no crdt-subscribe reached the
+    // leader, so neither the server catch-up nor the leader's cached-
+    // snapshot replay ever fired). Same stranding applied to rooms
+    // (presence), observed entities, and pending mutations — mirror
+    // the request-sub-replay bundle.
+    this.subscriptions.replayForwardedSubs();
+    for (const roomId of this.rooms.roomIds()) {
+      this.broadcastToTabs({ type: "room-sub-register", room: roomId });
+    }
+    for (const entity of this.observedEntities) {
+      this.broadcastToTabs({ type: "entity-observe", entity });
+    }
+    const pending = this.mutations.pending();
+    if (pending.length > 0) {
+      this.broadcastToTabs({ type: "mutations", ops: pending });
+    }
   }
 
   /** Broadcast a payload to other tabs in this origin. Delegates to
@@ -2316,6 +2381,7 @@ export class SyncEngine {
       }
       this.store.notify();
     }
+    this.evictCrdtFrame(entity, rowId);
     for (const listener of this.rowEvictionListeners) {
       listener(entity, rowId);
     }
@@ -3317,6 +3383,19 @@ export class SyncEngine {
    *  follower tab forwarded a CRDT sub, mirror over the multi-tab
    *  channel so followers see Loro updates too. */
   private dispatchBinaryFrame(bytes: Uint8Array): void {
+    // Remember the latest snapshot per row for follower catch-up (see
+    // lastCrdtFrames). Header-only peek; the payload stays opaque.
+    const key = crdtFrameKey(bytes);
+    if (key !== null) {
+      if (!this.lastCrdtFrames.has(key) &&
+          this.lastCrdtFrames.size >= LAST_CRDT_FRAMES_MAX) {
+        const oldest = this.lastCrdtFrames.keys().next().value;
+        if (oldest !== undefined) this.lastCrdtFrames.delete(oldest);
+      }
+      // Re-insert to refresh FIFO position.
+      this.lastCrdtFrames.delete(key);
+      this.lastCrdtFrames.set(key, bytes);
+    }
     for (const handler of this.binaryHandlers) {
       try {
         handler(bytes);
@@ -3340,6 +3419,12 @@ export class SyncEngine {
     }
   }
 
+  /** Drop the cached snapshot for a revoked row so it can't be
+   *  replayed to a late-joining tab after policy said no. */
+  private evictCrdtFrame(entity: string, rowId: string): void {
+    this.lastCrdtFrames.delete(`${entity}|${rowId}`);
+  }
+
   private async request<T>(method: string, path: string, body?: unknown): Promise<T> {
     const headers: Record<string, string> = {};
     if (body) headers["Content-Type"] = "application/json";
@@ -3448,6 +3533,39 @@ export async function getServerData(
  * varies by mutation path on the server). Recursive on objects only;
  * arrays and primitives use their natural shape.
  */
+/** Cap on cached per-row CRDT snapshots (see lastCrdtFrames). 64 rows of
+ *  collaborative state is far beyond what one browser session edits at
+ *  once; FIFO eviction keeps a long-lived leader tab bounded. */
+export const LAST_CRDT_FRAMES_MAX = 64;
+
+/**
+ * Header-only peek at a binary CRDT frame: returns `"entity|rowId"` for
+ * a SNAPSHOT frame, null for anything else (updates, foreign binary,
+ * truncated bytes). Mirrors the layout of
+ * `crates/router::encode_crdt_frame` / `@pylonsync/loro`'s wire.ts:
+ *
+ *   [type: u8] [entity_len: u16 BE] [entity utf8]
+ *   [row_id_len: u16 BE] [row_id utf8] [payload]
+ *
+ * The engine still treats the PAYLOAD as opaque — this reads only the
+ * routing header so the follower-catch-up cache can be keyed per row.
+ */
+export function crdtFrameKey(bytes: Uint8Array): string | null {
+  const SNAPSHOT_TYPE = 0x10;
+  if (bytes.length < 5 || bytes[0] !== SNAPSHOT_TYPE) return null;
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const entityLen = view.getUint16(1, false);
+  const entityEnd = 3 + entityLen;
+  if (entityEnd + 2 > bytes.length) return null;
+  const rowIdLen = view.getUint16(entityEnd, false);
+  const rowIdEnd = entityEnd + 2 + rowIdLen;
+  if (rowIdEnd > bytes.length) return null;
+  const decoder = new TextDecoder();
+  const entity = decoder.decode(bytes.subarray(3, entityEnd));
+  const rowId = decoder.decode(bytes.subarray(entityEnd + 2, rowIdEnd));
+  return `${entity}|${rowId}`;
+}
+
 function rowsDiffer(a: Row, b: Row): boolean {
   return stableStringify(a) !== stableStringify(b);
 }

package/src/room-subscriptions.test.ts

@@ -181,6 +181,26 @@ describe("RoomSubscriptions: snapshot / update / error", () => {
     expect(h.rooms.members("channel:foo")).toHaveLength(1);
   });
 
+  test("applyUpdate presence applies the envelope's data slot", () => {
+    // The server ships new presence in the DATA slot; `member` carries
+    // only user_id. Pre-fix the handler merged `member` alone — a no-op
+    // that froze remote presence (live cursors, typing indicators) at
+    // whatever the join carried.
+    const h = makeHarness();
+    h.rooms.register("channel:foo", () => {});
+    h.rooms.applySnapshot("channel:foo", [
+      { user_id: "u1", joined_at: "t1", data: { caret: 1 } },
+    ]);
+    h.rooms.applyUpdate(
+      "channel:foo",
+      "presence",
+      { user_id: "u1", joined_at: "t1" },
+      { caret: 42, name: "Ada" },
+    );
+    const members = h.rooms.members("channel:foo")!;
+    expect(members[0].data).toEqual({ caret: 42, name: "Ada" });
+  });
+
   test("applyUpdate leave removes the matching user", () => {
     const h = makeHarness();
     h.rooms.register("channel:foo", () => {});

package/src/room-subscriptions.ts

@@ -264,8 +264,18 @@ export class RoomSubscriptions {
       case "presence": {
         if (!member) break;
         // Swap the matching member's data while preserving join order.
+        // The server ships the NEW presence in the envelope's data slot
+        // (`member` carries only the user_id) — merging `member` alone
+        // is a no-op that drops every live presence update: remote
+        // cursors/typing indicators freeze at whatever the join carried.
+        const presence =
+          _data && typeof _data === "object"
+            ? (_data as Record<string, unknown>)
+            : undefined;
         entry.members = entry.members.map((m) =>
-          m.user_id === member.user_id ? { ...m, ...member } : m,
+          m.user_id === member.user_id
+            ? { ...m, ...member, ...(presence ? { data: presence } : {}) }
+            : m,
         );
         break;
       }

package/src/subscription-coordinator.ts

@@ -53,6 +53,13 @@ const OWN_TAB = "__self__";
 export interface SubscriptionCoordinatorContext {
   isLeader(): boolean;
   broadcastToTabs(payload: unknown): void;
+  /** Leader-side replay of the cached CRDT snapshot for a row (see
+   *  SyncEngine.lastCrdtFrames). Called when a follower registers
+   *  interest in a row whose WS subscription is ALREADY alive — the
+   *  server only ships its catch-up snapshot on a fresh
+   *  `crdt-subscribe`, so without the replay the new tab's doc stays
+   *  empty until the next live edit. */
+  replayCrdtFrame?(entity: string, rowId: string): void;
 }
 
 interface ReactiveSpec {
@@ -282,6 +289,12 @@ export class SubscriptionCoordinator {
           entity,
           rowId,
         });
+      } else {
+        // Subscription already alive → the server won't send a fresh
+        // catch-up snapshot. Replay the leader's cached one so the
+        // follower converges immediately instead of staying empty
+        // until the next live edit.
+        this.ctx.replayCrdtFrame?.(entity, rowId);
       }
       return;
     }