@pylonsync/sync 0.22.40 → 0.22.43

package/dist/index.d.ts

@@ -2,7 +2,7 @@ import { LocalStore } from "./local-store";
 import { MutationQueue } from "./mutation-queue";
 import { type RoomError, type RoomMember, type RoomSubscriber } from "./room-subscriptions";
 import { SessionResolver } from "./session-resolver";
-export { RoomSubscriptions, type RoomError, type RoomErrorCode, type RoomMember, type RoomMessage, type RoomMessageSubscriber, type RoomSubscriber, } from "./room-subscriptions";
+export { RoomSubscriptions, rowRoom, type RoomError, type RoomErrorCode, type RoomMember, type RoomMessage, type RoomMessageSubscriber, type RoomSubscriber, } from "./room-subscriptions";
 export { IndexedDBPersistence, persistChange } from "./persistence";
 export type { ReplicaPersistence } from "./persistence";
 export { BEARER_REJECTED_HEADER, bearerRejected, buildRequest, clearRejectedToken, pylonFetch, pylonFetchRaw, PylonHttpError, resolveBaseUrl, } from "./transport";
@@ -733,6 +733,35 @@ export declare class SyncEngine {
      */
     observeEntity(entity: string): void;
     reconcile(entities?: string[]): Promise<void>;
+    /** Entities a server hint (`sync-reconcile`) asked to reconcile that
+     *  have not reconciled yet. */
+    private hintPending;
+    private hintTimer;
+    private hintRetries;
+    private lastHintReconcileAt;
+    /** Hints that arrive this close together share one reconcile. */
+    private static readonly HINT_DEBOUNCE_MS;
+    /** At most one hinted reconcile per this interval. */
+    private static readonly HINT_MIN_INTERVAL_MS;
+    /** Backoff for entities whose fetch raced a newer change: doubling from
+     *  the base, capped. */
+    private static readonly HINT_RETRY_BASE_MS;
+    private static readonly HINT_RETRY_MAX_MS;
+    /**
+     * The server said a change elsewhere (a share, an unshare, a folder
+     * membership) changed which rows of `entities` this user may read
+     * (`sync-reconcile`). No other event will tell this replica, so the
+     * entities stay pending until a reconcile of them completes: hints are
+     * merged, run at most once a second, and an entity whose fetch raced a
+     * newer change is retried with backoff until it lands or the engine
+     * stops.
+     */
+    private queueHintReconcile;
+    private scheduleHintReconcile;
+    private flushHintReconcile;
+    /** `reconcile`, resolving with the entities skipped because the cursor
+     *  moved during their fetch. */
+    private reconcileReportingSkips;
     /** Scoped reconcile batch that has not started running yet. */
     private scopedReconcileBatch;
     private scopedReconcileSeq;

package/dist/room-subscriptions.d.ts

@@ -4,7 +4,18 @@ export interface RoomMember {
     data?: Record<string, unknown>;
 }
 /** Reason codes the SDK exposes on a room subscription error. */
-export type RoomErrorCode = "NOT_IN_ROOM" | "UNKNOWN";
+/**
+ * - `NOT_IN_ROOM`: subscribed before joining (rejoin and resubscribe).
+ * - `ROOM_FORBIDDEN`: a room tied to a row the caller may not read.
+ * - `ROOM_REVOKED`: the caller lost access to that row while in the room.
+ */
+export type RoomErrorCode = "NOT_IN_ROOM" | "ROOM_FORBIDDEN" | "ROOM_REVOKED" | "UNKNOWN";
+/**
+ * The room tied to a row: its members are the people who may read the row,
+ * and the server checks that on every room operation. `@` starts only these
+ * names.
+ */
+export declare function rowRoom(entity: string, id: string): string;
 export interface RoomError {
     code: RoomErrorCode;
     message?: string;

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.22.40",
+  "version": "0.22.43",
   "type": "module",
   "main": "./src/index.ts",
   "types": "./dist/index.d.ts",

package/src/index.ts

@@ -22,6 +22,7 @@ import { OpQueue } from "./op-queue";
 import {
   RoomSubscriptions,
   type RoomError,
+  type RoomErrorCode,
   type RoomMember,
   type RoomSubscriber,
 } from "./room-subscriptions";
@@ -37,6 +38,7 @@ import {
 import { generateClientId, generateId } from "./ids";
 export {
   RoomSubscriptions,
+  rowRoom,
   type RoomError,
   type RoomErrorCode,
   type RoomMember,
@@ -1649,6 +1651,12 @@ export class SyncEngine {
   /** Stop the sync engine. */
   stop(): void {
     this.running = false;
+    if (this.hintTimer !== null) {
+      clearTimeout(this.hintTimer);
+      this.hintTimer = null;
+    }
+    this.hintPending.clear();
+    this.hintRetries = 0;
     if (this.identityRetryTimer !== null) {
       clearTimeout(this.identityRetryTimer);
       this.identityRetryTimer = null;
@@ -2301,10 +2309,75 @@ export class SyncEngine {
   }
 
   async reconcile(entities?: string[]): Promise<void> {
+    await this.reconcileReportingSkips(entities);
+  }
+
+  /** Entities a server hint (`sync-reconcile`) asked to reconcile that
+   *  have not reconciled yet. */
+  private hintPending = new Set<string>();
+  private hintTimer: ReturnType<typeof setTimeout> | null = null;
+  private hintRetries = 0;
+  private lastHintReconcileAt = 0;
+  /** Hints that arrive this close together share one reconcile. */
+  private static readonly HINT_DEBOUNCE_MS = 100;
+  /** At most one hinted reconcile per this interval. */
+  private static readonly HINT_MIN_INTERVAL_MS = 1_000;
+  /** Backoff for entities whose fetch raced a newer change: doubling from
+   *  the base, capped. */
+  private static readonly HINT_RETRY_BASE_MS = 250;
+  private static readonly HINT_RETRY_MAX_MS = 10_000;
+
+  /**
+   * The server said a change elsewhere (a share, an unshare, a folder
+   * membership) changed which rows of `entities` this user may read
+   * (`sync-reconcile`). No other event will tell this replica, so the
+   * entities stay pending until a reconcile of them completes: hints are
+   * merged, run at most once a second, and an entity whose fetch raced a
+   * newer change is retried with backoff until it lands or the engine
+   * stops.
+   */
+  private queueHintReconcile(entities: string[]): void {
+    for (const e of entities) this.hintPending.add(e);
+    this.scheduleHintReconcile(SyncEngine.HINT_DEBOUNCE_MS);
+  }
+
+  private scheduleHintReconcile(delayMs: number): void {
+    if (this.hintTimer !== null || !this.running) return;
+    const wait = Math.max(delayMs, this.lastHintReconcileAt + SyncEngine.HINT_MIN_INTERVAL_MS - Date.now());
+    this.hintTimer = setTimeout(() => {
+      this.hintTimer = null;
+      void this.flushHintReconcile();
+    }, wait);
+  }
+
+  private async flushHintReconcile(): Promise<void> {
+    if (!this.running || this.hintPending.size === 0) return;
+    const entities = [...this.hintPending];
+    this.hintPending.clear();
+    this.lastHintReconcileAt = Date.now();
+    let skipped: string[];
+    try {
+      skipped = await this.reconcileReportingSkips(entities);
+    } catch {
+      skipped = entities;
+    }
+    for (const e of skipped) this.hintPending.add(e);
+    this.hintRetries = skipped.length > 0 ? this.hintRetries + 1 : 0;
+    if (this.hintPending.size === 0) return;
+    const delay =
+      skipped.length > 0
+        ? Math.min(SyncEngine.HINT_RETRY_BASE_MS * 2 ** (this.hintRetries - 1), SyncEngine.HINT_RETRY_MAX_MS)
+        : SyncEngine.HINT_DEBOUNCE_MS;
+    this.scheduleHintReconcile(delay);
+  }
+
+  /** `reconcile`, resolving with the entities skipped because the cursor
+   *  moved during their fetch. */
+  private async reconcileReportingSkips(entities?: string[]): Promise<string[]> {
     const minIntervalMs = this.config.reconcileMinIntervalMs ?? 2_000;
     const now = Date.now();
     if (entities === undefined && now - this.lastReconcileAt < minIntervalMs) {
-      return;
+      return [];
     }
     // Reconcile waits behind any in-flight pull / refresh / push so it
     // can't apply rows captured under a stale session.
@@ -2313,7 +2386,7 @@ export class SyncEngine {
       // the same window share one fetch per entity.
       return this.opQueue.enqueue("reconcile", async () => {
         try {
-          await this.reconcileInner();
+          return await this.reconcileInner();
         } finally {
           this.lastReconcileAt = Date.now();
         }
@@ -2328,9 +2401,9 @@ export class SyncEngine {
       for (const e of entities) open.entities.add(e);
       return open.promise;
     }
-    const batch: { entities: Set<string>; promise: Promise<void> } = {
+    const batch: { entities: Set<string>; promise: Promise<string[]> } = {
       entities: new Set(entities),
-      promise: Promise.resolve(),
+      promise: Promise.resolve([]),
     };
     this.scopedReconcileBatch = batch;
     this.scopedReconcileSeq += 1;
@@ -2339,7 +2412,7 @@ export class SyncEngine {
       async () => {
         if (this.scopedReconcileBatch === batch) this.scopedReconcileBatch = null;
         try {
-          await this.reconcileInner([...batch.entities]);
+          return await this.reconcileInner([...batch.entities]);
         } finally {
           this.lastReconcileAt = Date.now();
         }
@@ -2351,14 +2424,14 @@ export class SyncEngine {
   /** Scoped reconcile batch that has not started running yet. */
   private scopedReconcileBatch: {
     entities: Set<string>;
-    promise: Promise<void>;
+    promise: Promise<string[]>;
   } | null = null;
   private scopedReconcileSeq = 0;
 
-  private async reconcileInner(entities?: string[]): Promise<void> {
+  private async reconcileInner(entities?: string[]): Promise<string[]> {
     // Same reasoning as pullInner: the leader reconciles, broadcasts
     // results, and follower replicas converge via the channel.
-    if (!this.isMultiTabLeader) return;
+    if (!this.isMultiTabLeader) return [];
     // Sweep entities with local rows PLUS entities the app has observed
     // via useQuery (even when empty locally). Without the observed set,
     // a server row in a never-cached entity is never reconciled and
@@ -2366,7 +2439,8 @@ export class SyncEngine {
     const names =
       entities ??
       [...new Set([...this.store.entityNames(), ...this.observedEntities])];
-    if (names.length === 0) return;
+    if (names.length === 0) return [];
+    const skipped: string[] = [];
     // Tombstone seq for any local row the server doesn't return. Using
     // the current cursor means future inserts (which have higher seqs)
     // bypass the tombstone — re-creation server-side still propagates.
@@ -2442,11 +2516,16 @@ export class SyncEngine {
             const streak = (this.reconcile403Streak.get(entity) ?? 0) + 1;
             if (streak < 2) {
               this.reconcile403Streak.set(entity, streak);
+              skipped.push(entity);
               return;
             }
             this.reconcile403Streak.delete(entity);
             await this.dropEntity(entity, tombstoneSeq);
+            return;
           }
+          // Not reconciled (offline, a 5xx): reported, so a server hint
+          // stays pending and retries.
+          skipped.push(entity);
           return;
         }
         // Successful fetch — the entity is readable, so any prior 403 streak
@@ -2458,7 +2537,9 @@ export class SyncEngine {
           // for a row our snapshot just captured. Bail out for this
           // entity; reconcile() is triggered again on visibility-change
           // and reconnect, and the WS event already carried the latest
-          // state for the affected row.
+          // state for the affected row. Reported, so a server hint can
+          // retry (flushHintReconcile).
+          skipped.push(entity);
           return;
         }
         if (this.session.signature() !== sessionBeforeFetch) {
@@ -2477,6 +2558,7 @@ export class SyncEngine {
         );
       }),
     );
+    return skipped;
   }
 
   /** Fetch every row for an entity. Uses cursor pagination so big tables
@@ -4103,6 +4185,15 @@ export class SyncEngine {
     // fresh /api/auth/me read which updates the cached session AND,
     // on tenant flip, resets the replica so stale rows from the
     // previous tenant disappear.
+    // A change elsewhere (a share, an unshare, a folder membership)
+    // changed which rows of these entities this user may read, without a
+    // change event for the rows themselves. Reconcile them.
+    if (msg.type === "sync-reconcile" && Array.isArray(msg.entities)) {
+      const entities = msg.entities.filter((e): e is string => typeof e === "string");
+      if (entities.length > 0) this.queueHintReconcile(entities);
+      return;
+    }
+
     if (msg.type === "session-changed") {
       void this.refreshResolvedSession();
       return;
@@ -4152,6 +4243,20 @@ export class SyncEngine {
       }
       return;
     }
+    // The server removed us from a room tied to a row we may no longer
+    // read. It stops sending the room's events; clear the members and tell
+    // the hook, which stops rejoining.
+    if (msg.type === "room-revoked" && typeof msg.room === "string") {
+      const room = msg.room;
+      const error: RoomError = { code: "ROOM_REVOKED" };
+      this.rooms.applySnapshot(room, []);
+      this.rooms.applyError(room, error);
+      if ((this.roomForwarders.get(room)?.size ?? 0) > 0) {
+        this.broadcastToTabs({ type: "room-fanout-snapshot", room, members: [] });
+        this.broadcastToTabs({ type: "room-fanout-error", room, error });
+      }
+      return;
+    }
     if (
       msg.type === "error" &&
       typeof msg.room === "string" &&
@@ -4163,7 +4268,8 @@ export class SyncEngine {
       // path right now, but the dispatch is keyed by `code` so future
       // error codes (rate-limited, room-full, etc.) route here too.
       const room = msg.room;
-      const code = msg.code === "NOT_IN_ROOM" ? "NOT_IN_ROOM" : "UNKNOWN";
+      const code: RoomErrorCode =
+        msg.code === "NOT_IN_ROOM" || msg.code === "ROOM_FORBIDDEN" ? msg.code : "UNKNOWN";
       const error: RoomError = {
         code,
         message: typeof msg.message === "string" ? msg.message : undefined,

package/src/reconcile.test.ts

@@ -668,3 +668,148 @@ describe("SyncEngine cookie auth", () => {
     expect(notifications).toBe(1);
   });
 });
+
+// The server sends `sync-reconcile` when a row of another entity changed
+// which rows of these entities the user may read (a share, an unshare):
+// no change event of the rows themselves will come.
+describe("sync-reconcile hint", () => {
+  let restore: (() => void) | null = null;
+
+  afterEach(() => {
+    restore?.();
+    restore = null;
+  });
+
+  type Internals = {
+    dispatchInboundMessage(m: Record<string, unknown>): void;
+    running: boolean;
+    cursor: { last_seq: number };
+    hintPending: Set<string>;
+  };
+  const inner = (engine: SyncEngine) => engine as unknown as Internals;
+  const deliver = (engine: SyncEngine, msg: Record<string, unknown>) => inner(engine).dispatchInboundMessage(msg);
+  const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+  function documentServer(onFetch: (n: number) => void = () => {}) {
+    let fetches = 0;
+    restore = installFetch(async (url) => {
+      if (url.includes("/api/entities/Document/cursor")) {
+        fetches += 1;
+        onFetch(fetches);
+        return {
+          status: 200,
+          body: { data: [{ id: "shared", title: "Now shared" }], next_cursor: null, has_more: false },
+        };
+      }
+      return { status: 404, body: {} };
+    });
+    return () => fetches;
+  }
+
+  test("reconciles the named entities: the shared row arrives, the unshared one leaves", async () => {
+    const fetches = documentServer();
+    const engine = makeEngine();
+    inner(engine).running = true;
+    seedStore(engine, "Document", [{ id: "unshared", title: "No longer shared" }]);
+
+    deliver(engine, { type: "sync-reconcile", entities: ["Document"] });
+    await sleep(250);
+
+    expect(fetches()).toBe(1);
+    expect(engine.store.list("Document").map((r) => (r as { id: string }).id)).toEqual(["shared"]);
+    engine.stop();
+  });
+
+  test("hints that arrive together share one fetch", async () => {
+    const fetches = documentServer();
+    const engine = makeEngine();
+    inner(engine).running = true;
+    for (let i = 0; i < 5; i++) deliver(engine, { type: "sync-reconcile", entities: ["Document"] });
+    await sleep(250);
+    expect(fetches()).toBe(1);
+    engine.stop();
+  });
+
+  test("keeps retrying while newer changes race the fetch, then lands", async () => {
+    const engine = makeEngine();
+    // The first two fetches race a change event: the cursor moves.
+    const fetches = documentServer((n) => {
+      if (n <= 2) inner(engine).cursor.last_seq += 1;
+    });
+    inner(engine).running = true;
+
+    deliver(engine, { type: "sync-reconcile", entities: ["Document"] });
+    await sleep(250);
+    expect(fetches()).toBe(1);
+    expect(engine.store.list("Document")).toEqual([]);
+    expect([...inner(engine).hintPending]).toEqual(["Document"]);
+
+    // Retries run at most once a second.
+    await sleep(2300);
+    expect(fetches()).toBe(3);
+    expect(engine.store.list("Document").map((r) => (r as { id: string }).id)).toEqual(["shared"]);
+    expect(inner(engine).hintPending.size).toBe(0);
+    engine.stop();
+  });
+
+  test("stop drops pending hints", async () => {
+    const fetches = documentServer();
+    const engine = makeEngine();
+    inner(engine).running = true;
+    deliver(engine, { type: "sync-reconcile", entities: ["Document"] });
+    engine.stop();
+    await sleep(250);
+    expect(fetches()).toBe(0);
+    expect(inner(engine).hintPending.size).toBe(0);
+  });
+
+  test("ignores a hint without entity names", async () => {
+    const fetches = documentServer();
+    const engine = makeEngine();
+    inner(engine).running = true;
+    deliver(engine, { type: "sync-reconcile", entities: [42, null] });
+    deliver(engine, { type: "sync-reconcile" });
+    await sleep(250);
+    expect(fetches()).toBe(0);
+    engine.stop();
+  });
+});
+
+describe("sync-reconcile hint after a failed fetch", () => {
+  let restore: (() => void) | null = null;
+  afterEach(() => {
+    restore?.();
+    restore = null;
+  });
+
+  test("a 5xx keeps the hint pending and the retry lands", async () => {
+    let fetches = 0;
+    restore = installFetch(async (url) => {
+      if (url.includes("/api/entities/Document/cursor")) {
+        fetches += 1;
+        if (fetches === 1) return { status: 503, body: {} };
+        return { status: 200, body: { data: [], next_cursor: null, has_more: false } };
+      }
+      return { status: 404, body: {} };
+    });
+    const engine = makeEngine();
+    const inner = engine as unknown as {
+      running: boolean;
+      hintPending: Set<string>;
+      dispatchInboundMessage(m: Record<string, unknown>): void;
+    };
+    inner.running = true;
+    seedStore(engine, "Document", [{ id: "unshared", title: "No longer shared" }]);
+
+    inner.dispatchInboundMessage({ type: "sync-reconcile", entities: ["Document"] });
+    await new Promise((r) => setTimeout(r, 250));
+    expect(fetches).toBe(1);
+    expect([...inner.hintPending]).toEqual(["Document"]);
+    expect(engine.store.list("Document").length).toBe(1);
+
+    await new Promise((r) => setTimeout(r, 1200));
+    expect(fetches).toBe(2);
+    expect(engine.store.list("Document")).toEqual([]);
+    engine.stop();
+  });
+});

package/src/room-subscriptions.test.ts

@@ -299,3 +299,15 @@ describe("RoomSubscriptions: resubscribe after a rejoin", () => {
     expect(sent).toEqual([{ type: "room-sub-resubscribe", room: "channel:a" }]);
   });
 });
+
+test("presence from someone not in the list adds them back", () => {
+  const { rooms } = makeHarness();
+  rooms.register("doc", () => {});
+  rooms.applySnapshot("doc", [{ user_id: "alice", joined_at: "t", data: {} }]);
+  // A leave relayed from another machine, though Alice is still connected
+  // through this one.
+  rooms.applyUpdate("doc", "leave", { user_id: "alice", joined_at: "", data: {} }, undefined);
+  expect(rooms.members("doc")).toEqual([]);
+  rooms.applyUpdate("doc", "presence", { user_id: "alice", joined_at: "", data: {} }, { cursor: 4 });
+  expect(rooms.members("doc")).toEqual([{ user_id: "alice", joined_at: "", data: { cursor: 4 } }]);
+});

package/src/room-subscriptions.ts

@@ -51,7 +51,21 @@ export interface RoomMember {
 }
 
 /** Reason codes the SDK exposes on a room subscription error. */
-export type RoomErrorCode = "NOT_IN_ROOM" | "UNKNOWN";
+/**
+ * - `NOT_IN_ROOM`: subscribed before joining (rejoin and resubscribe).
+ * - `ROOM_FORBIDDEN`: a room tied to a row the caller may not read.
+ * - `ROOM_REVOKED`: the caller lost access to that row while in the room.
+ */
+export type RoomErrorCode = "NOT_IN_ROOM" | "ROOM_FORBIDDEN" | "ROOM_REVOKED" | "UNKNOWN";
+
+/**
+ * The room tied to a row: its members are the people who may read the row,
+ * and the server checks that on every room operation. `@` starts only these
+ * names.
+ */
+export function rowRoom(entity: string, id: string): string {
+  return `@${entity}/${id}`;
+}
 
 export interface RoomError {
   code: RoomErrorCode;
@@ -272,11 +286,22 @@ export class RoomSubscriptions {
           _data && typeof _data === "object"
             ? (_data as Record<string, unknown>)
             : undefined;
-        entry.members = entry.members.map((m) =>
-          m.user_id === member.user_id
-            ? { ...m, ...member, ...(presence ? { data: presence } : {}) }
-            : m,
-        );
+        if (entry.members.some((m) => m.user_id === member.user_id)) {
+          entry.members = entry.members.map((m) =>
+            m.user_id === member.user_id
+              ? { ...m, ...member, ...(presence ? { data: presence } : {}) }
+              : m,
+          );
+        } else {
+          // Presence from someone not in the list is in the room: on a
+          // cluster, a leave relayed from one machine can drop a person
+          // still connected through another. Their next presence puts them
+          // back.
+          entry.members = [
+            ...entry.members,
+            { ...member, joined_at: member.joined_at ?? "", ...(presence ? { data: presence } : {}) },
+          ];
+        }
         break;
       }
       case "broadcast": {