@multiplatform.one/frappe 6.1.0 → 6.4.0

package/src/sync/realtime.ts

@@ -10,16 +10,34 @@ export class RealtimeHandler {
   private reconnectAttempts = 0;
   private reconnectTimer: ReturnType<typeof setTimeout> | undefined;
   private hasConnected = false;
+  /** Live socket state: true between `connected` and `disconnect` events. */
+  private connected = false;
   private connectionConfig: Required<ConnectionConfig>;
+  /** Refcount of active subscriptions per doctype — drives doctype room membership. */
+  private watchedDoctypes = new Map<string, number>();
+  /** In-flight authoritative fetches, deduping concurrent doc_update/list_update pairs. */
+  private inflightFetches = new Map<string, Promise<void>>();
+  private readonly db: DbModule;
+  private readonly realtime: RealtimeModule | undefined;
+  private readonly normalizedCache: NormalizedCache;
+  private readonly subscriptionManager: SubscriptionManager;
+  private readonly rehydrationManager: RehydrationManager | undefined;
+  private readonly options: SyncOptions;
 
   constructor(
-    private readonly db: DbModule,
-    private readonly realtime: RealtimeModule | undefined,
-    private readonly normalizedCache: NormalizedCache,
-    private readonly subscriptionManager: SubscriptionManager,
-    private readonly rehydrationManager: RehydrationManager | undefined,
-    private readonly options: SyncOptions = {},
+    db: DbModule,
+    realtime: RealtimeModule | undefined,
+    normalizedCache: NormalizedCache,
+    subscriptionManager: SubscriptionManager,
+    rehydrationManager: RehydrationManager | undefined,
+    options: SyncOptions = {},
   ) {
+    this.db = db;
+    this.realtime = realtime;
+    this.normalizedCache = normalizedCache;
+    this.subscriptionManager = subscriptionManager;
+    this.rehydrationManager = rehydrationManager;
+    this.options = options;
     this.connectionConfig = {
       reconnectBaseDelay: options.connectionConfig?.reconnectBaseDelay ?? 1000,
       reconnectMaxDelay: options.connectionConfig?.reconnectMaxDelay ?? 30000,
@@ -32,10 +50,50 @@ export class RealtimeHandler {
 
   /**
    * Sets up realtime event handlers for document changes.
+   *
+   * Real Frappe (v15) wire protocol, verified live against a stock bench
+   * (see .test-evidence/storybook-rubric/p-results/frappe-realtime-live-e2e.md):
+   * - `doc_update`  `{doctype, name, modified}` — emitted to the per-doc room
+   *   `doc:{doctype}/{name}` on every save.
+   * - `list_update` `{doctype, name, user}` — emitted to the doctype room
+   *   `doctype:{doctype}` on every save of a non-single, non-child doctype.
+   * - Deletion emits the SAME two events (payload indistinguishable from an
+   *   update) — there is NO `doc_deleted` event. Deletions are detected here
+   *   opportunistically when the authoritative re-fetch 404s, but the durable
+   *   deletion signal remains the tombstone/CDC backfill (`Deleted Document`
+   *   doctype maintained by the `live` app's on_trash hook).
+   *
+   * Both real events funnel into the same fetch-and-reconcile path, deduped
+   * so a doc_update/list_update pair for the same doc costs one fetch.
    */
   private setupRealtimeHandlers() {
     if (!this.realtime) return;
 
+    this.realtime?.on("doc_update", async (payload) => {
+      if (this.options.onRealtimeEvent)
+        this.options.onRealtimeEvent({
+          type: RealtimeEventType.DocUpdate,
+          payload,
+        });
+      return this.onDocChanged(payload as { doctype: string; name: string });
+    });
+
+    this.realtime?.on("list_update", async (payload) => {
+      if (this.options.onRealtimeEvent)
+        this.options.onRealtimeEvent({
+          type: RealtimeEventType.ListUpdate,
+          payload,
+        });
+      return this.onDocChanged(payload as { doctype: string; name: string });
+    });
+
+    // `doc_deleted` is emitted by the `live` server app's on_trash hook
+    // (live.live.tombstones.notify_deletion) to the doctype room after
+    // commit — it gives subscribed clients an explicit deletion signal
+    // (payload {doctype, name, deleted_at}) without the 404 probe below.
+    // `doc_changed` remains a forward-compat no-op from the designed wire
+    // protocol (src/core/wire.ts); stock frappe never emits it and the
+    // dedup map absorbs doubles if a server ever does.
     this.realtime?.on("doc_changed", async (payload) => {
       if (this.options.onRealtimeEvent)
         this.options.onRealtimeEvent({
@@ -87,6 +145,7 @@ export class RealtimeHandler {
    */
   private onConnected() {
     this.hasConnected = true;
+    this.connected = true;
     this.reconnectAttempts = 0;
 
     // Clear any pending reconnection timers
@@ -104,6 +163,9 @@ export class RealtimeHandler {
       this.options.onBackfillAll();
     }
 
+    // Let store subscribers re-read connection state (reconnect indicators)
+    this.options.onStoreChange?.();
+
     console.debug("[RealtimeHandler] Connected to realtime server");
   }
 
@@ -112,6 +174,8 @@ export class RealtimeHandler {
    */
   private onDisconnected() {
     console.debug("[RealtimeHandler] Disconnected from realtime server");
+    this.connected = false;
+    this.options.onStoreChange?.();
 
     // Only attempt reconnection if we've successfully connected before
     if (this.hasConnected) {
@@ -120,16 +184,18 @@ export class RealtimeHandler {
   }
 
   /**
-   * Attempts to connect to the realtime server.
+   * No-op placeholder retained for the disconnect backoff schedule below.
+   * Transport reconnect is owned by Socket.IO inside RealtimeModule; this
+   * layer only tracks attempts for logging / future health hooks.
    */
   private attemptConnection() {
     if (!this.realtime) return;
-    // RealtimeModule handles reconnection internally via its reconnection config.
-    // Handlers are already attached in the constructor via setupRealtimeHandlers().
   }
 
   /**
-   * Schedules a reconnection attempt with exponential backoff and jitter.
+   * Schedules a reconnection *watchdog* tick with exponential backoff and jitter.
+   * Does not open sockets — RealtimeModule already reconnects. Kept so disconnect
+   * storms remain observable (and existing unit tests stay meaningful).
    */
   private scheduleReconnection() {
     // Clear any existing timer
@@ -175,12 +241,67 @@ export class RealtimeHandler {
     );
   }
 
+  /**
+   * Join the realtime doctype room for a doctype (refcounted). Called by the
+   * sync module whenever a subscription for the doctype is added. Room
+   * membership survives reconnects — RealtimeModule re-joins tracked rooms on
+   * every `connect`.
+   *
+   * Note: only doctype-level rooms are joined. `list_update` fires for every
+   * save of a regular doctype, which covers doc updates too; per-doc rooms
+   * would only add coverage for single/read-only doctypes.
+   */
+  watchDoctype(doctype: string): void {
+    const count = (this.watchedDoctypes.get(doctype) ?? 0) + 1;
+    this.watchedDoctypes.set(doctype, count);
+    if (count === 1 && this.realtime) {
+      Promise.resolve(this.realtime.subscribeList(doctype)).catch((err) =>
+        console.warn(`[RealtimeHandler] Failed to join doctype room ${doctype}:`, err),
+      );
+    }
+  }
+
+  /** Release one reference to a doctype room; leaves the room on the last one. */
+  unwatchDoctype(doctype: string): void {
+    const count = (this.watchedDoctypes.get(doctype) ?? 0) - 1;
+    if (count > 0) {
+      this.watchedDoctypes.set(doctype, count);
+      return;
+    }
+    this.watchedDoctypes.delete(doctype);
+    this.realtime?.unsubscribeList(doctype);
+  }
+
+  /** Leave all doctype rooms (used by cache resets/disposal). */
+  unwatchAll(): void {
+    if (this.realtime) {
+      for (const doctype of this.watchedDoctypes.keys()) {
+        this.realtime.unsubscribeList(doctype);
+      }
+    }
+    this.watchedDoctypes.clear();
+  }
+
   /**
    * Realtime handler: authoritative document changed.
    * Fetches document via REST with current user context to respect permissions,
    * updates cache, and reconciles subscription membership.
+   *
+   * Concurrent events for the same document (frappe emits doc_update AND
+   * list_update per save) share a single in-flight fetch.
    */
-  private async onDocChanged(payload: { doctype: string; name: string }) {
+  private onDocChanged(payload: { doctype: string; name: string }): Promise<void> {
+    const key = `${payload?.doctype}\0${payload?.name}`;
+    const inflight = this.inflightFetches.get(key);
+    if (inflight) return inflight;
+    const fetchPromise = this.fetchAndReconcile(payload).finally(() => {
+      this.inflightFetches.delete(key);
+    });
+    this.inflightFetches.set(key, fetchPromise);
+    return fetchPromise;
+  }
+
+  private async fetchAndReconcile(payload: { doctype: string; name: string }) {
     try {
       // Fetch authoritative document to respect permissions and server-only fields
       const document = await this.db.getDoc<any>(payload.doctype, payload.name);
@@ -195,9 +316,21 @@ export class RealtimeHandler {
       for (const subscription of subscriptionsNeedingRehydration) {
         this.rehydrationManager?.scheduleRehydration(subscription, RehydrationReason.RemoteCreate);
       }
+
+      // Cache/membership changed without a local mutation or backfill —
+      // notify the store so live queries re-render the patched rows.
+      this.options.onStoreChange?.();
     } catch (err) {
-      // If fetch fails (permission, network), skip this update
-      // The next backfill will catch any missed changes
+      // Frappe emits a plain doc_update/list_update on deletion (no doc_deleted
+      // event exists) — a 404 on the authoritative re-fetch means the document
+      // is gone (deleted, or read access revoked): evict it. The tombstone/CDC
+      // backfill remains the durable deletion signal for events missed offline.
+      if ((err as { status?: number })?.status === 404) {
+        await this.onDocDeleted(payload);
+        return;
+      }
+      // Other failures (network, transient server errors): skip this update —
+      // the next backfill will catch any missed changes
       console.warn(
         `Failed to fetch document ${payload?.doctype}/${payload?.name} during realtime update:`,
         err,
@@ -227,6 +360,9 @@ export class RealtimeHandler {
       for (const subscription of affectedSubscriptions) {
         this.rehydrationManager?.scheduleRehydration(subscription, RehydrationReason.Delete);
       }
+
+      // Row evicted — notify the store so live queries drop it immediately.
+      this.options.onStoreChange?.();
     } catch (err) {
       console.warn(`Failed to process doc_deleted for ${payload?.doctype}/${payload?.name}:`, err);
     }
@@ -243,6 +379,16 @@ export class RealtimeHandler {
     return this.hasConnected;
   }
 
+  /**
+   * Live connection snapshot for reconnect indicators.
+   * `connected` tracks the socket right now; `everConnected` distinguishes
+   * "reconnecting" (was up, dropped) from "never had realtime" (fixtures,
+   * SSR, no realtime module).
+   */
+  getConnectionState(): { connected: boolean; everConnected: boolean } {
+    return { connected: this.connected, everConnected: this.hasConnected };
+  }
+
   /**
    * Disposes of resources and clears timers.
    */
@@ -252,6 +398,8 @@ export class RealtimeHandler {
       this.reconnectTimer = undefined;
     }
 
+    this.unwatchAll();
+
     if (this.realtime) {
       this.realtime?.disconnect();
     }

package/src/sync/rehydration.ts

@@ -32,13 +32,26 @@ export class RehydrationManager {
   private readonly processDebouncer = new Debouncer(() => this.processQueue(), { wait: 100 });
   /** Maps temporary IDs to server-assigned IDs for optimistic creates */
   private temporaryIdMap = new Map<string, string>();
+  private readonly db: DbModule;
+  private readonly cache: NormalizedCache;
+  private readonly subscriptionManager: SubscriptionManager;
+  private readonly mutationJournal: MutationJournal;
+  /** Notifies the sync store after rehydration mutates cache/membership. */
+  private readonly onChange?: () => void;
 
   constructor(
-    private readonly db: DbModule,
-    private readonly cache: NormalizedCache,
-    private readonly subscriptionManager: SubscriptionManager,
-    private readonly mutationJournal: MutationJournal,
-  ) {}
+    db: DbModule,
+    cache: NormalizedCache,
+    subscriptionManager: SubscriptionManager,
+    mutationJournal: MutationJournal,
+    onChange?: () => void,
+  ) {
+    this.db = db;
+    this.cache = cache;
+    this.subscriptionManager = subscriptionManager;
+    this.mutationJournal = mutationJournal;
+    this.onChange = onChange;
+  }
 
   /**
    * Schedules a rehydration task for a subscription based on the reason.
@@ -171,15 +184,20 @@ export class RehydrationManager {
     });
 
     // Process each task
+    let anyExecuted = false;
     for (const task of tasks) {
       try {
         await this.executeRehydration(task);
         this.rehydrationQueue.delete(task.subscriptionId);
+        anyExecuted = true;
       } catch (error) {
         console.error(`Rehydration failed for ${task.subscriptionId}:`, error);
         // Keep in queue for retry
       }
     }
+    // Rehydration refreshed cache/membership outside a mutate/backfill call —
+    // notify store listeners so live queries pick up the reconciled page.
+    if (anyExecuted) this.onChange?.();
   }
 
   /**

package/src/sync/storage.ts

@@ -73,11 +73,12 @@ export class PlatformStorageKV implements KVStorage {
  */
 export class IndexedDBKV implements KVStorage {
   private dbPromise: Promise<IDBDatabase>;
+  private readonly dbName: string;
+  private readonly store: string;
 
-  constructor(
-    private readonly dbName = "frappe-sync",
-    private readonly store = "kv",
-  ) {
+  constructor(dbName = "frappe-sync", store = "kv") {
+    this.dbName = dbName;
+    this.store = store;
     this.dbPromise = new Promise((resolve, reject) => {
       if (typeof indexedDB === "undefined") {
         reject(new Error("IndexedDB not available in this environment"));

package/src/sync/subscriptionManager.spec.ts

@@ -534,6 +534,100 @@ describe("SubscriptionManager", () => {
       expect(subscription.tombstones?.has("doc2")).toBe(true);
     });
 
+    it("should drain follow-up pages while the server reports has_more", async () => {
+      const result = await subscriptionManager.subscribe({
+        doctype: "TestDoc",
+        fields: ["*"],
+      });
+      const subscription = subscriptionManager.getSubscription(result.subscriptionId)!;
+      (filters.matchesFilters as any).mockReturnValue(true);
+
+      mockDb.backfill
+        .mockResolvedValueOnce({
+          rows: [{ name: "doc4", doctype: "TestDoc", modified: "2024-01-02T10:00:00" }],
+          deleted: [],
+          checkpoint: "2024-01-02T10:00:00|doc4",
+          has_more: true,
+        })
+        .mockResolvedValueOnce({
+          rows: [{ name: "doc5", doctype: "TestDoc", modified: "2024-01-02T11:00:00" }],
+          deleted: [{ name: "doc2" }],
+          checkpoint: "2024-01-02T11:00:00|doc5",
+          has_more: true,
+        })
+        .mockResolvedValueOnce({
+          rows: [],
+          deleted: [],
+          checkpoint: "2024-01-02T11:00:00|doc5",
+          has_more: false,
+        });
+
+      await subscriptionManager.backfillSub(subscription);
+
+      expect(mockDb.backfill).toHaveBeenCalledTimes(3);
+      // Each follow-up call resumes from the cursor of the previous page
+      expect(mockDb.backfill.mock.calls[1][0].modifiedAfter).toBe("2024-01-02T10:00:00|doc4");
+      expect(mockDb.backfill.mock.calls[2][0].modifiedAfter).toBe("2024-01-02T11:00:00|doc5");
+      // All drained pages are applied: rows joined, tombstone evicted
+      expect(subscription.docIds).toEqual(expect.arrayContaining(["doc4", "doc5"]));
+      expect(subscription.tombstones?.has("doc2")).toBe(true);
+      expect(subscription.checkpoint).toBe("2024-01-02T11:00:00|doc5");
+    });
+
+    it("should stop draining at the configured page cap", async () => {
+      subscriptionManager = new SubscriptionManager(mockDb, mockNormalizedCache, {
+        backfillMaxPages: 2,
+      });
+      const result = await subscriptionManager.subscribe({
+        doctype: "TestDoc",
+        fields: ["*"],
+      });
+      const subscription = subscriptionManager.getSubscription(result.subscriptionId)!;
+
+      let page = 0;
+      mockDb.backfill.mockImplementation(async () => {
+        page++;
+        return {
+          rows: [{ name: `doc-${page}`, doctype: "TestDoc", modified: `2024-01-0${page}` }],
+          deleted: [],
+          checkpoint: `2024-01-0${page}|doc-${page}`,
+          has_more: true,
+        };
+      });
+
+      await subscriptionManager.backfillSub(subscription);
+
+      expect(mockDb.backfill).toHaveBeenCalledTimes(2);
+    });
+
+    it("should plumb backfillMaxPages into the BackfillManager drain path", () => {
+      subscriptionManager = new SubscriptionManager(mockDb, mockNormalizedCache, {
+        backfillMaxPages: 7,
+      });
+
+      // One knob bounds both drain paths: the option must reach the
+      // BackfillManager (incrementalBackfill) as its maxPages cap.
+      const backfillManager = (subscriptionManager as any).backfillManager;
+      expect(backfillManager.options.maxPages).toBe(7);
+    });
+
+    it("should not drain when the server omits has_more (legacy servers)", async () => {
+      const result = await subscriptionManager.subscribe({
+        doctype: "TestDoc",
+        fields: ["*"],
+      });
+      const subscription = subscriptionManager.getSubscription(result.subscriptionId)!;
+
+      mockDb.backfill.mockResolvedValue({
+        rows: [{ name: "doc4", doctype: "TestDoc", modified: "2024-01-02T10:00:00" }],
+        checkpoint: "2024-01-02T10:00:00",
+      });
+
+      await subscriptionManager.backfillSub(subscription);
+
+      expect(mockDb.backfill).toHaveBeenCalledTimes(1);
+    });
+
     it("should reconcile membership with filters", async () => {
       const result = await subscriptionManager.subscribe({
         doctype: "TestDoc",