@pylonsync/sync 0.3.212 → 0.3.215

package/src/index.ts

@@ -202,6 +202,24 @@ export class SyncEngine {
     return this._hydrated;
   }
 
+  /**
+   * True when the engine drained at least one row OR a saved cursor
+   * out of IndexedDB during `start()`. Distinguishes a returning user
+   * (cached replica may contain rows the server has since deleted) from
+   * a true first-time user (cache empty, pull-from-0 IS canonical
+   * truth).
+   *
+   * Used by the WS `onConnected` fast-path: `lastPullStartedFromZero`
+   * only fires the reconcile-skip when this flag is ALSO false. A
+   * returning user whose IDB cursor somehow rolled back to 0 (rare:
+   * partial wipe, corrupt write) must still get the reconcile pass —
+   * otherwise rows deleted on the server while the tab was closed
+   * survive forever.
+   *
+   * Read-only after start() observes the IDB load.
+   */
+  private _hadCachedReplica = false;
+
   readonly store: LocalStore;
   readonly mutations: MutationQueue;
 
@@ -433,12 +451,38 @@ export class SyncEngine {
     const shouldPersist = this.config.persist !== false && typeof indexedDB !== "undefined";
     if (shouldPersist) {
       try {
-        const { IndexedDBPersistence, persistChange } = await import("./persistence");
+        const { IndexedDBPersistence } = await import("./persistence");
         this.persistence = new IndexedDBPersistence(this.config.appName);
         await this.persistence.open();
 
-        // Load cached data into the store.
-        const cached = await this.persistence.loadAllEntities();
+        // Warm-load entities + cursor in ONE readonly transaction so
+        // the hydrated rows and the cursor we'll advance from are a
+        // consistent snapshot. Separate reads could (in a multi-tab
+        // race) interleave a mid-load save and read (rows@C, cursor@C+1)
+        // — the pull would then skip seqs we never applied. The
+        // post-load timing log surfaces cold-IDB pages so a regression
+        // (50MB cache, slow disk) is observable.
+        const idbLoadStart =
+          typeof performance !== "undefined" ? performance.now() : Date.now();
+        const { entities: cached, cursor: cachedCursor, hadCache } =
+          await this.persistence.loadSnapshot();
+        const idbLoadMs =
+          (typeof performance !== "undefined" ? performance.now() : Date.now()) -
+          idbLoadStart;
+        if (idbLoadMs > 100) {
+          console.warn(
+            `[persistence] cold IDB load took ${idbLoadMs.toFixed(0)}ms (${
+              Object.keys(cached).length
+            } entities)`,
+          );
+        }
+        // Record whether IDB had a prior session's state. The cold-load
+        // fast-path in onConnected (skip post-pull reconcile when the
+        // pull was a full snapshot from cursor=0) is only safe when
+        // there was no cached replica to begin with — a returning user
+        // whose pull-from-cursor misses an offline server-side delete
+        // depends on that reconcile pass to catch the ghost row.
+        this._hadCachedReplica = hadCache;
         let hydrated = false;
         for (const [entity, rows] of Object.entries(cached)) {
           for (const row of rows) {
@@ -459,10 +503,13 @@ export class SyncEngine {
         else this.store.notify(); // notify even on empty cache so useQuery
         // sees `isHydrated()` flip and can drop its initial loading state.
 
-        // Load cursor.
-        const savedCursor = await this.persistence.loadCursor();
-        if (savedCursor) {
-          this.cursor = savedCursor;
+        // Apply the cached cursor BEFORE pull so the first pull is a
+        // delta against where we left off, not a full re-snapshot.
+        // Already part of the single loadAll() tx above — assigning
+        // here can't race a concurrent save because pull/push haven't
+        // started yet (initMultiTab is still ahead).
+        if (cachedCursor) {
+          this.cursor = cachedCursor;
         }
 
         // Auto-save changes to IndexedDB. Returns a Promise so the async
@@ -512,12 +559,30 @@ export class SyncEngine {
     // applied changes broadcast by the leader. The election settles
     // in ~250ms; if the broker is unavailable (no BroadcastChannel)
     // every tab is implicitly its own leader.
-    await this.initMultiTab();
+    //
+    // Bootstrap parallelization: the election (~250ms) and
+    // /api/auth/me (~60ms) are independent — kick both off, then
+    // await election first. If we lose the election we discard the
+    // session result and let the leader broadcast its session over
+    // the multi-tab channel; the "leader-only network writes"
+    // invariant is preserved because no peers have observed our
+    // pending /api/auth/me request and no apply has happened yet.
+    const electionPromise = this.initMultiTab();
+    const sessionPromise = this.fetchSessionBootstrap().catch(() => null);
+    await electionPromise;
 
     if (!this.isMultiTabLeader) {
       // Follower path: rely on the leader's broadcasts for session +
       // applied changes. Nothing else to do here — the broker is
-      // wired to forward inbound messages into the engine.
+      // wired to forward inbound messages into the engine. The
+      // sessionPromise we kicked off above resolves into the void;
+      // the leader's broadcast will deliver the authoritative view.
+      // Swallow any pending error so it doesn't surface as an
+      // unhandled rejection.
+      void sessionPromise.then(
+        () => {},
+        () => {},
+      );
       return;
     }
 
@@ -533,15 +598,29 @@ export class SyncEngine {
     // Seed the server-resolved session before the first pull so
     // `useSession` subscribers see the right tenant from frame one,
     // and the resolver's lastSeenTenant is populated before any
-    // subsequent flip can race with it.
-    await this.refreshResolvedSession();
+    // subsequent flip can race with it. We pre-fired the HTTP fetch
+    // above (in parallel with election); apply its result now.
+    // Falls through to a normal refresh on network/parse error so
+    // we don't get stuck without a session.
+    const bootstrapSession = await sessionPromise;
+    if (bootstrapSession !== null) {
+      await this.applySessionTransition(bootstrapSession, /* broadcast */ true);
+    } else {
+      await this.refreshResolvedSession();
+    }
 
     // Pull from server, then connect real-time transport.
     await this.pull();
 
-    // Save cursor after pull.
+    // Save cursor after pull. Fire-and-forget on bootstrap — the
+    // enqueueApply path already persists per-batch as pull lands rows,
+    // so this final save is belt-and-braces. Awaiting it adds 5-30ms
+    // of IDB tail latency to the critical path before transport.start
+    // runs; the apply path's idempotent op_id-keyed merge handles the
+    // worst case (one re-applied batch on next cold pull if the tab
+    // crashes between this line and the saveCursor task completing).
     if (this.persistence) {
-      await this.persistence.saveCursor(this.cursor);
+      void this.persistence.saveCursor(this.cursor);
     }
 
     // First-load reconciliation pass — closes the "phantom row" gap when
@@ -919,6 +998,14 @@ export class SyncEngine {
   private async resetReplicaInner(): Promise<void> {
     this.cursor = { last_seq: 0 };
     this.store.clearAll();
+    // The cache is now empty. The next pull will start from 0 and
+    // return a full snapshot — that's a true cold start, so the
+    // onConnected fast-path may skip the post-pull reconcile. Without
+    // this flip, a sign-out → sign-in inside the same tab would
+    // forever re-run reconcile after every pull because
+    // `_hadCachedReplica` was set to true at start() time and never
+    // cleared.
+    this._hadCachedReplica = false;
     if (this.persistence) {
       try {
         await this.persistence.clear();
@@ -1046,6 +1133,12 @@ export class SyncEngine {
       void this.refreshResolvedSession();
     }
 
+    // Capture whether this pull started from cursor=0 BEFORE the
+    // snapshot loop mutates the cursor. On successful exhaustion the
+    // WS onConnected hook reads the flag to skip the redundant
+    // bootstrap reconcile (the snapshot path already returned every
+    // policy-visible row, per-entity refetch right after is waste).
+    const startedFromZero = this.cursor.last_seq === 0;
     try {
       // Snapshot pagination: when the cursor is 0 and the server's
       // table is larger than a single batch, the response carries
@@ -1080,6 +1173,11 @@ export class SyncEngine {
           break;
         }
       }
+      // Snapshot+tail loop exhausted without throwing: if we started
+      // from cursor=0 we just hydrated the full replica from server
+      // truth. Record it so onConnected skips the reconcile that would
+      // otherwise re-fetch every entity via cursor pagination.
+      this.lastPullStartedFromZero = startedFromZero;
     } catch (err) {
       // Swallow network + transient errors so the poll/reconnect loop
       // keeps trying — but on 429 bump the backoff counter so the next
@@ -1137,6 +1235,17 @@ export class SyncEngine {
    *  that doesn't throw a 410. */
   private consecutive_410s = 0;
 
+  /** Set by pullInner whenever the just-completed pull started with
+   *  `cursor.last_seq === 0` (cold load OR post-reset). The WS
+   *  onConnected hook reads this to skip the reconcile() that would
+   *  otherwise fire immediately after the bootstrap pull — the
+   *  snapshot path of pull already returned every row visible under
+   *  current policy, so per-entity reconcile fetches right after are
+   *  pure waste (~300ms on the critical path). One-shot: the flag is
+   *  cleared on read so a subsequent reconnect-after-disconnect still
+   *  runs reconcile normally. */
+  private lastPullStartedFromZero = false;
+
   /** Timestamp of the last `reconcile()` invocation. Used to debounce —
    *  reconcile runs on connect, WS reconnect, AND visibility-change, so
    *  a quick tab-flick after a normal reconnect shouldn't refetch every
@@ -1207,64 +1316,73 @@ export class SyncEngine {
     // the current cursor means future inserts (which have higher seqs)
     // bypass the tombstone — re-creation server-side still propagates.
     const tombstoneSeq = this.cursor.last_seq;
-    for (const entity of names) {
-      // Capture cursor + resolved session BEFORE the fetch so we can
-      // detect drift mid-reconcile. Two distinct races:
-      //
-      //   1. Cursor moves: a WS event for this (or another) entity
-      //      landed while the page-paginated fetch was in flight. Our
-      //      snapshot is stale; applying it would clobber the fresher
-      //      WS-delivered row.
-      //
-      //   2. Session flips: the resolved tenant/user changed while
-      //      the fetch was in flight (e.g., the app called
-      //      /api/auth/select-org just after we issued the fetch).
-      //      The server filtered the response under the OLD tenant
-      //      context, so applying the result would tombstone rows
-      //      that ARE visible under the NEW tenant. This is the
-      //      "dashboard flashes data away on first load" bug — the
-      //      engine starts before the app calls selectOrg, fetches
-      //      under tenant=null, returns 0 rows, then the apply pass
-      //      nukes every locally-cached row. Skip the apply when
-      //      the session signature changed; the next reconcile
-      //      (triggered by session-changed envelope) will re-fetch
-      //      under the new context.
-      const cursorBeforeFetch = this.cursor.last_seq;
-      const sessionBeforeFetch = this.session.signature();
-      let serverRows: Row[];
-      try {
-        serverRows = await this.fetchEntityRows(entity);
-      } catch (err) {
-        // Network errors are expected (offline, transient 5xx). Skip
-        // this entity; the next reconcile trigger will retry.
-        const status = (err as { status?: number })?.status;
-        if (status === 403 || status === 404) {
-          // Entity is no longer readable (policy revoked) or removed
-          // from the manifest. Drop every local row for it — keeping
-          // them around just leaks invisible state.
-          await this.dropEntity(entity, tombstoneSeq);
+    // Fan out the per-entity fetches in parallel. Bootstrap reconcile
+    // used to serialize 5 entities × ~60ms each → 300ms of dead time
+    // on the critical path before channels render. The per-entity
+    // drift checks (cursor + session signature) are captured inside
+    // each task's closure, so each entity still bails individually
+    // if its OWN fetch raced a WS event or a session flip — parallel
+    // fan-out doesn't weaken either guard.
+    await Promise.all(
+      names.map(async (entity) => {
+        // Capture cursor + resolved session BEFORE the fetch so we can
+        // detect drift mid-reconcile. Two distinct races:
+        //
+        //   1. Cursor moves: a WS event for this (or another) entity
+        //      landed while the page-paginated fetch was in flight. Our
+        //      snapshot is stale; applying it would clobber the fresher
+        //      WS-delivered row.
+        //
+        //   2. Session flips: the resolved tenant/user changed while
+        //      the fetch was in flight (e.g., the app called
+        //      /api/auth/select-org just after we issued the fetch).
+        //      The server filtered the response under the OLD tenant
+        //      context, so applying the result would tombstone rows
+        //      that ARE visible under the NEW tenant. This is the
+        //      "dashboard flashes data away on first load" bug — the
+        //      engine starts before the app calls selectOrg, fetches
+        //      under tenant=null, returns 0 rows, then the apply pass
+        //      nukes every locally-cached row. Skip the apply when
+        //      the session signature changed; the next reconcile
+        //      (triggered by session-changed envelope) will re-fetch
+        //      under the new context.
+        const cursorBeforeFetch = this.cursor.last_seq;
+        const sessionBeforeFetch = this.session.signature();
+        let serverRows: Row[];
+        try {
+          serverRows = await this.fetchEntityRows(entity);
+        } catch (err) {
+          // Network errors are expected (offline, transient 5xx). Skip
+          // this entity; the next reconcile trigger will retry.
+          const status = (err as { status?: number })?.status;
+          if (status === 403 || status === 404) {
+            // Entity is no longer readable (policy revoked) or removed
+            // from the manifest. Drop every local row for it — keeping
+            // them around just leaks invisible state.
+            await this.dropEntity(entity, tombstoneSeq);
+          }
+          return;
         }
-        continue;
-      }
-      if (this.cursor.last_seq !== cursorBeforeFetch) {
-        // Cursor moved during fetch — at least one WS event for this
-        // (or another) entity landed and might have a fresher value
-        // for a row our snapshot just captured. Bail out for this
-        // entity; reconcile() is triggered again on visibility-change
-        // and reconnect, and the WS event already carried the latest
-        // state for the affected row.
-        continue;
-      }
-      if (this.session.signature() !== sessionBeforeFetch) {
-        // Session changed (token flipped, tenant switched, user
-        // signed out → in, etc.). The rows we fetched reflect the
-        // OLD session's policy view; applying them now would
-        // tombstone rows visible under the NEW session. Bail and let
-        // the session-changed envelope drive the next reconcile.
-        continue;
-      }
-      await this.applyEntityReconcile(entity, serverRows, tombstoneSeq);
-    }
+        if (this.cursor.last_seq !== cursorBeforeFetch) {
+          // Cursor moved during fetch — at least one WS event for this
+          // (or another) entity landed and might have a fresher value
+          // for a row our snapshot just captured. Bail out for this
+          // entity; reconcile() is triggered again on visibility-change
+          // and reconnect, and the WS event already carried the latest
+          // state for the affected row.
+          return;
+        }
+        if (this.session.signature() !== sessionBeforeFetch) {
+          // Session changed (token flipped, tenant switched, user
+          // signed out → in, etc.). The rows we fetched reflect the
+          // OLD session's policy view; applying them now would
+          // tombstone rows visible under the NEW session. Bail and let
+          // the session-changed envelope drive the next reconcile.
+          return;
+        }
+        await this.applyEntityReconcile(entity, serverRows, tombstoneSeq);
+      }),
+    );
   }
 
   /** Fetch every row for an entity. Uses cursor pagination so big tables
@@ -1375,17 +1493,36 @@ export class SyncEngine {
     // broadcasts the result, which `handleMultiTabMessage` routes
     // into the resolver.
     if (!this.isMultiTabLeader) return;
-    let next: ResolvedSession;
+    const next = await this.fetchSessionBootstrap();
+    if (next === null) return;
+    await this.applySessionTransition(next, /* broadcast */ true);
+  }
+
+  /**
+   * Pure HTTP fetch of /api/auth/me → ResolvedSession. Unlike
+   * `refreshResolvedSession`, this does NOT gate on `isMultiTabLeader`
+   * — bootstrap callers in `start()` fire this in PARALLEL with the
+   * multi-tab election to overlap two independent latency windows
+   * (election ~250ms || auth/me ~60ms). At that point no other tabs'
+   * messages have been observed yet, so there's no broadcast-policy
+   * violation; the caller is responsible for discarding the result
+   * if it lost the election.
+   *
+   * Returns null on HTTP error / network failure / parse error — the
+   * caller's next pull cycle (or the WS `session-changed` envelope)
+   * will retry. Errors must not abort bootstrap.
+   */
+  private async fetchSessionBootstrap(): Promise<ResolvedSession | null> {
     try {
       const res = await this.rawFetch("/api/auth/me");
-      if (!res.ok) return;
+      if (!res.ok) return null;
       const raw = (await res.json()) as {
         user_id?: string | null;
         tenant_id?: string | null;
         is_admin?: boolean;
         roles?: string[];
       };
-      next = {
+      return {
         userId: raw.user_id ?? null,
         tenantId: raw.tenant_id ?? null,
         isAdmin: raw.is_admin ?? false,
@@ -1394,9 +1531,8 @@ export class SyncEngine {
     } catch {
       // Swallow — /api/auth/me errors are transient and the next pull
       // will retry. Don't take down the sync loop for this.
-      return;
+      return null;
     }
-    await this.applySessionTransition(next, /* broadcast */ true);
   }
 
   /**
@@ -2057,7 +2193,31 @@ export class SyncEngine {
         // opening. Reconcile fires after the pull since pull is the
         // cheap incremental path; reconcile is the server-truth
         // backstop for anything pull couldn't replay.
-        void this.pull().then(() => this.reconcile());
+        //
+        // Cold-load fast path: if pull just hydrated a full snapshot
+        // from cursor=0, the snapshot already returned every row
+        // visible under current policy. The reconcile pass that would
+        // normally follow is pure waste — same rows, second time,
+        // ~60ms × N entities. Skip it once; visibility-change and
+        // reconnect-after-disconnect paths invoke reconcile() directly
+        // (not gated by this flag) so the safety net still triggers.
+        void this.pull().then(() => {
+          // Cold-load fast-path: skip reconcile only when this WAS a
+          // true cold start (no IDB cache → the pull-from-0 returned
+          // every visible row, reconcile would refetch the same set).
+          // A returning user whose pull happened to start from 0
+          // (cursor rolled back, partial cache wipe) MUST still run
+          // reconcile to catch rows deleted on the server while the
+          // tab was closed — the snapshot path only returns currently-
+          // visible rows, never tombstones, so ghost rows on the
+          // cached side persist without the reconcile pass.
+          if (this.lastPullStartedFromZero && !this._hadCachedReplica) {
+            this.lastPullStartedFromZero = false;
+            return;
+          }
+          this.lastPullStartedFromZero = false;
+          return this.reconcile();
+        });
       },
       onDisconnected: () => {
         /* Engine has no work on disconnect — the transport's own

package/src/persistence.ts

@@ -162,6 +162,61 @@ export class IndexedDBPersistence {
     });
   }
 
+  /**
+   * Atomic warm-load: returns entities + cursor in a single IDB
+   * read transaction. Used by `SyncEngine.start()` to hydrate the
+   * in-memory replica BEFORE the network pull resolves so React
+   * hooks see real data on first render (no empty-then-populated
+   * flash on returning visits).
+   *
+   * `hadCache` is true when at least one row OR a saved cursor
+   * was found. The engine uses it to distinguish "true cold start
+   * — pull-from-0 IS a full snapshot, skip the post-snapshot
+   * reconcile" from "returning user with cached state — pull-from-
+   * cursor may miss server-side deletes that happened offline, the
+   * onConnected reconcile MUST run". Without that distinction, a
+   * returning user whose cursor somehow rolled back to 0 (rare:
+   * IDB partial corruption, cleared-by-mistake) would end up with
+   * ghost rows that survive forever.
+   *
+   * Single readonly tx is intentional — two separate reads could
+   * race a mid-load saveCursor/saveRow from another tab's apply
+   * pipeline and read an inconsistent (cursor C', rows for cursor C)
+   * pair. One tx guarantees a consistent snapshot.
+   */
+  async loadSnapshot(): Promise<{
+    entities: Record<string, Row[]>;
+    cursor: SyncCursor | null;
+    hadCache: boolean;
+  }> {
+    if (!this.db) return { entities: {}, cursor: null, hadCache: false };
+    const tx = this.db.transaction([STORE_NAME, CURSOR_STORE], "readonly");
+    const entitiesReq = tx.objectStore(STORE_NAME).getAll();
+    const cursorReq = tx.objectStore(CURSOR_STORE).get("cursor");
+    return new Promise((resolve) => {
+      tx.oncomplete = () => {
+        const entities: Record<string, Row[]> = {};
+        for (const item of (entitiesReq.result ?? []) as {
+          entity: string;
+          id: string;
+          data: Row;
+        }[]) {
+          if (!entities[item.entity]) entities[item.entity] = [];
+          entities[item.entity].push({ id: item.id, ...item.data });
+        }
+        const cursorRec = cursorReq.result as { last_seq?: number } | undefined;
+        const cursor: SyncCursor | null = cursorRec
+          ? { last_seq: cursorRec.last_seq ?? 0 }
+          : null;
+        const hadCache =
+          Object.keys(entities).length > 0 || cursor !== null;
+        resolve({ entities, cursor, hadCache });
+      };
+      tx.onerror = () => resolve({ entities: {}, cursor: null, hadCache: false });
+      tx.onabort = () => resolve({ entities: {}, cursor: null, hadCache: false });
+    });
+  }
+
   /** Save the sync cursor. */
   async saveCursor(cursor: SyncCursor): Promise<void> {
     if (!this.db) return;