@pylonsync/sync 0.22.47 → 0.22.58

package/src/index.ts

@@ -16,8 +16,25 @@ import {
   type TransportConfig,
 } from "./transport";
 import { LocalStore } from "./local-store";
-import { MutationQueue, type PendingMutation } from "./mutation-queue";
+import {
+  byCreatedAt,
+  MutationQueue,
+  MutationRejectedError,
+  toOutboxItem,
+  type Outbox,
+  type OutboxItem,
+  type PendingMutation,
+} from "./mutation-queue";
 import { MultiTabOrchestrator } from "./multi-tab-orchestrator";
+import {
+  IndexedDBUploadStore,
+  UploadManager,
+  uploadOutboxItem,
+  type QueuedUpload,
+  type QueuedUploadHandle,
+  type UploadAttachTarget,
+} from "./upload-queue";
+import type { FileVisibility } from "./upload";
 import { OpQueue } from "./op-queue";
 import {
   RoomSubscriptions,
@@ -64,9 +81,53 @@ export {
   MutationQueue,
   MutationRejectedError,
   type MutationQueuePersistence,
+  type Outbox,
+  type OutboxItem,
   type PendingMutation,
 } from "./mutation-queue";
+
+/** The store channel the outbox notifies on. `store.subscribe(cb,
+ *  OUTBOX_CHANNEL)` hears outbox changes and no entity changes. */
+export const OUTBOX_CHANNEL = "$outbox";
+
+/** The store channel `syncProgress()` changes notify on. */
+export const SYNC_PROGRESS_CHANNEL = "$sync-progress";
+
+/** How far the current pull has got. See `SyncEngine.syncProgress()`. */
+export interface SyncProgress {
+  /** A pull is running. */
+  pulling: boolean;
+  /** The pull is a snapshot of current rows (first sync, or a far-behind
+   *  catch-up) rather than the change log. */
+  snapshot: boolean;
+  /** Rows received by this pull so far. */
+  rows: number;
+  /** Pages received by this pull so far. */
+  pages: number;
+  /** A pull has completed against the server since the last replica
+   *  reset (`isSynced()`). */
+  synced: boolean;
+}
 export { generateId } from "./ids";
+export {
+  uploadBytes,
+  type FileVisibility,
+  type UploadBytesOptions,
+  type UploadedFile,
+} from "./upload";
+export {
+  DEFAULT_MAX_QUEUED_UPLOAD_BYTES,
+  IndexedDBUploadStore,
+  LOCAL_FILE_PREFIX,
+  localUploadId,
+  MemoryUploadStore,
+  UploadQueueFullError,
+  UploadRejectedError,
+  type QueuedUpload,
+  type QueuedUploadHandle,
+  type UploadAttachTarget,
+  type UploadStore,
+} from "./upload-queue";
 export type {
   ChangeEvent,
   ClientChange,
@@ -163,6 +224,19 @@ export interface SyncEngineConfig {
    * becomes visible).
    */
   reconcileOnVisibility?: boolean;
+  /**
+   * The automatic sweep (on connect and when the tab becomes visible)
+   * downloads every replicated row to catch changes the change log does not
+   * carry. It runs only when the app was offline (the connection dropped,
+   * or it started from a cached replica) and the replica last matched the
+   * server (a complete snapshot, or a full sweep) at least this long ago;
+   * or after a snapshot that could not remove deleted rows itself. It
+   * never runs right after a complete snapshot or merely because the tab
+   * came to the foreground. Server hints (`sync-reconcile`) reconcile at
+   * once, and an explicit `reconcile()` call always runs. Default 300000
+   * (5 minutes).
+   */
+  autoReconcileIntervalMs?: number;
   /**
    * Keep the local replica across ORG switches (same user). Default
    * `true`: a tenant flip wipes the replica and re-bootstraps, because
@@ -204,6 +278,9 @@ export interface SyncEngineConfig {
    * its own leader (the pre-multi-tab semantics).
    */
   multiTab?: boolean;
+  /** Cap on the bytes `queueUpload` keeps on the device. Default 200 MB.
+   *  Past it `queueUpload` throws `UploadQueueFullError`. */
+  maxQueuedUploadBytes?: number;
 }
 
 /**
@@ -305,10 +382,46 @@ export class SyncEngine {
     return this._synced;
   }
 
+  private _progress: SyncProgress = {
+    pulling: false,
+    snapshot: false,
+    rows: 0,
+    pages: 0,
+    synced: false,
+  };
+
+  /**
+   * How far the current pull has got: whether one is running, whether it
+   * is a snapshot, and the rows and pages received. Apps render from the
+   * session as soon as it resolves and show this while the replica fills.
+   * A new object on every change, so it works as a `useSyncExternalStore`
+   * snapshot. Follower tabs report `pulling: false` (the leader pulls).
+   */
+  syncProgress(): SyncProgress {
+    return this._progress;
+  }
+
+  private setProgress(patch: Partial<SyncProgress>): void {
+    const next = { ...this._progress, ...patch, synced: this._synced };
+    const prev = this._progress;
+    if (
+      next.pulling === prev.pulling &&
+      next.snapshot === prev.snapshot &&
+      next.rows === prev.rows &&
+      next.pages === prev.pages &&
+      next.synced === prev.synced
+    ) {
+      return;
+    }
+    this._progress = next;
+    this.store.notify(SYNC_PROGRESS_CHANNEL);
+  }
+
   /** Flip `_synced` true (idempotent), notify, and tell follower tabs. */
   private markSynced(): void {
     if (this._synced) return;
     this._synced = true;
+    this.setProgress({});
     this.store.notify();
     if (this.isMultiTabLeader) this.broadcastToTabs({ type: "synced" });
   }
@@ -383,6 +496,8 @@ export class SyncEngine {
 
   readonly store: LocalStore;
   readonly mutations: MutationQueue;
+  /** Files queued with `queueUpload`, sent when there is a network. */
+  readonly uploads: UploadManager;
 
   /**
    * Stable per-client identifier. Minted on first construction, not
@@ -604,6 +719,168 @@ export class SyncEngine {
     return this.mutations.pending().length;
   }
 
+  private outboxCache: { version: number; uploadVersion: number; value: Outbox } | null =
+    null;
+
+  /**
+   * Writes made here that the server has not applied yet (`pending`:
+   * offline, or waiting for a sign-in), and writes the server rejected
+   * (`failed`, with its error). Oldest first. The same object is
+   * returned until the queue changes, so it can back
+   * `useSyncExternalStore`. Changes notify store subscribers on
+   * `OUTBOX_CHANNEL`.
+   */
+  outbox(): Outbox {
+    const version = this.mutations.version();
+    const uploadVersion = this.uploads.version();
+    if (
+      this.outboxCache &&
+      this.outboxCache.version === version &&
+      this.outboxCache.uploadVersion === uploadVersion
+    ) {
+      return this.outboxCache.value;
+    }
+    const uploads = this.uploads.list();
+    const byTime = (a: OutboxItem, b: OutboxItem) => a.createdAt - b.createdAt;
+    const value: Outbox = {
+      pending: [
+        ...byCreatedAt(this.mutations.pending().slice()).map(toOutboxItem),
+        ...uploads.filter((u) => u.status === "pending").map(uploadOutboxItem),
+      ].sort(byTime),
+      failed: [
+        ...this.mutations.failed().map(toOutboxItem),
+        ...uploads.filter((u) => u.status === "failed").map(uploadOutboxItem),
+      ].sort(byTime),
+    };
+    this.outboxCache = { version, uploadVersion, value };
+    return value;
+  }
+
+  /**
+   * Send a failed write again, with `data` (when given) merged over the
+   * fields it set. The write is applied locally again at once (the
+   * rejection rolled it back) and queued under a new op id. Settles like
+   * `insert`: resolves with the new op id once the server applies it or
+   * it is queued offline, and rejects with a `MutationRejectedError` when
+   * the server rejects it again (it is then failed again, with the new
+   * error).
+   *
+   * A write is only ever sent as the user who made it. Throws
+   * `MutationRejectedError` with code `OWNER_MISMATCH` when someone else
+   * is signed in, and `OWNER_UNKNOWN` when it is not yet known who is
+   * signed in (or the write predates owner tracking). Dismiss those.
+   */
+  async retryWrite(id: string, data?: Row): Promise<string> {
+    if (this.uploads.get(id)) {
+      // An upload is sent as it was queued; `data` does not apply.
+      await this.uploads.retry(id);
+      return id;
+    }
+    const m = this.mutations.get(id);
+    if (!m || m.status !== "failed") {
+      throw new Error(`retryWrite: no failed write with id ${id}`);
+    }
+    const reject = (code: string, message: string) =>
+      new MutationRejectedError(message, {
+        code,
+        opId: id,
+        entity: m.change.entity,
+        rowId: m.change.row_id,
+        kind: m.change.kind,
+      });
+    const current = this.currentOwner();
+    if (current === undefined || m.owner === undefined || m.ownerPending) {
+      throw reject(
+        "OWNER_UNKNOWN",
+        "This write cannot be sent again: it is not known who made it or who is signed in.",
+      );
+    }
+    if (m.owner !== current) {
+      throw reject(
+        "OWNER_MISMATCH",
+        "This write was made by another user and cannot be sent as the signed-in user.",
+      );
+    }
+    const { entity, row_id: rowId, kind } = m.change;
+    const before = this.store.get(entity, rowId);
+    const prev = before ? { ...before } : null;
+    const newId = this.mutations.retry(id, data, kind === "insert" ? undefined : prev);
+    const change = this.mutations.get(newId)!.change;
+    if (kind === "insert") {
+      this.store.optimisticInsertWithId(entity, rowId, change.data ?? { id: rowId });
+    } else if (kind === "update") {
+      this.store.optimisticUpdate(entity, rowId, change.data ?? {});
+    } else {
+      this.store.optimisticDelete(entity, rowId);
+    }
+    this.broadcastToTabs({ type: "mutations-removed", opIds: [id] });
+    await this.sendAndAwait(newId);
+    return newId;
+  }
+
+  /** Drop a failed write after the user has seen it. Its local change
+   *  was already rolled back. Returns false when there is no failed write
+   *  with that id; a pending write cannot be dismissed. */
+  dismissWrite(id: string): boolean {
+    if (this.uploads.get(id)) return this.uploads.dismiss(id);
+    if (!this.mutations.dismiss(id)) return false;
+    this.broadcastToTabs({ type: "mutations-removed", opIds: [id] });
+    return true;
+  }
+
+  /**
+   * Keep a file on the device and upload it when there is a network, then
+   * set it on the record it belongs to. Returns at once.
+   *
+   * At once, `attach.entity`/`attach.id` gets `attach.field` (and
+   * `attach.urlField`, when given) = `local:<localId>` as a queued write,
+   * so the record shows the file from the device (`uploadLocalUrl`). When
+   * the upload finishes, a second queued write sets the file's id (and
+   * URL). Both are ordinary writes: policies and owner rules apply.
+   *
+   * The file is sent only as the user who queued it. Throws
+   * `UploadQueueFullError` past `maxQueuedUploadBytes`. Pending and failed
+   * uploads show in `outbox()` with kind `"upload"`; retry and dismiss them
+   * with `retryWrite` / `dismissWrite`.
+   */
+  queueUpload(
+    file: Blob,
+    opts: {
+      attach: UploadAttachTarget;
+      filename?: string;
+      contentType?: string;
+      visibility?: FileVisibility;
+    },
+  ): Promise<QueuedUploadHandle> {
+    return this.uploads.enqueue(file, opts);
+  }
+
+  /** An object URL for a queued file (`local:<localId>` without the
+   *  prefix), or null when no queued upload has that id. */
+  uploadLocalUrl(localId: string): string | null {
+    return this.uploads.localUrl(localId);
+  }
+
+  /** Apply the queued writes to the replica again after a reload. An
+   *  optimistic change lives only in memory, so without this a row
+   *  written offline vanished from the screen on reload (it was still
+   *  queued and still reached the server later). Oldest first, so an
+   *  update of a row inserted offline finds the row. */
+  private reapplyPendingWrites(): void {
+    for (const m of byCreatedAt(this.mutations.pending().slice())) {
+      const { entity, row_id: rowId, kind, data } = m.change;
+      if (kind === "insert") {
+        if (!this.store.get(entity, rowId)) {
+          this.store.optimisticInsertWithId(entity, rowId, data ?? { id: rowId });
+        }
+      } else if (kind === "update") {
+        this.store.optimisticUpdate(entity, rowId, data ?? {});
+      } else {
+        this.store.optimisticDelete(entity, rowId);
+      }
+    }
+  }
+
   /**
    * Mutate connection status + notify subscribers. Idempotent — same-
    * status calls are a no-op so the WS onopen → connected transition
@@ -613,6 +890,8 @@ export class SyncEngine {
     if (this._connectionStatus === next) return;
     this._connectionStatus = next;
     this.store.notify();
+    // Back online: send the files queued while offline.
+    if (next === "connected") void this.uploads.process();
   }
 
   /** Sync key-value adapter for hot-path state (token, client_id). */
@@ -622,6 +901,30 @@ export class SyncEngine {
     this.config = config;
     this.store = new LocalStore();
     this.mutations = new MutationQueue();
+    this.mutations.setChangeListener(() => this.store.notify(OUTBOX_CHANNEL));
+    this.uploads = new UploadManager(
+      {
+        transport: () => ({
+          baseUrl: this.config.baseUrl,
+          getToken: () => this.currentToken() ?? undefined,
+        }),
+        currentOwner: () =>
+          this.identityPending || !this.session.hasObserved()
+            ? undefined
+            : this.session.resolved().userId,
+        ownerForNewUpload: () => this.currentOwner(),
+        isLeader: () => this.isMultiTabLeader,
+        isRunning: () => this.running,
+        update: (entity, id, patch) => {
+          // An ordinary queued write: its outcome shows in the outbox.
+          this.update(entity, id, patch).catch(() => {});
+        },
+        changed: () => this.store.notify(OUTBOX_CHANNEL),
+        broadcast: () => this.broadcastToTabs({ type: "uploads-changed" }),
+        later: (ms, fn) => this.later(ms, fn),
+      },
+      config.maxQueuedUploadBytes,
+    );
     this.storage = config.storage ?? defaultStorage();
     this.clientId = generateClientId(this.storage);
     // ServerSubscriptions defers sending until the WS is open (the
@@ -746,6 +1049,9 @@ export class SyncEngine {
         // whose pull-from-cursor misses an offline server-side delete
         // depends on that reconcile pass to catch the ghost row.
         this._hadCachedReplica = hadCache;
+        // Rows from an earlier run: the server may have changed them while
+        // this app was closed, which counts as offline.
+        if (hadCache) this.offlineSinceMatch = true;
         // Which identity the hydrated rows belong to. Compared against the
         // resolved session in `guardReplicaIdentity` before the first pull
         // so a reload-time account switch drops the prior user's rows.
@@ -822,6 +1128,9 @@ export class SyncEngine {
             const mqPersistence = new IndexedDBMutationPersistence(persistence);
             this.mutations.attachPersistence(mqPersistence);
             await this.mutations.hydrate();
+            this.reapplyPendingWrites();
+            // Queued files live in their own database (see upload-queue.ts).
+            await this.uploads.attachStore(new IndexedDBUploadStore(this.config.appName));
           }
           // The hydrated offline writes are drained in the leader path
           // below (after `initMultiTab` settles), NOT here. We're still
@@ -954,6 +1263,9 @@ export class SyncEngine {
     // drops here on the normal fast path and the start() fallback is cancelled.
     await this.pull();
 
+    // Files queued in an earlier run (or by a tab that closed).
+    void this.uploads.process();
+
     // Save cursor after pull. Fire-and-forget on bootstrap — the
     // enqueueApply path already persists per-batch as pull lands rows,
     // so this final save is belt-and-braces. Awaiting it adds 5-30ms
@@ -1088,7 +1400,7 @@ export class SyncEngine {
           // DELETE the leader's still-valid row. The follower's prevRow
           // (its pre-edit value) equals the leader's canonical row, so
           // restoring it is correct on both tabs.
-          this.mutations.add(op.change, op.prevRow, op.owner, op.ownerPending);
+          this.mutations.add(op.change, op.prevRow, op.owner, op.ownerPending, op.createdAt);
         }
         void this.pushNew(ops.map((op) => op.id));
       },
@@ -1101,6 +1413,19 @@ export class SyncEngine {
         // and the leader retries them. Release this tab's callers.
         for (const id of opIds) this.mutations.settleQueued(id);
       },
+      onUploadsChanged: () => {
+        void this.uploads.reload().then(() => {
+          if (this.isMultiTabLeader) return this.uploads.process();
+        });
+      },
+      onMutationsRemoved: (opIds: string[]) => {
+        // Another tab retried (under a new op id) or dismissed these
+        // failed writes. Drop this tab's copy so neither tab shows or
+        // resends the old one.
+        for (const id of opIds) {
+          if (this.mutations.get(id)?.status === "failed") this.mutations.remove(id);
+        }
+      },
       onMutationsFailed: (
         ops: { opId: string; error: string; code?: string }[],
       ) => {
@@ -1238,7 +1563,7 @@ export class SyncEngine {
         if (!this.isMultiTabLeader) return;
         if (this.observedEntities.has(entity)) return;
         this.observedEntities.add(entity);
-        if (this.isHydrated() && this.store.list(entity).length === 0) {
+        if (this.isHydrated() && this.store.list(entity).length === 0 && this.emptyMayBeStale()) {
           void this.reconcile([entity]);
         }
       },
@@ -1387,7 +1712,7 @@ export class SyncEngine {
       // doesn't duplicate the cursor-catch-up work pull is already
       // doing. They're serial via the queue, not concurrent.
       void this.pull();
-      void this.reconcile();
+      if (this.autoReconcileDue()) void this.reconcile();
     };
     document.addEventListener("visibilitychange", this.visibilityHandler);
   }
@@ -1482,6 +1807,12 @@ export class SyncEngine {
    *  push whose current token differs holds owned writes: the token may
    *  belong to someone else. `undefined` until a session is committed. */
   private sessionToken: string | null | undefined = undefined;
+  /** The `/api/auth/me` request in flight, and the token it was sent with.
+   *  `refreshSessionIfStale` joins it instead of sending another. */
+  private sessionFetch: {
+    token: string | null;
+    promise: Promise<{ session: ResolvedSession; token: string | null } | null>;
+  } | null = null;
   private identityRetryTimer: ReturnType<typeof setTimeout> | null = null;
   /** Delayed push / pull retries scheduled through `later()`. `stop()`
    *  clears them, so a stopped engine makes no further requests. */
@@ -1805,6 +2136,7 @@ export class SyncEngine {
     // did nothing: an org switch dropped straight to the new org's empty list.
     this._initialSyncSettled = false;
     this._synced = false;
+    this.setProgress({});
     this.armInitialSyncFallback();
     this.store.clearAll();
     // Disk is about to be wiped + re-pulled from 0, so any prior
@@ -2013,6 +2345,7 @@ export class SyncEngine {
     // bootstrap reconcile (the snapshot path already returned every
     // policy-visible row, per-entity refetch right after is waste).
     const startedFromZero = this.cursor.last_seq === 0;
+    this.setProgress({ pulling: true, snapshot: false, rows: 0, pages: 0 });
     // Open the live-event hold for the pull's whole (possibly multi-page)
     // duration so a racing WS frame can't leapfrog the cursor — past the
     // snapshot's rows on a from-zero pull, or past the not-yet-pulled gap
@@ -2054,6 +2387,20 @@ export class SyncEngine {
       let hasMore = false;
       let firstPass = true;
       let nextSince = this.cursor.last_seq;
+      // Adaptive page size: a small first page paints quickly, then each
+      // full page doubles the next one up to the server's maximum, so a
+      // large catch-up costs a few round trips instead of hundreds.
+      let pageRows = SyncEngine.FIRST_PULL_PAGE_ROWS;
+      // Ids per entity seen in a snapshot (a pull from 0, or the server's
+      // answer to a far-behind cursor). When the snapshot completes, local
+      // rows it did not send are deleted: a snapshot has no tombstones.
+      let snapshotSeen: Map<string, Set<string>> | null = null;
+      // A snapshot replaced rows this replica already held (a catch-up, or a
+      // pull from 0 over a cached replica), and whether its last page said
+      // which entities it covers. Without that list the deletes can't be
+      // swept here, and the next automatic reconcile has to.
+      let replacedRows = startedFromZero && this._hadCachedReplica;
+      let snapshotCompleted = false;
       while (firstPass || snapshotAfter || hasMore) {
         firstPass = false;
         // `snapshot_after` is an OPAQUE cursor the server already URL-encoded
@@ -2064,17 +2411,44 @@ export class SyncEngine {
         // restart the snapshot from row 0 — an infinite re-snapshot loop for any
         // table larger than one page (SNAPSHOT_BATCH_LIMIT rows). `since` is a
         // plain integer, so it's safe to inline.
-        let query = `since=${nextSince}`;
+        // `snapshot_ok=1`: a far-behind cursor gets a snapshot of current
+        // rows (applied in place, then the log resumes from its seq)
+        // instead of a 410 that wipes the replica.
+        let query = `since=${nextSince}&limit=${pageRows}&snapshot_ok=1`;
         if (snapshotAfter) {
           query += `&snapshot_after=${snapshotAfter}`;
         }
-        const resp = await this.request<
-          PullResponse & { snapshot_after?: string | null }
-        >("GET", `/api/sync/pull?${query}`);
+        const resp = await this.request<PullResponse>("GET", `/api/sync/pull?${query}`);
         if (pendingApply) await pendingApply;
         pendingApply = this.enqueueApply(resp.changes, resp.cursor, {
           isPull: true,
         });
+        if (resp.snapshot && !startedFromZero) replacedRows = true;
+        if (resp.snapshot) {
+          snapshotSeen ??= new Map();
+          for (const ev of resp.changes) {
+            let ids = snapshotSeen.get(ev.entity);
+            if (!ids) snapshotSeen.set(ev.entity, (ids = new Set()));
+            ids.add(ev.row_id);
+          }
+          if (resp.snapshot_entities) {
+            snapshotCompleted = true;
+            const entities = resp.snapshot_entities;
+            const seen = snapshotSeen;
+            const tombstoneSeq = resp.cursor.last_seq;
+            snapshotSeen = null;
+            const applied = pendingApply;
+            pendingApply = applied.then(() => this.sweepAfterSnapshot(entities, seen, tombstoneSeq));
+          }
+        }
+        if (resp.changes.length >= pageRows) {
+          pageRows = Math.min(pageRows * 2, SyncEngine.MAX_PULL_PAGE_ROWS);
+        }
+        this.setProgress({
+          snapshot: resp.snapshot === true || this._progress.snapshot,
+          rows: this._progress.rows + resp.changes.length,
+          pages: this._progress.pages + 1,
+        });
         // Guard against a server that reports more pages without
         // advancing the cursor (a transient no-progress page): break
         // rather than refetch the same page forever. The next poll /
@@ -2086,6 +2460,7 @@ export class SyncEngine {
         hasMore = !snapshotAfter && resp.has_more && advanced;
       }
       if (pendingApply) await pendingApply;
+      if (replacedRows && !snapshotCompleted) this.sweepNeeded = true;
       // Clear the resync circuit breaker ONLY on a successful DELTA
       // pull — one that started from a real, non-zero cursor the server
       // honored. A snapshot pull from cursor=0 succeeding does NOT prove
@@ -2194,9 +2569,49 @@ export class SyncEngine {
       // already drained + nulled the hold, so this is a no-op there — and
       // the 410 recursion drained or re-owned it before we got here.
       if (this.pullHold !== null) this.pullHold = null;
+      this.setProgress({ pulling: false });
     }
   }
 
+  /** Rows a reconcile fetches per entity, at most. */
+  private static readonly RECONCILE_ROW_CAP = 20_000;
+  /** The server's page maximum for replication reads (`?sync=1`). */
+  private static readonly MAX_RECONCILE_PAGE_ROWS = 5_000;
+
+  /** Rows the first page of a pull asks for. Later pages double. */
+  private static readonly FIRST_PULL_PAGE_ROWS = 1000;
+  /** The server's page maximum (MAX_PAGE_ROWS in routes/sync.rs). */
+  private static readonly MAX_PULL_PAGE_ROWS = 10_000;
+
+  /**
+   * After a complete snapshot: delete local rows of `entities` that the
+   * snapshot did not send. Rows with writes not yet on the server are kept.
+   * The deletes are tombstoned at the snapshot's seq, so an older event
+   * cannot bring a row back.
+   */
+  private async sweepAfterSnapshot(
+    entities: string[],
+    seen: Map<string, Set<string>>,
+    tombstoneSeq: number,
+  ): Promise<void> {
+    const pendingKeys = this.mutations.pendingRowKeys();
+    for (const entity of entities) {
+      const ids = seen.get(entity);
+      const removalIds: string[] = [];
+      for (const row of this.store.list(entity)) {
+        const id = (row as { id?: unknown }).id;
+        if (typeof id !== "string") continue;
+        if (ids?.has(id) || pendingKeys.has(`${entity}/${id}`)) continue;
+        removalIds.push(id);
+      }
+      if (removalIds.length > 0) {
+        await this.enqueueReconcile(entity, [], removalIds, tombstoneSeq);
+      }
+    }
+    this.markMatchedServer();
+    this.snapshotEpoch = this.replicaEpoch;
+  }
+
   /** Consecutive 410 RESYNC_REQUIRED responses since the last successful
    *  pull. Used by the circuit breaker in pull() to bound the retry
    *  storm against a misconfigured server. Resets to 0 on any pull
@@ -2219,6 +2634,42 @@ export class SyncEngine {
    *  cleared on read so a subsequent reconnect-after-disconnect still
    *  runs reconcile normally. */
   private lastPullStartedFromZero = false;
+  /** When the replica last matched the server: a complete snapshot
+   *  (local rows it did not hold deleted) or a full reconcile. 0 = never
+   *  this run. */
+  private matchedServerAt = 0;
+  /** The replica epoch a complete snapshot filled. An entity with no
+   *  rows in it is empty on the server too. */
+  private snapshotEpoch = -1;
+
+  /** Whether an automatic sweep (connect, visibility) should run. */
+  /** The connection dropped (or this run started from a cached replica)
+   *  since the replica last matched the server. */
+  private offlineSinceMatch = false;
+  /** A snapshot replaced cached rows without saying which entities it
+   *  covers, so rows deleted on the server may still be here. */
+  private sweepNeeded = false;
+
+  /**
+   * Whether an automatic sweep (connect, visibility) should run: after a
+   * snapshot that could not sweep its own deletes, or when the app was
+   * offline since the replica last matched the server and the last match
+   * is `autoReconcileIntervalMs` old. Never right after a complete
+   * snapshot, and not merely because the app came to the foreground.
+   */
+  private autoReconcileDue(): boolean {
+    if (this.sweepNeeded) return true;
+    if (!this.offlineSinceMatch) return false;
+    const interval = this.config.autoReconcileIntervalMs ?? 300_000;
+    return this.matchedServerAt === 0 || Date.now() - this.matchedServerAt >= interval;
+  }
+
+  /** The replica matches the server now. */
+  private markMatchedServer(): void {
+    this.matchedServerAt = Date.now();
+    this.offlineSinceMatch = false;
+    this.sweepNeeded = false;
+  }
 
   /** Timestamp of the last `reconcile()` invocation. Used to debounce —
    *  reconcile runs on connect, WS reconnect, AND visibility-change, so
@@ -2309,13 +2760,22 @@ export class SyncEngine {
       this.broadcastToTabs({ type: "entity-observe", entity });
       return;
     }
-    if (this.isHydrated() && this.store.list(entity).length === 0) {
+    if (this.isHydrated() && this.store.list(entity).length === 0 && this.emptyMayBeStale()) {
       // Scoped reconcile bypasses the no-arg debounce and reuses the
       // session-flip / cursor-drift guards in reconcileInner.
       void this.reconcile([entity]);
     }
   }
 
+  /** Whether an entity with no local rows may still have rows on the
+   *  server that no pull will deliver. Not when a snapshot is coming (the
+   *  cursor is 0 and no pull has completed since the last reset) or one
+   *  already filled this replica: a snapshot sends every visible row. */
+  private emptyMayBeStale(): boolean {
+    const snapshotComing = this.cursor.last_seq === 0 && !this._synced;
+    return !snapshotComing && this.snapshotEpoch !== this.replicaEpoch;
+  }
+
   async reconcile(entities?: string[]): Promise<void> {
     const epoch = this.pendingReconcileEpoch;
     const { done, skipped, startedAt } = await this.reconcileReportingSkips(entities);
@@ -2619,6 +3079,9 @@ export class SyncEngine {
         done.push(entity);
       }),
     );
+    // A full sweep that reconciled every entity matched the replica to the
+    // server (see autoReconcileDue).
+    if (entities === undefined && skipped.length === 0) this.markMatchedServer();
     return { done, skipped, startedAt };
   }
 
@@ -2632,20 +3095,22 @@ export class SyncEngine {
   ): Promise<{ rows: Row[]; truncated: boolean }> {
     const out: Row[] = [];
     let cursor: string | null = null;
-    // 20 pages × 1000 per page = 20k rows. A `sync:true` entity larger than
-    // this should switch to `sync: false` + search/by-id — see useInfiniteQuery.
-    // 1000/page matches the sync-pull batch limits; the server only honors it
-    // on replication fetches (`sync=1`), and each page costs a full round
-    // trip, so the page size directly divides sweep latency (a 20k-row
-    // entity was 200 sequential requests; now 20).
-    for (let page = 0; page < 20; page++) {
+    // At most RECONCILE_ROW_CAP rows in 20 pages. A `sync:true` entity larger than this
+    // should switch to `sync: false` + search/by-id (see useInfiniteQuery).
+    // Pages start at 1000 rows and double up to the server's replication
+    // maximum: each page costs a round trip, so a 7k-row entity takes 3
+    // requests instead of 8.
+    // The page cap bounds requests when pages come back short (a sparse
+    // policy can leave few visible rows per page).
+    let limit = 1000;
+    for (let page = 0; page < 20 && out.length < SyncEngine.RECONCILE_ROW_CAP; page++) {
       // `sync=1` marks this as a REPLICATION fetch, which is what makes an
       // entity's `sync` scope apply. Without it the server treats the request
       // as an ordinary app read and returns unscoped rows — correct for a
       // direct read, wrong here, since these rows land in the replica.
       const qs: string = cursor
-        ? `?limit=1000&sync=1&after=${encodeURIComponent(cursor)}`
-        : `?limit=1000&sync=1`;
+        ? `?limit=${limit}&sync=1&after=${encodeURIComponent(cursor)}`
+        : `?limit=${limit}&sync=1`;
       const resp: {
         data: Row[];
         next_cursor: string | null;
@@ -2656,6 +3121,9 @@ export class SyncEngine {
         return { rows: out, truncated: false };
       }
       cursor = resp.next_cursor;
+      if (resp.data.length >= limit) {
+        limit = Math.min(limit * 2, SyncEngine.MAX_RECONCILE_PAGE_ROWS);
+      }
     }
     // Exhausted the page cap while the server still had more rows: the set
     // is incomplete. Surface it so reconcile upserts-only instead of
@@ -2703,7 +3171,7 @@ export class SyncEngine {
     const removalIds: string[] = [];
     if (truncated) {
       console.warn(
-        `[reconcile] "${entity}" exceeds the ${20_000}-row client-replication ` +
+        `[reconcile] "${entity}" exceeds the ${SyncEngine.RECONCILE_ROW_CAP}-row client-replication ` +
           `cap; skipping stale-row cleanup to avoid deleting un-fetched rows. ` +
           `Mark large server-queried entities \`sync: false\` and read them via ` +
           `search + by-id instead of replicating the whole table.`,
@@ -2778,11 +3246,23 @@ export class SyncEngine {
    * caller's next pull cycle (or the WS `session-changed` envelope)
    * will retry. Errors must not abort bootstrap.
    */
-  private async fetchSessionBootstrap(): Promise<
+  private fetchSessionBootstrap(): Promise<
     { session: ResolvedSession; token: string | null } | null
   > {
+    const token = this.currentToken();
+    const promise = this.fetchSession(token);
+    this.sessionFetch = { token, promise };
+    const clear = () => {
+      if (this.sessionFetch?.promise === promise) this.sessionFetch = null;
+    };
+    promise.then(clear, clear);
+    return promise;
+  }
+
+  private async fetchSession(
+    token: string | null,
+  ): Promise<{ session: ResolvedSession; token: string | null } | null> {
     try {
-      const token = this.currentToken();
       const res = await this.rawFetch("/api/auth/me");
       if (!res.ok) return null;
       const raw = (await res.json()) as {
@@ -2890,12 +3370,58 @@ export class SyncEngine {
     // need to handle) errors from a session refresh.
     this.sessionChain = prev.then(async () => {
       // Defer the null→X / X→Y / first-resolution distinction to the
-      // resolver, but DON'T commit the new resolved session until
-      // after the engine has finished acting on the verdict — that
-      // closes the brief window where useSession would report the
-      // new tenant while useQuery still has the old tenant's rows.
+      // resolver, but DON'T commit the new resolved session until the
+      // old identity's rows are gone — that closes the window where
+      // useSession would report the new tenant while useQuery still has
+      // the old tenant's rows. It does not wait for the new rows: the
+      // re-pull can take seconds on a slow phone.
       const verdict = this.session.inspectSession(next);
       const prevUserId = this.session.resolved().userId;
+      // Commit the session and act on it. Runs once: right after the wipe
+      // when there is one, so `useSession` and `sessionResolved()` update
+      // while the replica refills from the server (an app renders from the
+      // session and shows `useSyncProgress` for the data), else after the
+      // branch below.
+      let committed = false;
+      const commit = () => {
+        if (committed) return;
+        committed = true;
+        const firstResolution = !this.session.hasObserved();
+        this.session.commitObservation(next);
+        // The reset above tagged the replica with the outgoing user (the
+        // session was not committed yet). Re-tag it with the new owner.
+        if (userFlipped && this.isMultiTabLeader) this.persistReplicaIdentity();
+        // The session now says who the token belongs to: push writes
+        // that were waiting for it (their owner may be this user).
+        if (this.isMultiTabLeader) {
+          const wasPending = this.identityPending || firstResolution;
+          if (opts.token !== undefined) this.sessionToken = opts.token;
+          // Writes made before any session resolved belong to this one.
+          if (firstResolution) this.mutations.stampPendingOwner(next.userId);
+          this.identityPending = false;
+          this.identityRetryAttempts = 0;
+          if ((wasPending || userFlipped) && this.mutations.pending().length > 0) {
+            void this.push();
+          }
+          // Uploads follow the same owner rule: stamp the ones made before
+          // the session was known, then send (or, for another user's,
+          // discard) what is queued.
+          if (wasPending || userFlipped) {
+            const stamp = firstResolution
+              ? this.uploads.stampPendingOwner(next.userId)
+              : Promise.resolve();
+            void stamp.then(() => this.uploads.process());
+          }
+        }
+        if (verdict.identityChanged || firstResolution) {
+          // The first answer flips `sessionResolved()` even when the
+          // session itself matches the placeholder (anonymous caller).
+          this.store.notify();
+        }
+        if (broadcast && this.isMultiTabLeader) {
+          this.broadcastToTabs({ type: "session", resolved: next });
+        }
+      };
       // USER flip (sign-in, sign-out, account switch) observed on a
       // session refresh. The rows, the cursor, and the live socket all
       // belong to the previous identity, and a tenant verdict alone
@@ -2922,7 +3448,7 @@ export class SyncEngine {
           this.cycleTransport();
           // Queued writes stay: each carries its owner, and push drops
           // the ones that belong to someone else.
-          await this.resetAndPull({ wipeMutations: false });
+          await this.resetAndPull({ wipeMutations: false }, commit);
         }
       } else if (userFlipped) {
         // Follower: the leader's reset already wiped this tab.
@@ -2942,12 +3468,17 @@ export class SyncEngine {
           // Reset and re-pull in one op-queue slot (see resetAndPull).
           // The tenant's view changed → wipe the outgoing tenant's
           // pending offline writes too.
-          await this.resetAndPull({ wipeMutations: true });
+          await this.resetAndPull({ wipeMutations: true }, commit);
         }
         if (this.isMultiTabLeader) {
           // Only the leader pulls — followers receive subsequent
           // applied broadcasts that close the catch-up window.
-          if (!reset) await this.pull();
+          // No wipe: the replica is valid for the new tenant as it is
+          // (membership-scoped reads), so the session commits first.
+          if (!reset) {
+            commit();
+            await this.pull();
+          }
           if (skipReset) {
             // No wipe happened, so rows of an org the user only just
             // JOINED aren't in the replica and no change event will
@@ -2957,32 +3488,7 @@ export class SyncEngine {
           }
         }
       }
-      const firstResolution = !this.session.hasObserved();
-      this.session.commitObservation(next);
-      // The reset above tagged the replica with the outgoing user (the
-      // session was not committed yet). Re-tag it with the new owner.
-      if (userFlipped && this.isMultiTabLeader) this.persistReplicaIdentity();
-      // The session now says who the token belongs to: push writes
-      // that were waiting for it (their owner may be this user).
-      if (this.isMultiTabLeader) {
-        const wasPending = this.identityPending || firstResolution;
-        if (opts.token !== undefined) this.sessionToken = opts.token;
-        // Writes made before any session resolved belong to this one.
-        if (firstResolution) this.mutations.stampPendingOwner(next.userId);
-        this.identityPending = false;
-        this.identityRetryAttempts = 0;
-        if ((wasPending || userFlipped) && this.mutations.pending().length > 0) {
-          void this.push();
-        }
-      }
-      if (verdict.identityChanged || firstResolution) {
-        // The first answer flips `sessionResolved()` even when the
-        // session itself matches the placeholder (anonymous caller).
-        this.store.notify();
-      }
-      if (broadcast && this.isMultiTabLeader) {
-        this.broadcastToTabs({ type: "session", resolved: next });
-      }
+      commit();
     }).catch(() => {});
     return this.sessionChain;
   }
@@ -2990,9 +3496,15 @@ export class SyncEngine {
   /** Reset the replica and pull from zero in ONE op-queue slot, so no
    *  live frame can move the cursor between the reset and the pull's
    *  hold (the pull would then run as a delta and miss rows). */
-  private resetAndPull(opts: { wipeMutations: boolean }): Promise<void> {
+  private resetAndPull(
+    opts: { wipeMutations: boolean },
+    afterReset?: () => void,
+  ): Promise<void> {
     return this.opQueue.enqueue("reset-pull", async () => {
       await this.resetReplicaInner(opts);
+      // The replica is empty: the caller commits the new session here, so
+      // the app does not wait for the whole re-pull to learn who is signed in.
+      afterReset?.();
       await this.pullInner();
     }).then(() => {
       if (this.isMultiTabLeader) this.markInitialSyncSettled();
@@ -3097,6 +3609,29 @@ export class SyncEngine {
     return this.refreshResolvedSession();
   }
 
+  /**
+   * Refresh the session only when it is not known for the current token:
+   * no answer yet this run, or the token changed since the last answer.
+   * A request already in flight for the token is joined, not repeated.
+   * `useSession` calls this on mount, so a screen with many session
+   * readers makes at most one `/api/auth/me` request per resolve.
+   * After a server-side session change, use `notifySessionChanged`,
+   * which always fetches.
+   */
+  async refreshSessionIfStale(): Promise<void> {
+    if (!this.isMultiTabLeader) return;
+    const token = this.currentToken();
+    if (this.session.hasObserved() && this.sessionToken === token) return;
+    const inflight = this.sessionFetch;
+    if (inflight && inflight.token === token) {
+      // The request's owner (start() or a refresh) applies the answer.
+      await inflight.promise;
+      await this.sessionChain;
+      return;
+    }
+    await this.refreshResolvedSession();
+  }
+
   /**
    * Drop a row from the local replica because the server signaled
    * that the current subscriber's read policy was revoked for it.
@@ -4180,13 +4715,18 @@ export class SyncEngine {
             return;
           }
           this.lastPullStartedFromZero = false;
+          // A complete snapshot or a full reconcile in the last few
+          // minutes already matched the replica to the server, deletions
+          // included (see `autoReconcileIntervalMs`).
+          if (!this.autoReconcileDue()) return;
           return this.reconcile();
         });
       },
       onDisconnected: () => {
-        /* Engine has no work on disconnect — the transport's own
-         *  reconnect loop drives the recovery. The connection-status
-         *  flip already happened inside the transport. */
+        // The transport's own reconnect loop drives the recovery; the
+        // connection-status flip already happened inside the transport.
+        // Record the gap: the next automatic sweep may be due.
+        this.offlineSinceMatch = true;
       },
       setStatus: (s) => this.setConnectionStatus(s),
       performPollTick: async () => {