@pylonsync/sync 0.22.50 → 0.22.59

package/dist/index.d.ts

@@ -14,6 +14,23 @@ export { MutationQueue, MutationRejectedError, type MutationQueuePersistence, ty
 /** The store channel the outbox notifies on. `store.subscribe(cb,
  *  OUTBOX_CHANNEL)` hears outbox changes and no entity changes. */
 export declare const OUTBOX_CHANNEL = "$outbox";
+/** The store channel `syncProgress()` changes notify on. */
+export declare const SYNC_PROGRESS_CHANNEL = "$sync-progress";
+/** How far the current pull has got. See `SyncEngine.syncProgress()`. */
+export interface SyncProgress {
+    /** A pull is running. */
+    pulling: boolean;
+    /** The pull is a snapshot of current rows (first sync, or a far-behind
+     *  catch-up) rather than the change log. */
+    snapshot: boolean;
+    /** Rows received by this pull so far. */
+    rows: number;
+    /** Pages received by this pull so far. */
+    pages: number;
+    /** A pull has completed against the server since the last replica
+     *  reset (`isSynced()`). */
+    synced: boolean;
+}
 export { generateId } from "./ids";
 export { uploadBytes, type FileVisibility, type UploadBytesOptions, type UploadedFile, } from "./upload";
 export { DEFAULT_MAX_QUEUED_UPLOAD_BYTES, IndexedDBUploadStore, LOCAL_FILE_PREFIX, localUploadId, MemoryUploadStore, UploadQueueFullError, UploadRejectedError, type QueuedUpload, type QueuedUploadHandle, type UploadAttachTarget, type UploadStore, } from "./upload-queue";
@@ -71,6 +88,19 @@ export interface SyncEngineConfig {
      * becomes visible).
      */
     reconcileOnVisibility?: boolean;
+    /**
+     * The automatic sweep (on connect and when the tab becomes visible)
+     * downloads every replicated row to catch changes the change log does not
+     * carry. It runs only when the app was offline (the connection dropped,
+     * or it started from a cached replica) and the replica last matched the
+     * server (a complete snapshot, or a full sweep) at least this long ago;
+     * or after a snapshot that could not remove deleted rows itself. It
+     * never runs right after a complete snapshot or merely because the tab
+     * came to the foreground. Server hints (`sync-reconcile`) reconcile at
+     * once, and an explicit `reconcile()` call always runs. Default 300000
+     * (5 minutes).
+     */
+    autoReconcileIntervalMs?: number;
     /**
      * Keep the local replica across ORG switches (same user). Default
      * `true`: a tenant flip wipes the replica and re-bootstraps, because
@@ -169,6 +199,16 @@ export declare class SyncEngine {
      */
     private _synced;
     isSynced(): boolean;
+    private _progress;
+    /**
+     * How far the current pull has got: whether one is running, whether it
+     * is a snapshot, and the rows and pages received. Apps render from the
+     * session as soon as it resolves and show this while the replica fills.
+     * A new object on every change, so it works as a `useSyncExternalStore`
+     * snapshot. Follower tabs report `pulling: false` (the leader pulls).
+     */
+    syncProgress(): SyncProgress;
+    private setProgress;
     /** Flip `_synced` true (idempotent), notify, and tell follower tabs. */
     private markSynced;
     /** Flip `_initialSyncSettled` true (idempotent) + notify so `useQuery`
@@ -571,6 +611,9 @@ export declare class SyncEngine {
      *  push whose current token differs holds owned writes: the token may
      *  belong to someone else. `undefined` until a session is committed. */
     private sessionToken;
+    /** The `/api/auth/me` request in flight, and the token it was sent with.
+     *  `refreshSessionIfStale` joins it instead of sending another. */
+    private sessionFetch;
     private identityRetryTimer;
     /** Delayed push / pull retries scheduled through `later()`. `stop()`
      *  clears them, so a stopped engine makes no further requests. */
@@ -687,6 +730,21 @@ export declare class SyncEngine {
      *  queue. */
     pull(): Promise<void>;
     private pullInner;
+    /** Rows a reconcile fetches per entity, at most. */
+    private static readonly RECONCILE_ROW_CAP;
+    /** The server's page maximum for replication reads (`?sync=1`). */
+    private static readonly MAX_RECONCILE_PAGE_ROWS;
+    /** Rows the first page of a pull asks for. Later pages double. */
+    private static readonly FIRST_PULL_PAGE_ROWS;
+    /** The server's page maximum (MAX_PAGE_ROWS in routes/sync.rs). */
+    private static readonly MAX_PULL_PAGE_ROWS;
+    /**
+     * After a complete snapshot: delete local rows of `entities` that the
+     * snapshot did not send. Rows with writes not yet on the server are kept.
+     * The deletes are tombstoned at the snapshot's seq, so an older event
+     * cannot bring a row back.
+     */
+    private sweepAfterSnapshot;
     /** Consecutive 410 RESYNC_REQUIRED responses since the last successful
      *  pull. Used by the circuit breaker in pull() to bound the retry
      *  storm against a misconfigured server. Resets to 0 on any pull
@@ -707,6 +765,30 @@ export declare class SyncEngine {
      *  cleared on read so a subsequent reconnect-after-disconnect still
      *  runs reconcile normally. */
     private lastPullStartedFromZero;
+    /** When the replica last matched the server: a complete snapshot
+     *  (local rows it did not hold deleted) or a full reconcile. 0 = never
+     *  this run. */
+    private matchedServerAt;
+    /** The replica epoch a complete snapshot filled. An entity with no
+     *  rows in it is empty on the server too. */
+    private snapshotEpoch;
+    /** Whether an automatic sweep (connect, visibility) should run. */
+    /** The connection dropped (or this run started from a cached replica)
+     *  since the replica last matched the server. */
+    private offlineSinceMatch;
+    /** A snapshot replaced cached rows without saying which entities it
+     *  covers, so rows deleted on the server may still be here. */
+    private sweepNeeded;
+    /**
+     * Whether an automatic sweep (connect, visibility) should run: after a
+     * snapshot that could not sweep its own deletes, or when the app was
+     * offline since the replica last matched the server and the last match
+     * is `autoReconcileIntervalMs` old. Never right after a complete
+     * snapshot, and not merely because the app came to the foreground.
+     */
+    private autoReconcileDue;
+    /** The replica matches the server now. */
+    private markMatchedServer;
     /** Timestamp of the last `reconcile()` invocation. Used to debounce —
      *  reconcile runs on connect, WS reconnect, AND visibility-change, so
      *  a quick tab-flick after a normal reconnect shouldn't refetch every
@@ -780,6 +862,11 @@ export declare class SyncEngine {
      * entities where the client missed an insert get the row back.
      */
     observeEntity(entity: string): void;
+    /** Whether an entity with no local rows may still have rows on the
+     *  server that no pull will deliver. Not when a snapshot is coming (the
+     *  cursor is 0 and no pull has completed since the last reset) or one
+     *  already filled this replica: a snapshot sends every visible row. */
+    private emptyMayBeStale;
     reconcile(entities?: string[]): Promise<void>;
     /** Entities to reconcile that have not reconciled yet: named by a server
      *  hint (`sync-reconcile`), or skipped by an earlier reconcile. */
@@ -861,6 +948,7 @@ export declare class SyncEngine {
      * will retry. Errors must not abort bootstrap.
      */
     private fetchSessionBootstrap;
+    private fetchSession;
     /**
      * A stored token that /api/auth/me resolves to no user is either expired
      * or — the common footgun — a FOREIGN token: two Pylon apps sharing an
@@ -939,6 +1027,16 @@ export declare class SyncEngine {
      * before the WS round-trip lands.
      */
     notifySessionChanged(): Promise<void>;
+    /**
+     * Refresh the session only when it is not known for the current token:
+     * no answer yet this run, or the token changed since the last answer.
+     * A request already in flight for the token is joined, not repeated.
+     * `useSession` calls this on mount, so a screen with many session
+     * readers makes at most one `/api/auth/me` request per resolve.
+     * After a server-side session change, use `notifySessionChanged`,
+     * which always fetches.
+     */
+    refreshSessionIfStale(): Promise<void>;
     /**
      * Drop a row from the local replica because the server signaled
      * that the current subscriber's read policy was revoked for it.

package/dist/test-harness/env.d.ts

@@ -28,6 +28,8 @@ export interface CreateTestEnvOptions {
      *  wall-clock waiting on every reconnect scenario; pass a small
      *  value when the test specifically exercises the reconnect path. */
     reconnectDelay?: number;
+    /** Passed through to the engine config (SyncEngineConfig). */
+    autoReconcileIntervalMs?: number;
 }
 export interface TestEnv {
     server: TestServer;

package/dist/test-harness/server.d.ts

@@ -211,6 +211,26 @@ export declare class TestServer {
      *  the response the engine sees, matching what a real server does
      *  when the session mutates mid-request. */
     listEntityRows(entity: string, token: string | undefined): Promise<Row[]>;
+    /** When true, /api/sync/pull answers like the real server's snapshot
+     *  path: a pull from 0 (or a delta pull with `snapshot_ok=1` and a cursor
+     *  `farBehind` accepts) gets the current visible rows as insert events,
+     *  paged by `limit`, flagged `snapshot`, with `snapshot_entities` on the
+     *  last page. Off by default: older tests expect the log replay. */
+    snapshotMode: boolean;
+    /** In `snapshotMode`: delta cursors answered with a snapshot. */
+    farBehind: (since: number) => boolean;
+    /** In `snapshotMode`: leave `snapshot_entities` off the last page, like a
+     *  server from before it was added. */
+    omitSnapshotEntities: boolean;
+    /** One snapshot page for `snapshotMode`. `offset` is the continuation. */
+    snapshotPage(token: string | undefined, offset: number, limit: number): {
+        changes: ChangeEvent[];
+        cursor: SyncCursor;
+        has_more: boolean;
+        snapshot: true;
+        snapshot_after: string | null;
+        snapshot_entities?: string[];
+    };
     /** /api/sync/pull — every visible change since `since`. */
     pull(token: string | undefined, since: number): Promise<{
         changes: ChangeEvent[];

package/dist/types.d.ts

@@ -13,6 +13,16 @@ export interface PullResponse {
     changes: ChangeEvent[];
     cursor: SyncCursor;
     has_more: boolean;
+    /** Set on every page of a snapshot (current rows, not the log): a pull
+     *  from 0, or a far-behind catch-up the client accepted with
+     *  `snapshot_ok=1`. */
+    snapshot?: boolean;
+    /** Continuation of a snapshot that spans pages. Opaque; send it back
+     *  verbatim. */
+    snapshot_after?: string | null;
+    /** On a snapshot's last page: the entities it holds every visible row
+     *  of. Rows of these entities that it did not send are gone. */
+    snapshot_entities?: string[];
 }
 /**
  * Server-resolved auth/session state. Shape mirrors what `/api/auth/me`

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.22.50",
+  "version": "0.22.59",
   "type": "module",
   "main": "./src/index.ts",
   "types": "./dist/index.d.ts",

package/src/first-sync-requests.test.ts

@@ -0,0 +1,147 @@
+// A first sync is one snapshot. Measured on the Veritas build: after a
+// 12-page snapshot, the client fetched every entity a second time through
+// /api/entities/<E>/cursor (28 requests, 6.5 MB), because the connect
+// reconcile ran after a later delta pull reset the "started from zero"
+// flag, and empty observed entities each started their own reconcile.
+
+import { afterEach, expect, test } from "bun:test";
+import { createTestEnv, type TestEnv } from "./test-harness";
+
+let env: TestEnv | null = null;
+let restoreFetch: (() => void) | null = null;
+afterEach(async () => {
+  restoreFetch?.();
+  restoreFetch = null;
+  if (env) {
+    await env.dispose();
+    env = null;
+  }
+});
+
+function recordUrls(): string[] {
+  const urls: string[] = [];
+  const inner = globalThis.fetch;
+  globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => {
+    urls.push(typeof input === "string" ? input : input instanceof URL ? input.href : input.url);
+    return inner(input, init);
+  }) as typeof fetch;
+  restoreFetch = () => {
+    globalThis.fetch = inner;
+  };
+  return urls;
+}
+
+test("a cold start with a websocket makes no cursor requests after the snapshot", async () => {
+  env = await createTestEnv({ transport: "websocket" });
+  env.server.snapshotMode = true;
+  env.signIn({ userId: "u1" });
+  env.server.seed("Property", Array.from({ length: 50 }, (_, i) => ({ id: `p${i}` })));
+  env.server.seed("Visit", Array.from({ length: 20 }, (_, i) => ({ id: `v${i}` })));
+  const urls = recordUrls();
+  // Screens observe entities before and during the first sync, including
+  // ones with no rows.
+  env.engine.observeEntity("Property");
+  env.engine.observeEntity("Member");
+  await env.engine.start();
+  env.engine.observeEntity("Task");
+  await env.flush();
+  await env.engine.pull();
+  await env.flush();
+
+  expect(env.engine.store.list("Property").length).toBe(50);
+  expect(env.engine.store.list("Visit").length).toBe(20);
+  expect(urls.filter((u) => u.includes("/cursor"))).toEqual([]);
+});
+
+test("a reconnect within the interval skips the sweep; after it, the sweep runs", async () => {
+  env = await createTestEnv({ transport: "websocket", autoReconcileIntervalMs: 50, reconnectDelay: 5 });
+  env.server.snapshotMode = true;
+  env.signIn({ userId: "u1" });
+  env.server.seed("Property", [{ id: "p1" }]);
+  await env.engine.start();
+  await env.flush();
+  const urls = recordUrls();
+
+  env.transport.closeLatestWs();
+  await env.flush();
+  expect(urls.filter((u) => u.includes("/cursor"))).toEqual([]);
+
+  await new Promise((r) => setTimeout(r, 60));
+  env.transport.closeLatestWs();
+  await env.flush();
+  await env.flush(100);
+  expect(urls.some((u) => u.includes("/api/entities/Property/cursor"))).toBe(true);
+});
+
+/** A minimal `document` so the engine attaches its visibility listener. */
+function installDocument() {
+  const target = new EventTarget() as EventTarget & { visibilityState: string };
+  target.visibilityState = "visible";
+  const g = globalThis as { document?: unknown };
+  const previous = g.document;
+  g.document = target;
+  return {
+    foreground() {
+      target.dispatchEvent(new Event("visibilitychange"));
+    },
+    restore() {
+      if (previous === undefined) delete g.document;
+      else g.document = previous;
+    },
+  };
+}
+
+test("coming to the foreground does not sweep unless the app was offline", async () => {
+  const doc = installDocument();
+  try {
+    env = await createTestEnv({ transport: "websocket", autoReconcileIntervalMs: 20, reconnectDelay: 5 });
+    env.server.snapshotMode = true;
+    env.signIn({ userId: "u1" });
+    env.server.seed("Property", [{ id: "p1" }]);
+    await env.engine.start();
+    await env.flush();
+    const urls = recordUrls();
+
+    // Long past the interval, but the connection never dropped.
+    await new Promise((r) => setTimeout(r, 40));
+    doc.foreground();
+    await env.flush();
+    expect(urls.filter((u) => u.includes("/cursor"))).toEqual([]);
+
+    // The connection drops (a phone in a pocket), then the app comes back.
+    env.transport.closeLatestWs();
+    await env.flush(50);
+    doc.foreground();
+    await env.flush(50);
+    expect(urls.some((u) => u.includes("/api/entities/Property/cursor"))).toBe(true);
+
+    // Swept: the next foreground does not sweep again.
+    const swept = urls.length;
+    doc.foreground();
+    await env.flush();
+    expect(urls.slice(swept).filter((u) => u.includes("/cursor"))).toEqual([]);
+  } finally {
+    doc.restore();
+  }
+});
+
+test("a catch-up snapshot that could not remove deletes is followed by a sweep", async () => {
+  env = await createTestEnv({ transport: "websocket", reconnectDelay: 5 });
+  env.server.snapshotMode = true;
+  env.signIn({ userId: "u1" });
+  env.server.seed("Lead", [{ id: "a" }, { id: "b" }]);
+  await env.engine.start();
+  await env.flush();
+
+  // An older server: its snapshot does not list the entities it covers.
+  env.server.omitSnapshotEntities = true;
+  env.server.farBehind = () => true;
+  env.server.delete("Lead", "b");
+  const urls = recordUrls();
+  await env.engine.pull();
+  // The snapshot could not drop b.
+  env.transport.closeLatestWs();
+  await env.flush(50);
+  expect(urls.some((u) => u.includes("/api/entities/Lead/cursor"))).toBe(true);
+  expect(env.engine.store.list("Lead").map((r) => r.id)).toEqual(["a"]);
+});