@pylonsync/sync 0.3.226 → 0.3.228

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.3.226",
+  "version": "0.3.228",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/index.ts

@@ -1296,7 +1296,6 @@ export class SyncEngine {
         const resp = await this.request<
           PullResponse & { snapshot_after?: string | null }
         >("GET", `/api/sync/pull?${params.toString()}`);
-        this.consecutive_410s = 0;
         await this.enqueueApply(resp.changes, resp.cursor);
         // `snapshot_after` is only set when the server is mid-snapshot.
         // Continue paginating in the same loop iteration so we don't
@@ -1311,6 +1310,25 @@ export class SyncEngine {
           break;
         }
       }
+      // Clear the resync circuit breaker ONLY on a successful DELTA
+      // pull — one that started from a real, non-zero cursor the server
+      // honored. A snapshot pull from cursor=0 succeeding does NOT prove
+      // our cursor is stable, so it must NOT clear the breaker.
+      //
+      // Why this matters (the bug this replaced): the reset used to fire
+      // on every successful page, including the cursor=0 snapshot that a
+      // 410 triggers. So a stale-cursor 410 → full snapshot → fresh-
+      // cursor 410 ping-pong — a client bouncing between cluster
+      // instances whose in-memory change logs diverge, with no shared
+      // persistent log to serve the delta — reset the breaker every
+      // cycle. The exponential backoff below could never engage, and the
+      // client re-ran a full `select *` snapshot of EVERY entity roughly
+      // every 3 seconds, indefinitely. That drove a ~280GB PlanetScale
+      // egress bill. Resetting only on a delta means repeated resyncs
+      // escalate into the backoff instead of melting egress.
+      if (!startedFromZero) {
+        this.consecutive_410s = 0;
+      }
       // Snapshot+tail loop exhausted without throwing: if we started
       // from cursor=0 we just hydrated the full replica from server
       // truth. Record it so onConnected skips the reconcile that would
@@ -1335,31 +1353,37 @@ export class SyncEngine {
       // re-pull from seq=0. The server replays all current entity rows as
       // seed events on startup so the fresh pull reconstructs state.
       //
-      // Circuit breaker: if the immediate re-pull ALSO 410s, accept it.
-      // Don't recurse — that's the infinite loop we used to ship before
-      // the cursor=0 server fix landed (or against an old server binary
-      // that hasn't been rebuilt yet). Track 410 retries against an
-      // exponential backoff so a misconfigured server can't melt our CPU.
+      // Circuit breaker. The first resync in an episode snapshots
+      // immediately (good UX: a cursor that genuinely fell off retention
+      // recovers in one round trip). But a SECOND 410 with no successful
+      // delta pull in between means re-snapshotting isn't converging —
+      // the cursor we just minted from the snapshot is itself stale
+      // (instance ping-pong, or a server that can't serve our delta). At
+      // that point each snapshot is a full `select *` of every entity,
+      // so we MUST back off instead of looping. The breaker only clears
+      // on a successful delta pull (see the `!startedFromZero` reset
+      // above) — NOT on the snapshot itself, which is what made this
+      // loop unbounded and burned ~280GB of egress.
       if (status === 410) {
         const attempt = this.consecutive_410s;
         this.consecutive_410s += 1;
         if (attempt === 0) {
-          // Bypass the queue here — we ARE the pull op holding the
-          // queue slot. Calling the public pull() would re-enqueue and
-          // share our own promise back to us (deadlock).
+          // First resync of the episode — snapshot now. Bypass the queue
+          // (we ARE the pull op holding the slot; the public pull()
+          // would re-enqueue and share our own promise back → deadlock).
           await this.resetReplicaInner();
           await this.pullInner();
         } else {
-          // Already retried once and still 410. Stop. Schedule a
-          // back-off retry tied to the WS reconnect path so we don't
-          // spam the server. Resets when any pull succeeds.
+          // Snapshotted once and still 410 → not converging. Back off
+          // exponentially instead of re-snapshotting. Clears only when a
+          // delta pull finally succeeds (cursor stabilised).
           const delayMs = Math.min(30_000, 1000 * 2 ** Math.min(attempt, 5));
           console.warn(
             `[pylon] persistent 410 RESYNC_REQUIRED (attempt ${attempt + 1}); backing off ${delayMs}ms`,
           );
           setTimeout(() => {
-            // Trigger one more attempt; either it succeeds (which resets
-            // the counter) or it 410s again (which extends the backoff).
+            // Retry after the delay; a delta success resets the counter,
+            // a repeat 410 extends the backoff (no snapshot).
             void this.pull();
           }, delayMs);
         }
@@ -1390,6 +1414,17 @@ export class SyncEngine {
    *  entity twice within seconds. Configurable via `reconcileMinIntervalMs`. */
   private lastReconcileAt = 0;
 
+  /** Entities the app has subscribed to via `useQuery` / `useQueryOne`,
+   *  even ones the local replica has zero rows for. The reconcile
+   *  safety net defaults to `store.entityNames()` — entities with at
+   *  least one local row — so a server row in a NEVER-cached entity (a
+   *  row created on another surface, or a freshly-added entity) stayed
+   *  invisible until a full snapshot / cache clear: `useQuery` reads
+   *  the local store and a delta `pull()` can't recover a row created
+   *  before the cursor. Tracking observed entities lets the no-arg
+   *  reconcile sweep them too. See `observeEntity`. */
+  private observedEntities = new Set<string>();
+
   /**
    * Reconcile the local replica against server truth.
    *
@@ -1423,8 +1458,38 @@ export class SyncEngine {
    *
    * Pass an explicit entity list to scope the reconcile (callers like
    * `db.useQueryOne` that know what they care about). When called with
-   * no arg, every entity with local rows is checked.
+   * no arg, every entity with local rows OR observed via `useQuery`
+   * (see `observeEntity`) is checked.
    */
+  /**
+   * Register interest in an entity — called by `useQuery` /
+   * `useQueryOne` on mount. Two effects:
+   *
+   *   1. Adds the entity to the reconcile sweep so the safety net
+   *      covers it even with zero local rows (see `observedEntities`).
+   *   2. The FIRST time an entity is observed while the replica is
+   *      hydrated and that entity is locally empty, fires a one-shot
+   *      scoped reconcile so a server row this client never cached
+   *      appears on page-open — instead of waiting for the next
+   *      reconnect / visibility-change trigger. Bounded: at most once
+   *      per entity per engine (the `observedEntities` guard).
+   *
+   * Genuinely-empty entities just pay one cheap policy-filtered fetch;
+   * entities where the client missed an insert get the row back.
+   */
+  observeEntity(entity: string): void {
+    if (this.observedEntities.has(entity)) return;
+    this.observedEntities.add(entity);
+    // Only the leader talks to the network; follower tabs converge via
+    // the multi-tab channel once the leader reconciles.
+    if (!this.isMultiTabLeader) return;
+    if (this.isHydrated() && this.store.list(entity).length === 0) {
+      // Scoped reconcile bypasses the no-arg debounce and reuses the
+      // session-flip / cursor-drift guards in reconcileInner.
+      void this.reconcile([entity]);
+    }
+  }
+
   async reconcile(entities?: string[]): Promise<void> {
     const minIntervalMs = this.config.reconcileMinIntervalMs ?? 2_000;
     const now = Date.now();
@@ -1448,7 +1513,13 @@ export class SyncEngine {
     // Same reasoning as pullInner: the leader reconciles, broadcasts
     // results, and follower replicas converge via the channel.
     if (!this.isMultiTabLeader) return;
-    const names = entities ?? this.store.entityNames();
+    // Sweep entities with local rows PLUS entities the app has observed
+    // via useQuery (even when empty locally). Without the observed set,
+    // a server row in a never-cached entity is never reconciled and
+    // stays invisible until a full snapshot.
+    const names =
+      entities ??
+      [...new Set([...this.store.entityNames(), ...this.observedEntities])];
     if (names.length === 0) return;
     // Tombstone seq for any local row the server doesn't return. Using
     // the current cursor means future inserts (which have higher seqs)

package/src/scenarios.test.ts

@@ -452,6 +452,79 @@ describe("sync scenarios", () => {
     expect(env.engine.store.get("Note", "n2")).not.toBeNull();
   });
 
+  // EGRESS STORM GUARD (pins the circuit-breaker fix). When every delta
+  // pull 410s but snapshots succeed — a client bouncing between cluster
+  // instances whose in-memory change logs diverge — the client must
+  // snapshot ONCE then back off, NOT re-snapshot on every pull.
+  //
+  // The bug: the breaker reset `consecutive_410s = 0` on every
+  // successful pull, including the cursor=0 snapshot a 410 triggers. So
+  // a 410 → full snapshot → 410 ping-pong reset the breaker each cycle
+  // and the backoff never engaged. The result was a full `select *`
+  // snapshot of EVERY entity ~every 3 seconds, indefinitely — a ~280GB
+  // PlanetScale egress bill. The fix clears the breaker only on a
+  // successful DELTA pull (cursor stable), never on the snapshot itself.
+  test("repeated delta 410s back off instead of re-snapshotting (egress storm)", async () => {
+    env = createTestEnv({ transport: "poll" });
+    env.signIn({ userId: "u1" });
+    env.server.seed("Note", [{ id: "n1", title: "x" }]);
+    await env.start();
+    await env.flush();
+
+    // Baseline: start() did exactly one snapshot pull.
+    const before = env.server.snapshotPullCount;
+
+    // Now every delta pull 410s (snapshots still succeed).
+    env.server.force410OnDelta = true;
+    for (let i = 0; i < 6; i++) {
+      await env.engine.pull();
+      await env.flush();
+    }
+
+    // Exactly ONE additional snapshot (the first resync). Every pull
+    // after that 410s on the delta and routes to exponential backoff —
+    // no further full snapshots. Pre-fix this was 6 (one per pull).
+    expect(env.server.snapshotPullCount - before).toBe(1);
+  });
+
+  // EMPTY-ENTITY RECONCILE GAP (pins observeEntity). A server row in an
+  // entity the local replica has NEVER cached stays invisible: useQuery
+  // reads the empty local store, a delta pull can't recover a row
+  // created before the cursor, and the no-arg reconcile sweeps only
+  // entities with ≥1 local row (store.entityNames()) — so it skips the
+  // empty entity entirely. The user hit this as "I attached the domain
+  // but the Domains list is empty"; clearing IndexedDB (cursor→0→
+  // snapshot) was the only recovery. observeEntity (called by useQuery
+  // on mount) adds the entity to the sweep + fires a one-shot fetch.
+  test("observing an entity recovers a server row the local cache never had", async () => {
+    env = createTestEnv({ transport: "poll" });
+    env.signIn({ userId: "u1" });
+    // Client has rows for Note, but none for Domain.
+    env.server.seed("Note", [{ id: "n1", title: "x" }]);
+    await env.start();
+    await env.flush();
+    expect(env.engine.store.list("Domain")).toHaveLength(0);
+
+    // A Domain row exists server-side but was never delivered to this
+    // client (created on another surface / a missed insert). Inserted
+    // after start so the initial snapshot didn't include it; poll
+    // transport means no auto-delivery.
+    env.server.insert("Domain", { id: "d1", host: "chat.example.com" });
+
+    // A no-arg reconcile sweeps only entities with local rows (Note),
+    // so it never touches Domain — the row stays invisible. The bug.
+    await env.engine.reconcile(["Note"]);
+    await env.flush();
+    expect(env.engine.store.get("Domain", "d1")).toBeNull();
+
+    // observeEntity (what useQuery now calls on mount) adds Domain to
+    // the sweep and fires a one-shot scoped reconcile — the row appears
+    // without a cache clear.
+    env.engine.observeEntity("Domain");
+    await env.flush();
+    expect(env.engine.store.get("Domain", "d1")).not.toBeNull();
+  });
+
   // Row-revoked envelope: server pushes `row-revoked` to a
   // subscriber whose read policy was revoked for a specific row.
   // The engine must drop the row from the local replica and

package/src/test-harness/server.ts

@@ -91,6 +91,17 @@ export class TestServer {
   /** When set, the next pull() returns this status instead of normal.
    *  Used to simulate 410 RESYNC_REQUIRED and similar transient errors. */
   private nextPullStatus: number | null = null;
+  /** When true, every DELTA pull (since > 0) 410s, but a snapshot pull
+   *  (since = 0) succeeds. Simulates a horizontally-scaled deployment
+   *  where a client bounces between instances whose in-memory change
+   *  logs diverge (no shared persistent log) — every cursor is "stale"
+   *  on the instance it lands on. This is the condition that drove the
+   *  280GB egress storm; the test asserts the client backs off instead
+   *  of re-snapshotting forever. */
+  force410OnDelta = false;
+  /** Count of snapshot pulls served (since = 0). The egress storm was a
+   *  runaway count here; the regression test bounds it. */
+  snapshotPullCount = 0;
   /** Captured outbound WS messages from clients — tests assert against
    *  this to verify `reactive-subscribe`, `crdt-subscribe`, etc., were
    *  actually sent over the wire. */
@@ -299,6 +310,7 @@ export class TestServer {
   }> {
     const auth = this.authContextFor(token);
     if (this.beforePullHook) await this.beforePullHook(auth, since);
+    if (since === 0) this.snapshotPullCount += 1;
     const visibleSet = (entity: string) => {
       const filtered = this.visible(
         entity,

package/src/test-harness/transport.ts

@@ -250,6 +250,14 @@ async function handle(
       };
     }
     const since = Number(new URL(url, "http://test").searchParams.get("since") ?? "0");
+    // Cluster-divergence sim: a delta pull lands on an instance that
+    // can't serve our cursor → 410. A snapshot (since=0) still succeeds.
+    if (since > 0 && server.force410OnDelta) {
+      return {
+        status: 410,
+        body: { error: { code: "RESYNC_REQUIRED" } },
+      };
+    }
     const resp = await server.pull(token, since);
     return { status: 200, body: resp };
   }