@pylonsync/sync 0.3.225 → 0.3.227

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.3.225",
+  "version": "0.3.227",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/index.ts

@@ -1296,7 +1296,6 @@ export class SyncEngine {
         const resp = await this.request<
           PullResponse & { snapshot_after?: string | null }
         >("GET", `/api/sync/pull?${params.toString()}`);
-        this.consecutive_410s = 0;
         await this.enqueueApply(resp.changes, resp.cursor);
         // `snapshot_after` is only set when the server is mid-snapshot.
         // Continue paginating in the same loop iteration so we don't
@@ -1311,6 +1310,25 @@ export class SyncEngine {
           break;
         }
       }
+      // Clear the resync circuit breaker ONLY on a successful DELTA
+      // pull — one that started from a real, non-zero cursor the server
+      // honored. A snapshot pull from cursor=0 succeeding does NOT prove
+      // our cursor is stable, so it must NOT clear the breaker.
+      //
+      // Why this matters (the bug this replaced): the reset used to fire
+      // on every successful page, including the cursor=0 snapshot that a
+      // 410 triggers. So a stale-cursor 410 → full snapshot → fresh-
+      // cursor 410 ping-pong — a client bouncing between cluster
+      // instances whose in-memory change logs diverge, with no shared
+      // persistent log to serve the delta — reset the breaker every
+      // cycle. The exponential backoff below could never engage, and the
+      // client re-ran a full `select *` snapshot of EVERY entity roughly
+      // every 3 seconds, indefinitely. That drove a ~280GB PlanetScale
+      // egress bill. Resetting only on a delta means repeated resyncs
+      // escalate into the backoff instead of melting egress.
+      if (!startedFromZero) {
+        this.consecutive_410s = 0;
+      }
       // Snapshot+tail loop exhausted without throwing: if we started
       // from cursor=0 we just hydrated the full replica from server
       // truth. Record it so onConnected skips the reconcile that would
@@ -1335,31 +1353,37 @@ export class SyncEngine {
       // re-pull from seq=0. The server replays all current entity rows as
       // seed events on startup so the fresh pull reconstructs state.
       //
-      // Circuit breaker: if the immediate re-pull ALSO 410s, accept it.
-      // Don't recurse — that's the infinite loop we used to ship before
-      // the cursor=0 server fix landed (or against an old server binary
-      // that hasn't been rebuilt yet). Track 410 retries against an
-      // exponential backoff so a misconfigured server can't melt our CPU.
+      // Circuit breaker. The first resync in an episode snapshots
+      // immediately (good UX: a cursor that genuinely fell off retention
+      // recovers in one round trip). But a SECOND 410 with no successful
+      // delta pull in between means re-snapshotting isn't converging —
+      // the cursor we just minted from the snapshot is itself stale
+      // (instance ping-pong, or a server that can't serve our delta). At
+      // that point each snapshot is a full `select *` of every entity,
+      // so we MUST back off instead of looping. The breaker only clears
+      // on a successful delta pull (see the `!startedFromZero` reset
+      // above) — NOT on the snapshot itself, which is what made this
+      // loop unbounded and burned ~280GB of egress.
       if (status === 410) {
         const attempt = this.consecutive_410s;
         this.consecutive_410s += 1;
         if (attempt === 0) {
-          // Bypass the queue here — we ARE the pull op holding the
-          // queue slot. Calling the public pull() would re-enqueue and
-          // share our own promise back to us (deadlock).
+          // First resync of the episode — snapshot now. Bypass the queue
+          // (we ARE the pull op holding the slot; the public pull()
+          // would re-enqueue and share our own promise back → deadlock).
           await this.resetReplicaInner();
           await this.pullInner();
         } else {
-          // Already retried once and still 410. Stop. Schedule a
-          // back-off retry tied to the WS reconnect path so we don't
-          // spam the server. Resets when any pull succeeds.
+          // Snapshotted once and still 410 → not converging. Back off
+          // exponentially instead of re-snapshotting. Clears only when a
+          // delta pull finally succeeds (cursor stabilised).
           const delayMs = Math.min(30_000, 1000 * 2 ** Math.min(attempt, 5));
           console.warn(
             `[pylon] persistent 410 RESYNC_REQUIRED (attempt ${attempt + 1}); backing off ${delayMs}ms`,
           );
           setTimeout(() => {
-            // Trigger one more attempt; either it succeeds (which resets
-            // the counter) or it 410s again (which extends the backoff).
+            // Retry after the delay; a delta success resets the counter,
+            // a repeat 410 extends the backoff (no snapshot).
             void this.pull();
           }, delayMs);
         }

package/src/scenarios.test.ts

@@ -452,6 +452,41 @@ describe("sync scenarios", () => {
     expect(env.engine.store.get("Note", "n2")).not.toBeNull();
   });
 
+  // EGRESS STORM GUARD (pins the circuit-breaker fix). When every delta
+  // pull 410s but snapshots succeed — a client bouncing between cluster
+  // instances whose in-memory change logs diverge — the client must
+  // snapshot ONCE then back off, NOT re-snapshot on every pull.
+  //
+  // The bug: the breaker reset `consecutive_410s = 0` on every
+  // successful pull, including the cursor=0 snapshot a 410 triggers. So
+  // a 410 → full snapshot → 410 ping-pong reset the breaker each cycle
+  // and the backoff never engaged. The result was a full `select *`
+  // snapshot of EVERY entity ~every 3 seconds, indefinitely — a ~280GB
+  // PlanetScale egress bill. The fix clears the breaker only on a
+  // successful DELTA pull (cursor stable), never on the snapshot itself.
+  test("repeated delta 410s back off instead of re-snapshotting (egress storm)", async () => {
+    env = createTestEnv({ transport: "poll" });
+    env.signIn({ userId: "u1" });
+    env.server.seed("Note", [{ id: "n1", title: "x" }]);
+    await env.start();
+    await env.flush();
+
+    // Baseline: start() did exactly one snapshot pull.
+    const before = env.server.snapshotPullCount;
+
+    // Now every delta pull 410s (snapshots still succeed).
+    env.server.force410OnDelta = true;
+    for (let i = 0; i < 6; i++) {
+      await env.engine.pull();
+      await env.flush();
+    }
+
+    // Exactly ONE additional snapshot (the first resync). Every pull
+    // after that 410s on the delta and routes to exponential backoff —
+    // no further full snapshots. Pre-fix this was 6 (one per pull).
+    expect(env.server.snapshotPullCount - before).toBe(1);
+  });
+
   // Row-revoked envelope: server pushes `row-revoked` to a
   // subscriber whose read policy was revoked for a specific row.
   // The engine must drop the row from the local replica and

package/src/test-harness/server.ts

@@ -91,6 +91,17 @@ export class TestServer {
   /** When set, the next pull() returns this status instead of normal.
    *  Used to simulate 410 RESYNC_REQUIRED and similar transient errors. */
   private nextPullStatus: number | null = null;
+  /** When true, every DELTA pull (since > 0) 410s, but a snapshot pull
+   *  (since = 0) succeeds. Simulates a horizontally-scaled deployment
+   *  where a client bounces between instances whose in-memory change
+   *  logs diverge (no shared persistent log) — every cursor is "stale"
+   *  on the instance it lands on. This is the condition that drove the
+   *  280GB egress storm; the test asserts the client backs off instead
+   *  of re-snapshotting forever. */
+  force410OnDelta = false;
+  /** Count of snapshot pulls served (since = 0). The egress storm was a
+   *  runaway count here; the regression test bounds it. */
+  snapshotPullCount = 0;
   /** Captured outbound WS messages from clients — tests assert against
    *  this to verify `reactive-subscribe`, `crdt-subscribe`, etc., were
    *  actually sent over the wire. */
@@ -299,6 +310,7 @@ export class TestServer {
   }> {
     const auth = this.authContextFor(token);
     if (this.beforePullHook) await this.beforePullHook(auth, since);
+    if (since === 0) this.snapshotPullCount += 1;
     const visibleSet = (entity: string) => {
       const filtered = this.visible(
         entity,

package/src/test-harness/transport.ts

@@ -250,6 +250,14 @@ async function handle(
       };
     }
     const since = Number(new URL(url, "http://test").searchParams.get("since") ?? "0");
+    // Cluster-divergence sim: a delta pull lands on an instance that
+    // can't serve our cursor → 410. A snapshot (since=0) still succeeds.
+    if (since > 0 && server.force410OnDelta) {
+      return {
+        status: 410,
+        body: { error: { code: "RESYNC_REQUIRED" } },
+      };
+    }
     const resp = await server.pull(token, since);
     return { status: 200, body: resp };
   }