@pylonsync/sync 0.3.275 → 0.3.276

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.3.275",
+  "version": "0.3.276",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/index.ts

@@ -1033,6 +1033,20 @@ export class SyncEngine {
           this.broadcastToTabs({ type: "entity-observe", entity });
         }
       },
+      onReplayForwardedMutations: () => {
+        // Follower path: re-forward our pending batch to the new leader.
+        // If we'd forwarded an op to the previous (now-dead) leader, it
+        // died before acking, so the op is still pending here as an
+        // optimistic ghost with nothing pushing it. The new leader only
+        // drains its OWN queue on promotion, so without this re-forward the
+        // op is silently lost until our next local write. The leader's
+        // `onMutationsForwarded` re-adds by op_id (idempotent) and pushes.
+        if (this.isMultiTabLeader) return;
+        const pending = this.mutations.pending();
+        if (pending.length > 0) {
+          this.broadcastToTabs({ type: "mutations", ops: pending });
+        }
+      },
     };
   }
 
@@ -1672,6 +1686,12 @@ export class SyncEngine {
    *  reconcile sweep them too. See `observeEntity`. */
   private observedEntities = new Set<string>();
 
+  /** Per-entity count of CONSECUTIVE 403s seen during reconcile, reset on
+   *  any successful fetch. A single 403 can be transient (a bearer caught
+   *  mid-refresh, a momentary policy blip) and must NOT wipe the local cache;
+   *  we only drop an entity's rows after two in a row. See `reconcile`. */
+  private reconcile403Streak = new Map<string, number>();
+
   /**
    * Reconcile the local replica against server truth.
    *
@@ -1819,14 +1839,41 @@ export class SyncEngine {
           // Network errors are expected (offline, transient 5xx). Skip
           // this entity; the next reconcile trigger will retry.
           const status = (err as { status?: number })?.status;
-          if (status === 403 || status === 404) {
-            // Entity is no longer readable (policy revoked) or removed
-            // from the manifest. Drop every local row for it — keeping
-            // them around just leaks invisible state.
+          if (status === 404) {
+            // Entity removed from the manifest — definitive. Drop its rows.
+            this.reconcile403Streak.delete(entity);
+            await this.dropEntity(entity, tombstoneSeq);
+            return;
+          }
+          if (status === 403) {
+            // A 403 can be TRANSIENT — a bearer caught mid-refresh, a tenant
+            // flip that raced the fetch, or a momentary server-side policy
+            // blip. Nuking every local row on the first one makes the data
+            // flash away and reappear next reconcile (the "rows vanish on
+            // load" bug, on the error path this time). Two gates before we
+            // drop:
+            //   1. If the session flipped during the fetch, the 403 reflects
+            //      the OLD context — never drop; the next reconcile under the
+            //      new session decides (mirrors the success-path guard below).
+            if (this.session.signature() !== sessionBeforeFetch) {
+              return;
+            }
+            //   2. Otherwise require TWO consecutive 403s for this entity, so
+            //      a one-off blip can't wipe the cache. A successful fetch
+            //      (below) resets the streak.
+            const streak = (this.reconcile403Streak.get(entity) ?? 0) + 1;
+            if (streak < 2) {
+              this.reconcile403Streak.set(entity, streak);
+              return;
+            }
+            this.reconcile403Streak.delete(entity);
             await this.dropEntity(entity, tombstoneSeq);
           }
           return;
         }
+        // Successful fetch — the entity is readable, so any prior 403 streak
+        // is broken (even if a drift guard below makes us skip the apply).
+        this.reconcile403Streak.delete(entity);
         if (this.cursor.last_seq !== cursorBeforeFetch) {
           // Cursor moved during fetch — at least one WS event for this
           // (or another) entity landed and might have a fresher value

package/src/multi-tab-orchestrator.test.ts

@@ -37,6 +37,7 @@ function makeRig(opts: { isLeader?: boolean } = {}) {
       onMutationsFailed: record("mutationsFailed"),
       onBinaryReceived: record("binary"),
       onPeerLeft: record("peerLeft"),
+      onReplayForwardedMutations: record("replayForwardedMutations"),
     },
   );
   return { orch, events, serverSubs, subs };
@@ -89,6 +90,26 @@ describe("MultiTabOrchestrator dispatch", () => {
     expect(follower.events.length).toBe(0);
   });
 
+  test("request-sub-replay re-forwards a follower's pending mutations (#341)", () => {
+    // A new leader broadcasts request-sub-replay after a handoff. Followers
+    // must re-forward their pending mutations — an op forwarded to the
+    // previous (now-dead) leader is otherwise stranded, since the new leader
+    // only drains its OWN queue on promotion. makeRig's orchestrator never
+    // ran init(), so _isLeader is false → it acts as a follower here.
+    const { orch, events } = makeRig();
+    orch.handleMessage({ type: "request-sub-replay" }, "new-leader");
+    expect(events.map((e) => e.kind)).toContain("replayForwardedMutations");
+  });
+
+  test("request-sub-replay does NOT re-forward on the leader (#341)", () => {
+    const { orch, events } = makeRig();
+    // Promote this rig to leader; a leader receiving the replay request (its
+    // own echo, or a stale broadcast) owns the network and must not act.
+    (orch as unknown as { _isLeader: boolean })._isLeader = true;
+    orch.handleMessage({ type: "request-sub-replay" }, "x");
+    expect(events.map((e) => e.kind)).not.toContain("replayForwardedMutations");
+  });
+
   test("mutations-acked + mutations-failed both fire their hooks", () => {
     const { orch, events } = makeRig();
     orch.handleMessage({ type: "mutations-acked", opIds: ["a", "b"] }, "x");

package/src/multi-tab-orchestrator.ts

@@ -125,6 +125,12 @@ export interface MultiTabOrchestratorHooks {
   /** New leader → followers: re-declare your observed entities so the
    *  new leader's reconcile sweep covers them. */
   onReplayObservedEntities?(): void;
+  /** New leader → followers: re-forward your pending mutation batch. A
+   *  mutation a follower forwarded to a leader that died before acking is
+   *  otherwise stranded — the new leader only drains its OWN queue on
+   *  promotion, and the other replay hooks cover subs/rooms/entities but
+   *  not mutations. op_id makes the re-forward idempotent. */
+  onReplayForwardedMutations?(): void;
 }
 
 export interface MultiTabOrchestratorConfig {
@@ -404,6 +410,11 @@ export class MultiTabOrchestrator {
         // ...and re-declare observed entities so the new leader's
         // reconcile sweep covers them.
         this.hooks.onReplayObservedEntities?.();
+        // ...and re-forward any pending mutations. A mutation forwarded to
+        // a now-dead leader (which never acked) is otherwise stranded as an
+        // optimistic ghost until this follower's next write — the new
+        // leader only drained its own queue on promotion.
+        this.hooks.onReplayForwardedMutations?.();
         break;
       }
       case "entity-observe": {

package/src/reconcile.test.ts

@@ -247,6 +247,52 @@ describe("SyncEngine.reconcile", () => {
     await engine.reconcile(["DeletedEntity"]);
     expect(engine.store.list("DeletedEntity").length).toBe(0);
   });
+
+  test("transient 403 keeps rows; two consecutive 403s drop them (#343)", async () => {
+    // A single 403 can be a bearer caught mid-refresh or a momentary policy
+    // blip; nuking the cache on the first one made rows flash away and return
+    // next reconcile. The first 403 must keep the rows.
+    const status = 403;
+    restore = installFetch(async () => ({ status, body: {} }));
+    const engine = makeEngine();
+    seedStore(engine, "Recording", [{ id: "r1", title: "alive" }]);
+
+    await engine.reconcile(["Recording"]);
+    expect(engine.store.list("Recording").length).toBe(1);
+
+    // A second consecutive 403 confirms access is really gone — now drop.
+    await engine.reconcile(["Recording"]);
+    expect(engine.store.list("Recording").length).toBe(0);
+  });
+
+  test("a successful fetch resets the 403 streak (#343)", async () => {
+    let status = 403;
+    restore = installFetch(async () =>
+      status === 200
+        ? {
+            status: 200,
+            body: {
+              data: [{ id: "r1", title: "alive" }],
+              next_cursor: null,
+              has_more: false,
+            },
+          }
+        : { status: 403, body: {} },
+    );
+    const engine = makeEngine();
+    seedStore(engine, "Recording", [{ id: "r1", title: "alive" }]);
+
+    await engine.reconcile(["Recording"]); // 403 #1 — kept
+    expect(engine.store.list("Recording").length).toBe(1);
+
+    status = 200;
+    await engine.reconcile(["Recording"]); // success — streak resets
+    expect(engine.store.list("Recording").length).toBe(1);
+
+    status = 403;
+    await engine.reconcile(["Recording"]); // a fresh lone 403 must NOT drop
+    expect(engine.store.list("Recording").length).toBe(1);
+  });
 });
 
 describe("LocalStore.reconcileRemove", () => {

package/src/round6-codex.test.ts

@@ -483,3 +483,88 @@ describe("sync hardening: multi-tab follower coverage", () => {
     expect(row?.title).toBe("keep");
   });
 });
+
+describe("#341: leader-handoff re-forwards a follower's pending mutations", () => {
+  let env: TestEnv | null = null;
+  afterEach(async () => {
+    if (env) await env.dispose();
+    env = null;
+  });
+
+  // The engine-side half of the fix: a follower's onReplayForwardedMutations
+  // hook (fired when a new leader broadcasts request-sub-replay) re-forwards
+  // its pending batch. Pre-fix the hook didn't exist, so an op forwarded to a
+  // now-dead leader was stranded until the follower's next local write.
+  test("a follower re-forwards its pending batch on replay", async () => {
+    env = createTestEnv();
+    env.signIn({ userId: "u1" });
+    await env.start();
+
+    const engine = env.engine as unknown as {
+      isMultiTabLeader: boolean;
+      multiTabHooks(): { onReplayForwardedMutations?: () => void };
+      broadcastToTabs(payload: unknown): void;
+      mutations: { add(change: unknown): string };
+    };
+
+    // Become a follower holding an op we'd forwarded to a leader that died
+    // before acking — it's still pending here with nothing pushing it.
+    engine.isMultiTabLeader = false;
+    engine.mutations.add({
+      op_id: "op-stranded",
+      entity: "Todo",
+      row_id: "x",
+      kind: "insert",
+      data: { id: "x", text: "stranded" },
+    });
+
+    const forwarded: { ops: { change: { op_id: string } }[] }[] = [];
+    const orig = engine.broadcastToTabs.bind(engine);
+    engine.broadcastToTabs = (p: unknown) => {
+      if ((p as { type?: string }).type === "mutations") {
+        forwarded.push(p as { ops: { change: { op_id: string } }[] });
+      }
+      orig(p);
+    };
+
+    // New leader → followers: re-forward your subs/rooms/entities AND your
+    // pending mutations. This is the hook the request-sub-replay handler now
+    // invokes.
+    engine.multiTabHooks().onReplayForwardedMutations?.();
+
+    expect(forwarded.length).toBe(1);
+    expect(forwarded[0].ops.map((o) => o.change.op_id)).toContain("op-stranded");
+  });
+
+  test("the leader does NOT re-forward on replay (it owns the network)", async () => {
+    env = createTestEnv(); // multiTab:false → this engine is the leader
+    env.signIn({ userId: "u1" });
+    await env.start();
+
+    const engine = env.engine as unknown as {
+      isMultiTabLeader: boolean;
+      multiTabHooks(): { onReplayForwardedMutations?: () => void };
+      broadcastToTabs(payload: unknown): void;
+      mutations: { add(change: unknown): string };
+    };
+
+    expect(engine.isMultiTabLeader).toBe(true);
+    engine.mutations.add({
+      op_id: "op-leader",
+      entity: "Todo",
+      row_id: "y",
+      kind: "insert",
+      data: { id: "y", text: "mine" },
+    });
+
+    const forwarded: unknown[] = [];
+    const orig = engine.broadcastToTabs.bind(engine);
+    engine.broadcastToTabs = (p: unknown) => {
+      if ((p as { type?: string }).type === "mutations") forwarded.push(p);
+      orig(p);
+    };
+
+    engine.multiTabHooks().onReplayForwardedMutations?.();
+    expect(forwarded.length).toBe(0);
+  });
+});