@rindle/remote 0.4.4 → 0.6.3

package/src/optimistic-source.ts

@@ -16,6 +16,7 @@
 import { LMID_QUERY_NAME } from "@rindle/client";
 import type {
   MutationEnvelope,
+  MutationOutcomeFrame,
   NormalizedEvent,
   NormalizedTableSchema,
   OptimisticSource,
@@ -24,7 +25,9 @@ import type {
   RemoteQuery,
 } from "@rindle/client";
 
+import type { AffinityTicketStore } from "./affinity.ts";
 import { NormalizedSubscriber } from "./normalized.ts";
+import { retryDelayMs } from "./query-error.ts";
 import type { NormalizedBatch, NormalizedHello } from "./normalized.ts";
 import { ProtocolError } from "./protocol.ts";
 import type { ServerMsg } from "./protocol.ts";
@@ -48,6 +51,16 @@ interface QState {
   resubscribing: boolean;
   /** Monotonic token that cancels stale async lease resolutions. */
   subscribeTicket: number;
+  /** Pending retryable-error re-subscribe timer (FOLLOWER-LAG-SHED §6.3), if any. */
+  retryTimer: ReturnType<typeof setTimeout> | undefined;
+  /** Consecutive retryable errors without a successful hello — the backoff exponent. */
+  retryAttempt: number;
+  /** Whether the LAST subscribe sent for this query presented a `leaseToken` — i.e. its hello
+   *  proves the connection is (re-)AUTHENTICATED (a room shell sets the socket's subject from the
+   *  first verified token; `pushMutation` requires it). After a reconnect on a lease-auth session,
+   *  queued pushes flush at the first such hello, never earlier — an envelope racing ahead of the
+   *  token subscribe would be refused by the shell's subject gate (H-v §7.5 rule 3). */
+  authed: boolean;
 }
 
 /** Build a transport to a follower's public ws endpoint (READ-ROUTER-DESIGN.md §2.3). Default
@@ -71,6 +84,11 @@ export interface RemoteOptimisticSourceOptions {
   resolveSubscribe?: SubscribeResolver;
   /** Override named-mutator delivery, e.g. POST envelopes to the app API server. */
   pushMutation?: MutationEnvelopeSender;
+  /** Follower-affinity mode (FOLLOWER-AFFINITY-DESIGN.md §3): the shared ticket store. When set, the
+   *  source records the follower's minted ticket from the `{t:"affinity"}` frame and CLEARS it on a
+   *  sustained outage so the next (ticketless) reconnect anycasts to a live follower and re-pins
+   *  (§8). Absent ⇒ affinity off (today's behavior). */
+  affinity?: AffinityTicketStore;
 }
 
 export class RemoteOptimisticSource implements OptimisticSource {
@@ -83,9 +101,23 @@ export class RemoteOptimisticSource implements OptimisticSource {
   private readonly clientID: string;
   private readonly resolveSubscribe: SubscribeResolver;
   private readonly pushMutationSender?: MutationEnvelopeSender;
+  /** The affinity ticket store (undefined ⇒ affinity off). See {@link RemoteOptimisticSourceOptions.affinity}. */
+  private readonly affinityStore?: AffinityTicketStore;
   private handler: (qid: QueryId, ev: NormalizedEvent) => void = () => {};
   private progressHandler: (frame: ProgressFrame) => void = () => {};
   private restartHandler: () => void = () => {};
+  private outcomeHandler: (frame: MutationOutcomeFrame) => void = () => {};
+  private resyncHandler: () => void = () => {};
+  /** Set by {@link resync} when this is a LEASE-AUTH session (some sub presented a `leaseToken`):
+   *  transport pushes queue in {@link pendingPushes} until the first authenticated re-subscribe's
+   *  hello re-establishes the socket's subject, then flush (see QState.authed). Never set on a
+   *  token-less (embedded/rindled) session — its pushes need no subject and go straight out. */
+  private awaitingAuthedHello = false;
+  /** Envelopes held while {@link awaitingAuthedHello} (H-v §7.5 rule 3). Every entry corresponds
+   *  to a still-pending backend mutation (the re-send reconstructs from pending entries; app
+   *  invokes in the window are pending by definition), so a superseding resync may CLEAR this —
+   *  its own re-send regenerates whatever still matters. */
+  private pendingPushes: MutationEnvelope[] = [];
   private readonly subs = new Map<QueryId, QState>();
   /** Queries whose subscribe is waiting for a transport to exist — endpoint-less subscribes issued
    *  in pure-lazy mode before any lease opens a transport (the lmid system query is registered by
@@ -110,6 +142,7 @@ export class RemoteOptimisticSource implements OptimisticSource {
     this.clientID = clientID;
     this.resolveSubscribe = opts.resolveSubscribe ?? defaultSubscribeTarget;
     this.pushMutationSender = opts.pushMutation;
+    this.affinityStore = opts.affinity;
     if (isTransport(connection)) {
       this.transportFactory = undefined;
       this.bringUp(connection, undefined);
@@ -126,7 +159,13 @@ export class RemoteOptimisticSource implements OptimisticSource {
   private attach(transport: Transport): void {
     transport.onMessage((msg) => this.onServerMsg(msg));
     // Heal a dropped/restarted connection (same endpoint): on reconnect, replay init + re-subscribe.
-    transport.onReconnect?.(() => this.resync());
+    transport.onReconnect?.(() => {
+      // The held ticket was useful for routing this ws handshake, but HTTP re-leases must wait for
+      // THIS connection's first affinity frame. Otherwise an expired/rotated persisted ticket can
+      // independently re-pin the control leg before the fresh frame arrives.
+      this.affinityStore?.connectionPending();
+      this.resync();
+    });
     // Sustained outage on this endpoint: re-lease (the router may move us off a dead follower, §3).
     transport.onDown?.(() => this.onDown());
   }
@@ -135,6 +174,10 @@ export class RemoteOptimisticSource implements OptimisticSource {
   private bringUp(transport: Transport, endpoint: string | undefined): void {
     this.transport = transport;
     this.currentEndpoint = endpoint;
+    // `WsTransport` has already evaluated the ticket thunk while constructing this connection.
+    // From this point the old/persisted ticket is handshake-only until the follower confirms the
+    // selected machine with its first affinity frame.
+    this.affinityStore?.connectionPending();
     this.attach(transport);
     transport.send({ t: "init", clientID: this.clientID });
     this.flushDeferred();
@@ -187,7 +230,16 @@ export class RemoteOptimisticSource implements OptimisticSource {
    *  lease ever carried a `wsEndpoint`) — there is nowhere to move, so we keep the pre-router
    *  behavior and let the transport's own reconnect→resync recover. */
   private onDown(): void {
-    if (this.closed || !this.sawRoutedEndpoint) return;
+    if (this.closed) return;
+    if (this.affinityStore) {
+      // Affinity: the pinned follower is gone (sustained outage). Drop the ticket so the transport's
+      // ongoing reconnects go TICKETLESS — the Fly edge then anycasts to a live follower, which mints
+      // a fresh ticket, and that reconnect's `onReconnect` → resync re-leases there (FOLLOWER-AFFINITY
+      // §8, one bounded reassignment). Nothing to migrate: the ws host is fixed, Fly routes by ticket.
+      this.affinityStore.clear();
+      return;
+    }
+    if (!this.sawRoutedEndpoint) return;
     this.resubscribeAll();
   }
 
@@ -196,8 +248,12 @@ export class RemoteOptimisticSource implements OptimisticSource {
   close(): void {
     this.closed = true;
     this.transport?.close();
+    for (const s of this.subs.values()) {
+      if (s.retryTimer !== undefined) clearTimeout(s.retryTimer);
+    }
     this.subs.clear();
     this.deferred.clear();
+    this.pendingPushes.length = 0;
   }
 
   /** Register a handler fired when the DAEMON restarts (a new boot id) — the backend resets its
@@ -211,11 +267,22 @@ export class RemoteOptimisticSource implements OptimisticSource {
   }
 
   registerQuery(qid: QueryId, remote: RemoteQuery): void {
-    this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0 });
+    this.subs.set(qid, {
+      remote,
+      subscriber: null,
+      epoch: 0,
+      resubscribing: false,
+      subscribeTicket: 0,
+      retryTimer: undefined,
+      retryAttempt: 0,
+      authed: false,
+    });
     this.subscribe(qid, remote);
   }
 
   unregisterQuery(qid: QueryId): void {
+    const s = this.subs.get(qid);
+    if (s?.retryTimer !== undefined) clearTimeout(s.retryTimer);
     this.subs.delete(qid);
     this.deferred.delete(qid);
     this.transport?.send({ t: "unsubscribe", queryId: qid });
@@ -223,6 +290,13 @@ export class RemoteOptimisticSource implements OptimisticSource {
 
   pushMutation(envelope: MutationEnvelope): Promise<void> {
     if (this.pushMutationSender) return Promise.resolve(this.pushMutationSender(envelope));
+    // A lease-auth session that just reconnected is not yet re-authenticated (the shell's
+    // `pushMutation` subject gate would refuse) — hold the envelope until the first token
+    // re-subscribe's hello, then flush in order (H-v §7.5 rule 3).
+    if (this.awaitingAuthedHello) {
+      this.pendingPushes.push(envelope);
+      return Promise.resolve();
+    }
     this.transport?.send({ t: "pushMutation", envelope });
     return Promise.resolve();
   }
@@ -235,11 +309,33 @@ export class RemoteOptimisticSource implements OptimisticSource {
     this.progressHandler = handler;
   }
 
+  /** The room deopt handshake's verdict stream (H-v). Dispatched OUT-OF-BAND on arrival — see
+   *  {@link onServerMsg}'s `mutationOutcome` arm for why it must never wait behind the cv buffer. */
+  onMutationOutcome(handler: (frame: MutationOutcomeFrame) => void): void {
+    this.outcomeHandler = handler;
+  }
+
+  /** Fired once per re-established session, SYNCHRONOUSLY inside {@link resync} — before any
+   *  post-reconnect frame can release (the §7.5 rule-3 window: a replayed lmid snapshot must not
+   *  retire an entry whose outcome frame died with the old socket before the re-send captured
+   *  it). The backend re-sends the domain's unconfirmed pending envelopes with their original
+   *  mids; on a lease-auth session their DELIVERY is deferred until the first token hello
+   *  re-authenticates the socket ({@link pendingPushes}). */
+  onResync(handler: () => void): void {
+    this.resyncHandler = handler;
+  }
+
   // --- internals ---------------------------------------------------------------
 
   private subscribe(qid: QueryId, remote: RemoteQuery): void {
     const s = this.subs.get(qid);
     if (!s) return;
+    // A fresh subscribe (reconnect resync, gap recovery, the retry timer itself) supersedes any
+    // scheduled retryable-error retry — never leave two subscribe paths racing for one query.
+    if (s.retryTimer !== undefined) {
+      clearTimeout(s.retryTimer);
+      s.retryTimer = undefined;
+    }
     const request = { queryId: qid, remote, mode: "normalized" as const };
     const ticket = ++s.subscribeTicket;
     const send = (target: SubscribeTarget) => {
@@ -247,6 +343,7 @@ export class RemoteOptimisticSource implements OptimisticSource {
       const cur = this.subs.get(qid);
       if (cur !== s || cur.subscribeTicket !== ticket) return;
       const endpoint = "leaseToken" in target ? target.wsEndpoint : undefined;
+      s.authed = "leaseToken" in target; // a token subscribe (re-)authenticates the socket (H-v)
       if (endpoint !== undefined) this.sawRoutedEndpoint = true;
       if (this.transport) {
         if (endpoint !== undefined && endpoint !== this.currentEndpoint && this.transportFactory) {
@@ -291,13 +388,34 @@ export class RemoteOptimisticSource implements OptimisticSource {
   }
 
   private onServerMsg(msg: ServerMsg): void {
+    if (msg.t === "affinity") {
+      // Connection-level: the follower minted/refreshed this connection's placement ticket. Persist
+      // it (via the store) so the next connect offers it as a subprotocol and the lease POST forwards
+      // it — both legs then pin THIS follower (§4). Off ⇒ no store ⇒ dropped.
+      this.affinityStore?.set(msg.ticket);
+      return;
+    }
     if (msg.t === "progress") {
       this.progressHandler(msg.frame);
       return;
     }
+    if (msg.t === "mutationOutcome") {
+      // OUT-OF-BAND BY DESIGN (H-v): the frame has no `cv`, so it must NEVER be routed through the
+      // backend's cv buffer — dispatch immediately. A deopt has to migrate its pending entry to
+      // the daemon stream BEFORE the buffered lmid release that would otherwise retire it as a
+      // success (silence + lmid coverage ⇒ applied), and the §7.3 hold-back trigger — keyed on the
+      // entry's confirming domain — would then park its staged writes the wrong way.
+      this.outcomeHandler({
+        mid: msg.mid,
+        kind: msg.kind,
+        ...(msg.reason !== undefined ? { reason: msg.reason } : {}),
+        ...(msg.name !== undefined ? { name: msg.name } : {}),
+        ...("args" in msg ? { args: msg.args } : {}),
+      });
+      return;
+    }
     if (msg.t === "queryError") {
-      this.subs.delete(msg.queryId);
-      console.error(`[rindle-remote] optimistic query ${msg.queryId} subscription rejected: ${msg.message}`);
+      this.onQueryError(msg.queryId, msg);
       return;
     }
     if (msg.t !== "nhello" && msg.t !== "nbatch") return;
@@ -310,11 +428,26 @@ export class RemoteOptimisticSource implements OptimisticSource {
     else this.applyBatch(msg.queryId, s, msg.batch);
   }
 
-  /** On reconnect: re-announce identity and re-subscribe every live query (each re-resolves its
-   *  lease, so a restarted daemon re-materializes + re-leases on the fly). */
+  /** On reconnect: re-announce identity, fire the `onResync` re-send, and re-subscribe every live
+   *  query (each re-resolves its lease, so a restarted daemon re-materializes + re-leases on the
+   *  fly). The re-send fires HERE — synchronously, before any post-reconnect frame can be
+   *  processed — because the §7.5 rule-3 window closes fast: the re-subscribed lmid stream's
+   *  fresh snapshot may cover a mid whose outcome frame died with the OLD socket, and once the
+   *  release retires that entry as an apparent success there is nothing left to re-send (the
+   *  lost-deopt write would silently vanish). Firing now captures the in-flight set intact; on a
+   *  lease-auth session the envelopes themselves are HELD ({@link pendingPushes}) until the first
+   *  token re-subscribe's hello re-authenticates the socket, then flush in order — so the shell's
+   *  subject gate never refuses them, and its re-answer (a recorded outcome for any non-applied
+   *  mid) resolves even an already-retired entry via the handshake's not-found arm. */
   private resync(): void {
     if (this.closed) return;
     this.transport?.send({ t: "init", clientID: this.clientID });
+    // Lease-auth session ⇒ hold pushes until re-authed. A stale queue from a superseded resync is
+    // cleared first: every held envelope maps to a still-pending mutation, and THIS resync's
+    // re-send below regenerates whatever still matters (no loss, no stale duplicates).
+    this.pendingPushes.length = 0;
+    this.awaitingAuthedHello = [...this.subs.values()].some((s) => s.authed);
+    this.resyncHandler();
     this.resubscribeAll();
   }
 
@@ -325,11 +458,55 @@ export class RemoteOptimisticSource implements OptimisticSource {
     this.lastBootId = bootId;
   }
 
+  /** Route a `queryError` by its 101 §5 classification: retryable ⇒ keep the QState (and the
+   *  rows already folded downstream — 101 §6) and re-subscribe after a jittered backoff
+   *  honoring `retryAfterMs`; terminal (or pre-classification servers) ⇒ drop the
+   *  subscription, exactly as before. Fixes the stranded-client gap: a worker fault's or a
+   *  shedding follower's error now heals end-to-end (FOLLOWER-LAG-SHED §6.3). */
+  private onQueryError(qid: QueryId, err: { message: string; code?: string; retryable?: boolean; retryAfterMs?: number }): void {
+    const s = this.subs.get(qid);
+    if (!s) return;
+    if (err.retryable !== true) {
+      if (s.retryTimer !== undefined) clearTimeout(s.retryTimer);
+      this.subs.delete(qid);
+      console.error(`[rindle-remote] optimistic query ${qid} subscription rejected: ${err.message}`);
+      return;
+    }
+    if (s.retryTimer !== undefined) return; // a retry is already scheduled — don't stack them
+    s.subscriber = null; // stop validating the dead epoch; recovery is a fresh seq-0 hydrate
+    s.resubscribing = true;
+    const delay = retryDelayMs(s.retryAttempt++, err.retryAfterMs);
+    console.warn(
+      `[rindle-remote] optimistic query ${qid} ${err.code ?? "error"} (retryable): re-subscribing in ${delay}ms: ${err.message}`,
+    );
+    const timer = setTimeout(() => {
+      const cur = this.subs.get(qid);
+      if (cur !== s) return;
+      s.retryTimer = undefined;
+      this.subscribe(qid, s.remote);
+    }, delay);
+    // Node returns a Timeout (unref keeps a retiring process from being pinned by a retry);
+    // browsers return a number, where the optional call is a no-op.
+    (timer as { unref?: () => void }).unref?.();
+    s.retryTimer = timer;
+  }
+
   private openSubscriber(qid: QueryId, s: QState, hello: NormalizedHello): void {
     try {
       s.subscriber = new NormalizedSubscriber(hello, (ev) => this.handler(qid, ev), this.clientTables);
       s.epoch = hello.epoch;
       s.resubscribing = false;
+      s.retryAttempt = 0; // a successful hello resets the retryable-error backoff
+      // H-v §7.5 rule 3: the first AUTHENTICATED hello after a reconnect proves the socket is
+      // re-authorized (the shell set its subject from the verified token — pushMutation-ready):
+      // flush the held envelopes, in order. A token-less hello (the lmid system query) does not
+      // qualify — an envelope racing ahead of the lease-token subscribe would be refused.
+      if (this.awaitingAuthedHello && s.authed) {
+        this.awaitingAuthedHello = false;
+        for (const envelope of this.pendingPushes.splice(0)) {
+          this.transport?.send({ t: "pushMutation", envelope });
+        }
+      }
     } catch (e) {
       // A comparator/fp mismatch at hello is unrecoverable (a code-contract divergence).
       s.subscriber = null;

package/src/protocol.ts

@@ -135,8 +135,24 @@ export type ServerMsg =
   // or `cv` state — so the client must force a clean re-hydrate (reset its `cv` watermark).
   | { t: "nhello"; queryId: number; hello: NormalizedHello; bootId?: string }
   | { t: "nbatch"; queryId: number; batch: NormalizedBatch }
-  | { t: "queryError"; queryId: number; message: string }
+  // `code`/`retryable`/`retryAfterMs` classify the error per 101-QUERY-ERRORS §5 (absent from
+  // older servers ⇒ terminal): a retryable error (`code:"shed"` from a load-shedding follower,
+  // `code:"faulted"` from a worker fault) schedules a backed-off re-subscribe instead of
+  // stranding the subscription (FOLLOWER-LAG-SHED §6.3).
+  | { t: "queryError"; queryId: number; message: string; code?: string; retryable?: boolean; retryAfterMs?: number }
   | { t: "progress"; frame: ProgressFrame }
+  // The follower-affinity ticket (FOLLOWER-AFFINITY-DESIGN.md §3), minted by the fleet follower as
+  // the FIRST frame after the ws opens (`rust/rindle-server/src/net.rs` `serve_ws_conn`). Opaque +
+  // connection-level (no `queryId`, no `cv`): the client persists it and offers it as a subprotocol
+  // on the next connect + forwards it on the lease POST so both legs pin the same follower. Absent
+  // on a single/affinity-off daemon (never sent) — old clients drop the unknown `t`.
+  | { t: "affinity"; ticket: string }
+  // The room deopt handshake's verdict frame (RINDLE-REALTIME-QUERY-ENABLEMENT §3.3, H-iv-b):
+  // sent on the author's socket for every NON-applied mutation, BEFORE the lmid ack that burns
+  // the mid; `name`/`args` ride `kind:"deopt"` only (self-contained re-invoke). Connection-level
+  // and cv-less — the client dispatches it OUT-OF-BAND, never behind the cv buffer. Additive:
+  // old clients drop unknown `t`.
+  | { t: "mutationOutcome"; mid: number; kind: "deopt" | "rejected"; reason?: string; name?: string; args?: unknown }
   // A per-connection error reply: the server could not process this client's message (bad
   // table/AST, a commit/derive failure). Sent INSTEAD of crashing the process — the connection
   // stays open and every other connection is unaffected. Existing clients ignore unknown `t`.

package/src/query-error.ts

@@ -0,0 +1,42 @@
+// Retryable-vs-terminal `queryError` handling (101-QUERY-ERRORS-DESIGN.md §5, the client half
+// of FOLLOWER-LAG-SHED-DESIGN.md §6.3 — the load-bearing contract fix).
+//
+// A `queryError` used to be terminal on every source: `subs.delete(queryId)`, no re-subscribe,
+// so a server-side worker fault or a follower shedding load STRANDED the subscription forever
+// (only a transport reconnect or a seq gap ever re-subscribed). Per 101 §5 the server now
+// classifies: a frame carrying `retryable: true` (a shed follower's `code:"shed"`, a worker
+// fault's `code:"faulted"`, a lease expiry) means the subscription is still WANTED and the
+// server expects the client to come back — with jittered backoff honoring `retryAfterMs`.
+//
+// Rows already folded downstream are deliberately NOT cleared on a retryable error (101 §6 —
+// row lifecycle stays server-driven): the degraded UX is a frozen but fully-rendered consistent
+// view, the optimistic layer still absorbing local writes, then a fresh seq-0 hydrate on
+// recovery. An absent/false `retryable` keeps today's terminal behavior.
+
+/** The wire shape of a `queryError` frame (the optional fields ride 101 §4's reason shape). */
+export interface QueryErrorFields {
+  message: string;
+  /** A machine code (`"shed"`, `"faulted"`, …); absent from older servers. */
+  code?: string;
+  /** True ⇒ schedule a re-subscribe; absent/false ⇒ terminal (today's behavior). */
+  retryable?: boolean;
+  /** The server's suggested floor for the first retry delay. */
+  retryAfterMs?: number;
+}
+
+/** Cap on the exponential backoff between re-subscribe attempts. */
+const MAX_RETRY_DELAY_MS = 30_000;
+
+/** Default first-attempt delay when the server suggests none. */
+const DEFAULT_RETRY_DELAY_MS = 500;
+
+/** The `attempt`-th (0-based) re-subscribe delay: exponential from the server's suggested
+ *  `retryAfterMs` (or 500 ms when it suggests none), capped at 30 s, with up-to-25 % upward
+ *  jitter so a shed follower's re-admission gate is not hit by a thundering herd of
+ *  synchronized retries. Never returns less than the server's suggestion — the floor is
+ *  honored exactly, and the server picked it knowing its own re-admission rate. */
+export function retryDelayMs(attempt: number, retryAfterMs: number | undefined): number {
+  const base = Math.max(retryAfterMs ?? DEFAULT_RETRY_DELAY_MS, 1);
+  const backoff = Math.min(base * 2 ** attempt, MAX_RETRY_DELAY_MS);
+  return Math.round(backoff * (1 + Math.random() * 0.25));
+}

package/src/remote-source.ts

@@ -18,6 +18,7 @@ import type {
 } from "@rindle/client";
 
 import { NormalizedSubscriber } from "./normalized.ts";
+import { retryDelayMs } from "./query-error.ts";
 import type { NormalizedBatch, NormalizedHello } from "./normalized.ts";
 import { ProtocolError } from "./protocol.ts";
 import type { ServerMsg } from "./protocol.ts";
@@ -41,6 +42,10 @@ interface QState {
   resubscribing: boolean;
   /** Monotonic token that cancels stale async lease resolutions. */
   subscribeTicket: number;
+  /** Pending retryable-error re-subscribe timer (FOLLOWER-LAG-SHED §6.3), if any. */
+  retryTimer: ReturnType<typeof setTimeout> | undefined;
+  /** Consecutive retryable errors without a successful hello — the backoff exponent. */
+  retryAttempt: number;
 }
 
 export interface RemoteNormalizedSourceOptions {
@@ -71,11 +76,13 @@ export class RemoteNormalizedSource implements NormalizedSource {
   }
 
   registerQuery(qid: QueryId, remote: RemoteQuery): void {
-    this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0 });
+    this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0, retryTimer: undefined, retryAttempt: 0 });
     this.subscribe(qid, remote);
   }
 
   unregisterQuery(qid: QueryId): void {
+    const s = this.subs.get(qid);
+    if (s?.retryTimer !== undefined) clearTimeout(s.retryTimer);
     this.subs.delete(qid);
     this.transport.send({ t: "unsubscribe", queryId: qid });
   }
@@ -95,6 +102,12 @@ export class RemoteNormalizedSource implements NormalizedSource {
   private subscribe(qid: QueryId, remote: RemoteQuery): void {
     const s = this.subs.get(qid);
     if (!s) return;
+    // A fresh subscribe (gap recovery, the retry timer itself) supersedes any scheduled
+    // retryable-error retry — never leave two subscribe paths racing for one query.
+    if (s.retryTimer !== undefined) {
+      clearTimeout(s.retryTimer);
+      s.retryTimer = undefined;
+    }
     const request = { queryId: qid, remote, mode: "normalized" as const };
     const ticket = ++s.subscribeTicket;
     const send = (target: SubscribeTarget) => {
@@ -121,8 +134,7 @@ export class RemoteNormalizedSource implements NormalizedSource {
 
   private onServerMsg(msg: ServerMsg): void {
     if (msg.t === "queryError") {
-      this.subs.delete(msg.queryId);
-      console.error(`[rindle-remote] normalized query ${msg.queryId} subscription rejected: ${msg.message}`);
+      this.onQueryError(msg.queryId, msg);
       return;
     }
     // This source is normalized-only; it sees `nhello`/`nbatch` (flat frames are ignored).
@@ -133,11 +145,42 @@ export class RemoteNormalizedSource implements NormalizedSource {
     else this.applyBatch(msg.queryId, s, msg.batch);
   }
 
+  /** Route a `queryError` by its 101 §5 classification: retryable ⇒ keep the QState (and the
+   *  rows already folded downstream — 101 §6) and re-subscribe after a jittered backoff
+   *  honoring `retryAfterMs`; terminal (or pre-classification servers) ⇒ drop the
+   *  subscription, exactly as before (FOLLOWER-LAG-SHED §6.3). */
+  private onQueryError(qid: QueryId, err: { message: string; code?: string; retryable?: boolean; retryAfterMs?: number }): void {
+    const s = this.subs.get(qid);
+    if (!s) return;
+    if (err.retryable !== true) {
+      if (s.retryTimer !== undefined) clearTimeout(s.retryTimer);
+      this.subs.delete(qid);
+      console.error(`[rindle-remote] normalized query ${qid} subscription rejected: ${err.message}`);
+      return;
+    }
+    if (s.retryTimer !== undefined) return; // a retry is already scheduled — don't stack them
+    s.subscriber = null; // stop validating the dead epoch; recovery is a fresh seq-0 hydrate
+    s.resubscribing = true;
+    const delay = retryDelayMs(s.retryAttempt++, err.retryAfterMs);
+    console.warn(
+      `[rindle-remote] normalized query ${qid} ${err.code ?? "error"} (retryable): re-subscribing in ${delay}ms: ${err.message}`,
+    );
+    const timer = setTimeout(() => {
+      const cur = this.subs.get(qid);
+      if (cur !== s) return;
+      s.retryTimer = undefined;
+      this.subscribe(qid, s.remote);
+    }, delay);
+    (timer as { unref?: () => void }).unref?.();
+    s.retryTimer = timer;
+  }
+
   private openSubscriber(qid: QueryId, s: QState, hello: NormalizedHello): void {
     try {
       s.subscriber = new NormalizedSubscriber(hello, (ev) => this.handler(qid, ev), this.clientTables);
       s.epoch = hello.epoch;
       s.resubscribing = false;
+      s.retryAttempt = 0; // a successful hello resets the retryable-error backoff
     } catch (e) {
       // A comparator/fp mismatch at hello is unrecoverable (a code-contract divergence).
       s.subscriber = null;

package/src/transport.ts

@@ -29,6 +29,11 @@ export interface Transport {
  *  it reopens and fires `onReconnect` so the source rebuilds its subscriptions. */
 export class WsTransport implements Transport {
   private readonly url: string;
+  /** Reads the subprotocols to offer at each (re)connect — in affinity mode, `["rindle.v1", "aff.…"]`
+   *  with the CURRENT ticket (FOLLOWER-AFFINITY-DESIGN.md §5). Undefined ⇒ offer none (today's
+   *  single-daemon behavior, byte-identical). Evaluated per connect so a reconnect presents the
+   *  freshest (or freshly cleared) ticket. */
+  private readonly subprotocols?: () => string[];
   private ws: WebSocket;
   private handler: (msg: ServerMsg) => void = () => {};
   private reconnectHandler: () => void = () => {};
@@ -46,14 +51,18 @@ export class WsTransport implements Transport {
    *  storm while a follower is gone). */
   private downFired = false;
 
-  constructor(url: string, opts: { downThreshold?: number } = {}) {
+  constructor(url: string, opts: { downThreshold?: number; subprotocols?: () => string[] } = {}) {
     this.url = url;
     this.downThreshold = opts.downThreshold ?? 4;
+    this.subprotocols = opts.subprotocols;
     this.ws = this.connect();
   }
 
   private connect(): WebSocket {
-    const ws = new WebSocket(this.url);
+    // Offer the affinity subprotocols (base + current ticket) when configured; otherwise open bare,
+    // exactly as before. An empty list is treated as bare (never send `Sec-WebSocket-Protocol: `).
+    const protocols = this.subprotocols?.();
+    const ws = protocols && protocols.length > 0 ? new WebSocket(this.url, protocols) : new WebSocket(this.url);
     ws.addEventListener("open", () => {
       this.open = true;
       this.attempt = 0;