@rindle/remote 0.4.4 → 0.5.0

package/src/query-error.ts

@@ -0,0 +1,42 @@
+// Retryable-vs-terminal `queryError` handling (101-QUERY-ERRORS-DESIGN.md §5, the client half
+// of FOLLOWER-LAG-SHED-DESIGN.md §6.3 — the load-bearing contract fix).
+//
+// A `queryError` used to be terminal on every source: `subs.delete(queryId)`, no re-subscribe,
+// so a server-side worker fault or a follower shedding load STRANDED the subscription forever
+// (only a transport reconnect or a seq gap ever re-subscribed). Per 101 §5 the server now
+// classifies: a frame carrying `retryable: true` (a shed follower's `code:"shed"`, a worker
+// fault's `code:"faulted"`, a lease expiry) means the subscription is still WANTED and the
+// server expects the client to come back — with jittered backoff honoring `retryAfterMs`.
+//
+// Rows already folded downstream are deliberately NOT cleared on a retryable error (101 §6 —
+// row lifecycle stays server-driven): the degraded UX is a frozen but fully-rendered consistent
+// view, the optimistic layer still absorbing local writes, then a fresh seq-0 hydrate on
+// recovery. An absent/false `retryable` keeps today's terminal behavior.
+
+/** The wire shape of a `queryError` frame (the optional fields ride 101 §4's reason shape). */
+export interface QueryErrorFields {
+  message: string;
+  /** A machine code (`"shed"`, `"faulted"`, …); absent from older servers. */
+  code?: string;
+  /** True ⇒ schedule a re-subscribe; absent/false ⇒ terminal (today's behavior). */
+  retryable?: boolean;
+  /** The server's suggested floor for the first retry delay. */
+  retryAfterMs?: number;
+}
+
+/** Cap on the exponential backoff between re-subscribe attempts. */
+const MAX_RETRY_DELAY_MS = 30_000;
+
+/** Default first-attempt delay when the server suggests none. */
+const DEFAULT_RETRY_DELAY_MS = 500;
+
+/** The `attempt`-th (0-based) re-subscribe delay: exponential from the server's suggested
+ *  `retryAfterMs` (or 500 ms when it suggests none), capped at 30 s, with up-to-25 % upward
+ *  jitter so a shed follower's re-admission gate is not hit by a thundering herd of
+ *  synchronized retries. Never returns less than the server's suggestion — the floor is
+ *  honored exactly, and the server picked it knowing its own re-admission rate. */
+export function retryDelayMs(attempt: number, retryAfterMs: number | undefined): number {
+  const base = Math.max(retryAfterMs ?? DEFAULT_RETRY_DELAY_MS, 1);
+  const backoff = Math.min(base * 2 ** attempt, MAX_RETRY_DELAY_MS);
+  return Math.round(backoff * (1 + Math.random() * 0.25));
+}

package/src/remote-source.ts

@@ -18,6 +18,7 @@ import type {
 } from "@rindle/client";
 
 import { NormalizedSubscriber } from "./normalized.ts";
+import { retryDelayMs } from "./query-error.ts";
 import type { NormalizedBatch, NormalizedHello } from "./normalized.ts";
 import { ProtocolError } from "./protocol.ts";
 import type { ServerMsg } from "./protocol.ts";
@@ -41,6 +42,10 @@ interface QState {
   resubscribing: boolean;
   /** Monotonic token that cancels stale async lease resolutions. */
   subscribeTicket: number;
+  /** Pending retryable-error re-subscribe timer (FOLLOWER-LAG-SHED §6.3), if any. */
+  retryTimer: ReturnType<typeof setTimeout> | undefined;
+  /** Consecutive retryable errors without a successful hello — the backoff exponent. */
+  retryAttempt: number;
 }
 
 export interface RemoteNormalizedSourceOptions {
@@ -71,11 +76,13 @@ export class RemoteNormalizedSource implements NormalizedSource {
   }
 
   registerQuery(qid: QueryId, remote: RemoteQuery): void {
-    this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0 });
+    this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0, retryTimer: undefined, retryAttempt: 0 });
     this.subscribe(qid, remote);
   }
 
   unregisterQuery(qid: QueryId): void {
+    const s = this.subs.get(qid);
+    if (s?.retryTimer !== undefined) clearTimeout(s.retryTimer);
     this.subs.delete(qid);
     this.transport.send({ t: "unsubscribe", queryId: qid });
   }
@@ -95,6 +102,12 @@ export class RemoteNormalizedSource implements NormalizedSource {
   private subscribe(qid: QueryId, remote: RemoteQuery): void {
     const s = this.subs.get(qid);
     if (!s) return;
+    // A fresh subscribe (gap recovery, the retry timer itself) supersedes any scheduled
+    // retryable-error retry — never leave two subscribe paths racing for one query.
+    if (s.retryTimer !== undefined) {
+      clearTimeout(s.retryTimer);
+      s.retryTimer = undefined;
+    }
     const request = { queryId: qid, remote, mode: "normalized" as const };
     const ticket = ++s.subscribeTicket;
     const send = (target: SubscribeTarget) => {
@@ -121,8 +134,7 @@ export class RemoteNormalizedSource implements NormalizedSource {
 
   private onServerMsg(msg: ServerMsg): void {
     if (msg.t === "queryError") {
-      this.subs.delete(msg.queryId);
-      console.error(`[rindle-remote] normalized query ${msg.queryId} subscription rejected: ${msg.message}`);
+      this.onQueryError(msg.queryId, msg);
       return;
     }
     // This source is normalized-only; it sees `nhello`/`nbatch` (flat frames are ignored).
@@ -133,11 +145,42 @@ export class RemoteNormalizedSource implements NormalizedSource {
     else this.applyBatch(msg.queryId, s, msg.batch);
   }
 
+  /** Route a `queryError` by its 101 §5 classification: retryable ⇒ keep the QState (and the
+   *  rows already folded downstream — 101 §6) and re-subscribe after a jittered backoff
+   *  honoring `retryAfterMs`; terminal (or pre-classification servers) ⇒ drop the
+   *  subscription, exactly as before (FOLLOWER-LAG-SHED §6.3). */
+  private onQueryError(qid: QueryId, err: { message: string; code?: string; retryable?: boolean; retryAfterMs?: number }): void {
+    const s = this.subs.get(qid);
+    if (!s) return;
+    if (err.retryable !== true) {
+      if (s.retryTimer !== undefined) clearTimeout(s.retryTimer);
+      this.subs.delete(qid);
+      console.error(`[rindle-remote] normalized query ${qid} subscription rejected: ${err.message}`);
+      return;
+    }
+    if (s.retryTimer !== undefined) return; // a retry is already scheduled — don't stack them
+    s.subscriber = null; // stop validating the dead epoch; recovery is a fresh seq-0 hydrate
+    s.resubscribing = true;
+    const delay = retryDelayMs(s.retryAttempt++, err.retryAfterMs);
+    console.warn(
+      `[rindle-remote] normalized query ${qid} ${err.code ?? "error"} (retryable): re-subscribing in ${delay}ms: ${err.message}`,
+    );
+    const timer = setTimeout(() => {
+      const cur = this.subs.get(qid);
+      if (cur !== s) return;
+      s.retryTimer = undefined;
+      this.subscribe(qid, s.remote);
+    }, delay);
+    (timer as { unref?: () => void }).unref?.();
+    s.retryTimer = timer;
+  }
+
   private openSubscriber(qid: QueryId, s: QState, hello: NormalizedHello): void {
     try {
       s.subscriber = new NormalizedSubscriber(hello, (ev) => this.handler(qid, ev), this.clientTables);
       s.epoch = hello.epoch;
       s.resubscribing = false;
+      s.retryAttempt = 0; // a successful hello resets the retryable-error backoff
     } catch (e) {
       // A comparator/fp mismatch at hello is unrecoverable (a code-contract divergence).
       s.subscriber = null;