@rindle/remote 0.4.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,17 @@
1
+ /** The wire shape of a `queryError` frame (the optional fields ride 101 §4's reason shape). */
2
+ export interface QueryErrorFields {
3
+ message: string;
4
+ /** A machine code (`"shed"`, `"faulted"`, …); absent from older servers. */
5
+ code?: string;
6
+ /** True ⇒ schedule a re-subscribe; absent/false ⇒ terminal (today's behavior). */
7
+ retryable?: boolean;
8
+ /** The server's suggested floor for the first retry delay. */
9
+ retryAfterMs?: number;
10
+ }
11
+ /** The `attempt`-th (0-based) re-subscribe delay: exponential from the server's suggested
12
+ * `retryAfterMs` (or 500 ms when it suggests none), capped at 30 s, with up-to-25 % upward
13
+ * jitter so a shed follower's re-admission gate is not hit by a thundering herd of
14
+ * synchronized retries. Never returns less than the server's suggestion — the floor is
15
+ * honored exactly, and the server picked it knowing its own re-admission rate. */
16
+ export declare function retryDelayMs(attempt: number, retryAfterMs: number | undefined): number;
17
+ //# sourceMappingURL=query-error.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"query-error.d.ts","sourceRoot":"","sources":["../src/query-error.ts"],"names":[],"mappings":"AAeA,+FAA+F;AAC/F,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,4EAA4E;IAC5E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kFAAkF;IAClF,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,8DAA8D;IAC9D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAQD;;;;mFAImF;AACnF,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAItF"}
@@ -0,0 +1,29 @@
1
+ // Retryable-vs-terminal `queryError` handling (101-QUERY-ERRORS-DESIGN.md §5, the client half
2
+ // of FOLLOWER-LAG-SHED-DESIGN.md §6.3 — the load-bearing contract fix).
3
+ //
4
+ // A `queryError` used to be terminal on every source: `subs.delete(queryId)`, no re-subscribe,
5
+ // so a server-side worker fault or a follower shedding load STRANDED the subscription forever
6
+ // (only a transport reconnect or a seq gap ever re-subscribed). Per 101 §5 the server now
7
+ // classifies: a frame carrying `retryable: true` (a shed follower's `code:"shed"`, a worker
8
+ // fault's `code:"faulted"`, a lease expiry) means the subscription is still WANTED and the
9
+ // server expects the client to come back — with jittered backoff honoring `retryAfterMs`.
10
+ //
11
+ // Rows already folded downstream are deliberately NOT cleared on a retryable error (101 §6 —
12
+ // row lifecycle stays server-driven): the degraded UX is a frozen but fully-rendered consistent
13
+ // view, the optimistic layer still absorbing local writes, then a fresh seq-0 hydrate on
14
+ // recovery. An absent/false `retryable` keeps today's terminal behavior.
15
+ /** Cap on the exponential backoff between re-subscribe attempts. */
16
+ const MAX_RETRY_DELAY_MS = 30_000;
17
+ /** Default first-attempt delay when the server suggests none. */
18
+ const DEFAULT_RETRY_DELAY_MS = 500;
19
+ /** The `attempt`-th (0-based) re-subscribe delay: exponential from the server's suggested
20
+ * `retryAfterMs` (or 500 ms when it suggests none), capped at 30 s, with up-to-25 % upward
21
+ * jitter so a shed follower's re-admission gate is not hit by a thundering herd of
22
+ * synchronized retries. Never returns less than the server's suggestion — the floor is
23
+ * honored exactly, and the server picked it knowing its own re-admission rate. */
24
+ export function retryDelayMs(attempt, retryAfterMs) {
25
+ const base = Math.max(retryAfterMs ?? DEFAULT_RETRY_DELAY_MS, 1);
26
+ const backoff = Math.min(base * 2 ** attempt, MAX_RETRY_DELAY_MS);
27
+ return Math.round(backoff * (1 + Math.random() * 0.25));
28
+ }
29
+ //# sourceMappingURL=query-error.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"query-error.js","sourceRoot":"","sources":["../src/query-error.ts"],"names":[],"mappings":"AAAA,8FAA8F;AAC9F,wEAAwE;AACxE,EAAE;AACF,+FAA+F;AAC/F,8FAA8F;AAC9F,0FAA0F;AAC1F,4FAA4F;AAC5F,2FAA2F;AAC3F,0FAA0F;AAC1F,EAAE;AACF,6FAA6F;AAC7F,gGAAgG;AAChG,yFAAyF;AACzF,yEAAyE;AAazE,oEAAoE;AACpE,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,iEAAiE;AACjE,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC;;;;mFAImF;AACnF,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,YAAgC;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,IAAI,sBAAsB,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAClE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;AAC1D,CAAC"}
@@ -23,6 +23,11 @@ export declare class RemoteNormalizedSource implements NormalizedSource {
23
23
  onNormalized(handler: (qid: QueryId, ev: NormalizedEvent) => void): void;
24
24
  private subscribe;
25
25
  private onServerMsg;
26
+ /** Route a `queryError` by its 101 §5 classification: retryable ⇒ keep the QState (and the
27
+ * rows already folded downstream — 101 §6) and re-subscribe after a jittered backoff
28
+ * honoring `retryAfterMs`; terminal (or pre-classification servers) ⇒ drop the
29
+ * subscription, exactly as before (FOLLOWER-LAG-SHED §6.3). */
30
+ private onQueryError;
26
31
  private openSubscriber;
27
32
  private applyBatch;
28
33
  }
@@ -1 +1 @@
1
- {"version":3,"file":"remote-source.d.ts","sourceRoot":"","sources":["../src/remote-source.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EACV,QAAQ,EACR,eAAe,EACf,gBAAgB,EAChB,qBAAqB,EACrB,OAAO,EACP,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAMxB,OAAO,EAIL,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EAEvB,MAAM,gBAAgB,CAAC;AAExB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAahD,MAAM,WAAW,6BAA6B;IAC5C,wFAAwF;IACxF,gBAAgB,CAAC,EAAE,iBAAiB,CAAC;IACrC,8EAA8E;IAC9E,YAAY,CAAC,EAAE,iBAAiB,CAAC;CAClC;AAED,qBAAa,sBAAuB,YAAW,gBAAgB;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoB;IACrD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAoB;IAClD,OAAO,CAAC,OAAO,CAAyD;IACxE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA8B;IACnD,mGAAmG;IACnG,OAAO,CAAC,YAAY,CAAsC;gBAE9C,SAAS,EAAE,SAAS,EAAE,IAAI,GAAE,6BAAkC;IAO1E,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,EAAE,GAAG,IAAI;IAIzD,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI;IAKtD,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IAKnC,MAAM,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM5C,YAAY,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,eAAe,KAAK,IAAI,GAAG,IAAI;IAMxE,OAAO,CAAC,SAAS;IA2BjB,OAAO,CAAC,WAAW;IAcnB,OAAO,CAAC,cAAc;IAYtB,OAAO,CAAC,UAAU;CAcnB;AAED,mFAAmF;AACnF,wBAAgB,4BAA4B,CAC1C,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,IAAI,GAAE,6BAAkC,GACvC,sBAAsB,CAGxB"}
1
+ {"version":3,"file":"remote-source.d.ts","sourceRoot":"","sources":["../src/remote-source.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EACV,QAAQ,EACR,eAAe,EACf,gBAAgB,EAChB,qBAAqB,EACrB,OAAO,EACP,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAOxB,OAAO,EAIL,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EAEvB,MAAM,gBAAgB,CAAC;AAExB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAiBhD,MAAM,WAAW,6BAA6B;IAC5C,wFAAwF;IACxF,gBAAgB,CAAC,EAAE,iBAAiB,CAAC;IACrC,8EAA8E;IAC9E,YAAY,CAAC,EAAE,iBAAiB,CAAC;CAClC;AAED,qBAAa,sBAAuB,YAAW,gBAAgB;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoB;IACrD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAoB;IAClD,OAAO,CAAC,OAAO,CAAyD;IACxE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA8B;IACnD,mGAAmG;IACnG,OAAO,CAAC,YAAY,CAAsC;gBAE9C,SAAS,EAAE,SAAS,EAAE,IAAI,GAAE,6BAAkC;IAO1E,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,EAAE,GAAG,IAAI;IAIzD,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI;IAKtD,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IAOnC,MAAM,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM5C,YAAY,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,eAAe,KAAK,IAAI,GAAG,IAAI;IAMxE,OAAO,CAAC,SAAS;IAiCjB,OAAO,CAAC,WAAW;IAanB;;;oEAGgE;IAChE,OAAO,CAAC,YAAY;IA0BpB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,UAAU;CAcnB;AAED,mFAAmF;AACnF,wBAAgB,4BAA4B,CAC1C,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,IAAI,GAAE,6BAAkC,GACvC,sBAAsB,CAGxB"}
@@ -8,6 +8,7 @@
8
8
  // epoch and replies with a fresh hello + snapshot, and `NormalizedSync` diffs the new footprint
9
9
  // against the old (the set-analogue re-hydrate, §5.3).
10
10
  import { NormalizedSubscriber } from "./normalized.js";
11
+ import { retryDelayMs } from "./query-error.js";
11
12
  import { ProtocolError } from "./protocol.js";
12
13
  import { defaultSubscribeTarget, isThenable, subscribeMessage, } from "./subscribe.js";
13
14
  import { WsTransport } from "./transport.js";
@@ -29,10 +30,13 @@ export class RemoteNormalizedSource {
29
30
  this.clientTables = tables;
30
31
  }
31
32
  registerQuery(qid, remote) {
32
- this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0 });
33
+ this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0, retryTimer: undefined, retryAttempt: 0 });
33
34
  this.subscribe(qid, remote);
34
35
  }
35
36
  unregisterQuery(qid) {
37
+ const s = this.subs.get(qid);
38
+ if (s?.retryTimer !== undefined)
39
+ clearTimeout(s.retryTimer);
36
40
  this.subs.delete(qid);
37
41
  this.transport.send({ t: "unsubscribe", queryId: qid });
38
42
  }
@@ -50,6 +54,12 @@ export class RemoteNormalizedSource {
50
54
  const s = this.subs.get(qid);
51
55
  if (!s)
52
56
  return;
57
+ // A fresh subscribe (gap recovery, the retry timer itself) supersedes any scheduled
58
+ // retryable-error retry — never leave two subscribe paths racing for one query.
59
+ if (s.retryTimer !== undefined) {
60
+ clearTimeout(s.retryTimer);
61
+ s.retryTimer = undefined;
62
+ }
53
63
  const request = { queryId: qid, remote, mode: "normalized" };
54
64
  const ticket = ++s.subscribeTicket;
55
65
  const send = (target) => {
@@ -78,8 +88,7 @@ export class RemoteNormalizedSource {
78
88
  }
79
89
  onServerMsg(msg) {
80
90
  if (msg.t === "queryError") {
81
- this.subs.delete(msg.queryId);
82
- console.error(`[rindle-remote] normalized query ${msg.queryId} subscription rejected: ${msg.message}`);
91
+ this.onQueryError(msg.queryId, msg);
83
92
  return;
84
93
  }
85
94
  // This source is normalized-only; it sees `nhello`/`nbatch` (flat frames are ignored).
@@ -93,11 +102,43 @@ export class RemoteNormalizedSource {
93
102
  else
94
103
  this.applyBatch(msg.queryId, s, msg.batch);
95
104
  }
105
+ /** Route a `queryError` by its 101 §5 classification: retryable ⇒ keep the QState (and the
106
+ * rows already folded downstream — 101 §6) and re-subscribe after a jittered backoff
107
+ * honoring `retryAfterMs`; terminal (or pre-classification servers) ⇒ drop the
108
+ * subscription, exactly as before (FOLLOWER-LAG-SHED §6.3). */
109
+ onQueryError(qid, err) {
110
+ const s = this.subs.get(qid);
111
+ if (!s)
112
+ return;
113
+ if (err.retryable !== true) {
114
+ if (s.retryTimer !== undefined)
115
+ clearTimeout(s.retryTimer);
116
+ this.subs.delete(qid);
117
+ console.error(`[rindle-remote] normalized query ${qid} subscription rejected: ${err.message}`);
118
+ return;
119
+ }
120
+ if (s.retryTimer !== undefined)
121
+ return; // a retry is already scheduled — don't stack them
122
+ s.subscriber = null; // stop validating the dead epoch; recovery is a fresh seq-0 hydrate
123
+ s.resubscribing = true;
124
+ const delay = retryDelayMs(s.retryAttempt++, err.retryAfterMs);
125
+ console.warn(`[rindle-remote] normalized query ${qid} ${err.code ?? "error"} (retryable): re-subscribing in ${delay}ms: ${err.message}`);
126
+ const timer = setTimeout(() => {
127
+ const cur = this.subs.get(qid);
128
+ if (cur !== s)
129
+ return;
130
+ s.retryTimer = undefined;
131
+ this.subscribe(qid, s.remote);
132
+ }, delay);
133
+ timer.unref?.();
134
+ s.retryTimer = timer;
135
+ }
96
136
  openSubscriber(qid, s, hello) {
97
137
  try {
98
138
  s.subscriber = new NormalizedSubscriber(hello, (ev) => this.handler(qid, ev), this.clientTables);
99
139
  s.epoch = hello.epoch;
100
140
  s.resubscribing = false;
141
+ s.retryAttempt = 0; // a successful hello resets the retryable-error backoff
101
142
  }
102
143
  catch (e) {
103
144
  // A comparator/fp mismatch at hello is unrecoverable (a code-contract divergence).
@@ -1 +1 @@
1
- {"version":3,"file":"remote-source.js","sourceRoot":"","sources":["../src/remote-source.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAC5F,8FAA8F;AAC9F,4FAA4F;AAC5F,6FAA6F;AAC7F,0FAA0F;AAC1F,EAAE;AACF,+FAA+F;AAC/F,gGAAgG;AAChG,uDAAuD;AAWvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,EACL,sBAAsB,EACtB,UAAU,EACV,gBAAgB,GAIjB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAqB7C,MAAM,OAAO,sBAAsB;IAChB,SAAS,CAAY;IACrB,gBAAgB,CAAoB;IACpC,YAAY,CAAqB;IAC1C,OAAO,GAAgD,GAAG,EAAE,GAAE,CAAC,CAAC;IACvD,IAAI,GAAG,IAAI,GAAG,EAAmB,CAAC;IACnD,mGAAmG;IAC3F,YAAY,CAAsC;IAE1D,YAAY,SAAoB,EAAE,OAAsC,EAAE;QACxE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,sBAAsB,CAAC;QACxE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,kBAAkB,CAAC,MAA+B;QAChD,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;IAC7B,CAAC;IAED,aAAa,CAAC,GAAY,EAAE,MAAmB;QAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED,eAAe,CAAC,GAAY;QAC1B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,aAAa,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,SAAqB;QAC1B,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5E,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAChD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,YAAY,CAAC,OAAoD;QAC/D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,gFAAgF;IAExE,SAAS,CAAC,GAAY,EAAE,MAAmB;QACjD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,YAAqB,EAAE,CAAC;QACtE,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,eAAe,CAAC;QACnC,MAAM,IAAI,GAAG,CAAC,MAAuB,EAAE,EAAE;YACvC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,eAAe,KAAK,MAAM;gBAAE,OAAO;YACxD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC;QACF,MAAM,IAAI,GAAG,CAAC,GAAY,EAAE,EAAE;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,eAAe,KAAK,MAAM;gBAAE,OAAO;YACxD,CAAC,CAAC,aAAa,GAAG,KAAK,CAAC;YACxB,OAAO,CAAC,KAAK,CACX,oCAAoC,GAAG,iCAAiC,MAAM,CAAE,GAAa,EAAE,OAAO,IAAI,GAAG,CAAC,EAAE,CACjH,CAAC;QACJ,CAAC,CAAC;QACF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC9C,IAAI,UAAU,CAAC,MAAM,CAAC;gBAAE,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;;gBAChD,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,GAAc;QAChC,IAAI,GAAG,CAAC,CAAC,KAAK,YAAY,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,CAAC,KAAK,CAAC,oCAAoC,GAAG,CAAC,OAAO,2BAA2B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YACvG,OAAO;QACT,CAAC;QACD,uFAAuF;QACvF,IAAI,GAAG,CAAC,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,CAAC,KAAK,QAAQ;YAAE,OAAO;QACrD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,CAAC;YAAE,OAAO,CAAC,+BAA+B;QAC/C,IAAI,GAAG,CAAC,CAAC,KAAK,QAAQ;YAAE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;;YAClE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IAClD,CAAC;IAEO,cAAc,CAAC,GAAY,EAAE,CAAS,EAAE,KAAsB;QACpE,IAAI,CAAC;YACH,CAAC,CAAC,UAAU,GAAG,IAAI,oBAAoB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YACjG,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YACtB,CAAC,CAAC,aAAa,GAAG,KAAK,CAAC;QAC1B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,mFAAmF;YACnF,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,oCAAoC,GAAG,2BAA4B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1G,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,GAAY,EAAE,CAAS,EAAE,KAAsB;QAChE,IAAI,CAAC,CAAC,CAAC,UAAU;YAAE,OAAO,CAAC,mCAAmC;QAC9D,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;YAAE,OAAO,CAAC,+CAA+C;QAClF,IAAI,CAAC;YACH,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,CAAC,CAAC,YAAY,aAAa,CAAC;gBAAE,MAAM,CAAC,CAAC;YAC3C,IAAI,CAAC,CAAC,aAAa;gBAAE,OAAO,CAAC,qBAAqB;YAClD,oFAAoF;YACpF,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF;AAED,mFAAmF;AACnF,MAAM,UAAU,4BAA4B,CAC1C,cAAkC,EAClC,OAAsC,EAAE;IAExC,MAAM,SAAS,GAAG,OAAO,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;IACxG,OAAO,IAAI,sBAAsB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AACrD,CAAC"}
1
+ {"version":3,"file":"remote-source.js","sourceRoot":"","sources":["../src/remote-source.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAC5F,8FAA8F;AAC9F,4FAA4F;AAC5F,6FAA6F;AAC7F,0FAA0F;AAC1F,EAAE;AACF,+FAA+F;AAC/F,gGAAgG;AAChG,uDAAuD;AAWvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEhD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,EACL,sBAAsB,EACtB,UAAU,EACV,gBAAgB,GAIjB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAyB7C,MAAM,OAAO,sBAAsB;IAChB,SAAS,CAAY;IACrB,gBAAgB,CAAoB;IACpC,YAAY,CAAqB;IAC1C,OAAO,GAAgD,GAAG,EAAE,GAAE,CAAC,CAAC;IACvD,IAAI,GAAG,IAAI,GAAG,EAAmB,CAAC;IACnD,mGAAmG;IAC3F,YAAY,CAAsC;IAE1D,YAAY,SAAoB,EAAE,OAAsC,EAAE;QACxE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,sBAAsB,CAAC;QACxE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,kBAAkB,CAAC,MAA+B;QAChD,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;IAC7B,CAAC;IAED,aAAa,CAAC,GAAY,EAAE,MAAmB;QAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7I,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED,eAAe,CAAC,GAAY;QAC1B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,EAAE,UAAU,KAAK,SAAS;YAAE,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,aAAa,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,SAAqB;QAC1B,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5E,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAChD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,YAAY,CAAC,OAAoD;QAC/D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,gFAAgF;IAExE,SAAS,CAAC,GAAY,EAAE,MAAmB;QACjD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,oFAAoF;QACpF,gFAAgF;QAChF,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC/B,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAC3B,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC;QAC3B,CAAC;QACD,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,YAAqB,EAAE,CAAC;QACtE,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,eAAe,CAAC;QACnC,MAAM,IAAI,GAAG,CAAC,MAAuB,EAAE,EAAE;YACvC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,eAAe,KAAK,MAAM;gBAAE,OAAO;YACxD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC;QACF,MAAM,IAAI,GAAG,CAAC,GAAY,EAAE,EAAE;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,eAAe,KAAK,MAAM;gBAAE,OAAO;YACxD,CAAC,CAAC,aAAa,GAAG,KAAK,CAAC;YACxB,OAAO,CAAC,KAAK,CACX,oCAAoC,GAAG,iCAAiC,MAAM,CAAE,GAAa,EAAE,OAAO,IAAI,GAAG,CAAC,EAAE,CACjH,CAAC;QACJ,CAAC,CAAC;QACF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC9C,IAAI,UAAU,CAAC,MAAM,CAAC;gBAAE,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;;gBAChD,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,GAAc;QAChC,IAAI,GAAG,CAAC,CAAC,KAAK,YAAY,EAAE,CAAC;YAC3B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACpC,OAAO;QACT,CAAC;QACD,uFAAuF;QACvF,IAAI,GAAG,CAAC,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,CAAC,KAAK,QAAQ;YAAE,OAAO;QACrD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,CAAC;YAAE,OAAO,CAAC,+BAA+B;QAC/C,IAAI,GAAG,CAAC,CAAC,KAAK,QAAQ;YAAE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;;YAClE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IAClD,CAAC;IAED;;;oEAGgE;IACxD,YAAY,CAAC,GAAY,EAAE,GAAmF;QACpH,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS;gBAAE,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAC3D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,oCAAoC,GAAG,2BAA2B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/F,OAAO;QACT,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS;YAAE,OAAO,CAAC,kDAAkD;QAC1F,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,oEAAoE;QACzF,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC;QACvB,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,EAAE,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;QAC/D,OAAO,CAAC,IAAI,CACV,oCAAoC,GAAG,IAAI,GAAG,CAAC,IAAI,IAAI,OAAO,mCAAmC,KAAK,OAAO,GAAG,CAAC,OAAO,EAAE,CAC3H,CAAC;QACF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,GAAG,KAAK,CAAC;gBAAE,OAAO;YACtB,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC;YACzB,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC,EAAE,KAAK,CAAC,CAAC;QACT,KAAgC,CAAC,KAAK,EAAE,EAAE,CAAC;QAC5C,CAAC,CAAC,UAAU,GAAG,KAAK,CAAC;IACvB,CAAC;IAEO,cAAc,CAAC,GAAY,EAAE,CAAS,EAAE,KAAsB;QACpE,IAAI,CAAC;YACH,CAAC,CAAC,UAAU,GAAG,IAAI,oBAAoB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YACjG,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YACtB,CAAC,CAAC,aAAa,GAAG,KAAK,CAAC;YACxB,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,wDAAwD;QAC9E,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,mFAAmF;YACnF,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,oCAAoC,GAAG,2BAA4B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1G,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,GAAY,EAAE,CAAS,EAAE,KAAsB;QAChE,IAAI,CAAC,CAAC,CAAC,UAAU;YAAE,OAAO,CAAC,mCAAmC;QAC9D,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;YAAE,OAAO,CAAC,+CAA+C;QAClF,IAAI,CAAC;YACH,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,CAAC,CAAC,YAAY,aAAa,CAAC;gBAAE,MAAM,CAAC,CAAC;YAC3C,IAAI,CAAC,CAAC,aAAa;gBAAE,OAAO,CAAC,qBAAqB;YAClD,oFAAoF;YACpF,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF;AAED,mFAAmF;AACnF,MAAM,UAAU,4BAA4B,CAC1C,cAAkC,EAClC,OAAsC,EAAE;IAExC,MAAM,SAAS,GAAG,OAAO,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;IACxG,OAAO,IAAI,sBAAsB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AACrD,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@rindle/remote",
3
- "version": "0.4.4",
3
+ "version": "0.5.0",
4
4
  "license": "Apache-2.0",
5
5
  "repository": {
6
6
  "type": "git",
@@ -25,7 +25,7 @@
25
25
  "README.md"
26
26
  ],
27
27
  "dependencies": {
28
- "@rindle/client": "0.4.4"
28
+ "@rindle/client": "0.5.0"
29
29
  },
30
30
  "devDependencies": {
31
31
  "@types/node": "^22.10.0",
package/src/backend.ts CHANGED
@@ -20,6 +20,7 @@ import {
20
20
  type SubscribeResolver,
21
21
  } from "./subscribe.ts";
22
22
  import { WsTransport } from "./transport.ts";
23
+ import { retryDelayMs } from "./query-error.ts";
23
24
  import type { Transport } from "./transport.ts";
24
25
 
25
26
  interface QState {
@@ -31,6 +32,10 @@ interface QState {
31
32
  resubscribing: boolean;
32
33
  /** Monotonic token that cancels stale async lease resolutions. */
33
34
  subscribeTicket: number;
35
+ /** Pending retryable-error re-subscribe timer (FOLLOWER-LAG-SHED §6.3), if any. */
36
+ retryTimer: ReturnType<typeof setTimeout> | undefined;
37
+ /** Consecutive retryable errors without a successful hello — the backoff exponent. */
38
+ retryAttempt: number;
34
39
  }
35
40
 
36
41
  export interface RemoteBackendOptions {
@@ -60,11 +65,13 @@ export class RemoteBackend implements Backend {
60
65
  console.error(`[rindle-remote] flat query ${qid} has no named remote identity`);
61
66
  return;
62
67
  }
63
- this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0 });
68
+ this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0, retryTimer: undefined, retryAttempt: 0 });
64
69
  this.subscribe(qid, remote);
65
70
  }
66
71
 
67
72
  unregisterQuery(qid: QueryId): void {
73
+ const s = this.subs.get(qid);
74
+ if (s?.retryTimer !== undefined) clearTimeout(s.retryTimer);
68
75
  this.subs.delete(qid);
69
76
  this.transport.send({ t: "unsubscribe", queryId: qid });
70
77
  }
@@ -93,6 +100,12 @@ export class RemoteBackend implements Backend {
93
100
  private subscribe(qid: QueryId, remote: RemoteQuery): void {
94
101
  const s = this.subs.get(qid);
95
102
  if (!s) return;
103
+ // A fresh subscribe (gap recovery, the retry timer itself) supersedes any scheduled
104
+ // retryable-error retry — never leave two subscribe paths racing for one query.
105
+ if (s.retryTimer !== undefined) {
106
+ clearTimeout(s.retryTimer);
107
+ s.retryTimer = undefined;
108
+ }
96
109
  const request = { queryId: qid, remote, mode: "flat" as const };
97
110
  const ticket = ++s.subscribeTicket;
98
111
  const send = (target: ReturnType<typeof defaultSubscribeTarget>) => {
@@ -117,8 +130,7 @@ export class RemoteBackend implements Backend {
117
130
 
118
131
  private onServerMsg(msg: ServerMsg): void {
119
132
  if (msg.t === "queryError") {
120
- this.subs.delete(msg.queryId);
121
- console.error(`[rindle-remote] query ${msg.queryId} subscription rejected: ${msg.message}`);
133
+ this.onQueryError(msg.queryId, msg);
122
134
  return;
123
135
  }
124
136
  // This backend is flat-only; it ignores normalized frames (`nhello`/`nbatch`).
@@ -129,6 +141,36 @@ export class RemoteBackend implements Backend {
129
141
  else this.applyBatch(msg.queryId, s, msg.batch);
130
142
  }
131
143
 
144
+ /** Route a `queryError` by its 101 §5 classification: retryable ⇒ keep the QState (and the
145
+ * rows already folded downstream — 101 §6) and re-subscribe after a jittered backoff
146
+ * honoring `retryAfterMs`; terminal (or pre-classification servers) ⇒ drop the
147
+ * subscription, exactly as before (FOLLOWER-LAG-SHED §6.3). */
148
+ private onQueryError(qid: QueryId, err: { message: string; code?: string; retryable?: boolean; retryAfterMs?: number }): void {
149
+ const s = this.subs.get(qid);
150
+ if (!s) return;
151
+ if (err.retryable !== true) {
152
+ if (s.retryTimer !== undefined) clearTimeout(s.retryTimer);
153
+ this.subs.delete(qid);
154
+ console.error(`[rindle-remote] query ${qid} subscription rejected: ${err.message}`);
155
+ return;
156
+ }
157
+ if (s.retryTimer !== undefined) return; // a retry is already scheduled — don't stack them
158
+ s.subscriber = null; // stop validating the dead epoch; recovery is a fresh seq-0 hydrate
159
+ s.resubscribing = true;
160
+ const delay = retryDelayMs(s.retryAttempt++, err.retryAfterMs);
161
+ console.warn(
162
+ `[rindle-remote] query ${qid} ${err.code ?? "error"} (retryable): re-subscribing in ${delay}ms: ${err.message}`,
163
+ );
164
+ const timer = setTimeout(() => {
165
+ const cur = this.subs.get(qid);
166
+ if (cur !== s) return;
167
+ s.retryTimer = undefined;
168
+ this.subscribe(qid, s.remote);
169
+ }, delay);
170
+ (timer as { unref?: () => void }).unref?.();
171
+ s.retryTimer = timer;
172
+ }
173
+
132
174
  private openSubscriber(qid: QueryId, s: QState, hello: Hello): void {
133
175
  try {
134
176
  // The Subscriber ctor validates the comparator + fingerprint and emits the `hello`
@@ -136,6 +178,7 @@ export class RemoteBackend implements Backend {
136
178
  s.subscriber = new Subscriber(hello, (ev) => this.emitServerEvent(qid, ev));
137
179
  s.epoch = hello.epoch;
138
180
  s.resubscribing = false;
181
+ s.retryAttempt = 0; // a successful hello resets the retryable-error backoff
139
182
  } catch (e) {
140
183
  // A comparator/schema mismatch at hello is unrecoverable (a code-contract divergence) —
141
184
  // leave the view pending and report it; do not loop.
@@ -16,6 +16,7 @@
16
16
  import { LMID_QUERY_NAME } from "@rindle/client";
17
17
  import type {
18
18
  MutationEnvelope,
19
+ MutationOutcomeFrame,
19
20
  NormalizedEvent,
20
21
  NormalizedTableSchema,
21
22
  OptimisticSource,
@@ -25,6 +26,7 @@ import type {
25
26
  } from "@rindle/client";
26
27
 
27
28
  import { NormalizedSubscriber } from "./normalized.ts";
29
+ import { retryDelayMs } from "./query-error.ts";
28
30
  import type { NormalizedBatch, NormalizedHello } from "./normalized.ts";
29
31
  import { ProtocolError } from "./protocol.ts";
30
32
  import type { ServerMsg } from "./protocol.ts";
@@ -48,6 +50,16 @@ interface QState {
48
50
  resubscribing: boolean;
49
51
  /** Monotonic token that cancels stale async lease resolutions. */
50
52
  subscribeTicket: number;
53
+ /** Pending retryable-error re-subscribe timer (FOLLOWER-LAG-SHED §6.3), if any. */
54
+ retryTimer: ReturnType<typeof setTimeout> | undefined;
55
+ /** Consecutive retryable errors without a successful hello — the backoff exponent. */
56
+ retryAttempt: number;
57
+ /** Whether the LAST subscribe sent for this query presented a `leaseToken` — i.e. its hello
58
+ * proves the connection is (re-)AUTHENTICATED (a room shell sets the socket's subject from the
59
+ * first verified token; `pushMutation` requires it). After a reconnect on a lease-auth session,
60
+ * queued pushes flush at the first such hello, never earlier — an envelope racing ahead of the
61
+ * token subscribe would be refused by the shell's subject gate (H-v §7.5 rule 3). */
62
+ authed: boolean;
51
63
  }
52
64
 
53
65
  /** Build a transport to a follower's public ws endpoint (READ-ROUTER-DESIGN.md §2.3). Default
@@ -86,6 +98,19 @@ export class RemoteOptimisticSource implements OptimisticSource {
86
98
  private handler: (qid: QueryId, ev: NormalizedEvent) => void = () => {};
87
99
  private progressHandler: (frame: ProgressFrame) => void = () => {};
88
100
  private restartHandler: () => void = () => {};
101
+ private bootIdHandler: (bootId: string) => void = () => {};
102
+ private outcomeHandler: (frame: MutationOutcomeFrame) => void = () => {};
103
+ private resyncHandler: () => void = () => {};
104
+ /** Set by {@link resync} when this is a LEASE-AUTH session (some sub presented a `leaseToken`):
105
+ * transport pushes queue in {@link pendingPushes} until the first authenticated re-subscribe's
106
+ * hello re-establishes the socket's subject, then flush (see QState.authed). Never set on a
107
+ * token-less (embedded/rindled) session — its pushes need no subject and go straight out. */
108
+ private awaitingAuthedHello = false;
109
+ /** Envelopes held while {@link awaitingAuthedHello} (H-v §7.5 rule 3). Every entry corresponds
110
+ * to a still-pending backend mutation (the re-send reconstructs from pending entries; app
111
+ * invokes in the window are pending by definition), so a superseding resync may CLEAR this —
112
+ * its own re-send regenerates whatever still matters. */
113
+ private pendingPushes: MutationEnvelope[] = [];
89
114
  private readonly subs = new Map<QueryId, QState>();
90
115
  /** Queries whose subscribe is waiting for a transport to exist — endpoint-less subscribes issued
91
116
  * in pure-lazy mode before any lease opens a transport (the lmid system query is registered by
@@ -196,8 +221,12 @@ export class RemoteOptimisticSource implements OptimisticSource {
196
221
  close(): void {
197
222
  this.closed = true;
198
223
  this.transport?.close();
224
+ for (const s of this.subs.values()) {
225
+ if (s.retryTimer !== undefined) clearTimeout(s.retryTimer);
226
+ }
199
227
  this.subs.clear();
200
228
  this.deferred.clear();
229
+ this.pendingPushes.length = 0;
201
230
  }
202
231
 
203
232
  /** Register a handler fired when the DAEMON restarts (a new boot id) — the backend resets its
@@ -206,16 +235,36 @@ export class RemoteOptimisticSource implements OptimisticSource {
206
235
  this.restartHandler = handler;
207
236
  }
208
237
 
238
+ /** Register the observed boot-id stream (`301-ECHO-FENCE-DESIGN.md` §2.4): fired on the FIRST
239
+ * observed daemon boot id and on every change — the backend's client-local boot ordinals for
240
+ * the direction-B pin fence. A late registration is replayed the current id immediately, so
241
+ * ordinal 0 is never missed on an already-open connection. */
242
+ onBootId(handler: (bootId: string) => void): void {
243
+ this.bootIdHandler = handler;
244
+ if (this.lastBootId !== undefined) handler(this.lastBootId);
245
+ }
246
+
209
247
  expectClientSchema(tables: NormalizedTableSchema[]): void {
210
248
  this.clientTables = tables;
211
249
  }
212
250
 
213
251
  registerQuery(qid: QueryId, remote: RemoteQuery): void {
214
- this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0 });
252
+ this.subs.set(qid, {
253
+ remote,
254
+ subscriber: null,
255
+ epoch: 0,
256
+ resubscribing: false,
257
+ subscribeTicket: 0,
258
+ retryTimer: undefined,
259
+ retryAttempt: 0,
260
+ authed: false,
261
+ });
215
262
  this.subscribe(qid, remote);
216
263
  }
217
264
 
218
265
  unregisterQuery(qid: QueryId): void {
266
+ const s = this.subs.get(qid);
267
+ if (s?.retryTimer !== undefined) clearTimeout(s.retryTimer);
219
268
  this.subs.delete(qid);
220
269
  this.deferred.delete(qid);
221
270
  this.transport?.send({ t: "unsubscribe", queryId: qid });
@@ -223,6 +272,13 @@ export class RemoteOptimisticSource implements OptimisticSource {
223
272
 
224
273
  pushMutation(envelope: MutationEnvelope): Promise<void> {
225
274
  if (this.pushMutationSender) return Promise.resolve(this.pushMutationSender(envelope));
275
+ // A lease-auth session that just reconnected is not yet re-authenticated (the shell's
276
+ // `pushMutation` subject gate would refuse) — hold the envelope until the first token
277
+ // re-subscribe's hello, then flush in order (H-v §7.5 rule 3).
278
+ if (this.awaitingAuthedHello) {
279
+ this.pendingPushes.push(envelope);
280
+ return Promise.resolve();
281
+ }
226
282
  this.transport?.send({ t: "pushMutation", envelope });
227
283
  return Promise.resolve();
228
284
  }
@@ -235,11 +291,33 @@ export class RemoteOptimisticSource implements OptimisticSource {
235
291
  this.progressHandler = handler;
236
292
  }
237
293
 
294
+ /** The room deopt handshake's verdict stream (H-v). Dispatched OUT-OF-BAND on arrival — see
295
+ * {@link onServerMsg}'s `mutationOutcome` arm for why it must never wait behind the cv buffer. */
296
+ onMutationOutcome(handler: (frame: MutationOutcomeFrame) => void): void {
297
+ this.outcomeHandler = handler;
298
+ }
299
+
300
+ /** Fired once per re-established session, SYNCHRONOUSLY inside {@link resync} — before any
301
+ * post-reconnect frame can release (the §7.5 rule-3 window: a replayed lmid snapshot must not
302
+ * retire an entry whose outcome frame died with the old socket before the re-send captured
303
+ * it). The backend re-sends the domain's unconfirmed pending envelopes with their original
304
+ * mids; on a lease-auth session their DELIVERY is deferred until the first token hello
305
+ * re-authenticates the socket ({@link pendingPushes}). */
306
+ onResync(handler: () => void): void {
307
+ this.resyncHandler = handler;
308
+ }
309
+
238
310
  // --- internals ---------------------------------------------------------------
239
311
 
240
312
  private subscribe(qid: QueryId, remote: RemoteQuery): void {
241
313
  const s = this.subs.get(qid);
242
314
  if (!s) return;
315
+ // A fresh subscribe (reconnect resync, gap recovery, the retry timer itself) supersedes any
316
+ // scheduled retryable-error retry — never leave two subscribe paths racing for one query.
317
+ if (s.retryTimer !== undefined) {
318
+ clearTimeout(s.retryTimer);
319
+ s.retryTimer = undefined;
320
+ }
243
321
  const request = { queryId: qid, remote, mode: "normalized" as const };
244
322
  const ticket = ++s.subscribeTicket;
245
323
  const send = (target: SubscribeTarget) => {
@@ -247,6 +325,7 @@ export class RemoteOptimisticSource implements OptimisticSource {
247
325
  const cur = this.subs.get(qid);
248
326
  if (cur !== s || cur.subscribeTicket !== ticket) return;
249
327
  const endpoint = "leaseToken" in target ? target.wsEndpoint : undefined;
328
+ s.authed = "leaseToken" in target; // a token subscribe (re-)authenticates the socket (H-v)
250
329
  if (endpoint !== undefined) this.sawRoutedEndpoint = true;
251
330
  if (this.transport) {
252
331
  if (endpoint !== undefined && endpoint !== this.currentEndpoint && this.transportFactory) {
@@ -295,9 +374,23 @@ export class RemoteOptimisticSource implements OptimisticSource {
295
374
  this.progressHandler(msg.frame);
296
375
  return;
297
376
  }
377
+ if (msg.t === "mutationOutcome") {
378
+ // OUT-OF-BAND BY DESIGN (H-v): the frame has no `cv`, so it must NEVER be routed through the
379
+ // backend's cv buffer — dispatch immediately. A deopt has to migrate its pending entry to
380
+ // the daemon stream BEFORE the buffered lmid release that would otherwise retire it as a
381
+ // success (silence + lmid coverage ⇒ applied), and the §7.3 hold-back trigger — keyed on the
382
+ // entry's confirming domain — would then park its staged writes the wrong way.
383
+ this.outcomeHandler({
384
+ mid: msg.mid,
385
+ kind: msg.kind,
386
+ ...(msg.reason !== undefined ? { reason: msg.reason } : {}),
387
+ ...(msg.name !== undefined ? { name: msg.name } : {}),
388
+ ...("args" in msg ? { args: msg.args } : {}),
389
+ });
390
+ return;
391
+ }
298
392
  if (msg.t === "queryError") {
299
- this.subs.delete(msg.queryId);
300
- console.error(`[rindle-remote] optimistic query ${msg.queryId} subscription rejected: ${msg.message}`);
393
+ this.onQueryError(msg.queryId, msg);
301
394
  return;
302
395
  }
303
396
  if (msg.t !== "nhello" && msg.t !== "nbatch") return;
@@ -310,19 +403,72 @@ export class RemoteOptimisticSource implements OptimisticSource {
310
403
  else this.applyBatch(msg.queryId, s, msg.batch);
311
404
  }
312
405
 
313
- /** On reconnect: re-announce identity and re-subscribe every live query (each re-resolves its
314
- * lease, so a restarted daemon re-materializes + re-leases on the fly). */
406
+ /** On reconnect: re-announce identity, fire the `onResync` re-send, and re-subscribe every live
407
+ * query (each re-resolves its lease, so a restarted daemon re-materializes + re-leases on the
408
+ * fly). The re-send fires HERE — synchronously, before any post-reconnect frame can be
409
+ * processed — because the §7.5 rule-3 window closes fast: the re-subscribed lmid stream's
410
+ * fresh snapshot may cover a mid whose outcome frame died with the OLD socket, and once the
411
+ * release retires that entry as an apparent success there is nothing left to re-send (the
412
+ * lost-deopt write would silently vanish). Firing now captures the in-flight set intact; on a
413
+ * lease-auth session the envelopes themselves are HELD ({@link pendingPushes}) until the first
414
+ * token re-subscribe's hello re-authenticates the socket, then flush in order — so the shell's
415
+ * subject gate never refuses them, and its re-answer (a recorded outcome for any non-applied
416
+ * mid) resolves even an already-retired entry via the handshake's not-found arm. */
315
417
  private resync(): void {
316
418
  if (this.closed) return;
317
419
  this.transport?.send({ t: "init", clientID: this.clientID });
420
+ // Lease-auth session ⇒ hold pushes until re-authed. A stale queue from a superseded resync is
421
+ // cleared first: every held envelope maps to a still-pending mutation, and THIS resync's
422
+ // re-send below regenerates whatever still matters (no loss, no stale duplicates).
423
+ this.pendingPushes.length = 0;
424
+ this.awaitingAuthedHello = [...this.subs.values()].some((s) => s.authed);
425
+ this.resyncHandler();
318
426
  this.resubscribeAll();
319
427
  }
320
428
 
321
- /** Track the daemon's boot id; a change (after the first) means it restarted — fire onRestart. */
429
+ /** Track the daemon's boot id; a change (after the first) means it restarted — fire onRestart.
430
+ * Every NEW id (including the first) also feeds {@link onBootId} — the §301 boot-ordinal
431
+ * stream. */
322
432
  private observeBootId(bootId: string | undefined): void {
323
433
  if (!bootId) return;
324
434
  if (this.lastBootId !== undefined && bootId !== this.lastBootId) this.restartHandler();
325
- this.lastBootId = bootId;
435
+ if (bootId !== this.lastBootId) {
436
+ this.lastBootId = bootId;
437
+ this.bootIdHandler(bootId);
438
+ }
439
+ }
440
+
441
+ /** Route a `queryError` by its 101 §5 classification: retryable ⇒ keep the QState (and the
442
+ * rows already folded downstream — 101 §6) and re-subscribe after a jittered backoff
443
+ * honoring `retryAfterMs`; terminal (or pre-classification servers) ⇒ drop the
444
+ * subscription, exactly as before. Fixes the stranded-client gap: a worker fault's or a
445
+ * shedding follower's error now heals end-to-end (FOLLOWER-LAG-SHED §6.3). */
446
+ private onQueryError(qid: QueryId, err: { message: string; code?: string; retryable?: boolean; retryAfterMs?: number }): void {
447
+ const s = this.subs.get(qid);
448
+ if (!s) return;
449
+ if (err.retryable !== true) {
450
+ if (s.retryTimer !== undefined) clearTimeout(s.retryTimer);
451
+ this.subs.delete(qid);
452
+ console.error(`[rindle-remote] optimistic query ${qid} subscription rejected: ${err.message}`);
453
+ return;
454
+ }
455
+ if (s.retryTimer !== undefined) return; // a retry is already scheduled — don't stack them
456
+ s.subscriber = null; // stop validating the dead epoch; recovery is a fresh seq-0 hydrate
457
+ s.resubscribing = true;
458
+ const delay = retryDelayMs(s.retryAttempt++, err.retryAfterMs);
459
+ console.warn(
460
+ `[rindle-remote] optimistic query ${qid} ${err.code ?? "error"} (retryable): re-subscribing in ${delay}ms: ${err.message}`,
461
+ );
462
+ const timer = setTimeout(() => {
463
+ const cur = this.subs.get(qid);
464
+ if (cur !== s) return;
465
+ s.retryTimer = undefined;
466
+ this.subscribe(qid, s.remote);
467
+ }, delay);
468
+ // Node returns a Timeout (unref keeps a retiring process from being pinned by a retry);
469
+ // browsers return a number, where the optional call is a no-op.
470
+ (timer as { unref?: () => void }).unref?.();
471
+ s.retryTimer = timer;
326
472
  }
327
473
 
328
474
  private openSubscriber(qid: QueryId, s: QState, hello: NormalizedHello): void {
@@ -330,6 +476,17 @@ export class RemoteOptimisticSource implements OptimisticSource {
330
476
  s.subscriber = new NormalizedSubscriber(hello, (ev) => this.handler(qid, ev), this.clientTables);
331
477
  s.epoch = hello.epoch;
332
478
  s.resubscribing = false;
479
+ s.retryAttempt = 0; // a successful hello resets the retryable-error backoff
480
+ // H-v §7.5 rule 3: the first AUTHENTICATED hello after a reconnect proves the socket is
481
+ // re-authorized (the shell set its subject from the verified token — pushMutation-ready):
482
+ // flush the held envelopes, in order. A token-less hello (the lmid system query) does not
483
+ // qualify — an envelope racing ahead of the lease-token subscribe would be refused.
484
+ if (this.awaitingAuthedHello && s.authed) {
485
+ this.awaitingAuthedHello = false;
486
+ for (const envelope of this.pendingPushes.splice(0)) {
487
+ this.transport?.send({ t: "pushMutation", envelope });
488
+ }
489
+ }
333
490
  } catch (e) {
334
491
  // A comparator/fp mismatch at hello is unrecoverable (a code-contract divergence).
335
492
  s.subscriber = null;
package/src/protocol.ts CHANGED
@@ -135,8 +135,18 @@ export type ServerMsg =
135
135
  // or `cv` state — so the client must force a clean re-hydrate (reset its `cv` watermark).
136
136
  | { t: "nhello"; queryId: number; hello: NormalizedHello; bootId?: string }
137
137
  | { t: "nbatch"; queryId: number; batch: NormalizedBatch }
138
- | { t: "queryError"; queryId: number; message: string }
138
+ // `code`/`retryable`/`retryAfterMs` classify the error per 101-QUERY-ERRORS §5 (absent from
139
+ // older servers ⇒ terminal): a retryable error (`code:"shed"` from a load-shedding follower,
140
+ // `code:"faulted"` from a worker fault) schedules a backed-off re-subscribe instead of
141
+ // stranding the subscription (FOLLOWER-LAG-SHED §6.3).
142
+ | { t: "queryError"; queryId: number; message: string; code?: string; retryable?: boolean; retryAfterMs?: number }
139
143
  | { t: "progress"; frame: ProgressFrame }
144
+ // The room deopt handshake's verdict frame (RINDLE-REALTIME-QUERY-ENABLEMENT §3.3, H-iv-b):
145
+ // sent on the author's socket for every NON-applied mutation, BEFORE the lmid ack that burns
146
+ // the mid; `name`/`args` ride `kind:"deopt"` only (self-contained re-invoke). Connection-level
147
+ // and cv-less — the client dispatches it OUT-OF-BAND, never behind the cv buffer. Additive:
148
+ // old clients drop unknown `t`.
149
+ | { t: "mutationOutcome"; mid: number; kind: "deopt" | "rejected"; reason?: string; name?: string; args?: unknown }
140
150
  // A per-connection error reply: the server could not process this client's message (bad
141
151
  // table/AST, a commit/derive failure). Sent INSTEAD of crashing the process — the connection
142
152
  // stays open and every other connection is unaffected. Existing clients ignore unknown `t`.