@rindle/remote 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend.d.ts +5 -0
- package/dist/backend.d.ts.map +1 -1
- package/dist/backend.js +44 -3
- package/dist/backend.js.map +1 -1
- package/dist/optimistic-source.d.ts +49 -4
- package/dist/optimistic-source.d.ts.map +1 -1
- package/dist/optimistic-source.js +154 -7
- package/dist/optimistic-source.js.map +1 -1
- package/dist/protocol.d.ts +10 -0
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js.map +1 -1
- package/dist/query-error.d.ts +17 -0
- package/dist/query-error.d.ts.map +1 -0
- package/dist/query-error.js +29 -0
- package/dist/query-error.js.map +1 -0
- package/dist/remote-source.d.ts +5 -0
- package/dist/remote-source.d.ts.map +1 -1
- package/dist/remote-source.js +44 -3
- package/dist/remote-source.js.map +1 -1
- package/package.json +2 -2
- package/src/backend.ts +46 -3
- package/src/optimistic-source.ts +164 -7
- package/src/protocol.ts +11 -1
- package/src/query-error.ts +42 -0
- package/src/remote-source.ts +46 -3
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** The wire shape of a `queryError` frame (the optional fields ride 101 §4's reason shape). */
|
|
2
|
+
export interface QueryErrorFields {
|
|
3
|
+
message: string;
|
|
4
|
+
/** A machine code (`"shed"`, `"faulted"`, …); absent from older servers. */
|
|
5
|
+
code?: string;
|
|
6
|
+
/** True ⇒ schedule a re-subscribe; absent/false ⇒ terminal (today's behavior). */
|
|
7
|
+
retryable?: boolean;
|
|
8
|
+
/** The server's suggested floor for the first retry delay. */
|
|
9
|
+
retryAfterMs?: number;
|
|
10
|
+
}
|
|
11
|
+
/** The `attempt`-th (0-based) re-subscribe delay: exponential from the server's suggested
|
|
12
|
+
* `retryAfterMs` (or 500 ms when it suggests none), capped at 30 s, with up-to-25 % upward
|
|
13
|
+
* jitter so a shed follower's re-admission gate is not hit by a thundering herd of
|
|
14
|
+
* synchronized retries. Never returns less than the server's suggestion — the floor is
|
|
15
|
+
* honored exactly, and the server picked it knowing its own re-admission rate. */
|
|
16
|
+
export declare function retryDelayMs(attempt: number, retryAfterMs: number | undefined): number;
|
|
17
|
+
//# sourceMappingURL=query-error.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query-error.d.ts","sourceRoot":"","sources":["../src/query-error.ts"],"names":[],"mappings":"AAeA,+FAA+F;AAC/F,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,4EAA4E;IAC5E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kFAAkF;IAClF,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,8DAA8D;IAC9D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAQD;;;;mFAImF;AACnF,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAItF"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Retryable-vs-terminal `queryError` handling (101-QUERY-ERRORS-DESIGN.md §5, the client half
|
|
2
|
+
// of FOLLOWER-LAG-SHED-DESIGN.md §6.3 — the load-bearing contract fix).
|
|
3
|
+
//
|
|
4
|
+
// A `queryError` used to be terminal on every source: `subs.delete(queryId)`, no re-subscribe,
|
|
5
|
+
// so a server-side worker fault or a follower shedding load STRANDED the subscription forever
|
|
6
|
+
// (only a transport reconnect or a seq gap ever re-subscribed). Per 101 §5 the server now
|
|
7
|
+
// classifies: a frame carrying `retryable: true` (a shed follower's `code:"shed"`, a worker
|
|
8
|
+
// fault's `code:"faulted"`, a lease expiry) means the subscription is still WANTED and the
|
|
9
|
+
// server expects the client to come back — with jittered backoff honoring `retryAfterMs`.
|
|
10
|
+
//
|
|
11
|
+
// Rows already folded downstream are deliberately NOT cleared on a retryable error (101 §6 —
|
|
12
|
+
// row lifecycle stays server-driven): the degraded UX is a frozen but fully-rendered consistent
|
|
13
|
+
// view, the optimistic layer still absorbing local writes, then a fresh seq-0 hydrate on
|
|
14
|
+
// recovery. An absent/false `retryable` keeps today's terminal behavior.
|
|
15
|
+
/** Cap on the exponential backoff between re-subscribe attempts. */
|
|
16
|
+
const MAX_RETRY_DELAY_MS = 30_000;
|
|
17
|
+
/** Default first-attempt delay when the server suggests none. */
|
|
18
|
+
const DEFAULT_RETRY_DELAY_MS = 500;
|
|
19
|
+
/** The `attempt`-th (0-based) re-subscribe delay: exponential from the server's suggested
|
|
20
|
+
* `retryAfterMs` (or 500 ms when it suggests none), capped at 30 s, with up-to-25 % upward
|
|
21
|
+
* jitter so a shed follower's re-admission gate is not hit by a thundering herd of
|
|
22
|
+
* synchronized retries. Never returns less than the server's suggestion — the floor is
|
|
23
|
+
* honored exactly, and the server picked it knowing its own re-admission rate. */
|
|
24
|
+
export function retryDelayMs(attempt, retryAfterMs) {
|
|
25
|
+
const base = Math.max(retryAfterMs ?? DEFAULT_RETRY_DELAY_MS, 1);
|
|
26
|
+
const backoff = Math.min(base * 2 ** attempt, MAX_RETRY_DELAY_MS);
|
|
27
|
+
return Math.round(backoff * (1 + Math.random() * 0.25));
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=query-error.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query-error.js","sourceRoot":"","sources":["../src/query-error.ts"],"names":[],"mappings":"AAAA,8FAA8F;AAC9F,wEAAwE;AACxE,EAAE;AACF,+FAA+F;AAC/F,8FAA8F;AAC9F,0FAA0F;AAC1F,4FAA4F;AAC5F,2FAA2F;AAC3F,0FAA0F;AAC1F,EAAE;AACF,6FAA6F;AAC7F,gGAAgG;AAChG,yFAAyF;AACzF,yEAAyE;AAazE,oEAAoE;AACpE,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,iEAAiE;AACjE,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC;;;;mFAImF;AACnF,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,YAAgC;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,IAAI,sBAAsB,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAClE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;AAC1D,CAAC"}
|
package/dist/remote-source.d.ts
CHANGED
|
@@ -23,6 +23,11 @@ export declare class RemoteNormalizedSource implements NormalizedSource {
|
|
|
23
23
|
onNormalized(handler: (qid: QueryId, ev: NormalizedEvent) => void): void;
|
|
24
24
|
private subscribe;
|
|
25
25
|
private onServerMsg;
|
|
26
|
+
/** Route a `queryError` by its 101 §5 classification: retryable ⇒ keep the QState (and the
|
|
27
|
+
* rows already folded downstream — 101 §6) and re-subscribe after a jittered backoff
|
|
28
|
+
* honoring `retryAfterMs`; terminal (or pre-classification servers) ⇒ drop the
|
|
29
|
+
* subscription, exactly as before (FOLLOWER-LAG-SHED §6.3). */
|
|
30
|
+
private onQueryError;
|
|
26
31
|
private openSubscriber;
|
|
27
32
|
private applyBatch;
|
|
28
33
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"remote-source.d.ts","sourceRoot":"","sources":["../src/remote-source.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EACV,QAAQ,EACR,eAAe,EACf,gBAAgB,EAChB,qBAAqB,EACrB,OAAO,EACP,WAAW,EACZ,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"remote-source.d.ts","sourceRoot":"","sources":["../src/remote-source.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EACV,QAAQ,EACR,eAAe,EACf,gBAAgB,EAChB,qBAAqB,EACrB,OAAO,EACP,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAOxB,OAAO,EAIL,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EAEvB,MAAM,gBAAgB,CAAC;AAExB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAiBhD,MAAM,WAAW,6BAA6B;IAC5C,wFAAwF;IACxF,gBAAgB,CAAC,EAAE,iBAAiB,CAAC;IACrC,8EAA8E;IAC9E,YAAY,CAAC,EAAE,iBAAiB,CAAC;CAClC;AAED,qBAAa,sBAAuB,YAAW,gBAAgB;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoB;IACrD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAoB;IAClD,OAAO,CAAC,OAAO,CAAyD;IACxE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA8B;IACnD,mGAAmG;IACnG,OAAO,CAAC,YAAY,CAAsC;gBAE9C,SAAS,EAAE,SAAS,EAAE,IAAI,GAAE,6BAAkC;IAO1E,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,EAAE,GAAG,IAAI;IAIzD,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI;IAKtD,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IAOnC,MAAM,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM5C,YAAY,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,eAAe,KAAK,IAAI,GAAG,IAAI;IAMxE,OAAO,CAAC,SAAS;IAiCjB,OAAO,CAAC,WAAW;IAanB;;;oEAGgE;IAChE,OAAO,CAAC,YAAY;IA0BpB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,UAAU;CAcnB;AAED,mFAAmF;AACnF,wBAAgB,4BAA4B,CAC1C,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,IAAI,GAAE,6BAAkC,GACvC,sBAAsB,CAGxB"}
|
package/dist/remote-source.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
// epoch and replies with a fresh hello + snapshot, and `NormalizedSync` diffs the new footprint
|
|
9
9
|
// against the old (the set-analogue re-hydrate, §5.3).
|
|
10
10
|
import { NormalizedSubscriber } from "./normalized.js";
|
|
11
|
+
import { retryDelayMs } from "./query-error.js";
|
|
11
12
|
import { ProtocolError } from "./protocol.js";
|
|
12
13
|
import { defaultSubscribeTarget, isThenable, subscribeMessage, } from "./subscribe.js";
|
|
13
14
|
import { WsTransport } from "./transport.js";
|
|
@@ -29,10 +30,13 @@ export class RemoteNormalizedSource {
|
|
|
29
30
|
this.clientTables = tables;
|
|
30
31
|
}
|
|
31
32
|
registerQuery(qid, remote) {
|
|
32
|
-
this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0 });
|
|
33
|
+
this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0, retryTimer: undefined, retryAttempt: 0 });
|
|
33
34
|
this.subscribe(qid, remote);
|
|
34
35
|
}
|
|
35
36
|
unregisterQuery(qid) {
|
|
37
|
+
const s = this.subs.get(qid);
|
|
38
|
+
if (s?.retryTimer !== undefined)
|
|
39
|
+
clearTimeout(s.retryTimer);
|
|
36
40
|
this.subs.delete(qid);
|
|
37
41
|
this.transport.send({ t: "unsubscribe", queryId: qid });
|
|
38
42
|
}
|
|
@@ -50,6 +54,12 @@ export class RemoteNormalizedSource {
|
|
|
50
54
|
const s = this.subs.get(qid);
|
|
51
55
|
if (!s)
|
|
52
56
|
return;
|
|
57
|
+
// A fresh subscribe (gap recovery, the retry timer itself) supersedes any scheduled
|
|
58
|
+
// retryable-error retry — never leave two subscribe paths racing for one query.
|
|
59
|
+
if (s.retryTimer !== undefined) {
|
|
60
|
+
clearTimeout(s.retryTimer);
|
|
61
|
+
s.retryTimer = undefined;
|
|
62
|
+
}
|
|
53
63
|
const request = { queryId: qid, remote, mode: "normalized" };
|
|
54
64
|
const ticket = ++s.subscribeTicket;
|
|
55
65
|
const send = (target) => {
|
|
@@ -78,8 +88,7 @@ export class RemoteNormalizedSource {
|
|
|
78
88
|
}
|
|
79
89
|
onServerMsg(msg) {
|
|
80
90
|
if (msg.t === "queryError") {
|
|
81
|
-
this.
|
|
82
|
-
console.error(`[rindle-remote] normalized query ${msg.queryId} subscription rejected: ${msg.message}`);
|
|
91
|
+
this.onQueryError(msg.queryId, msg);
|
|
83
92
|
return;
|
|
84
93
|
}
|
|
85
94
|
// This source is normalized-only; it sees `nhello`/`nbatch` (flat frames are ignored).
|
|
@@ -93,11 +102,43 @@ export class RemoteNormalizedSource {
|
|
|
93
102
|
else
|
|
94
103
|
this.applyBatch(msg.queryId, s, msg.batch);
|
|
95
104
|
}
|
|
105
|
+
/** Route a `queryError` by its 101 §5 classification: retryable ⇒ keep the QState (and the
|
|
106
|
+
* rows already folded downstream — 101 §6) and re-subscribe after a jittered backoff
|
|
107
|
+
* honoring `retryAfterMs`; terminal (or pre-classification servers) ⇒ drop the
|
|
108
|
+
* subscription, exactly as before (FOLLOWER-LAG-SHED §6.3). */
|
|
109
|
+
onQueryError(qid, err) {
|
|
110
|
+
const s = this.subs.get(qid);
|
|
111
|
+
if (!s)
|
|
112
|
+
return;
|
|
113
|
+
if (err.retryable !== true) {
|
|
114
|
+
if (s.retryTimer !== undefined)
|
|
115
|
+
clearTimeout(s.retryTimer);
|
|
116
|
+
this.subs.delete(qid);
|
|
117
|
+
console.error(`[rindle-remote] normalized query ${qid} subscription rejected: ${err.message}`);
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
if (s.retryTimer !== undefined)
|
|
121
|
+
return; // a retry is already scheduled — don't stack them
|
|
122
|
+
s.subscriber = null; // stop validating the dead epoch; recovery is a fresh seq-0 hydrate
|
|
123
|
+
s.resubscribing = true;
|
|
124
|
+
const delay = retryDelayMs(s.retryAttempt++, err.retryAfterMs);
|
|
125
|
+
console.warn(`[rindle-remote] normalized query ${qid} ${err.code ?? "error"} (retryable): re-subscribing in ${delay}ms: ${err.message}`);
|
|
126
|
+
const timer = setTimeout(() => {
|
|
127
|
+
const cur = this.subs.get(qid);
|
|
128
|
+
if (cur !== s)
|
|
129
|
+
return;
|
|
130
|
+
s.retryTimer = undefined;
|
|
131
|
+
this.subscribe(qid, s.remote);
|
|
132
|
+
}, delay);
|
|
133
|
+
timer.unref?.();
|
|
134
|
+
s.retryTimer = timer;
|
|
135
|
+
}
|
|
96
136
|
openSubscriber(qid, s, hello) {
|
|
97
137
|
try {
|
|
98
138
|
s.subscriber = new NormalizedSubscriber(hello, (ev) => this.handler(qid, ev), this.clientTables);
|
|
99
139
|
s.epoch = hello.epoch;
|
|
100
140
|
s.resubscribing = false;
|
|
141
|
+
s.retryAttempt = 0; // a successful hello resets the retryable-error backoff
|
|
101
142
|
}
|
|
102
143
|
catch (e) {
|
|
103
144
|
// A comparator/fp mismatch at hello is unrecoverable (a code-contract divergence).
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"remote-source.js","sourceRoot":"","sources":["../src/remote-source.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAC5F,8FAA8F;AAC9F,4FAA4F;AAC5F,6FAA6F;AAC7F,0FAA0F;AAC1F,EAAE;AACF,+FAA+F;AAC/F,gGAAgG;AAChG,uDAAuD;AAWvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"remote-source.js","sourceRoot":"","sources":["../src/remote-source.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAC5F,8FAA8F;AAC9F,4FAA4F;AAC5F,6FAA6F;AAC7F,0FAA0F;AAC1F,EAAE;AACF,+FAA+F;AAC/F,gGAAgG;AAChG,uDAAuD;AAWvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEhD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,EACL,sBAAsB,EACtB,UAAU,EACV,gBAAgB,GAIjB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAyB7C,MAAM,OAAO,sBAAsB;IAChB,SAAS,CAAY;IACrB,gBAAgB,CAAoB;IACpC,YAAY,CAAqB;IAC1C,OAAO,GAAgD,GAAG,EAAE,GAAE,CAAC,CAAC;IACvD,IAAI,GAAG,IAAI,GAAG,EAAmB,CAAC;IACnD,mGAAmG;IAC3F,YAAY,CAAsC;IAE1D,YAAY,SAAoB,EAAE,OAAsC,EAAE;QACxE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,sBAAsB,CAAC;QACxE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,kBAAkB,CAAC,MAA+B;QAChD,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;IAC7B,CAAC;IAED,aAAa,CAAC,GAAY,EAAE,MAAmB;QAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7I,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED,eAAe,CAAC,GAAY;QAC1B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,EAAE,UAAU,KAAK,SAAS;YAAE,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,aAAa,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,SAAqB;QAC1B,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5E,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAChD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,YAAY,CAAC,OAAoD;QAC/D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,gFAAgF;IAExE,SAAS,CAAC,GAAY,EAAE,MAAmB;QACjD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,oFAAoF;QACpF,gFAAgF;QAChF,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC/B,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAC3B,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC;QAC3B,CAAC;QACD,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,YAAqB,EAAE,CAAC;QACtE,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,eAAe,CAAC;QACnC,MAAM,IAAI,GAAG,CAAC,MAAuB,EAAE,EAAE;YACvC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,eAAe,KAAK,MAAM;gBAAE,OAAO;YACxD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC;QACF,MAAM,IAAI,GAAG,CAAC,GAAY,EAAE,EAAE;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,eAAe,KAAK,MAAM;gBAAE,OAAO;YACxD,CAAC,CAAC,aAAa,GAAG,KAAK,CAAC;YACxB,OAAO,CAAC,KAAK,CACX,oCAAoC,GAAG,iCAAiC,MAAM,CAAE,GAAa,EAAE,OAAO,IAAI,GAAG,CAAC,EAAE,CACjH,CAAC;QACJ,CAAC,CAAC;QACF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC9C,IAAI,UAAU,CAAC,MAAM,CAAC;gBAAE,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;;gBAChD,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,GAAc;QAChC,IAAI,GAAG,CAAC,CAAC,KAAK,YAAY,EAAE,CAAC;YAC3B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACpC,OAAO;QACT,CAAC;QACD,uFAAuF;QACvF,IAAI,GAAG,CAAC,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,CAAC,KAAK,QAAQ;YAAE,OAAO;QACrD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,CAAC;YAAE,OAAO,CAAC,+BAA+B;QAC/C,IAAI,GAAG,CAAC,CAAC,KAAK,QAAQ;YAAE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;;YAClE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IAClD,CAAC;IAED;;;oEAGgE;IACxD,YAAY,CAAC,GAAY,EAAE,GAAmF;QACpH,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS;gBAAE,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAC3D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,oCAAoC,GAAG,2BAA2B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/F,OAAO;QACT,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS;YAAE,OAAO,CAAC,kDAAkD;QAC1F,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,oEAAoE;QACzF,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC;QACvB,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,EAAE,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;QAC/D,OAAO,CAAC,IAAI,CACV,oCAAoC,GAAG,IAAI,GAAG,CAAC,IAAI,IAAI,OAAO,mCAAmC,KAAK,OAAO,GAAG,CAAC,OAAO,EAAE,CAC3H,CAAC;QACF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,GAAG,KAAK,CAAC;gBAAE,OAAO;YACtB,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC;YACzB,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC,EAAE,KAAK,CAAC,CAAC;QACT,KAAgC,CAAC,KAAK,EAAE,EAAE,CAAC;QAC5C,CAAC,CAAC,UAAU,GAAG,KAAK,CAAC;IACvB,CAAC;IAEO,cAAc,CAAC,GAAY,EAAE,CAAS,EAAE,KAAsB;QACpE,IAAI,CAAC;YACH,CAAC,CAAC,UAAU,GAAG,IAAI,oBAAoB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YACjG,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YACtB,CAAC,CAAC,aAAa,GAAG,KAAK,CAAC;YACxB,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,wDAAwD;QAC9E,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,mFAAmF;YACnF,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,oCAAoC,GAAG,2BAA4B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1G,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,GAAY,EAAE,CAAS,EAAE,KAAsB;QAChE,IAAI,CAAC,CAAC,CAAC,UAAU;YAAE,OAAO,CAAC,mCAAmC;QAC9D,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;YAAE,OAAO,CAAC,+CAA+C;QAClF,IAAI,CAAC;YACH,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,CAAC,CAAC,YAAY,aAAa,CAAC;gBAAE,MAAM,CAAC,CAAC;YAC3C,IAAI,CAAC,CAAC,aAAa;gBAAE,OAAO,CAAC,qBAAqB;YAClD,oFAAoF;YACpF,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF;AAED,mFAAmF;AACnF,MAAM,UAAU,4BAA4B,CAC1C,cAAkC,EAClC,OAAsC,EAAE;IAExC,MAAM,SAAS,GAAG,OAAO,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;IACxG,OAAO,IAAI,sBAAsB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AACrD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rindle/remote",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"README.md"
|
|
26
26
|
],
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@rindle/client": "0.
|
|
28
|
+
"@rindle/client": "0.5.0"
|
|
29
29
|
},
|
|
30
30
|
"devDependencies": {
|
|
31
31
|
"@types/node": "^22.10.0",
|
package/src/backend.ts
CHANGED
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
type SubscribeResolver,
|
|
21
21
|
} from "./subscribe.ts";
|
|
22
22
|
import { WsTransport } from "./transport.ts";
|
|
23
|
+
import { retryDelayMs } from "./query-error.ts";
|
|
23
24
|
import type { Transport } from "./transport.ts";
|
|
24
25
|
|
|
25
26
|
interface QState {
|
|
@@ -31,6 +32,10 @@ interface QState {
|
|
|
31
32
|
resubscribing: boolean;
|
|
32
33
|
/** Monotonic token that cancels stale async lease resolutions. */
|
|
33
34
|
subscribeTicket: number;
|
|
35
|
+
/** Pending retryable-error re-subscribe timer (FOLLOWER-LAG-SHED §6.3), if any. */
|
|
36
|
+
retryTimer: ReturnType<typeof setTimeout> | undefined;
|
|
37
|
+
/** Consecutive retryable errors without a successful hello — the backoff exponent. */
|
|
38
|
+
retryAttempt: number;
|
|
34
39
|
}
|
|
35
40
|
|
|
36
41
|
export interface RemoteBackendOptions {
|
|
@@ -60,11 +65,13 @@ export class RemoteBackend implements Backend {
|
|
|
60
65
|
console.error(`[rindle-remote] flat query ${qid} has no named remote identity`);
|
|
61
66
|
return;
|
|
62
67
|
}
|
|
63
|
-
this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0 });
|
|
68
|
+
this.subs.set(qid, { remote, subscriber: null, epoch: 0, resubscribing: false, subscribeTicket: 0, retryTimer: undefined, retryAttempt: 0 });
|
|
64
69
|
this.subscribe(qid, remote);
|
|
65
70
|
}
|
|
66
71
|
|
|
67
72
|
unregisterQuery(qid: QueryId): void {
|
|
73
|
+
const s = this.subs.get(qid);
|
|
74
|
+
if (s?.retryTimer !== undefined) clearTimeout(s.retryTimer);
|
|
68
75
|
this.subs.delete(qid);
|
|
69
76
|
this.transport.send({ t: "unsubscribe", queryId: qid });
|
|
70
77
|
}
|
|
@@ -93,6 +100,12 @@ export class RemoteBackend implements Backend {
|
|
|
93
100
|
private subscribe(qid: QueryId, remote: RemoteQuery): void {
|
|
94
101
|
const s = this.subs.get(qid);
|
|
95
102
|
if (!s) return;
|
|
103
|
+
// A fresh subscribe (gap recovery, the retry timer itself) supersedes any scheduled
|
|
104
|
+
// retryable-error retry — never leave two subscribe paths racing for one query.
|
|
105
|
+
if (s.retryTimer !== undefined) {
|
|
106
|
+
clearTimeout(s.retryTimer);
|
|
107
|
+
s.retryTimer = undefined;
|
|
108
|
+
}
|
|
96
109
|
const request = { queryId: qid, remote, mode: "flat" as const };
|
|
97
110
|
const ticket = ++s.subscribeTicket;
|
|
98
111
|
const send = (target: ReturnType<typeof defaultSubscribeTarget>) => {
|
|
@@ -117,8 +130,7 @@ export class RemoteBackend implements Backend {
|
|
|
117
130
|
|
|
118
131
|
private onServerMsg(msg: ServerMsg): void {
|
|
119
132
|
if (msg.t === "queryError") {
|
|
120
|
-
this.
|
|
121
|
-
console.error(`[rindle-remote] query ${msg.queryId} subscription rejected: ${msg.message}`);
|
|
133
|
+
this.onQueryError(msg.queryId, msg);
|
|
122
134
|
return;
|
|
123
135
|
}
|
|
124
136
|
// This backend is flat-only; it ignores normalized frames (`nhello`/`nbatch`).
|
|
@@ -129,6 +141,36 @@ export class RemoteBackend implements Backend {
|
|
|
129
141
|
else this.applyBatch(msg.queryId, s, msg.batch);
|
|
130
142
|
}
|
|
131
143
|
|
|
144
|
+
/** Route a `queryError` by its 101 §5 classification: retryable ⇒ keep the QState (and the
|
|
145
|
+
* rows already folded downstream — 101 §6) and re-subscribe after a jittered backoff
|
|
146
|
+
* honoring `retryAfterMs`; terminal (or pre-classification servers) ⇒ drop the
|
|
147
|
+
* subscription, exactly as before (FOLLOWER-LAG-SHED §6.3). */
|
|
148
|
+
private onQueryError(qid: QueryId, err: { message: string; code?: string; retryable?: boolean; retryAfterMs?: number }): void {
|
|
149
|
+
const s = this.subs.get(qid);
|
|
150
|
+
if (!s) return;
|
|
151
|
+
if (err.retryable !== true) {
|
|
152
|
+
if (s.retryTimer !== undefined) clearTimeout(s.retryTimer);
|
|
153
|
+
this.subs.delete(qid);
|
|
154
|
+
console.error(`[rindle-remote] query ${qid} subscription rejected: ${err.message}`);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
if (s.retryTimer !== undefined) return; // a retry is already scheduled — don't stack them
|
|
158
|
+
s.subscriber = null; // stop validating the dead epoch; recovery is a fresh seq-0 hydrate
|
|
159
|
+
s.resubscribing = true;
|
|
160
|
+
const delay = retryDelayMs(s.retryAttempt++, err.retryAfterMs);
|
|
161
|
+
console.warn(
|
|
162
|
+
`[rindle-remote] query ${qid} ${err.code ?? "error"} (retryable): re-subscribing in ${delay}ms: ${err.message}`,
|
|
163
|
+
);
|
|
164
|
+
const timer = setTimeout(() => {
|
|
165
|
+
const cur = this.subs.get(qid);
|
|
166
|
+
if (cur !== s) return;
|
|
167
|
+
s.retryTimer = undefined;
|
|
168
|
+
this.subscribe(qid, s.remote);
|
|
169
|
+
}, delay);
|
|
170
|
+
(timer as { unref?: () => void }).unref?.();
|
|
171
|
+
s.retryTimer = timer;
|
|
172
|
+
}
|
|
173
|
+
|
|
132
174
|
private openSubscriber(qid: QueryId, s: QState, hello: Hello): void {
|
|
133
175
|
try {
|
|
134
176
|
// The Subscriber ctor validates the comparator + fingerprint and emits the `hello`
|
|
@@ -136,6 +178,7 @@ export class RemoteBackend implements Backend {
|
|
|
136
178
|
s.subscriber = new Subscriber(hello, (ev) => this.emitServerEvent(qid, ev));
|
|
137
179
|
s.epoch = hello.epoch;
|
|
138
180
|
s.resubscribing = false;
|
|
181
|
+
s.retryAttempt = 0; // a successful hello resets the retryable-error backoff
|
|
139
182
|
} catch (e) {
|
|
140
183
|
// A comparator/schema mismatch at hello is unrecoverable (a code-contract divergence) —
|
|
141
184
|
// leave the view pending and report it; do not loop.
|
package/src/optimistic-source.ts
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
import { LMID_QUERY_NAME } from "@rindle/client";
|
|
17
17
|
import type {
|
|
18
18
|
MutationEnvelope,
|
|
19
|
+
MutationOutcomeFrame,
|
|
19
20
|
NormalizedEvent,
|
|
20
21
|
NormalizedTableSchema,
|
|
21
22
|
OptimisticSource,
|
|
@@ -25,6 +26,7 @@ import type {
|
|
|
25
26
|
} from "@rindle/client";
|
|
26
27
|
|
|
27
28
|
import { NormalizedSubscriber } from "./normalized.ts";
|
|
29
|
+
import { retryDelayMs } from "./query-error.ts";
|
|
28
30
|
import type { NormalizedBatch, NormalizedHello } from "./normalized.ts";
|
|
29
31
|
import { ProtocolError } from "./protocol.ts";
|
|
30
32
|
import type { ServerMsg } from "./protocol.ts";
|
|
@@ -48,6 +50,16 @@ interface QState {
|
|
|
48
50
|
resubscribing: boolean;
|
|
49
51
|
/** Monotonic token that cancels stale async lease resolutions. */
|
|
50
52
|
subscribeTicket: number;
|
|
53
|
+
/** Pending retryable-error re-subscribe timer (FOLLOWER-LAG-SHED §6.3), if any. */
|
|
54
|
+
retryTimer: ReturnType<typeof setTimeout> | undefined;
|
|
55
|
+
/** Consecutive retryable errors without a successful hello — the backoff exponent. */
|
|
56
|
+
retryAttempt: number;
|
|
57
|
+
/** Whether the LAST subscribe sent for this query presented a `leaseToken` — i.e. its hello
|
|
58
|
+
* proves the connection is (re-)AUTHENTICATED (a room shell sets the socket's subject from the
|
|
59
|
+
* first verified token; `pushMutation` requires it). After a reconnect on a lease-auth session,
|
|
60
|
+
* queued pushes flush at the first such hello, never earlier — an envelope racing ahead of the
|
|
61
|
+
* token subscribe would be refused by the shell's subject gate (H-v §7.5 rule 3). */
|
|
62
|
+
authed: boolean;
|
|
51
63
|
}
|
|
52
64
|
|
|
53
65
|
/** Build a transport to a follower's public ws endpoint (READ-ROUTER-DESIGN.md §2.3). Default
|
|
@@ -86,6 +98,19 @@ export class RemoteOptimisticSource implements OptimisticSource {
|
|
|
86
98
|
private handler: (qid: QueryId, ev: NormalizedEvent) => void = () => {};
|
|
87
99
|
private progressHandler: (frame: ProgressFrame) => void = () => {};
|
|
88
100
|
private restartHandler: () => void = () => {};
|
|
101
|
+
private bootIdHandler: (bootId: string) => void = () => {};
|
|
102
|
+
private outcomeHandler: (frame: MutationOutcomeFrame) => void = () => {};
|
|
103
|
+
private resyncHandler: () => void = () => {};
|
|
104
|
+
/** Set by {@link resync} when this is a LEASE-AUTH session (some sub presented a `leaseToken`):
|
|
105
|
+
* transport pushes queue in {@link pendingPushes} until the first authenticated re-subscribe's
|
|
106
|
+
* hello re-establishes the socket's subject, then flush (see QState.authed). Never set on a
|
|
107
|
+
* token-less (embedded/rindled) session — its pushes need no subject and go straight out. */
|
|
108
|
+
private awaitingAuthedHello = false;
|
|
109
|
+
/** Envelopes held while {@link awaitingAuthedHello} (H-v §7.5 rule 3). Every entry corresponds
|
|
110
|
+
* to a still-pending backend mutation (the re-send reconstructs from pending entries; app
|
|
111
|
+
* invokes in the window are pending by definition), so a superseding resync may CLEAR this —
|
|
112
|
+
* its own re-send regenerates whatever still matters. */
|
|
113
|
+
private pendingPushes: MutationEnvelope[] = [];
|
|
89
114
|
private readonly subs = new Map<QueryId, QState>();
|
|
90
115
|
/** Queries whose subscribe is waiting for a transport to exist — endpoint-less subscribes issued
|
|
91
116
|
* in pure-lazy mode before any lease opens a transport (the lmid system query is registered by
|
|
@@ -196,8 +221,12 @@ export class RemoteOptimisticSource implements OptimisticSource {
|
|
|
196
221
|
close(): void {
|
|
197
222
|
this.closed = true;
|
|
198
223
|
this.transport?.close();
|
|
224
|
+
for (const s of this.subs.values()) {
|
|
225
|
+
if (s.retryTimer !== undefined) clearTimeout(s.retryTimer);
|
|
226
|
+
}
|
|
199
227
|
this.subs.clear();
|
|
200
228
|
this.deferred.clear();
|
|
229
|
+
this.pendingPushes.length = 0;
|
|
201
230
|
}
|
|
202
231
|
|
|
203
232
|
/** Register a handler fired when the DAEMON restarts (a new boot id) — the backend resets its
|
|
@@ -206,16 +235,36 @@ export class RemoteOptimisticSource implements OptimisticSource {
|
|
|
206
235
|
this.restartHandler = handler;
|
|
207
236
|
}
|
|
208
237
|
|
|
238
|
+
/** Register the observed boot-id stream (`301-ECHO-FENCE-DESIGN.md` §2.4): fired on the FIRST
|
|
239
|
+
* observed daemon boot id and on every change — the backend's client-local boot ordinals for
|
|
240
|
+
* the direction-B pin fence. A late registration is replayed the current id immediately, so
|
|
241
|
+
* ordinal 0 is never missed on an already-open connection. */
|
|
242
|
+
onBootId(handler: (bootId: string) => void): void {
|
|
243
|
+
this.bootIdHandler = handler;
|
|
244
|
+
if (this.lastBootId !== undefined) handler(this.lastBootId);
|
|
245
|
+
}
|
|
246
|
+
|
|
209
247
|
expectClientSchema(tables: NormalizedTableSchema[]): void {
|
|
210
248
|
this.clientTables = tables;
|
|
211
249
|
}
|
|
212
250
|
|
|
213
251
|
registerQuery(qid: QueryId, remote: RemoteQuery): void {
|
|
214
|
-
this.subs.set(qid, {
|
|
252
|
+
this.subs.set(qid, {
|
|
253
|
+
remote,
|
|
254
|
+
subscriber: null,
|
|
255
|
+
epoch: 0,
|
|
256
|
+
resubscribing: false,
|
|
257
|
+
subscribeTicket: 0,
|
|
258
|
+
retryTimer: undefined,
|
|
259
|
+
retryAttempt: 0,
|
|
260
|
+
authed: false,
|
|
261
|
+
});
|
|
215
262
|
this.subscribe(qid, remote);
|
|
216
263
|
}
|
|
217
264
|
|
|
218
265
|
unregisterQuery(qid: QueryId): void {
|
|
266
|
+
const s = this.subs.get(qid);
|
|
267
|
+
if (s?.retryTimer !== undefined) clearTimeout(s.retryTimer);
|
|
219
268
|
this.subs.delete(qid);
|
|
220
269
|
this.deferred.delete(qid);
|
|
221
270
|
this.transport?.send({ t: "unsubscribe", queryId: qid });
|
|
@@ -223,6 +272,13 @@ export class RemoteOptimisticSource implements OptimisticSource {
|
|
|
223
272
|
|
|
224
273
|
pushMutation(envelope: MutationEnvelope): Promise<void> {
|
|
225
274
|
if (this.pushMutationSender) return Promise.resolve(this.pushMutationSender(envelope));
|
|
275
|
+
// A lease-auth session that just reconnected is not yet re-authenticated (the shell's
|
|
276
|
+
// `pushMutation` subject gate would refuse) — hold the envelope until the first token
|
|
277
|
+
// re-subscribe's hello, then flush in order (H-v §7.5 rule 3).
|
|
278
|
+
if (this.awaitingAuthedHello) {
|
|
279
|
+
this.pendingPushes.push(envelope);
|
|
280
|
+
return Promise.resolve();
|
|
281
|
+
}
|
|
226
282
|
this.transport?.send({ t: "pushMutation", envelope });
|
|
227
283
|
return Promise.resolve();
|
|
228
284
|
}
|
|
@@ -235,11 +291,33 @@ export class RemoteOptimisticSource implements OptimisticSource {
|
|
|
235
291
|
this.progressHandler = handler;
|
|
236
292
|
}
|
|
237
293
|
|
|
294
|
+
/** The room deopt handshake's verdict stream (H-v). Dispatched OUT-OF-BAND on arrival — see
|
|
295
|
+
* {@link onServerMsg}'s `mutationOutcome` arm for why it must never wait behind the cv buffer. */
|
|
296
|
+
onMutationOutcome(handler: (frame: MutationOutcomeFrame) => void): void {
|
|
297
|
+
this.outcomeHandler = handler;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** Fired once per re-established session, SYNCHRONOUSLY inside {@link resync} — before any
|
|
301
|
+
* post-reconnect frame can release (the §7.5 rule-3 window: a replayed lmid snapshot must not
|
|
302
|
+
* retire an entry whose outcome frame died with the old socket before the re-send captured
|
|
303
|
+
* it). The backend re-sends the domain's unconfirmed pending envelopes with their original
|
|
304
|
+
* mids; on a lease-auth session their DELIVERY is deferred until the first token hello
|
|
305
|
+
* re-authenticates the socket ({@link pendingPushes}). */
|
|
306
|
+
onResync(handler: () => void): void {
|
|
307
|
+
this.resyncHandler = handler;
|
|
308
|
+
}
|
|
309
|
+
|
|
238
310
|
// --- internals ---------------------------------------------------------------
|
|
239
311
|
|
|
240
312
|
private subscribe(qid: QueryId, remote: RemoteQuery): void {
|
|
241
313
|
const s = this.subs.get(qid);
|
|
242
314
|
if (!s) return;
|
|
315
|
+
// A fresh subscribe (reconnect resync, gap recovery, the retry timer itself) supersedes any
|
|
316
|
+
// scheduled retryable-error retry — never leave two subscribe paths racing for one query.
|
|
317
|
+
if (s.retryTimer !== undefined) {
|
|
318
|
+
clearTimeout(s.retryTimer);
|
|
319
|
+
s.retryTimer = undefined;
|
|
320
|
+
}
|
|
243
321
|
const request = { queryId: qid, remote, mode: "normalized" as const };
|
|
244
322
|
const ticket = ++s.subscribeTicket;
|
|
245
323
|
const send = (target: SubscribeTarget) => {
|
|
@@ -247,6 +325,7 @@ export class RemoteOptimisticSource implements OptimisticSource {
|
|
|
247
325
|
const cur = this.subs.get(qid);
|
|
248
326
|
if (cur !== s || cur.subscribeTicket !== ticket) return;
|
|
249
327
|
const endpoint = "leaseToken" in target ? target.wsEndpoint : undefined;
|
|
328
|
+
s.authed = "leaseToken" in target; // a token subscribe (re-)authenticates the socket (H-v)
|
|
250
329
|
if (endpoint !== undefined) this.sawRoutedEndpoint = true;
|
|
251
330
|
if (this.transport) {
|
|
252
331
|
if (endpoint !== undefined && endpoint !== this.currentEndpoint && this.transportFactory) {
|
|
@@ -295,9 +374,23 @@ export class RemoteOptimisticSource implements OptimisticSource {
|
|
|
295
374
|
this.progressHandler(msg.frame);
|
|
296
375
|
return;
|
|
297
376
|
}
|
|
377
|
+
if (msg.t === "mutationOutcome") {
|
|
378
|
+
// OUT-OF-BAND BY DESIGN (H-v): the frame has no `cv`, so it must NEVER be routed through the
|
|
379
|
+
// backend's cv buffer — dispatch immediately. A deopt has to migrate its pending entry to
|
|
380
|
+
// the daemon stream BEFORE the buffered lmid release that would otherwise retire it as a
|
|
381
|
+
// success (silence + lmid coverage ⇒ applied), and the §7.3 hold-back trigger — keyed on the
|
|
382
|
+
// entry's confirming domain — would then park its staged writes the wrong way.
|
|
383
|
+
this.outcomeHandler({
|
|
384
|
+
mid: msg.mid,
|
|
385
|
+
kind: msg.kind,
|
|
386
|
+
...(msg.reason !== undefined ? { reason: msg.reason } : {}),
|
|
387
|
+
...(msg.name !== undefined ? { name: msg.name } : {}),
|
|
388
|
+
...("args" in msg ? { args: msg.args } : {}),
|
|
389
|
+
});
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
298
392
|
if (msg.t === "queryError") {
|
|
299
|
-
this.
|
|
300
|
-
console.error(`[rindle-remote] optimistic query ${msg.queryId} subscription rejected: ${msg.message}`);
|
|
393
|
+
this.onQueryError(msg.queryId, msg);
|
|
301
394
|
return;
|
|
302
395
|
}
|
|
303
396
|
if (msg.t !== "nhello" && msg.t !== "nbatch") return;
|
|
@@ -310,19 +403,72 @@ export class RemoteOptimisticSource implements OptimisticSource {
|
|
|
310
403
|
else this.applyBatch(msg.queryId, s, msg.batch);
|
|
311
404
|
}
|
|
312
405
|
|
|
313
|
-
/** On reconnect: re-announce identity and re-subscribe every live
|
|
314
|
-
* lease, so a restarted daemon re-materializes + re-leases on the
|
|
406
|
+
/** On reconnect: re-announce identity, fire the `onResync` re-send, and re-subscribe every live
|
|
407
|
+
* query (each re-resolves its lease, so a restarted daemon re-materializes + re-leases on the
|
|
408
|
+
* fly). The re-send fires HERE — synchronously, before any post-reconnect frame can be
|
|
409
|
+
* processed — because the §7.5 rule-3 window closes fast: the re-subscribed lmid stream's
|
|
410
|
+
* fresh snapshot may cover a mid whose outcome frame died with the OLD socket, and once the
|
|
411
|
+
* release retires that entry as an apparent success there is nothing left to re-send (the
|
|
412
|
+
* lost-deopt write would silently vanish). Firing now captures the in-flight set intact; on a
|
|
413
|
+
* lease-auth session the envelopes themselves are HELD ({@link pendingPushes}) until the first
|
|
414
|
+
* token re-subscribe's hello re-authenticates the socket, then flush in order — so the shell's
|
|
415
|
+
* subject gate never refuses them, and its re-answer (a recorded outcome for any non-applied
|
|
416
|
+
* mid) resolves even an already-retired entry via the handshake's not-found arm. */
|
|
315
417
|
private resync(): void {
|
|
316
418
|
if (this.closed) return;
|
|
317
419
|
this.transport?.send({ t: "init", clientID: this.clientID });
|
|
420
|
+
// Lease-auth session ⇒ hold pushes until re-authed. A stale queue from a superseded resync is
|
|
421
|
+
// cleared first: every held envelope maps to a still-pending mutation, and THIS resync's
|
|
422
|
+
// re-send below regenerates whatever still matters (no loss, no stale duplicates).
|
|
423
|
+
this.pendingPushes.length = 0;
|
|
424
|
+
this.awaitingAuthedHello = [...this.subs.values()].some((s) => s.authed);
|
|
425
|
+
this.resyncHandler();
|
|
318
426
|
this.resubscribeAll();
|
|
319
427
|
}
|
|
320
428
|
|
|
321
|
-
/** Track the daemon's boot id; a change (after the first) means it restarted — fire onRestart.
|
|
429
|
+
/** Track the daemon's boot id; a change (after the first) means it restarted — fire onRestart.
|
|
430
|
+
* Every NEW id (including the first) also feeds {@link onBootId} — the §301 boot-ordinal
|
|
431
|
+
* stream. */
|
|
322
432
|
private observeBootId(bootId: string | undefined): void {
|
|
323
433
|
if (!bootId) return;
|
|
324
434
|
if (this.lastBootId !== undefined && bootId !== this.lastBootId) this.restartHandler();
|
|
325
|
-
this.lastBootId
|
|
435
|
+
if (bootId !== this.lastBootId) {
|
|
436
|
+
this.lastBootId = bootId;
|
|
437
|
+
this.bootIdHandler(bootId);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/** Route a `queryError` by its 101 §5 classification: retryable ⇒ keep the QState (and the
|
|
442
|
+
* rows already folded downstream — 101 §6) and re-subscribe after a jittered backoff
|
|
443
|
+
* honoring `retryAfterMs`; terminal (or pre-classification servers) ⇒ drop the
|
|
444
|
+
* subscription, exactly as before. Fixes the stranded-client gap: a worker fault's or a
|
|
445
|
+
* shedding follower's error now heals end-to-end (FOLLOWER-LAG-SHED §6.3). */
|
|
446
|
+
private onQueryError(qid: QueryId, err: { message: string; code?: string; retryable?: boolean; retryAfterMs?: number }): void {
|
|
447
|
+
const s = this.subs.get(qid);
|
|
448
|
+
if (!s) return;
|
|
449
|
+
if (err.retryable !== true) {
|
|
450
|
+
if (s.retryTimer !== undefined) clearTimeout(s.retryTimer);
|
|
451
|
+
this.subs.delete(qid);
|
|
452
|
+
console.error(`[rindle-remote] optimistic query ${qid} subscription rejected: ${err.message}`);
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
if (s.retryTimer !== undefined) return; // a retry is already scheduled — don't stack them
|
|
456
|
+
s.subscriber = null; // stop validating the dead epoch; recovery is a fresh seq-0 hydrate
|
|
457
|
+
s.resubscribing = true;
|
|
458
|
+
const delay = retryDelayMs(s.retryAttempt++, err.retryAfterMs);
|
|
459
|
+
console.warn(
|
|
460
|
+
`[rindle-remote] optimistic query ${qid} ${err.code ?? "error"} (retryable): re-subscribing in ${delay}ms: ${err.message}`,
|
|
461
|
+
);
|
|
462
|
+
const timer = setTimeout(() => {
|
|
463
|
+
const cur = this.subs.get(qid);
|
|
464
|
+
if (cur !== s) return;
|
|
465
|
+
s.retryTimer = undefined;
|
|
466
|
+
this.subscribe(qid, s.remote);
|
|
467
|
+
}, delay);
|
|
468
|
+
// Node returns a Timeout (unref keeps a retiring process from being pinned by a retry);
|
|
469
|
+
// browsers return a number, where the optional call is a no-op.
|
|
470
|
+
(timer as { unref?: () => void }).unref?.();
|
|
471
|
+
s.retryTimer = timer;
|
|
326
472
|
}
|
|
327
473
|
|
|
328
474
|
private openSubscriber(qid: QueryId, s: QState, hello: NormalizedHello): void {
|
|
@@ -330,6 +476,17 @@ export class RemoteOptimisticSource implements OptimisticSource {
|
|
|
330
476
|
s.subscriber = new NormalizedSubscriber(hello, (ev) => this.handler(qid, ev), this.clientTables);
|
|
331
477
|
s.epoch = hello.epoch;
|
|
332
478
|
s.resubscribing = false;
|
|
479
|
+
s.retryAttempt = 0; // a successful hello resets the retryable-error backoff
|
|
480
|
+
// H-v §7.5 rule 3: the first AUTHENTICATED hello after a reconnect proves the socket is
|
|
481
|
+
// re-authorized (the shell set its subject from the verified token — pushMutation-ready):
|
|
482
|
+
// flush the held envelopes, in order. A token-less hello (the lmid system query) does not
|
|
483
|
+
// qualify — an envelope racing ahead of the lease-token subscribe would be refused.
|
|
484
|
+
if (this.awaitingAuthedHello && s.authed) {
|
|
485
|
+
this.awaitingAuthedHello = false;
|
|
486
|
+
for (const envelope of this.pendingPushes.splice(0)) {
|
|
487
|
+
this.transport?.send({ t: "pushMutation", envelope });
|
|
488
|
+
}
|
|
489
|
+
}
|
|
333
490
|
} catch (e) {
|
|
334
491
|
// A comparator/fp mismatch at hello is unrecoverable (a code-contract divergence).
|
|
335
492
|
s.subscriber = null;
|
package/src/protocol.ts
CHANGED
|
@@ -135,8 +135,18 @@ export type ServerMsg =
|
|
|
135
135
|
// or `cv` state — so the client must force a clean re-hydrate (reset its `cv` watermark).
|
|
136
136
|
| { t: "nhello"; queryId: number; hello: NormalizedHello; bootId?: string }
|
|
137
137
|
| { t: "nbatch"; queryId: number; batch: NormalizedBatch }
|
|
138
|
-
|
|
138
|
+
// `code`/`retryable`/`retryAfterMs` classify the error per 101-QUERY-ERRORS §5 (absent from
|
|
139
|
+
// older servers ⇒ terminal): a retryable error (`code:"shed"` from a load-shedding follower,
|
|
140
|
+
// `code:"faulted"` from a worker fault) schedules a backed-off re-subscribe instead of
|
|
141
|
+
// stranding the subscription (FOLLOWER-LAG-SHED §6.3).
|
|
142
|
+
| { t: "queryError"; queryId: number; message: string; code?: string; retryable?: boolean; retryAfterMs?: number }
|
|
139
143
|
| { t: "progress"; frame: ProgressFrame }
|
|
144
|
+
// The room deopt handshake's verdict frame (RINDLE-REALTIME-QUERY-ENABLEMENT §3.3, H-iv-b):
|
|
145
|
+
// sent on the author's socket for every NON-applied mutation, BEFORE the lmid ack that burns
|
|
146
|
+
// the mid; `name`/`args` ride `kind:"deopt"` only (self-contained re-invoke). Connection-level
|
|
147
|
+
// and cv-less — the client dispatches it OUT-OF-BAND, never behind the cv buffer. Additive:
|
|
148
|
+
// old clients drop unknown `t`.
|
|
149
|
+
| { t: "mutationOutcome"; mid: number; kind: "deopt" | "rejected"; reason?: string; name?: string; args?: unknown }
|
|
140
150
|
// A per-connection error reply: the server could not process this client's message (bad
|
|
141
151
|
// table/AST, a commit/derive failure). Sent INSTEAD of crashing the process — the connection
|
|
142
152
|
// stays open and every other connection is unaffected. Existing clients ignore unknown `t`.
|