@ultimat3/realtime 2.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +129 -20
- package/README.md +56 -16
- package/package.json +3 -3
- package/src/changefeed.ts +14 -2
- package/src/channel.ts +32 -5
- package/src/client-contract.ts +3 -2
- package/src/client-heartbeat.ts +4 -3
- package/src/cursor.ts +2 -2
- package/src/errors.ts +30 -0
- package/src/index.ts +9 -6
- package/src/json.ts +8 -48
- package/src/live-contract.ts +11 -18
- package/src/live-definition.ts +5 -5
- package/src/live-query.ts +3 -7
- package/src/pg-preflight.ts +117 -0
- package/src/pg-replication.ts +20 -66
- package/src/query-window.ts +52 -16
- package/src/rebase.ts +7 -2
- package/src/socket.ts +55 -20
- package/src/subscriber-gate.ts +18 -1
- package/src/sync-frames.ts +49 -12
- package/src/sync-node.ts +71 -14
- package/src/sync-protocol.ts +2 -2
- package/src/sync-upgrade.ts +20 -2
- package/src/thundering-herd.ts +10 -0
package/src/socket.ts
CHANGED
|
@@ -126,7 +126,13 @@ export class SyncSocket {
|
|
|
126
126
|
readonly frameBudget: AcceptBudget;
|
|
127
127
|
|
|
128
128
|
actor: Actor | null;
|
|
129
|
-
|
|
129
|
+
/**
|
|
130
|
+
* MONOTONIC milliseconds, not an instant — the idle sweep measures a duration, and a duration
|
|
131
|
+
* read off the wall clock is decided by whatever NTP last wrote. A step forward evicts sockets
|
|
132
|
+
* that are talking; a step backward makes `idleFor` negative and spares sockets that are dead.
|
|
133
|
+
* Named for the units so nobody hands it to `new Date()`; `openedAt` is the wall-clock one.
|
|
134
|
+
*/
|
|
135
|
+
lastSeenMonotonicMs: number;
|
|
130
136
|
droppedFrames = 0;
|
|
131
137
|
sentFrames = 0;
|
|
132
138
|
|
|
@@ -150,8 +156,10 @@ export class SyncSocket {
|
|
|
150
156
|
burst: options.frameBurst ?? DEFAULT_FRAME_BURST,
|
|
151
157
|
clock: this.#clock,
|
|
152
158
|
});
|
|
159
|
+
// Two clocks on purpose: `openedAt` is an instant a human reads, `lastSeenMonotonicMs` is the
|
|
160
|
+
// start of a duration only this process compares.
|
|
153
161
|
this.openedAt = this.#clock.now().getTime();
|
|
154
|
-
this.
|
|
162
|
+
this.lastSeenMonotonicMs = this.#clock.monotonic();
|
|
155
163
|
}
|
|
156
164
|
|
|
157
165
|
get actorId(): string | null {
|
|
@@ -206,11 +214,12 @@ export class SyncSocket {
|
|
|
206
214
|
}
|
|
207
215
|
|
|
208
216
|
touch(): void {
|
|
209
|
-
this.
|
|
217
|
+
this.lastSeenMonotonicMs = this.#clock.monotonic();
|
|
210
218
|
}
|
|
211
219
|
|
|
212
|
-
|
|
213
|
-
|
|
220
|
+
/** `nowMonotonicMs` comes from the SAME clock — `Clock.monotonic()`, never `now().getTime()`. */
|
|
221
|
+
idleFor(nowMonotonicMs: number): number {
|
|
222
|
+
return nowMonotonicMs - this.lastSeenMonotonicMs;
|
|
214
223
|
}
|
|
215
224
|
|
|
216
225
|
close(code: number = CLOSE.normal, reason = ''): void {
|
|
@@ -220,9 +229,28 @@ export class SyncSocket {
|
|
|
220
229
|
}
|
|
221
230
|
}
|
|
222
231
|
|
|
232
|
+
/**
|
|
233
|
+
* How long a socket may route no frame before `sync-node` evicts it. It is an APPLICATION
|
|
234
|
+
* inactivity budget and not Bun's transport one: Bun's `idleTimeout` is renewed by its own
|
|
235
|
+
* ping/pong, so a client whose TCP stack still answers pings while its frame loop is wedged holds
|
|
236
|
+
* its grant, its subscriptions and its topic membership forever. A beating client sends a `hello`
|
|
237
|
+
* every `DEFAULT_HEARTBEAT_MS` (15s), so this is eight missed beats.
|
|
238
|
+
*/
|
|
239
|
+
export const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* How often to ask. A quarter of the budget, floored at a second: a socket is evicted within 25%
|
|
243
|
+
* of its window of going quiet, and a node holding 50,000 of them pays one pass over the table
|
|
244
|
+
* four times per window rather than once a second. Derived rather than configured — a second knob
|
|
245
|
+
* is a second number that can disagree with the one it is a fraction of.
|
|
246
|
+
*/
|
|
247
|
+
export function idleSweepPeriodMs(idleTimeoutMs: number): number {
|
|
248
|
+
return Math.max(1_000, Math.floor(idleTimeoutMs / 4));
|
|
249
|
+
}
|
|
250
|
+
|
|
223
251
|
export interface SocketRegistryOptions {
|
|
224
252
|
readonly clock?: Clock;
|
|
225
|
-
/** Bun
|
|
253
|
+
/** Bun's own `idleTimeout` is renewed by its ping/pong; this budget counts routed FRAMES. */
|
|
226
254
|
readonly idleTimeoutMs?: number;
|
|
227
255
|
}
|
|
228
256
|
|
|
@@ -244,7 +272,7 @@ export class SocketRegistry {
|
|
|
244
272
|
|
|
245
273
|
constructor(options: SocketRegistryOptions = {}) {
|
|
246
274
|
this.#clock = options.clock ?? systemClock;
|
|
247
|
-
this.#idleTimeoutMs = options.idleTimeoutMs ??
|
|
275
|
+
this.#idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
|
|
248
276
|
}
|
|
249
277
|
|
|
250
278
|
/**
|
|
@@ -310,19 +338,26 @@ export class SocketRegistry {
|
|
|
310
338
|
return this.#sockets.size;
|
|
311
339
|
}
|
|
312
340
|
|
|
313
|
-
/**
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
341
|
+
/** The budget `idle()` answers against, so a caller can size its own sweep from one number. */
|
|
342
|
+
get idleTimeoutMs(): number {
|
|
343
|
+
return this.#idleTimeoutMs;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* Everything past the idle budget. A QUERY, and deliberately not an eviction: this table is three
|
|
348
|
+
* of the five things a socket holds, and the other two — its live subscriptions and its presence
|
|
349
|
+
* membership on the SHARED set — are only reachable from `sync-node`'s `teardown`. A sweep that
|
|
350
|
+
* closed and `remove`d here left a member every other node renders until its TTL and a
|
|
351
|
+
* `QueryEntry` whose `subscribers` map never empties. `sync-node` is the one caller and it
|
|
352
|
+
* releases each one the way the close callback does.
|
|
353
|
+
*/
|
|
354
|
+
idle(): SyncSocket[] {
|
|
355
|
+
// Monotonic, because this is the one comparison an operator's clock could otherwise decide:
|
|
356
|
+
// `sync-node` hands this registry and every socket it builds the same `Clock`.
|
|
357
|
+
const now = this.#clock.monotonic();
|
|
358
|
+
return [...this.#sockets.values()].filter(
|
|
359
|
+
(socket) => socket.idleFor(now) > this.#idleTimeoutMs,
|
|
360
|
+
);
|
|
326
361
|
}
|
|
327
362
|
|
|
328
363
|
/**
|
package/src/subscriber-gate.ts
CHANGED
|
@@ -129,7 +129,24 @@ export class SubscriberGate {
|
|
|
129
129
|
patch: RowPatch,
|
|
130
130
|
holds: boolean,
|
|
131
131
|
): Promise<RowPatch | null> {
|
|
132
|
-
|
|
132
|
+
// A delete carries no row, so there is nothing to put in front of the rule — `holds` IS the
|
|
133
|
+
// decision, the same one the two branches below take for a row a rule has just refused.
|
|
134
|
+
// Returned unconditionally it was a leak with no upper bound: the shared window is pre-policy,
|
|
135
|
+
// so every subscriber learned the id and the instant of every OTHER tenant's row as it was
|
|
136
|
+
// deleted, on a query whose `visible` rule had never let them see one of them.
|
|
137
|
+
//
|
|
138
|
+
// `holds` comes from `subscription.cursor.ids`, truncated at `CURSOR_ID_LIMIT` — so on a window
|
|
139
|
+
// wider than 512 rows a legitimate delete past position 512 is dropped and that row stays on
|
|
140
|
+
// screen until the subscriber re-snapshots. That is the trade the denied-update branch below
|
|
141
|
+
// already makes, and it is the right way round: a stale row is a bug, a row id leaked to
|
|
142
|
+
// another tenant is a breach.
|
|
143
|
+
if (patch.op === 'delete' || patch.row === null) {
|
|
144
|
+
if (holds) return patch;
|
|
145
|
+
// Counted, or a withheld delete is invisible in exactly the way `onRowDenied` exists to
|
|
146
|
+
// stop — and the rate of it is how an operator sees a window shared across tenants at all.
|
|
147
|
+
this.#denied(target.qid, who, patch.id);
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
133
150
|
const full = target.rows.find((row) => row.id === patch.id);
|
|
134
151
|
// No whole row means no decision to take. An update patch carries the changed columns only, so
|
|
135
152
|
// a rule reading `row.ownerId` on one reads `undefined` and answers as if the row had said so —
|
package/src/sync-frames.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// inbound surface the `sync` node exposes. Every dependency is injected, so the router is
|
|
3
3
|
// exercisable without a socket, a bus or a server.
|
|
4
4
|
|
|
5
|
+
import { logger } from '@ultimat3/core';
|
|
5
6
|
import type { ChannelHub } from './channel';
|
|
6
7
|
import { topic as makeTopic } from './channel';
|
|
7
8
|
import { FrameRateLimitError } from './errors';
|
|
@@ -9,7 +10,7 @@ import { FrameLanes, laneKeyOf } from './frame-lanes';
|
|
|
9
10
|
import type { JsonValue, Row } from './json';
|
|
10
11
|
import type { LiveQueryRegistry } from './live-query';
|
|
11
12
|
import { type PresenceRegistry, presenceFrame } from './presence';
|
|
12
|
-
import type
|
|
13
|
+
import { CLOSE, type SyncSocket } from './socket';
|
|
13
14
|
import { type Frame, PROTOCOL_VERSION, toWireError } from './sync-protocol';
|
|
14
15
|
|
|
15
16
|
/** Server-authoritative mutation execution. Injected: `sync` never owns business logic. */
|
|
@@ -104,7 +105,14 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
104
105
|
// Repeating the frame is therefore also the heartbeat — `join` re-`put`s the member.
|
|
105
106
|
if (presence) {
|
|
106
107
|
const roster = await presence.join(name, { id: socket.id, actorId: socket.actorId });
|
|
107
|
-
|
|
108
|
+
// The answer is read even though nothing here can repair it. A roster has no cursor and
|
|
109
|
+
// no re-send path of its own: the client renders an empty room until it repeats this
|
|
110
|
+
// very frame as its heartbeat, which re-joins and re-rosters. Membership on the shared
|
|
111
|
+
// set is already correct, so the drop costs one client one heartbeat of blank room —
|
|
112
|
+
// and the log is the only trace it leaves anywhere.
|
|
113
|
+
if (!socket.send(presenceFrame(name, 'sync', roster.members, roster.total))) {
|
|
114
|
+
logger.warn('sync.presence_roster_dropped', { topic: name, socketId: socket.id });
|
|
115
|
+
}
|
|
108
116
|
}
|
|
109
117
|
return;
|
|
110
118
|
}
|
|
@@ -121,12 +129,19 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
121
129
|
sid: frame.sid,
|
|
122
130
|
cursor: frame.target.cursor,
|
|
123
131
|
});
|
|
124
|
-
|
|
132
|
+
// `subscribe` has already seated the subscription and cleared its desync mark, so a reply
|
|
133
|
+
// the socket refuses leaves the server believing a client that holds no rows is in sync:
|
|
134
|
+
// the next change reaches it as a PATCH folded onto nothing, forever, on a socket that has
|
|
135
|
+
// since drained. Marked instead, which is the state it is actually in — the next delivery
|
|
136
|
+
// re-snapshots it out of the shared window, exactly as `live-fanout` does for a lost patch.
|
|
137
|
+
if (!socket.send(reply)) socket.markDesynced(frame.sid);
|
|
125
138
|
return;
|
|
126
139
|
}
|
|
127
140
|
case 'mutate': {
|
|
128
141
|
if (!options.onMutate) {
|
|
129
|
-
|
|
142
|
+
// A failure receipt is still a receipt: dropped, the client's mutation stays `inflight`
|
|
143
|
+
// and is neither rolled back nor retried, so this one reads the answer too.
|
|
144
|
+
const sent = socket.send({
|
|
130
145
|
type: 'ack',
|
|
131
146
|
v: PROTOCOL_VERSION,
|
|
132
147
|
ref: frame.key,
|
|
@@ -137,6 +152,7 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
137
152
|
fix: 'pass onMutate to createSyncNode({ onMutate })',
|
|
138
153
|
}),
|
|
139
154
|
});
|
|
155
|
+
if (!sent) undeliverable(socket, frame.key, 'ack');
|
|
140
156
|
return;
|
|
141
157
|
}
|
|
142
158
|
const result = await options.onMutate({
|
|
@@ -153,7 +169,7 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
153
169
|
// and no sequence to decide which later optimistic writes to replay over server truth.
|
|
154
170
|
// These are two frames on one socket, so the order is the only coordination there is.
|
|
155
171
|
if (result.entity !== undefined) {
|
|
156
|
-
socket.send({
|
|
172
|
+
const sent = socket.send({
|
|
157
173
|
type: 'rebase',
|
|
158
174
|
v: PROTOCOL_VERSION,
|
|
159
175
|
key: frame.key,
|
|
@@ -161,14 +177,22 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
161
177
|
strategy: 'server-wins',
|
|
162
178
|
row: result.row ?? null,
|
|
163
179
|
});
|
|
180
|
+
// The ack retires the client's rebase-log entry, so acking a rebase that never left is
|
|
181
|
+
// the divergence the ordering above exists to prevent — one frame later instead of one
|
|
182
|
+
// frame earlier. Nothing is acked; the mutation stays unsettled and is replayed.
|
|
183
|
+
if (!sent) return undeliverable(socket, frame.key, 'rebase');
|
|
184
|
+
}
|
|
185
|
+
if (
|
|
186
|
+
!socket.send({
|
|
187
|
+
type: 'ack',
|
|
188
|
+
v: PROTOCOL_VERSION,
|
|
189
|
+
ref: frame.key,
|
|
190
|
+
lsn: result.lsn ?? null,
|
|
191
|
+
error: null,
|
|
192
|
+
})
|
|
193
|
+
) {
|
|
194
|
+
undeliverable(socket, frame.key, 'ack');
|
|
164
195
|
}
|
|
165
|
-
socket.send({
|
|
166
|
-
type: 'ack',
|
|
167
|
-
v: PROTOCOL_VERSION,
|
|
168
|
-
ref: frame.key,
|
|
169
|
-
lsn: result.lsn ?? null,
|
|
170
|
-
error: null,
|
|
171
|
-
});
|
|
172
196
|
return;
|
|
173
197
|
}
|
|
174
198
|
// Server-authored frames are never received from a client.
|
|
@@ -183,3 +207,16 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
183
207
|
}
|
|
184
208
|
}
|
|
185
209
|
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* A settlement the socket refused. There is nothing on the node to mark — the mutation is applied
|
|
213
|
+
* and the server keeps no per-mutation state — and a client only returns an `inflight` mutation to
|
|
214
|
+
* its queue when the connection dies (`requeueInflight`), so a receipt dropped on a socket that
|
|
215
|
+
* stays up is a write the client neither retires nor retries, with the server believing it settled.
|
|
216
|
+
* Closing IS the repair: the queue hands the mutation back and the reconnect replays it under the
|
|
217
|
+
* same idempotency key.
|
|
218
|
+
*/
|
|
219
|
+
function undeliverable(socket: SyncSocket, key: string, kind: 'rebase' | 'ack'): void {
|
|
220
|
+
logger.warn('sync.settlement_dropped', { socketId: socket.id, key, kind });
|
|
221
|
+
socket.close(CLOSE.overloaded, 'settlement undeliverable');
|
|
222
|
+
}
|
package/src/sync-node.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { CHANGE_SUBJECT_PREFIX, parseEnvelope, SeqGapDetector } from './replicat
|
|
|
14
14
|
import {
|
|
15
15
|
CLOSE,
|
|
16
16
|
DEFAULT_MAX_BUFFERED_BYTES,
|
|
17
|
+
idleSweepPeriodMs,
|
|
17
18
|
SocketRegistry,
|
|
18
19
|
SyncSocket,
|
|
19
20
|
type WsLike,
|
|
@@ -22,7 +23,13 @@ import { GrantBook, type SyncAuthenticator, sweepGrants } from './sync-auth';
|
|
|
22
23
|
import { ackRefOf, createFrameRouter, type MutationHandler } from './sync-frames';
|
|
23
24
|
import { decode, type Frame, PROTOCOL_VERSION, toWireError } from './sync-protocol';
|
|
24
25
|
import { handleUpgrade, type UpgradeTarget, type WsData } from './sync-upgrade';
|
|
25
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
AcceptBudget,
|
|
28
|
+
type DrainedSocket,
|
|
29
|
+
drainPlan,
|
|
30
|
+
type Rng,
|
|
31
|
+
reconnectFrame,
|
|
32
|
+
} from './thundering-herd';
|
|
26
33
|
|
|
27
34
|
/** Declared with the upgrade that builds it — this file only ever reads one. */
|
|
28
35
|
export type { UpgradeTarget, WsData } from './sync-upgrade';
|
|
@@ -76,6 +83,12 @@ export interface SyncNodeOptions {
|
|
|
76
83
|
*/
|
|
77
84
|
readonly maxBufferedBytes?: number;
|
|
78
85
|
readonly maxDroppedFrames?: number;
|
|
86
|
+
/**
|
|
87
|
+
* How long a socket may route no frame before this node evicts it. Every ceiling on a socket
|
|
88
|
+
* `sync` builds has to be reachable from here, and this one was not: `SocketRegistry`'s default
|
|
89
|
+
* was only settable by constructing the registry yourself, and nothing swept it either way.
|
|
90
|
+
*/
|
|
91
|
+
readonly idleTimeoutMs?: number;
|
|
79
92
|
readonly onMutate?: MutationHandler;
|
|
80
93
|
/**
|
|
81
94
|
* Who is dialling. Injected for the same reason `onMutate` is: `sync` owns no business logic and
|
|
@@ -126,12 +139,16 @@ export interface SyncNode {
|
|
|
126
139
|
close(ws: SyncWs): void;
|
|
127
140
|
};
|
|
128
141
|
/** Sends every client a distinct reconnect delay, then closes. Returns the plan for tests/logs. */
|
|
129
|
-
drain(options?: { graceMs?: number }): Promise<readonly
|
|
142
|
+
drain(options?: { graceMs?: number }): Promise<readonly DrainedSocket[]>;
|
|
130
143
|
}
|
|
131
144
|
|
|
132
145
|
export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
133
146
|
const sockets =
|
|
134
|
-
options.sockets ??
|
|
147
|
+
options.sockets ??
|
|
148
|
+
new SocketRegistry({
|
|
149
|
+
...(options.clock ? { clock: options.clock } : {}),
|
|
150
|
+
...(options.idleTimeoutMs === undefined ? {} : { idleTimeoutMs: options.idleTimeoutMs }),
|
|
151
|
+
});
|
|
135
152
|
const clock = options.clock ?? systemClock;
|
|
136
153
|
const accept = options.accept ?? new AcceptBudget({ perSecond: 500, burst: 2000, clock });
|
|
137
154
|
const maxConnections = options.maxConnections ?? DEFAULT_MAX_CONNECTIONS;
|
|
@@ -143,6 +160,7 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
143
160
|
let changes: TransportSubscription | null = null;
|
|
144
161
|
let sweeping: ReturnType<typeof setInterval> | null = null;
|
|
145
162
|
let reauthing: ReturnType<typeof setInterval> | null = null;
|
|
163
|
+
let idling: ReturnType<typeof setInterval> | null = null;
|
|
146
164
|
|
|
147
165
|
/**
|
|
148
166
|
* Work nobody is waiting on — a presence leave from a synchronous close, a sweep on a timer, a
|
|
@@ -178,6 +196,8 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
178
196
|
sweeping = null;
|
|
179
197
|
if (reauthing !== null) clearInterval(reauthing);
|
|
180
198
|
reauthing = null;
|
|
199
|
+
if (idling !== null) clearInterval(idling);
|
|
200
|
+
idling = null;
|
|
181
201
|
gaps.forget();
|
|
182
202
|
};
|
|
183
203
|
|
|
@@ -200,6 +220,17 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
200
220
|
}
|
|
201
221
|
};
|
|
202
222
|
|
|
223
|
+
/**
|
|
224
|
+
* The node's one eviction: close, then release everything the socket held. Every path that ends
|
|
225
|
+
* a socket without a `close` callback behind it — the drain, the idle sweep — goes through it,
|
|
226
|
+
* because dropping the socket from the table is three of `teardown`'s five steps and the two it
|
|
227
|
+
* misses are the ones another node can see.
|
|
228
|
+
*/
|
|
229
|
+
const evict = (socket: SyncSocket, code: number, reason: string): void => {
|
|
230
|
+
socket.close(code, reason);
|
|
231
|
+
teardown(socket);
|
|
232
|
+
};
|
|
233
|
+
|
|
203
234
|
/**
|
|
204
235
|
* One pass over the grants whose window has closed. This is the half R2 was missing: `reauthorize`
|
|
205
236
|
* and `onActorChange` were both written and neither had a caller, so a socket that was accepted
|
|
@@ -222,8 +253,12 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
222
253
|
onRevoked: (socketId) => {
|
|
223
254
|
const socket = sockets.get(socketId);
|
|
224
255
|
if (!socket) return;
|
|
225
|
-
|
|
226
|
-
|
|
256
|
+
// Through `evict`, which closes BEFORE it releases. The other order was a close that never
|
|
257
|
+
// happened: `teardown` reaches `sockets.remove`, which closes the socket itself with
|
|
258
|
+
// `1001 connection closed`, and `SyncSocket.close` returns once `#closed` — so a revoked
|
|
259
|
+
// grant reached the client as a normal shutdown, which it retries against this node with
|
|
260
|
+
// the same dead credential, instead of the `1008` that tells it to re-dial with a new one.
|
|
261
|
+
evict(socket, CLOSE.policy, 'grant expired');
|
|
227
262
|
},
|
|
228
263
|
onRefreshFailed: (socketId, error) => {
|
|
229
264
|
// Not a denial: the grant is kept and retried next pass. Reported because a socket nobody
|
|
@@ -277,6 +312,14 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
277
312
|
);
|
|
278
313
|
sweeping.unref();
|
|
279
314
|
}
|
|
315
|
+
// The half-open connection Bun's own `idleTimeout` renews through its ping/pong: a client
|
|
316
|
+
// whose frame loop is wedged answers pings and keeps its grant, its subscriptions and its
|
|
317
|
+
// topic membership. `sweepIdle` was written for this and never called, so `touch()` and the
|
|
318
|
+
// 120s budget under it decided nothing.
|
|
319
|
+
idling = setInterval(() => {
|
|
320
|
+
for (const socket of sockets.idle()) evict(socket, CLOSE.idle, 'idle timeout');
|
|
321
|
+
}, idleSweepPeriodMs(sockets.idleTimeoutMs));
|
|
322
|
+
idling.unref();
|
|
280
323
|
if (options.authenticate) {
|
|
281
324
|
reauthing = setInterval(
|
|
282
325
|
() => detach(reauthenticate(), 'sync.reauthenticate'),
|
|
@@ -320,6 +363,9 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
320
363
|
newSocketId: () => uuid(),
|
|
321
364
|
authenticate: options.authenticate,
|
|
322
365
|
onGranted: (socketId, grant) => grants.set(socketId, grant),
|
|
366
|
+
// The other half of recording the grant before the upgrade: an upgrade that never took
|
|
367
|
+
// gets no `close` callback, so this is the only thing that can free its entry.
|
|
368
|
+
onUngranted: (socketId) => grants.delete(socketId),
|
|
323
369
|
},
|
|
324
370
|
request,
|
|
325
371
|
server,
|
|
@@ -407,23 +453,34 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
407
453
|
},
|
|
408
454
|
},
|
|
409
455
|
|
|
410
|
-
async drain(drainOptions = {}): Promise<readonly
|
|
456
|
+
async drain(drainOptions = {}): Promise<readonly DrainedSocket[]> {
|
|
411
457
|
ready = false;
|
|
412
458
|
const ids = [...sockets.all()].map((socket) => socket.id);
|
|
413
|
-
const
|
|
459
|
+
const spread = drainPlan(ids, {
|
|
414
460
|
spreadMs: options.drainSpreadMs ?? 30_000,
|
|
415
461
|
...(options.rng ? { rng: options.rng } : {}),
|
|
416
462
|
});
|
|
417
|
-
|
|
418
|
-
|
|
463
|
+
// The answer is read, not assumed: this frame IS the socket's slot, so a client that never
|
|
464
|
+
// received one reconnects on its own backoff — the herd the spread exists to break, minus
|
|
465
|
+
// that client. Nothing repairs it, so what the drop owes is a count.
|
|
466
|
+
const plan: DrainedSocket[] = spread.map((entry) => ({
|
|
467
|
+
...entry,
|
|
468
|
+
notified:
|
|
469
|
+
sockets.get(entry.socketId)?.send(reconnectFrame(entry.afterMs, 'drain')) === true,
|
|
470
|
+
}));
|
|
471
|
+
const notified = plan.reduce((total, entry) => total + (entry.notified ? 1 : 0), 0);
|
|
472
|
+
if (notified < plan.length) {
|
|
473
|
+
logger.warn('sync.drain_frames_dropped', { sockets: plan.length, notified });
|
|
419
474
|
}
|
|
420
475
|
const graceMs = drainOptions.graceMs ?? 5_000;
|
|
421
476
|
if (graceMs > 0) await new Promise((resolve) => setTimeout(resolve, graceMs));
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
477
|
+
// Through `evict`, never `sockets.remove` + `grants.delete`: those are three of `teardown`'s
|
|
478
|
+
// five steps, and the two they skip are the ones the rest of the fleet can see. A drained
|
|
479
|
+
// socket that never left its presence set is a member every other node renders for a full
|
|
480
|
+
// TTL — during a rolling restart, beside the same client's reconnection under a new id —
|
|
481
|
+
// and its live subscriptions stay in the registry, so `entry.subscribers` never empties and
|
|
482
|
+
// the matcher, the shared window and the retained ring are pinned for the process's life.
|
|
483
|
+
for (const socket of [...sockets.all()]) evict(socket, CLOSE.goingAway, 'drain');
|
|
427
484
|
// Released once the sockets are gone rather than at the top: a client is entitled to its
|
|
428
485
|
// patches for the whole grace window, and it is entitled to them *before* the hub the
|
|
429
486
|
// fanout writes through is closed.
|
package/src/sync-protocol.ts
CHANGED
|
@@ -72,7 +72,7 @@ export type SubscribeTarget =
|
|
|
72
72
|
* The opening frame, and the heartbeat's. It carries **no cursors**: resume is decided per
|
|
73
73
|
* subscription by `subscribe`, whose target already carries the cursor and whose `(name, input)`
|
|
74
74
|
* is what the node needs to authorize the read and reach the retained window at all. A cursor's
|
|
75
|
-
* `qid` is `
|
|
75
|
+
* `qid` is `queryHash(name, input)` — a digest, not an input — so a resume list here could never be
|
|
76
76
|
* more than a second, unauthorized restatement of that decision, and it cost every reconnect a
|
|
77
77
|
* duplicate copy of up to `CURSOR_ID_LIMIT` ids per subscription during the exact restart storm
|
|
78
78
|
* `thundering-herd.ts` exists to bound. Removing it needs no `PROTOCOL_VERSION` bump: `decode`
|
|
@@ -375,7 +375,7 @@ function list(obj: JsonObject, key: string, max: number, label = key): JsonValue
|
|
|
375
375
|
|
|
376
376
|
/**
|
|
377
377
|
* A client-supplied value, walked ITERATIVELY to its limits. Iteratively because the thing being
|
|
378
|
-
* refused is a stack overflow: `
|
|
378
|
+
* refused is a stack overflow: `queryHash` -> `canonicalJson` recurses over exactly this value, so a
|
|
379
379
|
* depth check that recursed would be the same crash one frame earlier.
|
|
380
380
|
*/
|
|
381
381
|
function bounded(value: JsonValue, label: string): JsonValue {
|
package/src/sync-upgrade.ts
CHANGED
|
@@ -37,8 +37,22 @@ export interface UpgradeDeps {
|
|
|
37
37
|
socketCount(): number;
|
|
38
38
|
newSocketId(): string;
|
|
39
39
|
readonly authenticate?: SyncAuthenticator | undefined;
|
|
40
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Recorded BEFORE `server.upgrade`, because Bun runs `websocket.open` synchronously inside it
|
|
42
|
+
* (measured on bun 1.3.14) and `open` is where the node reads this grant to build the socket's
|
|
43
|
+
* actor. Recorded after, every authenticated socket carried `actor: null` — the topic guard,
|
|
44
|
+
* `authorize`, `visible` and the per-tenant cap all deciding about nobody — and it never
|
|
45
|
+
* repaired, because the re-auth sweep only visits grants with an `expiresAt`.
|
|
46
|
+
*/
|
|
41
47
|
onGranted(socketId: string, grant: SyncGrant): void;
|
|
48
|
+
/**
|
|
49
|
+
* The grant given back on the one path that will never open a socket. Recording first is only
|
|
50
|
+
* safe because this exists: nothing but a `close` callback deletes a grant, and there is no
|
|
51
|
+
* callback for an upgrade that never took. Required, not optional — a host that reserves and
|
|
52
|
+
* cannot release is a leak the type refuses rather than a rule a reviewer has to remember. The
|
|
53
|
+
* same "reserve, then release" shape `channel.ts` uses for a topic slot.
|
|
54
|
+
*/
|
|
55
|
+
onUngranted(socketId: string): void;
|
|
42
56
|
}
|
|
43
57
|
|
|
44
58
|
/**
|
|
@@ -97,10 +111,14 @@ export async function handleUpgrade(
|
|
|
97
111
|
socketId: deps.newSocketId(),
|
|
98
112
|
clientBuildId: url.searchParams.get('build') ?? deps.buildId,
|
|
99
113
|
};
|
|
114
|
+
// Before the upgrade, never after: `server.upgrade` runs `websocket.open` synchronously and does
|
|
115
|
+
// not return until it has, so a grant recorded on the next line is one the socket was already
|
|
116
|
+
// built without.
|
|
117
|
+
if (grant) deps.onGranted(data.socketId, grant);
|
|
100
118
|
if (!server.upgrade(request, { data })) {
|
|
119
|
+
deps.onUngranted(data.socketId);
|
|
101
120
|
return new Response('expected websocket', { status: 426 });
|
|
102
121
|
}
|
|
103
|
-
if (grant) deps.onGranted(data.socketId, grant);
|
|
104
122
|
return undefined;
|
|
105
123
|
}
|
|
106
124
|
|
package/src/thundering-herd.ts
CHANGED
|
@@ -72,6 +72,16 @@ export interface DrainPlanEntry {
|
|
|
72
72
|
readonly afterMs: number;
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
+
/**
|
|
76
|
+
* A plan entry and what became of it — what `SyncNode.drain` returns. `notified: false` means
|
|
77
|
+
* backpressure dropped that socket's `reconnect` frame: the frame is what carries the slot, nothing
|
|
78
|
+
* re-sends it, so that client reconnects on its own backoff and the count is the only place a log
|
|
79
|
+
* can say how much of the spread actually shipped.
|
|
80
|
+
*/
|
|
81
|
+
export interface DrainedSocket extends DrainPlanEntry {
|
|
82
|
+
readonly notified: boolean;
|
|
83
|
+
}
|
|
84
|
+
|
|
75
85
|
export interface DrainPlanOptions {
|
|
76
86
|
/** Window across which reconnects are spread. Must exceed the node's own drain grace period. */
|
|
77
87
|
readonly spreadMs?: number;
|