@ultimat3/realtime 2.0.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/socket.ts CHANGED
@@ -126,7 +126,13 @@ export class SyncSocket {
126
126
  readonly frameBudget: AcceptBudget;
127
127
 
128
128
  actor: Actor | null;
129
- lastSeenAt: number;
129
+ /**
130
+ * MONOTONIC milliseconds, not an instant — the idle sweep measures a duration, and a duration
131
+ * read off the wall clock is decided by whatever NTP last wrote. A step forward evicts sockets
132
+ * that are talking; a step backward makes `idleFor` negative and spares sockets that are dead.
133
+ * Named for the units so nobody hands it to `new Date()`; `openedAt` is the wall-clock one.
134
+ */
135
+ lastSeenMonotonicMs: number;
130
136
  droppedFrames = 0;
131
137
  sentFrames = 0;
132
138
 
@@ -150,8 +156,10 @@ export class SyncSocket {
150
156
  burst: options.frameBurst ?? DEFAULT_FRAME_BURST,
151
157
  clock: this.#clock,
152
158
  });
159
+ // Two clocks on purpose: `openedAt` is an instant a human reads, `lastSeenMonotonicMs` is the
160
+ // start of a duration only this process compares.
153
161
  this.openedAt = this.#clock.now().getTime();
154
- this.lastSeenAt = this.openedAt;
162
+ this.lastSeenMonotonicMs = this.#clock.monotonic();
155
163
  }
156
164
 
157
165
  get actorId(): string | null {
@@ -206,11 +214,12 @@ export class SyncSocket {
206
214
  }
207
215
 
208
216
  touch(): void {
209
- this.lastSeenAt = this.#clock.now().getTime();
217
+ this.lastSeenMonotonicMs = this.#clock.monotonic();
210
218
  }
211
219
 
212
- idleFor(now: number): number {
213
- return now - this.lastSeenAt;
220
+ /** `nowMonotonicMs` comes from the SAME clock — `Clock.monotonic()`, never `now().getTime()`. */
221
+ idleFor(nowMonotonicMs: number): number {
222
+ return nowMonotonicMs - this.lastSeenMonotonicMs;
214
223
  }
215
224
 
216
225
  close(code: number = CLOSE.normal, reason = ''): void {
@@ -220,9 +229,28 @@ export class SyncSocket {
220
229
  }
221
230
  }
222
231
 
232
+ /**
233
+ * How long a socket may route no frame before `sync-node` evicts it. It is an APPLICATION
234
+ * inactivity budget and not Bun's transport one: Bun's `idleTimeout` is renewed by its own
235
+ * ping/pong, so a client whose TCP stack still answers pings while its frame loop is wedged holds
236
+ * its grant, its subscriptions and its topic membership forever. A beating client sends a `hello`
237
+ * every `DEFAULT_HEARTBEAT_MS` (15s), so this is eight missed beats.
238
+ */
239
+ export const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
240
+
241
+ /**
242
+ * How often to ask. A quarter of the budget, floored at a second: a socket is evicted within 25%
243
+ * of its window of going quiet, and a node holding 50,000 of them pays one pass over the table
244
+ * four times per window rather than once a second. Derived rather than configured — a second knob
245
+ * is a second number that can disagree with the one it is a fraction of.
246
+ */
247
+ export function idleSweepPeriodMs(idleTimeoutMs: number): number {
248
+ return Math.max(1_000, Math.floor(idleTimeoutMs / 4));
249
+ }
250
+
223
251
  export interface SocketRegistryOptions {
224
252
  readonly clock?: Clock;
225
- /** Bun also enforces its own `idleTimeout`; this sweep catches half-open connections. */
253
+ /** Bun's own `idleTimeout` is renewed by its ping/pong; this budget counts routed FRAMES. */
226
254
  readonly idleTimeoutMs?: number;
227
255
  }
228
256
 
@@ -244,7 +272,7 @@ export class SocketRegistry {
244
272
 
245
273
  constructor(options: SocketRegistryOptions = {}) {
246
274
  this.#clock = options.clock ?? systemClock;
247
- this.#idleTimeoutMs = options.idleTimeoutMs ?? 120_000;
275
+ this.#idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
248
276
  }
249
277
 
250
278
  /**
@@ -310,19 +338,26 @@ export class SocketRegistry {
310
338
  return this.#sockets.size;
311
339
  }
312
340
 
313
- /** Closes and returns everything past the idle budget. Called on an interval by `sync-node`. */
314
- sweepIdle(): SyncSocket[] {
315
- const now = this.#clock.now().getTime();
316
- const closed: SyncSocket[] = [];
317
- for (const socket of this.#sockets.values()) {
318
- if (socket.idleFor(now) > this.#idleTimeoutMs) {
319
- socket.close(CLOSE.idle, 'idle timeout');
320
- // Through `remove`, not the map: the sweep is exactly the abnormal close a gauge leaks on.
321
- this.remove(socket.id);
322
- closed.push(socket);
323
- }
324
- }
325
- return closed;
341
+ /** The budget `idle()` answers against, so a caller can size its own sweep from one number. */
342
+ get idleTimeoutMs(): number {
343
+ return this.#idleTimeoutMs;
344
+ }
345
+
346
+ /**
347
+ * Everything past the idle budget. A QUERY, and deliberately not an eviction: this table is three
348
+ * of the five things a socket holds, and the other two its live subscriptions and its presence
349
+ * membership on the SHARED set — are only reachable from `sync-node`'s `teardown`. A sweep that
350
+ * closed and `remove`d here left a member every other node renders until its TTL and a
351
+ * `QueryEntry` whose `subscribers` map never empties. `sync-node` is the one caller and it
352
+ * releases each one the way the close callback does.
353
+ */
354
+ idle(): SyncSocket[] {
355
+ // Monotonic, because this is the one comparison an operator's clock could otherwise decide:
356
+ // `sync-node` hands this registry and every socket it builds the same `Clock`.
357
+ const now = this.#clock.monotonic();
358
+ return [...this.#sockets.values()].filter(
359
+ (socket) => socket.idleFor(now) > this.#idleTimeoutMs,
360
+ );
326
361
  }
327
362
 
328
363
  /**
@@ -129,7 +129,24 @@ export class SubscriberGate {
129
129
  patch: RowPatch,
130
130
  holds: boolean,
131
131
  ): Promise<RowPatch | null> {
132
- if (patch.op === 'delete' || patch.row === null) return patch;
132
+ // A delete carries no row, so there is nothing to put in front of the rule — `holds` IS the
133
+ // decision, the same one the two branches below take for a row a rule has just refused.
134
+ // Returned unconditionally it was a leak with no upper bound: the shared window is pre-policy,
135
+ // so every subscriber learned the id and the instant of every OTHER tenant's row as it was
136
+ // deleted, on a query whose `visible` rule had never let them see one of them.
137
+ //
138
+ // `holds` comes from `subscription.cursor.ids`, truncated at `CURSOR_ID_LIMIT` — so on a window
139
+ // wider than 512 rows a legitimate delete past position 512 is dropped and that row stays on
140
+ // screen until the subscriber re-snapshots. That is the trade the denied-update branch below
141
+ // already makes, and it is the right way round: a stale row is a bug, a row id leaked to
142
+ // another tenant is a breach.
143
+ if (patch.op === 'delete' || patch.row === null) {
144
+ if (holds) return patch;
145
+ // Counted, or a withheld delete is invisible in exactly the way `onRowDenied` exists to
146
+ // stop — and the rate of it is how an operator sees a window shared across tenants at all.
147
+ this.#denied(target.qid, who, patch.id);
148
+ return null;
149
+ }
133
150
  const full = target.rows.find((row) => row.id === patch.id);
134
151
  // No whole row means no decision to take. An update patch carries the changed columns only, so
135
152
  // a rule reading `row.ownerId` on one reads `undefined` and answers as if the row had said so —
@@ -2,6 +2,7 @@
2
2
  // inbound surface the `sync` node exposes. Every dependency is injected, so the router is
3
3
  // exercisable without a socket, a bus or a server.
4
4
 
5
+ import { logger } from '@ultimat3/core';
5
6
  import type { ChannelHub } from './channel';
6
7
  import { topic as makeTopic } from './channel';
7
8
  import { FrameRateLimitError } from './errors';
@@ -9,7 +10,7 @@ import { FrameLanes, laneKeyOf } from './frame-lanes';
9
10
  import type { JsonValue, Row } from './json';
10
11
  import type { LiveQueryRegistry } from './live-query';
11
12
  import { type PresenceRegistry, presenceFrame } from './presence';
12
- import type { SyncSocket } from './socket';
13
+ import { CLOSE, type SyncSocket } from './socket';
13
14
  import { type Frame, PROTOCOL_VERSION, toWireError } from './sync-protocol';
14
15
 
15
16
  /** Server-authoritative mutation execution. Injected: `sync` never owns business logic. */
@@ -104,7 +105,14 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
104
105
  // Repeating the frame is therefore also the heartbeat — `join` re-`put`s the member.
105
106
  if (presence) {
106
107
  const roster = await presence.join(name, { id: socket.id, actorId: socket.actorId });
107
- socket.send(presenceFrame(name, 'sync', roster.members, roster.total));
108
+ // The answer is read even though nothing here can repair it. A roster has no cursor and
109
+ // no re-send path of its own: the client renders an empty room until it repeats this
110
+ // very frame as its heartbeat, which re-joins and re-rosters. Membership on the shared
111
+ // set is already correct, so the drop costs one client one heartbeat of blank room —
112
+ // and the log is the only trace it leaves anywhere.
113
+ if (!socket.send(presenceFrame(name, 'sync', roster.members, roster.total))) {
114
+ logger.warn('sync.presence_roster_dropped', { topic: name, socketId: socket.id });
115
+ }
108
116
  }
109
117
  return;
110
118
  }
@@ -121,12 +129,19 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
121
129
  sid: frame.sid,
122
130
  cursor: frame.target.cursor,
123
131
  });
124
- socket.send(reply);
132
+ // `subscribe` has already seated the subscription and cleared its desync mark, so a reply
133
+ // the socket refuses leaves the server believing a client that holds no rows is in sync:
134
+ // the next change reaches it as a PATCH folded onto nothing, forever, on a socket that has
135
+ // since drained. Marked instead, which is the state it is actually in — the next delivery
136
+ // re-snapshots it out of the shared window, exactly as `live-fanout` does for a lost patch.
137
+ if (!socket.send(reply)) socket.markDesynced(frame.sid);
125
138
  return;
126
139
  }
127
140
  case 'mutate': {
128
141
  if (!options.onMutate) {
129
- socket.send({
142
+ // A failure receipt is still a receipt: dropped, the client's mutation stays `inflight`
143
+ // and is neither rolled back nor retried, so this one reads the answer too.
144
+ const sent = socket.send({
130
145
  type: 'ack',
131
146
  v: PROTOCOL_VERSION,
132
147
  ref: frame.key,
@@ -137,6 +152,7 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
137
152
  fix: 'pass onMutate to createSyncNode({ onMutate })',
138
153
  }),
139
154
  });
155
+ if (!sent) undeliverable(socket, frame.key, 'ack');
140
156
  return;
141
157
  }
142
158
  const result = await options.onMutate({
@@ -153,7 +169,7 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
153
169
  // and no sequence to decide which later optimistic writes to replay over server truth.
154
170
  // These are two frames on one socket, so the order is the only coordination there is.
155
171
  if (result.entity !== undefined) {
156
- socket.send({
172
+ const sent = socket.send({
157
173
  type: 'rebase',
158
174
  v: PROTOCOL_VERSION,
159
175
  key: frame.key,
@@ -161,14 +177,22 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
161
177
  strategy: 'server-wins',
162
178
  row: result.row ?? null,
163
179
  });
180
+ // The ack retires the client's rebase-log entry, so acking a rebase that never left is
181
+ // the divergence the ordering above exists to prevent — one frame later instead of one
182
+ // frame earlier. Nothing is acked; the mutation stays unsettled and is replayed.
183
+ if (!sent) return undeliverable(socket, frame.key, 'rebase');
184
+ }
185
+ if (
186
+ !socket.send({
187
+ type: 'ack',
188
+ v: PROTOCOL_VERSION,
189
+ ref: frame.key,
190
+ lsn: result.lsn ?? null,
191
+ error: null,
192
+ })
193
+ ) {
194
+ undeliverable(socket, frame.key, 'ack');
164
195
  }
165
- socket.send({
166
- type: 'ack',
167
- v: PROTOCOL_VERSION,
168
- ref: frame.key,
169
- lsn: result.lsn ?? null,
170
- error: null,
171
- });
172
196
  return;
173
197
  }
174
198
  // Server-authored frames are never received from a client.
@@ -183,3 +207,16 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
183
207
  }
184
208
  }
185
209
  }
210
+
211
+ /**
212
+ * A settlement the socket refused. There is nothing on the node to mark — the mutation is applied
213
+ * and the server keeps no per-mutation state — and a client only returns an `inflight` mutation to
214
+ * its queue when the connection dies (`requeueInflight`), so a receipt dropped on a socket that
215
+ * stays up is a write the client neither retires nor retries, with the server believing it settled.
216
+ * Closing IS the repair: the queue hands the mutation back and the reconnect replays it under the
217
+ * same idempotency key.
218
+ */
219
+ function undeliverable(socket: SyncSocket, key: string, kind: 'rebase' | 'ack'): void {
220
+ logger.warn('sync.settlement_dropped', { socketId: socket.id, key, kind });
221
+ socket.close(CLOSE.overloaded, 'settlement undeliverable');
222
+ }
package/src/sync-node.ts CHANGED
@@ -14,6 +14,7 @@ import { CHANGE_SUBJECT_PREFIX, parseEnvelope, SeqGapDetector } from './replicat
14
14
  import {
15
15
  CLOSE,
16
16
  DEFAULT_MAX_BUFFERED_BYTES,
17
+ idleSweepPeriodMs,
17
18
  SocketRegistry,
18
19
  SyncSocket,
19
20
  type WsLike,
@@ -22,7 +23,13 @@ import { GrantBook, type SyncAuthenticator, sweepGrants } from './sync-auth';
22
23
  import { ackRefOf, createFrameRouter, type MutationHandler } from './sync-frames';
23
24
  import { decode, type Frame, PROTOCOL_VERSION, toWireError } from './sync-protocol';
24
25
  import { handleUpgrade, type UpgradeTarget, type WsData } from './sync-upgrade';
25
- import { AcceptBudget, drainPlan, type Rng, reconnectFrame } from './thundering-herd';
26
+ import {
27
+ AcceptBudget,
28
+ type DrainedSocket,
29
+ drainPlan,
30
+ type Rng,
31
+ reconnectFrame,
32
+ } from './thundering-herd';
26
33
 
27
34
  /** Declared with the upgrade that builds it — this file only ever reads one. */
28
35
  export type { UpgradeTarget, WsData } from './sync-upgrade';
@@ -76,6 +83,12 @@ export interface SyncNodeOptions {
76
83
  */
77
84
  readonly maxBufferedBytes?: number;
78
85
  readonly maxDroppedFrames?: number;
86
+ /**
87
+ * How long a socket may route no frame before this node evicts it. Every ceiling on a socket
88
+ * `sync` builds has to be reachable from here, and this one was not: `SocketRegistry`'s default
89
+ * was only settable by constructing the registry yourself, and nothing swept it either way.
90
+ */
91
+ readonly idleTimeoutMs?: number;
79
92
  readonly onMutate?: MutationHandler;
80
93
  /**
81
94
  * Who is dialling. Injected for the same reason `onMutate` is: `sync` owns no business logic and
@@ -126,12 +139,16 @@ export interface SyncNode {
126
139
  close(ws: SyncWs): void;
127
140
  };
128
141
  /** Sends every client a distinct reconnect delay, then closes. Returns the plan for tests/logs. */
129
- drain(options?: { graceMs?: number }): Promise<readonly { socketId: string; afterMs: number }[]>;
142
+ drain(options?: { graceMs?: number }): Promise<readonly DrainedSocket[]>;
130
143
  }
131
144
 
132
145
  export function createSyncNode(options: SyncNodeOptions): SyncNode {
133
146
  const sockets =
134
- options.sockets ?? new SocketRegistry({ ...(options.clock ? { clock: options.clock } : {}) });
147
+ options.sockets ??
148
+ new SocketRegistry({
149
+ ...(options.clock ? { clock: options.clock } : {}),
150
+ ...(options.idleTimeoutMs === undefined ? {} : { idleTimeoutMs: options.idleTimeoutMs }),
151
+ });
135
152
  const clock = options.clock ?? systemClock;
136
153
  const accept = options.accept ?? new AcceptBudget({ perSecond: 500, burst: 2000, clock });
137
154
  const maxConnections = options.maxConnections ?? DEFAULT_MAX_CONNECTIONS;
@@ -143,6 +160,7 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
143
160
  let changes: TransportSubscription | null = null;
144
161
  let sweeping: ReturnType<typeof setInterval> | null = null;
145
162
  let reauthing: ReturnType<typeof setInterval> | null = null;
163
+ let idling: ReturnType<typeof setInterval> | null = null;
146
164
 
147
165
  /**
148
166
  * Work nobody is waiting on — a presence leave from a synchronous close, a sweep on a timer, a
@@ -178,6 +196,8 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
178
196
  sweeping = null;
179
197
  if (reauthing !== null) clearInterval(reauthing);
180
198
  reauthing = null;
199
+ if (idling !== null) clearInterval(idling);
200
+ idling = null;
181
201
  gaps.forget();
182
202
  };
183
203
 
@@ -200,6 +220,17 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
200
220
  }
201
221
  };
202
222
 
223
+ /**
224
+ * The node's one eviction: close, then release everything the socket held. Every path that ends
225
+ * a socket without a `close` callback behind it — the drain, the idle sweep — goes through it,
226
+ * because dropping the socket from the table is three of `teardown`'s five steps and the two it
227
+ * misses are the ones another node can see.
228
+ */
229
+ const evict = (socket: SyncSocket, code: number, reason: string): void => {
230
+ socket.close(code, reason);
231
+ teardown(socket);
232
+ };
233
+
203
234
  /**
204
235
  * One pass over the grants whose window has closed. This is the half R2 was missing: `reauthorize`
205
236
  * and `onActorChange` were both written and neither had a caller, so a socket that was accepted
@@ -222,8 +253,12 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
222
253
  onRevoked: (socketId) => {
223
254
  const socket = sockets.get(socketId);
224
255
  if (!socket) return;
225
- teardown(socket);
226
- socket.close(CLOSE.policy, 'grant expired');
256
+ // Through `evict`, which closes BEFORE it releases. The other order was a close that never
257
+ // happened: `teardown` reaches `sockets.remove`, which closes the socket itself with
258
+ // `1001 connection closed`, and `SyncSocket.close` returns once `#closed` — so a revoked
259
+ // grant reached the client as a normal shutdown, which it retries against this node with
260
+ // the same dead credential, instead of the `1008` that tells it to re-dial with a new one.
261
+ evict(socket, CLOSE.policy, 'grant expired');
227
262
  },
228
263
  onRefreshFailed: (socketId, error) => {
229
264
  // Not a denial: the grant is kept and retried next pass. Reported because a socket nobody
@@ -277,6 +312,14 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
277
312
  );
278
313
  sweeping.unref();
279
314
  }
315
+ // The half-open connection Bun's own `idleTimeout` renews through its ping/pong: a client
316
+ // whose frame loop is wedged answers pings and keeps its grant, its subscriptions and its
317
+ // topic membership. `sweepIdle` was written for this and never called, so `touch()` and the
318
+ // 120s budget under it decided nothing.
319
+ idling = setInterval(() => {
320
+ for (const socket of sockets.idle()) evict(socket, CLOSE.idle, 'idle timeout');
321
+ }, idleSweepPeriodMs(sockets.idleTimeoutMs));
322
+ idling.unref();
280
323
  if (options.authenticate) {
281
324
  reauthing = setInterval(
282
325
  () => detach(reauthenticate(), 'sync.reauthenticate'),
@@ -320,6 +363,9 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
320
363
  newSocketId: () => uuid(),
321
364
  authenticate: options.authenticate,
322
365
  onGranted: (socketId, grant) => grants.set(socketId, grant),
366
+ // The other half of recording the grant before the upgrade: an upgrade that never took
367
+ // gets no `close` callback, so this is the only thing that can free its entry.
368
+ onUngranted: (socketId) => grants.delete(socketId),
323
369
  },
324
370
  request,
325
371
  server,
@@ -407,23 +453,34 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
407
453
  },
408
454
  },
409
455
 
410
- async drain(drainOptions = {}): Promise<readonly { socketId: string; afterMs: number }[]> {
456
+ async drain(drainOptions = {}): Promise<readonly DrainedSocket[]> {
411
457
  ready = false;
412
458
  const ids = [...sockets.all()].map((socket) => socket.id);
413
- const plan = drainPlan(ids, {
459
+ const spread = drainPlan(ids, {
414
460
  spreadMs: options.drainSpreadMs ?? 30_000,
415
461
  ...(options.rng ? { rng: options.rng } : {}),
416
462
  });
417
- for (const entry of plan) {
418
- sockets.get(entry.socketId)?.send(reconnectFrame(entry.afterMs, 'drain'));
463
+ // The answer is read, not assumed: this frame IS the socket's slot, so a client that never
464
+ // received one reconnects on its own backoff — the herd the spread exists to break, minus
465
+ // that client. Nothing repairs it, so what the drop owes is a count.
466
+ const plan: DrainedSocket[] = spread.map((entry) => ({
467
+ ...entry,
468
+ notified:
469
+ sockets.get(entry.socketId)?.send(reconnectFrame(entry.afterMs, 'drain')) === true,
470
+ }));
471
+ const notified = plan.reduce((total, entry) => total + (entry.notified ? 1 : 0), 0);
472
+ if (notified < plan.length) {
473
+ logger.warn('sync.drain_frames_dropped', { sockets: plan.length, notified });
419
474
  }
420
475
  const graceMs = drainOptions.graceMs ?? 5_000;
421
476
  if (graceMs > 0) await new Promise((resolve) => setTimeout(resolve, graceMs));
422
- for (const socket of [...sockets.all()]) {
423
- socket.close(CLOSE.goingAway, 'drain');
424
- sockets.remove(socket.id);
425
- grants.delete(socket.id);
426
- }
477
+ // Through `evict`, never `sockets.remove` + `grants.delete`: those are three of `teardown`'s
478
+ // five steps, and the two they skip are the ones the rest of the fleet can see. A drained
479
+ // socket that never left its presence set is a member every other node renders for a full
480
+ // TTL — during a rolling restart, beside the same client's reconnection under a new id
481
+ // and its live subscriptions stay in the registry, so `entry.subscribers` never empties and
482
+ // the matcher, the shared window and the retained ring are pinned for the process's life.
483
+ for (const socket of [...sockets.all()]) evict(socket, CLOSE.goingAway, 'drain');
427
484
  // Released once the sockets are gone rather than at the top: a client is entitled to its
428
485
  // patches for the whole grace window, and it is entitled to them *before* the hub the
429
486
  // fanout writes through is closed.
@@ -72,7 +72,7 @@ export type SubscribeTarget =
72
72
  * The opening frame, and the heartbeat's. It carries **no cursors**: resume is decided per
73
73
  * subscription by `subscribe`, whose target already carries the cursor and whose `(name, input)`
74
74
  * is what the node needs to authorize the read and reach the retained window at all. A cursor's
75
- * `qid` is `qidOf(name, input)` — a digest, not an input — so a resume list here could never be
75
+ * `qid` is `queryHash(name, input)` — a digest, not an input — so a resume list here could never be
76
76
  * more than a second, unauthorized restatement of that decision, and it cost every reconnect a
77
77
  * duplicate copy of up to `CURSOR_ID_LIMIT` ids per subscription during the exact restart storm
78
78
  * `thundering-herd.ts` exists to bound. Removing it needs no `PROTOCOL_VERSION` bump: `decode`
@@ -375,7 +375,7 @@ function list(obj: JsonObject, key: string, max: number, label = key): JsonValue
375
375
 
376
376
  /**
377
377
  * A client-supplied value, walked ITERATIVELY to its limits. Iteratively because the thing being
378
- * refused is a stack overflow: `qidOf` -> `canonicalJson` recurses over exactly this value, so a
378
+ * refused is a stack overflow: `queryHash` -> `canonicalJson` recurses over exactly this value, so a
379
379
  * depth check that recursed would be the same crash one frame earlier.
380
380
  */
381
381
  function bounded(value: JsonValue, label: string): JsonValue {
@@ -37,8 +37,22 @@ export interface UpgradeDeps {
37
37
  socketCount(): number;
38
38
  newSocketId(): string;
39
39
  readonly authenticate?: SyncAuthenticator | undefined;
40
- /** Recorded only after the upgrade took: a grant for a socket that never opened is never closed. */
40
+ /**
41
+ * Recorded BEFORE `server.upgrade`, because Bun runs `websocket.open` synchronously inside it
42
+ * (measured on bun 1.3.14) and `open` is where the node reads this grant to build the socket's
43
+ * actor. Recorded after, every authenticated socket carried `actor: null` — the topic guard,
44
+ * `authorize`, `visible` and the per-tenant cap all deciding about nobody — and it never
45
+ * repaired, because the re-auth sweep only visits grants with an `expiresAt`.
46
+ */
41
47
  onGranted(socketId: string, grant: SyncGrant): void;
48
+ /**
49
+ * The grant given back on the one path that will never open a socket. Recording first is only
50
+ * safe because this exists: nothing but a `close` callback deletes a grant, and there is no
51
+ * callback for an upgrade that never took. Required, not optional — a host that reserves and
52
+ * cannot release is a leak the type refuses rather than a rule a reviewer has to remember. The
53
+ * same "reserve, then release" shape `channel.ts` uses for a topic slot.
54
+ */
55
+ onUngranted(socketId: string): void;
42
56
  }
43
57
 
44
58
  /**
@@ -97,10 +111,14 @@ export async function handleUpgrade(
97
111
  socketId: deps.newSocketId(),
98
112
  clientBuildId: url.searchParams.get('build') ?? deps.buildId,
99
113
  };
114
+ // Before the upgrade, never after: `server.upgrade` runs `websocket.open` synchronously and does
115
+ // not return until it has, so a grant recorded on the next line is one the socket was already
116
+ // built without.
117
+ if (grant) deps.onGranted(data.socketId, grant);
100
118
  if (!server.upgrade(request, { data })) {
119
+ deps.onUngranted(data.socketId);
101
120
  return new Response('expected websocket', { status: 426 });
102
121
  }
103
- if (grant) deps.onGranted(data.socketId, grant);
104
122
  return undefined;
105
123
  }
106
124
 
@@ -72,6 +72,16 @@ export interface DrainPlanEntry {
72
72
  readonly afterMs: number;
73
73
  }
74
74
 
75
+ /**
76
+ * A plan entry and what became of it — what `SyncNode.drain` returns. `notified: false` means
77
+ * backpressure dropped that socket's `reconnect` frame: the frame is what carries the slot, nothing
78
+ * re-sends it, so that client reconnects on its own backoff and the count is the only place a log
79
+ * can say how much of the spread actually shipped.
80
+ */
81
+ export interface DrainedSocket extends DrainPlanEntry {
82
+ readonly notified: boolean;
83
+ }
84
+
75
85
  export interface DrainPlanOptions {
76
86
  /** Window across which reconnects are spread. Must exceed the node's own drain grace period. */
77
87
  readonly spreadMs?: number;