@ultimat3/realtime 22.15.0 → 24.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/CLAUDE.md +89 -102
  2. package/README.md +140 -20
  3. package/package.json +4 -4
  4. package/src/advisory-lock.ts +57 -0
  5. package/src/boot.ts +12 -7
  6. package/src/change-buffer.ts +96 -41
  7. package/src/changefeed-env.ts +2 -2
  8. package/src/changefeed.ts +39 -1
  9. package/src/channel-authz.ts +43 -12
  10. package/src/channel-latch.ts +52 -0
  11. package/src/channel-logs.ts +34 -4
  12. package/src/channel-records.ts +33 -2
  13. package/src/channel-sids.ts +37 -0
  14. package/src/channel.ts +143 -44
  15. package/src/client-channels.ts +10 -1
  16. package/src/client-contract.ts +4 -1
  17. package/src/client-frames.ts +34 -4
  18. package/src/client-heartbeat.ts +39 -9
  19. package/src/client.ts +1 -0
  20. package/src/cursor.ts +11 -0
  21. package/src/errors.ts +70 -19
  22. package/src/fanout.ts +13 -0
  23. package/src/index.ts +1 -1
  24. package/src/live-contract.ts +6 -2
  25. package/src/live-definition.ts +46 -11
  26. package/src/live-fanout.ts +70 -11
  27. package/src/live-floor.ts +35 -0
  28. package/src/live-query-options.ts +38 -0
  29. package/src/live-query.ts +119 -91
  30. package/src/live-refusal.ts +21 -0
  31. package/src/live-replicator.ts +6 -6
  32. package/src/live-resume.ts +74 -0
  33. package/src/live-tenant.ts +25 -0
  34. package/src/nats-client.ts +11 -2
  35. package/src/nats-open.ts +9 -0
  36. package/src/nats-transport.ts +23 -1
  37. package/src/offline-queue.ts +48 -5
  38. package/src/page-errors.ts +17 -0
  39. package/src/page-outbox.ts +121 -50
  40. package/src/pg-advisory-lock.ts +45 -1
  41. package/src/pg-auth.ts +9 -4
  42. package/src/pg-bytes.ts +4 -1
  43. package/src/pg-connection.ts +119 -21
  44. package/src/pg-entity-row.ts +27 -2
  45. package/src/pg-identity-tables.ts +25 -0
  46. package/src/pg-preflight.ts +50 -4
  47. package/src/pg-replication-row.ts +82 -0
  48. package/src/pg-replication.ts +44 -45
  49. package/src/pg-socket.ts +6 -4
  50. package/src/pg-tls.ts +18 -0
  51. package/src/pg-wire.ts +5 -3
  52. package/src/pgoutput.ts +19 -9
  53. package/src/presence.ts +85 -8
  54. package/src/query-window.ts +28 -7
  55. package/src/record-persister.ts +7 -2
  56. package/src/replication-errors.ts +8 -4
  57. package/src/replicator-envelope.ts +110 -0
  58. package/src/replicator-row.ts +75 -0
  59. package/src/replicator.ts +319 -186
  60. package/src/server.ts +11 -7
  61. package/src/socket-engine.ts +14 -2
  62. package/src/socket-idle.ts +3 -1
  63. package/src/socket-routes.ts +8 -3
  64. package/src/subscriber-gate.ts +11 -1
  65. package/src/sync-auth.ts +47 -2
  66. package/src/sync-frames.ts +73 -20
  67. package/src/sync-node-bounds.ts +15 -0
  68. package/src/sync-node-contract.ts +20 -2
  69. package/src/sync-node.ts +92 -17
  70. package/src/sync-origin.ts +26 -11
  71. package/src/sync-protocol.ts +13 -3
  72. package/src/sync-upgrade.ts +76 -8
  73. package/src/use-mutation.ts +33 -5
  74. package/src/wire-channel.ts +13 -4
@@ -173,6 +173,9 @@ export class SocketEngine {
173
173
  switch (frame.type) {
174
174
  case 'hello':
175
175
  this.#hello = frame;
176
+ // The node names the beat. Every tab hears the same number on its next beat (this frame is
177
+ // what `#helloReply` answers with), so the socket and the ports keep one rhythm.
178
+ if (frame.heartbeatMs !== undefined) this.#heartbeat.follow(frame.heartbeatMs);
176
179
  return;
177
180
  case 'snapshot':
178
181
  case 'patch':
@@ -301,12 +304,21 @@ export class SocketEngine {
301
304
  if (this.#reaper !== null) return;
302
305
  this.#reaper = this.#schedule(() => {
303
306
  this.#reaper = null;
304
- const cutoff = this.#now() - REAP_AFTER_BEATS * this.#beatMs;
307
+ const cutoff = this.#now() - REAP_AFTER_BEATS * this.#portBeatMs();
305
308
  for (const attached of [...this.#ports.values()]) {
306
309
  if (attached.lastSeen < cutoff) this.#reap(attached);
307
310
  }
308
311
  if (this.#ports.size > 0) this.#armReaper();
309
- }, this.#beatMs);
312
+ }, this.#portBeatMs());
313
+ }
314
+
315
+ /**
316
+ * The slowest a healthy tab can be beating: its own default until it hears the node's `hello`,
317
+ * the node's figure after. The LONGER of the two — a tab is never reaped for keeping a beat the
318
+ * node asked for, nor for not having heard a shorter one yet.
319
+ */
320
+ #portBeatMs(): number {
321
+ return Math.max(this.#beatMs, this.#heartbeat.intervalMs);
310
322
  }
311
323
 
312
324
  #beat(): void {
@@ -6,7 +6,9 @@
6
6
  * inactivity budget and not Bun's transport one: Bun's `idleTimeout` is renewed by its own
7
7
  * ping/pong, so a client whose TCP stack still answers pings while its frame loop is wedged holds
8
8
  * its grant, its subscriptions and its topic membership forever. A beating client sends a `hello`
9
- * every `DEFAULT_HEARTBEAT_MS` (15s), so this is eight missed beats.
9
+ * every `DEFAULT_HEARTBEAT_MS` (10s), so this is twelve missed beats — and it stays a multiple of
10
+ * the beat whatever either is set to: the node TELLS the client its beat (`hello.heartbeatMs`),
11
+ * never slower than a quarter of this budget (`clientHeartbeatMs`, `sync-node-bounds.ts`).
10
12
  */
11
13
  export const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
12
14
 
@@ -122,9 +122,14 @@ export class PortRouter {
122
122
  }
123
123
  if (held.ports.has(attached.id)) return;
124
124
  held.ports.add(attached.id);
125
- // The node never hears this add, so it cannot answer it with the `replay-gap` a fresh seat
126
- // gets; the engine does. With no epoch yet, the node's own answer is still on its way and now
127
- // reaches this port too.
125
+ // Forwarded as a BEAT (no `since`): to the node it is the seat it already holds, repeated, and
126
+ // what a seated socket is answered with is the room's roster — the only frame an events-only
127
+ // channel ever gets, so unforwarded this tab read `joining` until the next beat, and with no
128
+ // presence on the node, until the first app event.
129
+ this.#deps.send(withoutSince(held.add));
130
+ // A beat is never answered with the `replay-gap` a fresh seat gets, so the engine says it.
131
+ // With no epoch yet, the node's own answer to the first add is on its way and reaches this
132
+ // port too.
128
133
  if (held.epoch === null) return;
129
134
  const gap: Frame = {
130
135
  type: 'replay-gap',
@@ -7,6 +7,7 @@ import type { Actor } from '@ultimat3/core';
7
7
  import { isPolicyDenial } from './errors';
8
8
  import type { JsonValue, Row, RowPatch } from './json';
9
9
  import type { LiveQueryDefinition } from './live-contract';
10
+ import { narrowRow, projectionOf } from './matcher-bridge';
10
11
 
11
12
  /**
12
13
  * Who a decision is being made for. Every policy call in the live pipeline takes one, which is the
@@ -219,7 +220,16 @@ export class SubscriberGate {
219
220
  // subscriber is not entitled to keep, and one that holds it is told so.
220
221
  if (full === undefined) return holds ? withdrawn(patch) : null;
221
222
  const row: Row = { ...full, ...patch.row, id: patch.id };
222
- if (await this.#visible(target, who, row, 'patch')) return patch;
223
+ if (await this.#visible(target, who, row, 'patch')) {
224
+ // An update carries the changed columns only. A subscriber who was never sent this row —
225
+ // denied at its snapshot, admitted by this change — would fold them onto nothing, and its
226
+ // cursor would never record the id, so the row's later delete was withheld from it. It gets
227
+ // the whole row as an insert, which is also what makes `Holding` and `advance` count it.
228
+ if (patch.op === 'update' && !holds) {
229
+ return { ...patch, op: 'insert', row: narrowRow(row, projectionOf(target.rows)) };
230
+ }
231
+ return patch;
232
+ }
223
233
  this.#denied(target.qid, who, patch.id);
224
234
  return holds ? withdrawn(patch) : null;
225
235
  }
package/src/sync-auth.ts CHANGED
@@ -3,6 +3,16 @@
3
3
  // holds it, and the pass that re-decides one whose window has closed.
4
4
 
5
5
  import type { Actor, Clock } from '@ultimat3/core';
6
+ import { finiteOption } from '@ultimat3/core';
7
+ import { GrantRefreshTimeoutError } from './errors';
8
+ import { type Scheduler, timeoutScheduler } from './thundering-herd';
9
+
10
+ /**
11
+ * How long one grant's `refresh()` may hold the re-auth pass. The pass is serial and one at a
12
+ * time, so a single call that never settles stopped every later grant on the node from being
13
+ * renewed OR revoked. On by default with no "off": a non-positive value takes the default.
14
+ */
15
+ export const DEFAULT_GRANT_REFRESH_DEADLINE_MS = 10_000;
6
16
 
7
17
  /**
8
18
  * One connection's identity. A grant, not an `Actor`, because a websocket outlives every credential
@@ -76,8 +86,12 @@ export interface GrantSweepDeps {
76
86
  onActor: (socketId: string, actor: Actor) => Promise<void>;
77
87
  /** Nobody may hold this socket any longer. The caller closes it. */
78
88
  onRevoked: (socketId: string) => void;
79
- /** `refresh` raised instead of deciding. The grant is kept and retried on the next pass. */
89
+ /** `refresh` raised, or did not answer in time. The grant is kept and retried on the next pass. */
80
90
  onRefreshFailed?: (socketId: string, error: unknown) => void;
91
+ /** See `DEFAULT_GRANT_REFRESH_DEADLINE_MS`. */
92
+ readonly refreshDeadlineMs?: number | undefined;
93
+ /** Injected so that deadline is provable without waiting for one. Production uses `setTimeout`. */
94
+ readonly schedule?: Scheduler | undefined;
81
95
  }
82
96
 
83
97
  export interface GrantSweepResult {
@@ -98,18 +112,29 @@ export interface GrantSweepResult {
98
112
  */
99
113
  export async function sweepGrants(deps: GrantSweepDeps): Promise<GrantSweepResult> {
100
114
  const now = deps.clock.now().getTime();
115
+ const declared = finiteOption(
116
+ 'sweepGrants',
117
+ 'refreshDeadlineMs',
118
+ deps.refreshDeadlineMs ?? DEFAULT_GRANT_REFRESH_DEADLINE_MS,
119
+ );
120
+ const deadlineMs = declared > 0 ? declared : DEFAULT_GRANT_REFRESH_DEADLINE_MS;
121
+ const schedule = deps.schedule ?? timeoutScheduler;
101
122
  let refreshed = 0;
102
123
  let revoked = 0;
103
124
  let failed = 0;
104
125
  for (const [socketId, grant] of deps.grants.expired(now)) {
105
126
  let next: SyncGrant | null;
106
127
  try {
107
- next = grant.refresh ? await grant.refresh() : null;
128
+ next = grant.refresh ? await within(grant.refresh(), socketId, deadlineMs, schedule) : null;
108
129
  } catch (error) {
109
130
  failed += 1;
110
131
  deps.onRefreshFailed?.(socketId, error);
111
132
  continue;
112
133
  }
134
+ // Asked again, because `refresh` is the app's and awaiting it is awaiting a token service: the
135
+ // socket may have closed meanwhile (its grant deleted) or been re-granted. Written back
136
+ // regardless, a closed socket's grant stayed in the book and was refreshed forever.
137
+ if (deps.grants.get(socketId) !== grant) continue;
113
138
  if (next === null) {
114
139
  deps.grants.delete(socketId);
115
140
  deps.onRevoked(socketId);
@@ -122,3 +147,23 @@ export async function sweepGrants(deps: GrantSweepDeps): Promise<GrantSweepResul
122
147
  }
123
148
  return { refreshed, revoked, failed };
124
149
  }
150
+
151
+ /**
152
+ * `refresh()`, or a coded timeout. The call itself cannot be cancelled and this does not pretend
153
+ * to: it runs on with nobody listening, and `Promise.race` keeps its late rejection handled.
154
+ */
155
+ function within(
156
+ refreshing: Promise<SyncGrant | null>,
157
+ socketId: string,
158
+ afterMs: number,
159
+ schedule: Scheduler,
160
+ ): Promise<SyncGrant | null> {
161
+ let disarm: () => void = () => undefined;
162
+ const deadline = new Promise<never>((_resolve, reject) => {
163
+ disarm = schedule(() => reject(new GrantRefreshTimeoutError({ socketId, afterMs })), afterMs);
164
+ });
165
+ const settled = Promise.race([refreshing, deadline]);
166
+ // An armed timer per answered refresh is a leak, and keeps a draining process alive.
167
+ void settled.then(disarm, disarm);
168
+ return settled;
169
+ }
@@ -5,7 +5,8 @@
5
5
  import { logger } from '@ultimat3/core';
6
6
  import type { ChannelHub } from './channel';
7
7
  import type { Topic } from './channel-decl';
8
- import { FrameRateLimitError } from './errors';
8
+ import { ChannelSids } from './channel-sids';
9
+ import { FrameRateLimitError, ProtocolVersionError } from './errors';
9
10
  import { FrameLanes, laneKeyOf } from './frame-lanes';
10
11
  import type { LiveQueryRegistry } from './live-query';
11
12
  import { type PresenceRegistry, presenceFrame } from './presence';
@@ -17,8 +18,20 @@ export interface FrameRouterOptions {
17
18
  readonly registry: LiveQueryRegistry;
18
19
  readonly buildId: string;
19
20
  readonly presence?: PresenceRegistry | undefined;
21
+ /** What the `hello` reply tells a client to beat at — `clientHeartbeatMs` on a real node. */
22
+ readonly heartbeatMs?: number | undefined;
23
+ /** Shared with the node, which names a seat's sid when a re-auth drops it. */
24
+ readonly channelSids?: ChannelSids | undefined;
20
25
  }
21
26
 
27
+ /**
28
+ * The longest `sid` a subscribe frame may carry. A sid is retained — in the subscription book, in
29
+ * `ChannelSids`, and echoed in every ack — so its size is the client's to choose only up to here.
30
+ * The bundled client sends a uuid for a live query and `channel:<topic>` for a channel, so the
31
+ * ceiling sits well above a topic of several uuid params.
32
+ */
33
+ export const MAX_SID_LENGTH = 512;
34
+
22
35
  export type FrameRouter = (socket: SyncSocket, frame: Frame) => Promise<void>;
23
36
 
24
37
  /**
@@ -29,26 +42,39 @@ export type FrameRouter = (socket: SyncSocket, frame: Frame) => Promise<void>;
29
42
  */
30
43
  export function ackRefOf(frame: Frame | null, socketId: string): string {
31
44
  if (frame === null) return socketId;
32
- if (frame.type === 'subscribe') return frame.sid;
45
+ // An oversized sid is the refusal's reason, never its echo.
46
+ if (frame.type === 'subscribe' && frame.sid.length <= MAX_SID_LENGTH) return frame.sid;
33
47
  return socketId;
34
48
  }
35
49
 
36
50
  export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
37
51
  const presence = options.presence;
38
- /** Per socket: the topic each channel sid was joined under. Dies with the socket. */
39
- const channelTopics = new WeakMap<SyncSocket, Map<string, Topic>>();
52
+ const channelSids = options.channelSids ?? new ChannelSids();
40
53
 
41
54
  /**
42
55
  * Subscribing to a channel declared `events: true` IS joining its presence set: presence has no
43
56
  * frame of its own (it rides that channel's `events`), so a second round trip saying "and I am
44
- * here" would be a second way to do one thing. A records-only channel has no roster. Repeating the
45
- * frame is therefore also the heartbeat — `join` re-`put`s the member. The roster's answer is
46
- * read though nothing here can repair it: a dropped roster costs one heartbeat of blank room, and
47
- * the log is the only trace it leaves anywhere.
57
+ * here" would be a second way to do one thing. A records-only channel has no roster.
58
+ *
59
+ * Repeating the frame is the heartbeat. A beat renews the member's TTL (`heartbeat`) instead of
60
+ * joining again — that was a KV put and a fleet-wide `join` event per member per beat — and is
61
+ * still answered with the whole roster: a presence delta is an ephemeral frame, so the roster a
62
+ * beat brings back is what repairs a client that lost one. A member that had already expired is
63
+ * joined again in full.
64
+ *
65
+ * A node built WITHOUT presence still answers: an events-only channel is sent nothing else, so
66
+ * unanswered its subscriber read `joining` until the first app event. The answer is the roster
67
+ * this node can vouch for — nobody.
48
68
  */
49
- const joinPresence = async (socket: SyncSocket, name: Topic): Promise<void> => {
50
- if (!presence || options.hub.channelOf(name)?.channel.events !== true) return;
51
- const roster = await presence.join(name, { id: socket.id, actorId: socket.actorId });
69
+ const joinPresence = async (socket: SyncSocket, name: Topic, beat: boolean): Promise<void> => {
70
+ if (options.hub.channelOf(name)?.channel.events !== true) return;
71
+ const roster = !presence
72
+ ? { members: [], total: 0 }
73
+ : beat && (await presence.heartbeat(name, socket.id))
74
+ ? await presence.roster(name)
75
+ : await presence.join(name, { id: socket.id, actorId: socket.actorId });
76
+ // Read, though nothing here can repair it: the next beat re-sends the set, and the log is the
77
+ // only trace a dropped one leaves anywhere.
52
78
  if (!socket.send(presenceFrame(name, 'sync', roster.members, roster.total))) {
53
79
  logger.warn('sync.presence_roster_dropped', { topic: name, socketId: socket.id });
54
80
  }
@@ -90,6 +116,9 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
90
116
  // The actor the upgrade resolved, so a client can render who the server thinks it is
91
117
  // rather than who it thinks it sent.
92
118
  actorId: socket.actorId,
119
+ // The beat this node's REAL presence ttl and idle budget need. A client cannot read
120
+ // either, and one beating on its own default under a shorter ttl is a false leave.
121
+ ...(options.heartbeatMs === undefined ? {} : { heartbeatMs: options.heartbeatMs }),
93
122
  });
94
123
  if (socket.skewed) {
95
124
  socket.send({ type: 'update-available', v: PROTOCOL_VERSION, buildId: options.buildId });
@@ -97,23 +126,47 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
97
126
  return;
98
127
  }
99
128
  case 'subscribe': {
129
+ if (frame.sid.length > MAX_SID_LENGTH) {
130
+ throw new ProtocolVersionError({
131
+ got: PROTOCOL_VERSION,
132
+ expected: PROTOCOL_VERSION,
133
+ detail: `subscribe.sid is ${frame.sid.length} characters; the limit is ${MAX_SID_LENGTH}`,
134
+ });
135
+ }
100
136
  if (frame.target.kind === 'channel') {
101
- // The topic is the DECLARATION's to spell (`channel.topic(params)`), so a drop finds it
102
- // by the sid its add was answered under — never by re-deriving it from client data.
103
- const topics = channelTopics.get(socket) ?? new Map<string, Topic>();
104
- channelTopics.set(socket, topics);
105
137
  if (frame.op === 'drop') {
106
- const name = topics.get(frame.sid);
138
+ // The topic is the DECLARATION's to spell (`channel.topic(params)`), so a drop finds
139
+ // it by the sid its add was answered under — never by re-deriving it from client data.
140
+ const name = channelSids.topicOf(socket, frame.sid);
107
141
  if (name === undefined) return;
108
- topics.delete(frame.sid);
142
+ channelSids.delete(socket, name);
109
143
  const events = options.hub.channelOf(name)?.channel.events === true;
110
144
  options.hub.unsubscribe(socket, name);
111
145
  if (presence && events) await presence.leave(name, socket.id);
112
146
  return;
113
147
  }
114
- const name = await options.hub.subscribeChannel(socket, frame.target);
115
- topics.set(frame.sid, name);
116
- await joinPresence(socket, name);
148
+ // A seat this sid already names may be DENIED by this ask (a suspended seat re-decided):
149
+ // the hub drops it, and the sid and the presence member must not outlive it.
150
+ const held = channelSids.topicOf(socket, frame.sid);
151
+ const room = held !== undefined && options.hub.channelOf(held)?.channel.events === true;
152
+ let name: Topic;
153
+ try {
154
+ name = await options.hub.subscribeChannel(socket, frame.target);
155
+ } catch (error) {
156
+ if (held !== undefined && !options.hub.topicsOf(socket).includes(held)) {
157
+ channelSids.delete(socket, held);
158
+ if (presence && room) await presence.leave(held, socket.id);
159
+ }
160
+ throw error;
161
+ }
162
+ // Not seated: the socket died while the guard was answering (a presence member written
163
+ // now is one no close will ever remove), or the seat is SUSPENDED — its guard could not
164
+ // decide — and a roster is exactly the delivery a suspension withholds.
165
+ if (socket.closed || !socket.topics.has(name)) return;
166
+ // A seat this socket already held makes the frame its beat, never a second join.
167
+ const beat = channelSids.sidOf(socket, name) !== undefined;
168
+ channelSids.set(socket, name, frame.sid);
169
+ await joinPresence(socket, name, beat);
117
170
  return;
118
171
  }
119
172
  if (frame.op === 'drop') {
@@ -6,6 +6,7 @@
6
6
  // reads false — so `maxConnections: NaN` is a node that accepts without limit and says nothing.
7
7
 
8
8
  import { finiteOption } from '@ultimat3/core';
9
+ import { DEFAULT_HEARTBEAT_MS } from './client-heartbeat';
9
10
 
10
11
  const SUBJECT = 'the sync node';
11
12
 
@@ -127,3 +128,17 @@ export function socketCeilings(options: SocketCeilings): SocketCeilings {
127
128
  /** Per CALL, not per node: `drain({ graceMs })` is an argument, so it is refused where it arrives. */
128
129
  export const drainGraceMs = (graceMs: number | undefined): number =>
129
130
  finiteOption('the sync node drain', 'graceMs', graceMs ?? DEFAULT_DRAIN_GRACE_MS);
131
+
132
+ /**
133
+ * The beat a client is TOLD (`hello.heartbeatMs`): a third of the node's real presence ttl, so one
134
+ * lost beat is never a false leave — and never slower than a quarter of the idle budget, or a
135
+ * long ttl would have this node evict a healthy socket for the silence it asked for. With no
136
+ * presence there is no ttl, and the client's own default is the starting point.
137
+ */
138
+ export function clientHeartbeatMs(
139
+ presenceBeatMs: number | undefined,
140
+ idleTimeoutMs: number,
141
+ ): number {
142
+ const idleBound = Math.max(1_000, Math.floor(idleTimeoutMs / 4));
143
+ return Math.min(presenceBeatMs ?? DEFAULT_HEARTBEAT_MS, idleBound);
144
+ }
@@ -54,11 +54,29 @@ export interface SyncNodeOptions {
54
54
  */
55
55
  readonly authenticate?: SyncAuthenticator;
56
56
  /**
57
- * Exact origins a page may open a socket from, besides this node's own host name — the page's
58
- * origin when it is served on another host (`SYNC_URL` on a separate domain). Anything else is
57
+ * Exact origins a page may open a socket from — the app's public origin (`APP_URL`). Declared,
58
+ * they are the WHOLE list; with none, the node admits the origin it was reached on (its `Host`
59
+ * and port, and that host's https spelling when reached over plain http). Anything else is
59
60
  * refused `X_SOCKET_ORIGIN_REFUSED` before `authenticate` runs.
60
61
  */
61
62
  readonly allowedOrigins?: readonly string[];
63
+ /**
64
+ * Also admit the origin the node was reached on BESIDE a declared list. For a loopback dev
65
+ * server reached through a forwarded host; never a container, where `Host` is the caller's.
66
+ */
67
+ readonly admitReachedOrigin?: boolean;
68
+ /**
69
+ * Peers this node's own `/healthz` and `/readyz` tell the whole report to: address classes
70
+ * (`loopback`, `private`, …) or exact IP literals — the app's `http.healthDetailPeers`. Defaults
71
+ * to core's `DEFAULT_HEALTH_DETAIL_PEERS` (the box itself); anyone else, and any request that
72
+ * says it was forwarded, gets `{ state, ready, role }`.
73
+ */
74
+ readonly healthDetailPeers?: readonly string[];
75
+ /**
76
+ * How long one grant's `refresh()` may hold the re-auth pass before it is reported and skipped
77
+ * (`DEFAULT_GRANT_REFRESH_DEADLINE_MS`). The grant stays expired and is asked again next pass.
78
+ */
79
+ readonly grantRefreshDeadlineMs?: number;
62
80
  /** How often an expired grant is re-decided. The clock a socket's authority runs on. */
63
81
  readonly reauthenticateIntervalMs?: number;
64
82
  readonly clock?: Clock;
package/src/sync-node.ts CHANGED
@@ -4,18 +4,33 @@
4
4
  // client may reconnect to any node and resume from its cursor, which is why drain is allowed to
5
5
  // redistribute connections at all.
6
6
 
7
- import { logger, markReady, reportError, systemClock, uuid } from '@ultimat3/core';
7
+ import {
8
+ DEFAULT_HEALTH_DETAIL_PEERS,
9
+ logger,
10
+ markReady,
11
+ reportError,
12
+ systemClock,
13
+ uuid,
14
+ } from '@ultimat3/core';
8
15
  import type { Topic } from './channel';
16
+ import { ChannelSids } from './channel-sids';
9
17
  import { detach } from './detach';
10
18
  import { evictInChunks } from './drain-evictions';
11
- import { isClientFault } from './errors';
19
+ import { isClientFault, TopicForbiddenError } from './errors';
12
20
  import type { TransportSubscription } from './fanout';
13
- import { CHANGE_SUBJECT_ALL, parseEnvelope, SeqGapDetector } from './replicator';
21
+ import { refuseSubscription } from './live-refusal';
22
+ import { CHANGE_SUBJECT_ALL } from './replicator';
23
+ import { parseEnvelope, SeqGapDetector } from './replicator-envelope';
14
24
  import { CLOSE, DEFAULT_MAX_BUFFERED_BYTES, SocketRegistry, SyncSocket } from './socket';
15
25
  import { idleSweepPeriodMs } from './socket-idle';
16
26
  import { GrantBook, sweepGrants } from './sync-auth';
17
27
  import { ackRefOf, createFrameRouter } from './sync-frames';
18
- import { drainGraceMs, socketCeilings, syncNodeBounds } from './sync-node-bounds';
28
+ import {
29
+ clientHeartbeatMs,
30
+ drainGraceMs,
31
+ socketCeilings,
32
+ syncNodeBounds,
33
+ } from './sync-node-bounds';
19
34
  import type { SyncNode, SyncNodeOptions, SyncWs } from './sync-node-contract';
20
35
  import { decode, type Frame, PROTOCOL_VERSION, toWireError } from './sync-protocol';
21
36
  import { handleUpgrade, type UpgradeTarget } from './sync-upgrade';
@@ -52,6 +67,10 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
52
67
  const presence = options.presence;
53
68
  const grants = new GrantBook();
54
69
  const gaps = new SeqGapDetector();
70
+ const channelSids = new ChannelSids();
71
+ let reconnects: (() => void) | null = null;
72
+ /** The re-auth pass in flight, shared by every tick that lands while it runs. */
73
+ let reauthPass: Promise<void> | null = null;
55
74
  let ready = false;
56
75
  /** Resolved by `teardown` when the last socket leaves, for a drain that is waiting its grace. */
57
76
  let lastSocketLeft: (() => void) | null = null;
@@ -71,6 +90,8 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
71
90
  const release = (): void => {
72
91
  changes?.unsubscribe();
73
92
  changes = null;
93
+ reconnects?.();
94
+ reconnects = null;
74
95
  if (sweeping !== null) clearInterval(sweeping);
75
96
  sweeping = null;
76
97
  if (reauthing !== null) clearInterval(reauthing);
@@ -91,7 +112,10 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
91
112
  */
92
113
  const teardown = (socket: SyncSocket): readonly Promise<unknown>[] => {
93
114
  options.registry.unsubscribeSocket(socket.id);
94
- const topics = [...socket.topics] as Topic[];
115
+ // Suspended seats included, and the presence rooms read BEFORE the seats are given back: the
116
+ // hub forgets a topic's declaration with its last member.
117
+ const topics = options.hub.topicsOf(socket);
118
+ const rooms = topics.filter(hasRoster);
95
119
  for (const name of topics) options.hub.unsubscribe(socket, name);
96
120
  sockets.remove(socket.id);
97
121
  grants.delete(socket.id);
@@ -101,7 +125,9 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
101
125
  // bus and the close callback is synchronous, so it cannot be awaited here.
102
126
  const leaves: Promise<unknown>[] = [];
103
127
  if (presence) {
104
- for (const name of topics) {
128
+ // Only a channel declared `events: true` has a roster: a leave on any other was a KV read,
129
+ // a KV write and a presence event on a channel that never carried one.
130
+ for (const name of rooms) {
105
131
  const leave = presence.leave(name, socket.id);
106
132
  // Detached as well as returned: `detach` attaches the reporting catch, so a caller that
107
133
  // awaits this later is awaiting a promise whose rejection is already handled.
@@ -130,6 +156,11 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
130
156
  lastSocketLeft = done;
131
157
  });
132
158
 
159
+ /** Whether this topic's channel carries presence — asked while the hub still knows the topic. */
160
+ function hasRoster(name: Topic): boolean {
161
+ return options.hub.channelOf(name)?.channel.events === true;
162
+ }
163
+
133
164
  const evict = (socket: SyncSocket, code: number, reason: string): readonly Promise<unknown>[] => {
134
165
  socket.close(code, reason);
135
166
  return teardown(socket);
@@ -141,17 +172,47 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
141
172
  * was authorized for as long as it stayed open — and an active client's socket never idles out,
142
173
  * because every inbound frame touches it.
143
174
  */
144
- const reauthenticate = async (): Promise<void> => {
175
+ const reauthenticate = (): Promise<void> => {
176
+ // One pass at a time. The interval does not wait for the pass it started: after a deploy every
177
+ // grant expires in one window, a pass over them can outlast the interval, and each overlapping
178
+ // tick then refreshed the same grants again and re-snapshotted the same subscriptions.
179
+ reauthPass ??= reauthPassOnce().finally(() => {
180
+ reauthPass = null;
181
+ });
182
+ return reauthPass;
183
+ };
184
+
185
+ const reauthPassOnce = async (): Promise<void> => {
145
186
  await sweepGrants({
146
187
  grants,
147
188
  clock,
189
+ refreshDeadlineMs: options.grantRefreshDeadlineMs,
148
190
  onActor: async (socketId, actor) => {
149
191
  const socket = sockets.get(socketId);
150
192
  if (!socket) return;
193
+ const rooms = new Set(options.hub.topicsOf(socket).filter(hasRoster));
151
194
  // The hub sets `socket.actor` and drops the topics this actor may no longer read; the
152
- // registry re-decides every live subscription and desyncs the survivors, so the next
153
- // delivery re-snapshots them under the new authority rather than the old window.
154
- await options.hub.onActorChange(socket, actor);
195
+ // registry re-decides every live subscription (refusing each one it drops under its sid)
196
+ // and desyncs the survivors, so the next delivery re-snapshots them under the new
197
+ // authority rather than the old window.
198
+ for (const name of await options.hub.onActorChange(socket, actor)) {
199
+ // A dropped seat is SAID: unsaid, the client kept rendering the channel as live.
200
+ const sid = channelSids.sidOf(socket, name);
201
+ channelSids.delete(socket, name);
202
+ if (sid !== undefined) {
203
+ refuseSubscription(
204
+ socket,
205
+ sid,
206
+ new TopicForbiddenError({
207
+ topic: name,
208
+ actorId: socket.actorId,
209
+ reason: 'the session changed and the channel policy no longer admits it',
210
+ }),
211
+ );
212
+ }
213
+ if (presence && rooms.has(name))
214
+ detach(presence.leave(name, socket.id), 'presence.leave', name);
215
+ }
155
216
  await options.registry.reauthorize(socket);
156
217
  },
157
218
  onRevoked: (socketId) => {
@@ -180,6 +241,8 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
180
241
  registry: options.registry,
181
242
  buildId: options.buildId,
182
243
  presence,
244
+ channelSids,
245
+ heartbeatMs: clientHeartbeatMs(presence?.heartbeatMs, sockets.idleTimeoutMs),
183
246
  });
184
247
 
185
248
  return {
@@ -200,7 +263,14 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
200
263
  // subscribers are re-served on the next change to each query.
201
264
  if (gaps.observe(envelope)) {
202
265
  const marked = options.registry.invalidate();
203
- logger.warn('live.change_gap', { entity: envelope.change.entity, desynced: marked });
266
+ // Channels too: their seq is minted from what this node sees, so the hole is invisible
267
+ // to a ring that would otherwise replay across it as complete history.
268
+ const channelGaps = options.hub.invalidate();
269
+ logger.warn('live.change_gap', {
270
+ entity: envelope.change.entity,
271
+ desynced: marked,
272
+ channelGaps,
273
+ });
204
274
  }
205
275
  // Not awaited: the bus handler must return before the next change, and ordering is the
206
276
  // registry's — one serial lane per query id. What this call site owes is the failure. An
@@ -210,6 +280,15 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
210
280
  // hub turns this change into `records` frames on every channel it touches.
211
281
  options.hub.deliverChange(envelope.change);
212
282
  });
283
+ // A bus that reconnected is changes this node never saw, and the gap detector only notices
284
+ // on the NEXT message: with no later write, every window and cursor here stayed behind and
285
+ // every new subscriber joined the stale window. Said by the transport, repaired now.
286
+ reconnects = options.transport.onReconnect(() => {
287
+ gaps.forget();
288
+ const marked = options.registry.invalidate();
289
+ const channelGaps = options.hub.invalidate();
290
+ logger.warn('live.bus_reconnected', { desynced: marked, channelGaps });
291
+ });
213
292
  // One pass per heartbeat window: a member is swept only once it has actually missed its
214
293
  // window, and the interval never holds the process open — shutdown is the drain's job.
215
294
  if (presence) {
@@ -270,6 +349,8 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
270
349
  newSocketId: () => uuid(),
271
350
  authenticate: options.authenticate,
272
351
  allowedOrigins: options.allowedOrigins,
352
+ admitReachedOrigin: options.admitReachedOrigin,
353
+ healthDetailPeers: options.healthDetailPeers ?? DEFAULT_HEALTH_DETAIL_PEERS,
273
354
  onGranted: (socketId, grant) => grants.set(socketId, grant),
274
355
  // The other half of recording the grant before the upgrade: an upgrade that never took
275
356
  // gets no `close` callback, so this is the only thing that can free its entry.
@@ -403,9 +484,3 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
403
484
  },
404
485
  };
405
486
  }
406
-
407
- /**
408
- * An upgrade refused before a socket exists, rendered as the error contract rather than as a word.
409
- * There is no frame to carry it — the client never got a connection — so the body is the only
410
- * channel, and `--json` on every error means this one too.
411
- */
@@ -1,7 +1,7 @@
1
1
  // Single responsibility: whether a websocket upgrade may carry this app's ambient credential. No
2
- // CORS applies to a websocket and the session cookie rides it, so without this a page on a sibling
3
- // host (same-site, so `SameSite=Lax` does not stop it) opened a socket as its visitor. The rule is
4
- // core's `proveSameOrigin`, the one `@ultimat3/http`'s CSRF check asks too.
2
+ // CORS applies to a websocket and the session cookie rides it, so without this a page that is not
3
+ // this app opened a socket as its visitor. The rule is core's `proveSameOrigin`, the one
4
+ // `@ultimat3/http`'s CSRF check asks too.
5
5
 
6
6
  import { type OriginVerdict, proveSameOrigin } from '@ultimat3/core';
7
7
 
@@ -9,25 +9,40 @@ import { type OriginVerdict, proveSameOrigin } from '@ultimat3/core';
9
9
  * Admitted, in order:
10
10
  * - **no `Origin`** — RFC 6455 has a browser send one on every handshake, so a dial without it is
11
11
  * not a browser and no visitor's cookie can be riding it;
12
- * - **the node's own host name**, whatever the port or scheme — cookies are not port-isolated, so
13
- * the page on `:3000` dialling the Compose rung's `:3001` is the same credential boundary, and a
14
- * TLS-terminating ingress means the node cannot know the page's scheme;
15
- * - otherwise core's rule, with `allowedOrigins` as the exact list (`APP_URL`'s origin, from the
16
- * CLI, when the page is served on another host than the node).
12
+ * - **`allowedOrigins`, exactly, when any is declared** (`APP_URL`'s origin, from the CLI). A
13
+ * declaration IS the app's origin, so it is the whole list: the origin the node sees is derived
14
+ * from the `Host` header, which names whatever host the request was aimed at — a sibling
15
+ * subdomain pointed at this node is "its own origin" by that reading, and a `Domain=`-wide
16
+ * session cookie rides its socket;
17
+ * - **the node's own ORIGIN, only when nothing is declared** — host and port exactly as the
18
+ * request reached it (`x dev`, a combined-role container). A host name alone is not this app:
19
+ * cookies are not isolated by port. `admitReached` keeps it beside a declared list (`x dev`).
17
20
  */
18
21
  export function upgradeOrigin(
19
22
  request: Request,
20
23
  url: URL,
21
24
  allowedOrigins: readonly string[],
25
+ admitReached = false,
22
26
  ): OriginVerdict {
23
27
  const origin = request.headers.get('origin');
24
28
  if (origin === null) return { ok: true };
25
- if (URL.parse(origin)?.hostname === url.hostname) return { ok: true };
29
+ // Normalised once (`https://x/` and `https://x` are one origin), then matched as whole strings.
30
+ const admitted = allowedOrigins.map((allowed) => URL.parse(allowed)?.origin);
26
31
  return proveSameOrigin({
27
- selfOrigins: allowedOrigins,
32
+ selfOrigins: admitted.length > 0 && !admitReached ? [] : selfOrigins(url),
28
33
  origin,
29
34
  secFetchSite: request.headers.get('sec-fetch-site'),
30
- listed: () => false,
35
+ listed: (asked) => admitted.includes(asked),
31
36
  listName: 'APP_URL or createSyncNode({ allowedOrigins })',
32
37
  });
33
38
  }
39
+
40
+ /**
41
+ * The origins an UNDECLARED node is reached on. TLS usually ends at a proxy, so a node reached over
42
+ * plain http cannot know its pages' scheme: the https spelling of the same host and port is this
43
+ * app too. That also admits the plain-http page, which is why a deployment behind TLS declares
44
+ * `APP_URL` — the chart sets it from the ingress host.
45
+ */
46
+ function selfOrigins(url: URL): readonly string[] {
47
+ return url.protocol === 'http:' ? [url.origin, `https://${url.host}`] : [url.origin];
48
+ }