@ultimat3/realtime 22.15.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +89 -102
- package/README.md +140 -20
- package/package.json +4 -4
- package/src/advisory-lock.ts +57 -0
- package/src/boot.ts +12 -7
- package/src/change-buffer.ts +96 -41
- package/src/changefeed-env.ts +2 -2
- package/src/changefeed.ts +39 -1
- package/src/channel-authz.ts +43 -12
- package/src/channel-latch.ts +52 -0
- package/src/channel-logs.ts +34 -4
- package/src/channel-records.ts +33 -2
- package/src/channel-sids.ts +37 -0
- package/src/channel.ts +143 -44
- package/src/client-channels.ts +10 -1
- package/src/client-contract.ts +4 -1
- package/src/client-frames.ts +34 -4
- package/src/client-heartbeat.ts +39 -9
- package/src/client.ts +1 -0
- package/src/cursor.ts +11 -0
- package/src/errors.ts +70 -19
- package/src/fanout.ts +13 -0
- package/src/index.ts +1 -1
- package/src/live-contract.ts +6 -2
- package/src/live-definition.ts +46 -11
- package/src/live-fanout.ts +70 -11
- package/src/live-floor.ts +35 -0
- package/src/live-query-options.ts +38 -0
- package/src/live-query.ts +119 -91
- package/src/live-refusal.ts +21 -0
- package/src/live-replicator.ts +6 -6
- package/src/live-resume.ts +74 -0
- package/src/live-tenant.ts +25 -0
- package/src/nats-client.ts +11 -2
- package/src/nats-open.ts +9 -0
- package/src/nats-transport.ts +23 -1
- package/src/offline-queue.ts +48 -5
- package/src/page-errors.ts +17 -0
- package/src/page-outbox.ts +121 -50
- package/src/pg-advisory-lock.ts +45 -1
- package/src/pg-auth.ts +9 -4
- package/src/pg-bytes.ts +4 -1
- package/src/pg-connection.ts +119 -21
- package/src/pg-entity-row.ts +27 -2
- package/src/pg-identity-tables.ts +25 -0
- package/src/pg-preflight.ts +50 -4
- package/src/pg-replication-row.ts +82 -0
- package/src/pg-replication.ts +44 -45
- package/src/pg-socket.ts +6 -4
- package/src/pg-tls.ts +18 -0
- package/src/pg-wire.ts +5 -3
- package/src/pgoutput.ts +19 -9
- package/src/presence.ts +85 -8
- package/src/query-window.ts +28 -7
- package/src/record-persister.ts +7 -2
- package/src/replication-errors.ts +8 -4
- package/src/replicator-envelope.ts +110 -0
- package/src/replicator-row.ts +75 -0
- package/src/replicator.ts +319 -186
- package/src/server.ts +11 -7
- package/src/socket-engine.ts +14 -2
- package/src/socket-idle.ts +3 -1
- package/src/socket-routes.ts +8 -3
- package/src/subscriber-gate.ts +11 -1
- package/src/sync-auth.ts +47 -2
- package/src/sync-frames.ts +73 -20
- package/src/sync-node-bounds.ts +15 -0
- package/src/sync-node-contract.ts +20 -2
- package/src/sync-node.ts +92 -17
- package/src/sync-origin.ts +26 -11
- package/src/sync-protocol.ts +13 -3
- package/src/sync-upgrade.ts +76 -8
- package/src/use-mutation.ts +33 -5
- package/src/wire-channel.ts +13 -4
package/src/socket-engine.ts
CHANGED
|
@@ -173,6 +173,9 @@ export class SocketEngine {
|
|
|
173
173
|
switch (frame.type) {
|
|
174
174
|
case 'hello':
|
|
175
175
|
this.#hello = frame;
|
|
176
|
+
// The node names the beat. Every tab hears the same number on its next beat (this frame is
|
|
177
|
+
// what `#helloReply` answers with), so the socket and the ports keep one rhythm.
|
|
178
|
+
if (frame.heartbeatMs !== undefined) this.#heartbeat.follow(frame.heartbeatMs);
|
|
176
179
|
return;
|
|
177
180
|
case 'snapshot':
|
|
178
181
|
case 'patch':
|
|
@@ -301,12 +304,21 @@ export class SocketEngine {
|
|
|
301
304
|
if (this.#reaper !== null) return;
|
|
302
305
|
this.#reaper = this.#schedule(() => {
|
|
303
306
|
this.#reaper = null;
|
|
304
|
-
const cutoff = this.#now() - REAP_AFTER_BEATS * this.#
|
|
307
|
+
const cutoff = this.#now() - REAP_AFTER_BEATS * this.#portBeatMs();
|
|
305
308
|
for (const attached of [...this.#ports.values()]) {
|
|
306
309
|
if (attached.lastSeen < cutoff) this.#reap(attached);
|
|
307
310
|
}
|
|
308
311
|
if (this.#ports.size > 0) this.#armReaper();
|
|
309
|
-
}, this.#
|
|
312
|
+
}, this.#portBeatMs());
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* The slowest a healthy tab can be beating: its own default until it hears the node's `hello`,
|
|
317
|
+
* the node's figure after. The LONGER of the two — a tab is never reaped for keeping a beat the
|
|
318
|
+
* node asked for, nor for not having heard a shorter one yet.
|
|
319
|
+
*/
|
|
320
|
+
#portBeatMs(): number {
|
|
321
|
+
return Math.max(this.#beatMs, this.#heartbeat.intervalMs);
|
|
310
322
|
}
|
|
311
323
|
|
|
312
324
|
#beat(): void {
|
package/src/socket-idle.ts
CHANGED
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
* inactivity budget and not Bun's transport one: Bun's `idleTimeout` is renewed by its own
|
|
7
7
|
* ping/pong, so a client whose TCP stack still answers pings while its frame loop is wedged holds
|
|
8
8
|
* its grant, its subscriptions and its topic membership forever. A beating client sends a `hello`
|
|
9
|
-
* every `DEFAULT_HEARTBEAT_MS` (
|
|
9
|
+
* every `DEFAULT_HEARTBEAT_MS` (10s), so this is twelve missed beats — and it stays a multiple of
|
|
10
|
+
* the beat whatever either is set to: the node TELLS the client its beat (`hello.heartbeatMs`),
|
|
11
|
+
* never slower than a quarter of this budget (`clientHeartbeatMs`, `sync-node-bounds.ts`).
|
|
10
12
|
*/
|
|
11
13
|
export const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
|
|
12
14
|
|
package/src/socket-routes.ts
CHANGED
|
@@ -122,9 +122,14 @@ export class PortRouter {
|
|
|
122
122
|
}
|
|
123
123
|
if (held.ports.has(attached.id)) return;
|
|
124
124
|
held.ports.add(attached.id);
|
|
125
|
-
//
|
|
126
|
-
//
|
|
127
|
-
//
|
|
125
|
+
// Forwarded as a BEAT (no `since`): to the node it is the seat it already holds, repeated, and
|
|
126
|
+
// what a seated socket is answered with is the room's roster — the only frame an events-only
|
|
127
|
+
// channel ever gets, so unforwarded this tab read `joining` until the next beat, and with no
|
|
128
|
+
// presence on the node, until the first app event.
|
|
129
|
+
this.#deps.send(withoutSince(held.add));
|
|
130
|
+
// A beat is never answered with the `replay-gap` a fresh seat gets, so the engine says it.
|
|
131
|
+
// With no epoch yet, the node's own answer to the first add is on its way and reaches this
|
|
132
|
+
// port too.
|
|
128
133
|
if (held.epoch === null) return;
|
|
129
134
|
const gap: Frame = {
|
|
130
135
|
type: 'replay-gap',
|
package/src/subscriber-gate.ts
CHANGED
|
@@ -7,6 +7,7 @@ import type { Actor } from '@ultimat3/core';
|
|
|
7
7
|
import { isPolicyDenial } from './errors';
|
|
8
8
|
import type { JsonValue, Row, RowPatch } from './json';
|
|
9
9
|
import type { LiveQueryDefinition } from './live-contract';
|
|
10
|
+
import { narrowRow, projectionOf } from './matcher-bridge';
|
|
10
11
|
|
|
11
12
|
/**
|
|
12
13
|
* Who a decision is being made for. Every policy call in the live pipeline takes one, which is the
|
|
@@ -219,7 +220,16 @@ export class SubscriberGate {
|
|
|
219
220
|
// subscriber is not entitled to keep, and one that holds it is told so.
|
|
220
221
|
if (full === undefined) return holds ? withdrawn(patch) : null;
|
|
221
222
|
const row: Row = { ...full, ...patch.row, id: patch.id };
|
|
222
|
-
if (await this.#visible(target, who, row, 'patch'))
|
|
223
|
+
if (await this.#visible(target, who, row, 'patch')) {
|
|
224
|
+
// An update carries the changed columns only. A subscriber who was never sent this row —
|
|
225
|
+
// denied at its snapshot, admitted by this change — would fold them onto nothing, and its
|
|
226
|
+
// cursor would never record the id, so the row's later delete was withheld from it. It gets
|
|
227
|
+
// the whole row as an insert, which is also what makes `Holding` and `advance` count it.
|
|
228
|
+
if (patch.op === 'update' && !holds) {
|
|
229
|
+
return { ...patch, op: 'insert', row: narrowRow(row, projectionOf(target.rows)) };
|
|
230
|
+
}
|
|
231
|
+
return patch;
|
|
232
|
+
}
|
|
223
233
|
this.#denied(target.qid, who, patch.id);
|
|
224
234
|
return holds ? withdrawn(patch) : null;
|
|
225
235
|
}
|
package/src/sync-auth.ts
CHANGED
|
@@ -3,6 +3,16 @@
|
|
|
3
3
|
// holds it, and the pass that re-decides one whose window has closed.
|
|
4
4
|
|
|
5
5
|
import type { Actor, Clock } from '@ultimat3/core';
|
|
6
|
+
import { finiteOption } from '@ultimat3/core';
|
|
7
|
+
import { GrantRefreshTimeoutError } from './errors';
|
|
8
|
+
import { type Scheduler, timeoutScheduler } from './thundering-herd';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* How long one grant's `refresh()` may hold the re-auth pass. The pass is serial and one at a
|
|
12
|
+
* time, so a single call that never settles stopped every later grant on the node from being
|
|
13
|
+
* renewed OR revoked. On by default with no "off": a non-positive value takes the default.
|
|
14
|
+
*/
|
|
15
|
+
export const DEFAULT_GRANT_REFRESH_DEADLINE_MS = 10_000;
|
|
6
16
|
|
|
7
17
|
/**
|
|
8
18
|
* One connection's identity. A grant, not an `Actor`, because a websocket outlives every credential
|
|
@@ -76,8 +86,12 @@ export interface GrantSweepDeps {
|
|
|
76
86
|
onActor: (socketId: string, actor: Actor) => Promise<void>;
|
|
77
87
|
/** Nobody may hold this socket any longer. The caller closes it. */
|
|
78
88
|
onRevoked: (socketId: string) => void;
|
|
79
|
-
/** `refresh` raised
|
|
89
|
+
/** `refresh` raised, or did not answer in time. The grant is kept and retried on the next pass. */
|
|
80
90
|
onRefreshFailed?: (socketId: string, error: unknown) => void;
|
|
91
|
+
/** See `DEFAULT_GRANT_REFRESH_DEADLINE_MS`. */
|
|
92
|
+
readonly refreshDeadlineMs?: number | undefined;
|
|
93
|
+
/** Injected so that deadline is provable without waiting for one. Production uses `setTimeout`. */
|
|
94
|
+
readonly schedule?: Scheduler | undefined;
|
|
81
95
|
}
|
|
82
96
|
|
|
83
97
|
export interface GrantSweepResult {
|
|
@@ -98,18 +112,29 @@ export interface GrantSweepResult {
|
|
|
98
112
|
*/
|
|
99
113
|
export async function sweepGrants(deps: GrantSweepDeps): Promise<GrantSweepResult> {
|
|
100
114
|
const now = deps.clock.now().getTime();
|
|
115
|
+
const declared = finiteOption(
|
|
116
|
+
'sweepGrants',
|
|
117
|
+
'refreshDeadlineMs',
|
|
118
|
+
deps.refreshDeadlineMs ?? DEFAULT_GRANT_REFRESH_DEADLINE_MS,
|
|
119
|
+
);
|
|
120
|
+
const deadlineMs = declared > 0 ? declared : DEFAULT_GRANT_REFRESH_DEADLINE_MS;
|
|
121
|
+
const schedule = deps.schedule ?? timeoutScheduler;
|
|
101
122
|
let refreshed = 0;
|
|
102
123
|
let revoked = 0;
|
|
103
124
|
let failed = 0;
|
|
104
125
|
for (const [socketId, grant] of deps.grants.expired(now)) {
|
|
105
126
|
let next: SyncGrant | null;
|
|
106
127
|
try {
|
|
107
|
-
next = grant.refresh ? await grant.refresh() : null;
|
|
128
|
+
next = grant.refresh ? await within(grant.refresh(), socketId, deadlineMs, schedule) : null;
|
|
108
129
|
} catch (error) {
|
|
109
130
|
failed += 1;
|
|
110
131
|
deps.onRefreshFailed?.(socketId, error);
|
|
111
132
|
continue;
|
|
112
133
|
}
|
|
134
|
+
// Asked again, because `refresh` is the app's and awaiting it is awaiting a token service: the
|
|
135
|
+
// socket may have closed meanwhile (its grant deleted) or been re-granted. Written back
|
|
136
|
+
// regardless, a closed socket's grant stayed in the book and was refreshed forever.
|
|
137
|
+
if (deps.grants.get(socketId) !== grant) continue;
|
|
113
138
|
if (next === null) {
|
|
114
139
|
deps.grants.delete(socketId);
|
|
115
140
|
deps.onRevoked(socketId);
|
|
@@ -122,3 +147,23 @@ export async function sweepGrants(deps: GrantSweepDeps): Promise<GrantSweepResul
|
|
|
122
147
|
}
|
|
123
148
|
return { refreshed, revoked, failed };
|
|
124
149
|
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* `refresh()`, or a coded timeout. The call itself cannot be cancelled and this does not pretend
|
|
153
|
+
* to: it runs on with nobody listening, and `Promise.race` keeps its late rejection handled.
|
|
154
|
+
*/
|
|
155
|
+
function within(
|
|
156
|
+
refreshing: Promise<SyncGrant | null>,
|
|
157
|
+
socketId: string,
|
|
158
|
+
afterMs: number,
|
|
159
|
+
schedule: Scheduler,
|
|
160
|
+
): Promise<SyncGrant | null> {
|
|
161
|
+
let disarm: () => void = () => undefined;
|
|
162
|
+
const deadline = new Promise<never>((_resolve, reject) => {
|
|
163
|
+
disarm = schedule(() => reject(new GrantRefreshTimeoutError({ socketId, afterMs })), afterMs);
|
|
164
|
+
});
|
|
165
|
+
const settled = Promise.race([refreshing, deadline]);
|
|
166
|
+
// An armed timer per answered refresh is a leak, and keeps a draining process alive.
|
|
167
|
+
void settled.then(disarm, disarm);
|
|
168
|
+
return settled;
|
|
169
|
+
}
|
package/src/sync-frames.ts
CHANGED
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
import { logger } from '@ultimat3/core';
|
|
6
6
|
import type { ChannelHub } from './channel';
|
|
7
7
|
import type { Topic } from './channel-decl';
|
|
8
|
-
import {
|
|
8
|
+
import { ChannelSids } from './channel-sids';
|
|
9
|
+
import { FrameRateLimitError, ProtocolVersionError } from './errors';
|
|
9
10
|
import { FrameLanes, laneKeyOf } from './frame-lanes';
|
|
10
11
|
import type { LiveQueryRegistry } from './live-query';
|
|
11
12
|
import { type PresenceRegistry, presenceFrame } from './presence';
|
|
@@ -17,8 +18,20 @@ export interface FrameRouterOptions {
|
|
|
17
18
|
readonly registry: LiveQueryRegistry;
|
|
18
19
|
readonly buildId: string;
|
|
19
20
|
readonly presence?: PresenceRegistry | undefined;
|
|
21
|
+
/** What the `hello` reply tells a client to beat at — `clientHeartbeatMs` on a real node. */
|
|
22
|
+
readonly heartbeatMs?: number | undefined;
|
|
23
|
+
/** Shared with the node, which names a seat's sid when a re-auth drops it. */
|
|
24
|
+
readonly channelSids?: ChannelSids | undefined;
|
|
20
25
|
}
|
|
21
26
|
|
|
27
|
+
/**
|
|
28
|
+
* The longest `sid` a subscribe frame may carry. A sid is retained — in the subscription book, in
|
|
29
|
+
* `ChannelSids`, and echoed in every ack — so its size is the client's to choose only up to here.
|
|
30
|
+
* The bundled client sends a uuid for a live query and `channel:<topic>` for a channel, so the
|
|
31
|
+
* ceiling sits well above a topic of several uuid params.
|
|
32
|
+
*/
|
|
33
|
+
export const MAX_SID_LENGTH = 512;
|
|
34
|
+
|
|
22
35
|
export type FrameRouter = (socket: SyncSocket, frame: Frame) => Promise<void>;
|
|
23
36
|
|
|
24
37
|
/**
|
|
@@ -29,26 +42,39 @@ export type FrameRouter = (socket: SyncSocket, frame: Frame) => Promise<void>;
|
|
|
29
42
|
*/
|
|
30
43
|
export function ackRefOf(frame: Frame | null, socketId: string): string {
|
|
31
44
|
if (frame === null) return socketId;
|
|
32
|
-
|
|
45
|
+
// An oversized sid is the refusal's reason, never its echo.
|
|
46
|
+
if (frame.type === 'subscribe' && frame.sid.length <= MAX_SID_LENGTH) return frame.sid;
|
|
33
47
|
return socketId;
|
|
34
48
|
}
|
|
35
49
|
|
|
36
50
|
export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
37
51
|
const presence = options.presence;
|
|
38
|
-
|
|
39
|
-
const channelTopics = new WeakMap<SyncSocket, Map<string, Topic>>();
|
|
52
|
+
const channelSids = options.channelSids ?? new ChannelSids();
|
|
40
53
|
|
|
41
54
|
/**
|
|
42
55
|
* Subscribing to a channel declared `events: true` IS joining its presence set: presence has no
|
|
43
56
|
* frame of its own (it rides that channel's `events`), so a second round trip saying "and I am
|
|
44
|
-
* here" would be a second way to do one thing. A records-only channel has no roster.
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
57
|
+
* here" would be a second way to do one thing. A records-only channel has no roster.
|
|
58
|
+
*
|
|
59
|
+
* Repeating the frame is the heartbeat. A beat renews the member's TTL (`heartbeat`) instead of
|
|
60
|
+
* joining again — that was a KV put and a fleet-wide `join` event per member per beat — and is
|
|
61
|
+
* still answered with the whole roster: a presence delta is an ephemeral frame, so the roster a
|
|
62
|
+
* beat brings back is what repairs a client that lost one. A member that had already expired is
|
|
63
|
+
* joined again in full.
|
|
64
|
+
*
|
|
65
|
+
* A node built WITHOUT presence still answers: an events-only channel is sent nothing else, so
|
|
66
|
+
* unanswered its subscriber read `joining` until the first app event. The answer is the roster
|
|
67
|
+
* this node can vouch for — nobody.
|
|
48
68
|
*/
|
|
49
|
-
const joinPresence = async (socket: SyncSocket, name: Topic): Promise<void> => {
|
|
50
|
-
if (
|
|
51
|
-
const roster =
|
|
69
|
+
const joinPresence = async (socket: SyncSocket, name: Topic, beat: boolean): Promise<void> => {
|
|
70
|
+
if (options.hub.channelOf(name)?.channel.events !== true) return;
|
|
71
|
+
const roster = !presence
|
|
72
|
+
? { members: [], total: 0 }
|
|
73
|
+
: beat && (await presence.heartbeat(name, socket.id))
|
|
74
|
+
? await presence.roster(name)
|
|
75
|
+
: await presence.join(name, { id: socket.id, actorId: socket.actorId });
|
|
76
|
+
// Read, though nothing here can repair it: the next beat re-sends the set, and the log is the
|
|
77
|
+
// only trace a dropped one leaves anywhere.
|
|
52
78
|
if (!socket.send(presenceFrame(name, 'sync', roster.members, roster.total))) {
|
|
53
79
|
logger.warn('sync.presence_roster_dropped', { topic: name, socketId: socket.id });
|
|
54
80
|
}
|
|
@@ -90,6 +116,9 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
90
116
|
// The actor the upgrade resolved, so a client can render who the server thinks it is
|
|
91
117
|
// rather than who it thinks it sent.
|
|
92
118
|
actorId: socket.actorId,
|
|
119
|
+
// The beat this node's REAL presence ttl and idle budget need. A client cannot read
|
|
120
|
+
// either, and one beating on its own default under a shorter ttl is a false leave.
|
|
121
|
+
...(options.heartbeatMs === undefined ? {} : { heartbeatMs: options.heartbeatMs }),
|
|
93
122
|
});
|
|
94
123
|
if (socket.skewed) {
|
|
95
124
|
socket.send({ type: 'update-available', v: PROTOCOL_VERSION, buildId: options.buildId });
|
|
@@ -97,23 +126,47 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
97
126
|
return;
|
|
98
127
|
}
|
|
99
128
|
case 'subscribe': {
|
|
129
|
+
if (frame.sid.length > MAX_SID_LENGTH) {
|
|
130
|
+
throw new ProtocolVersionError({
|
|
131
|
+
got: PROTOCOL_VERSION,
|
|
132
|
+
expected: PROTOCOL_VERSION,
|
|
133
|
+
detail: `subscribe.sid is ${frame.sid.length} characters; the limit is ${MAX_SID_LENGTH}`,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
100
136
|
if (frame.target.kind === 'channel') {
|
|
101
|
-
// The topic is the DECLARATION's to spell (`channel.topic(params)`), so a drop finds it
|
|
102
|
-
// by the sid its add was answered under — never by re-deriving it from client data.
|
|
103
|
-
const topics = channelTopics.get(socket) ?? new Map<string, Topic>();
|
|
104
|
-
channelTopics.set(socket, topics);
|
|
105
137
|
if (frame.op === 'drop') {
|
|
106
|
-
|
|
138
|
+
// The topic is the DECLARATION's to spell (`channel.topic(params)`), so a drop finds
|
|
139
|
+
// it by the sid its add was answered under — never by re-deriving it from client data.
|
|
140
|
+
const name = channelSids.topicOf(socket, frame.sid);
|
|
107
141
|
if (name === undefined) return;
|
|
108
|
-
|
|
142
|
+
channelSids.delete(socket, name);
|
|
109
143
|
const events = options.hub.channelOf(name)?.channel.events === true;
|
|
110
144
|
options.hub.unsubscribe(socket, name);
|
|
111
145
|
if (presence && events) await presence.leave(name, socket.id);
|
|
112
146
|
return;
|
|
113
147
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
148
|
+
// A seat this sid already names may be DENIED by this ask (a suspended seat re-decided):
|
|
149
|
+
// the hub drops it, and the sid and the presence member must not outlive it.
|
|
150
|
+
const held = channelSids.topicOf(socket, frame.sid);
|
|
151
|
+
const room = held !== undefined && options.hub.channelOf(held)?.channel.events === true;
|
|
152
|
+
let name: Topic;
|
|
153
|
+
try {
|
|
154
|
+
name = await options.hub.subscribeChannel(socket, frame.target);
|
|
155
|
+
} catch (error) {
|
|
156
|
+
if (held !== undefined && !options.hub.topicsOf(socket).includes(held)) {
|
|
157
|
+
channelSids.delete(socket, held);
|
|
158
|
+
if (presence && room) await presence.leave(held, socket.id);
|
|
159
|
+
}
|
|
160
|
+
throw error;
|
|
161
|
+
}
|
|
162
|
+
// Not seated: the socket died while the guard was answering (a presence member written
|
|
163
|
+
// now is one no close will ever remove), or the seat is SUSPENDED — its guard could not
|
|
164
|
+
// decide — and a roster is exactly the delivery a suspension withholds.
|
|
165
|
+
if (socket.closed || !socket.topics.has(name)) return;
|
|
166
|
+
// A seat this socket already held makes the frame its beat, never a second join.
|
|
167
|
+
const beat = channelSids.sidOf(socket, name) !== undefined;
|
|
168
|
+
channelSids.set(socket, name, frame.sid);
|
|
169
|
+
await joinPresence(socket, name, beat);
|
|
117
170
|
return;
|
|
118
171
|
}
|
|
119
172
|
if (frame.op === 'drop') {
|
package/src/sync-node-bounds.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
// reads false — so `maxConnections: NaN` is a node that accepts without limit and says nothing.
|
|
7
7
|
|
|
8
8
|
import { finiteOption } from '@ultimat3/core';
|
|
9
|
+
import { DEFAULT_HEARTBEAT_MS } from './client-heartbeat';
|
|
9
10
|
|
|
10
11
|
const SUBJECT = 'the sync node';
|
|
11
12
|
|
|
@@ -127,3 +128,17 @@ export function socketCeilings(options: SocketCeilings): SocketCeilings {
|
|
|
127
128
|
/** Per CALL, not per node: `drain({ graceMs })` is an argument, so it is refused where it arrives. */
|
|
128
129
|
export const drainGraceMs = (graceMs: number | undefined): number =>
|
|
129
130
|
finiteOption('the sync node drain', 'graceMs', graceMs ?? DEFAULT_DRAIN_GRACE_MS);
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* The beat a client is TOLD (`hello.heartbeatMs`): a third of the node's real presence ttl, so one
|
|
134
|
+
* lost beat is never a false leave — and never slower than a quarter of the idle budget, or a
|
|
135
|
+
* long ttl would have this node evict a healthy socket for the silence it asked for. With no
|
|
136
|
+
* presence there is no ttl, and the client's own default is the starting point.
|
|
137
|
+
*/
|
|
138
|
+
export function clientHeartbeatMs(
|
|
139
|
+
presenceBeatMs: number | undefined,
|
|
140
|
+
idleTimeoutMs: number,
|
|
141
|
+
): number {
|
|
142
|
+
const idleBound = Math.max(1_000, Math.floor(idleTimeoutMs / 4));
|
|
143
|
+
return Math.min(presenceBeatMs ?? DEFAULT_HEARTBEAT_MS, idleBound);
|
|
144
|
+
}
|
|
@@ -54,11 +54,29 @@ export interface SyncNodeOptions {
|
|
|
54
54
|
*/
|
|
55
55
|
readonly authenticate?: SyncAuthenticator;
|
|
56
56
|
/**
|
|
57
|
-
* Exact origins a page may open a socket from
|
|
58
|
-
*
|
|
57
|
+
* Exact origins a page may open a socket from — the app's public origin (`APP_URL`). Declared,
|
|
58
|
+
* they are the WHOLE list; with none, the node admits the origin it was reached on (its `Host`
|
|
59
|
+
* and port, and that host's https spelling when reached over plain http). Anything else is
|
|
59
60
|
* refused `X_SOCKET_ORIGIN_REFUSED` before `authenticate` runs.
|
|
60
61
|
*/
|
|
61
62
|
readonly allowedOrigins?: readonly string[];
|
|
63
|
+
/**
|
|
64
|
+
* Also admit the origin the node was reached on BESIDE a declared list. For a loopback dev
|
|
65
|
+
* server reached through a forwarded host; never a container, where `Host` is the caller's.
|
|
66
|
+
*/
|
|
67
|
+
readonly admitReachedOrigin?: boolean;
|
|
68
|
+
/**
|
|
69
|
+
* Peers this node's own `/healthz` and `/readyz` tell the whole report to: address classes
|
|
70
|
+
* (`loopback`, `private`, …) or exact IP literals — the app's `http.healthDetailPeers`. Defaults
|
|
71
|
+
* to core's `DEFAULT_HEALTH_DETAIL_PEERS` (the box itself); anyone else, and any request that
|
|
72
|
+
* says it was forwarded, gets `{ state, ready, role }`.
|
|
73
|
+
*/
|
|
74
|
+
readonly healthDetailPeers?: readonly string[];
|
|
75
|
+
/**
|
|
76
|
+
* How long one grant's `refresh()` may hold the re-auth pass before it is reported and skipped
|
|
77
|
+
* (`DEFAULT_GRANT_REFRESH_DEADLINE_MS`). The grant stays expired and is asked again next pass.
|
|
78
|
+
*/
|
|
79
|
+
readonly grantRefreshDeadlineMs?: number;
|
|
62
80
|
/** How often an expired grant is re-decided. The clock a socket's authority runs on. */
|
|
63
81
|
readonly reauthenticateIntervalMs?: number;
|
|
64
82
|
readonly clock?: Clock;
|
package/src/sync-node.ts
CHANGED
|
@@ -4,18 +4,33 @@
|
|
|
4
4
|
// client may reconnect to any node and resume from its cursor, which is why drain is allowed to
|
|
5
5
|
// redistribute connections at all.
|
|
6
6
|
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
DEFAULT_HEALTH_DETAIL_PEERS,
|
|
9
|
+
logger,
|
|
10
|
+
markReady,
|
|
11
|
+
reportError,
|
|
12
|
+
systemClock,
|
|
13
|
+
uuid,
|
|
14
|
+
} from '@ultimat3/core';
|
|
8
15
|
import type { Topic } from './channel';
|
|
16
|
+
import { ChannelSids } from './channel-sids';
|
|
9
17
|
import { detach } from './detach';
|
|
10
18
|
import { evictInChunks } from './drain-evictions';
|
|
11
|
-
import { isClientFault } from './errors';
|
|
19
|
+
import { isClientFault, TopicForbiddenError } from './errors';
|
|
12
20
|
import type { TransportSubscription } from './fanout';
|
|
13
|
-
import {
|
|
21
|
+
import { refuseSubscription } from './live-refusal';
|
|
22
|
+
import { CHANGE_SUBJECT_ALL } from './replicator';
|
|
23
|
+
import { parseEnvelope, SeqGapDetector } from './replicator-envelope';
|
|
14
24
|
import { CLOSE, DEFAULT_MAX_BUFFERED_BYTES, SocketRegistry, SyncSocket } from './socket';
|
|
15
25
|
import { idleSweepPeriodMs } from './socket-idle';
|
|
16
26
|
import { GrantBook, sweepGrants } from './sync-auth';
|
|
17
27
|
import { ackRefOf, createFrameRouter } from './sync-frames';
|
|
18
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
clientHeartbeatMs,
|
|
30
|
+
drainGraceMs,
|
|
31
|
+
socketCeilings,
|
|
32
|
+
syncNodeBounds,
|
|
33
|
+
} from './sync-node-bounds';
|
|
19
34
|
import type { SyncNode, SyncNodeOptions, SyncWs } from './sync-node-contract';
|
|
20
35
|
import { decode, type Frame, PROTOCOL_VERSION, toWireError } from './sync-protocol';
|
|
21
36
|
import { handleUpgrade, type UpgradeTarget } from './sync-upgrade';
|
|
@@ -52,6 +67,10 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
52
67
|
const presence = options.presence;
|
|
53
68
|
const grants = new GrantBook();
|
|
54
69
|
const gaps = new SeqGapDetector();
|
|
70
|
+
const channelSids = new ChannelSids();
|
|
71
|
+
let reconnects: (() => void) | null = null;
|
|
72
|
+
/** The re-auth pass in flight, shared by every tick that lands while it runs. */
|
|
73
|
+
let reauthPass: Promise<void> | null = null;
|
|
55
74
|
let ready = false;
|
|
56
75
|
/** Resolved by `teardown` when the last socket leaves, for a drain that is waiting its grace. */
|
|
57
76
|
let lastSocketLeft: (() => void) | null = null;
|
|
@@ -71,6 +90,8 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
71
90
|
const release = (): void => {
|
|
72
91
|
changes?.unsubscribe();
|
|
73
92
|
changes = null;
|
|
93
|
+
reconnects?.();
|
|
94
|
+
reconnects = null;
|
|
74
95
|
if (sweeping !== null) clearInterval(sweeping);
|
|
75
96
|
sweeping = null;
|
|
76
97
|
if (reauthing !== null) clearInterval(reauthing);
|
|
@@ -91,7 +112,10 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
91
112
|
*/
|
|
92
113
|
const teardown = (socket: SyncSocket): readonly Promise<unknown>[] => {
|
|
93
114
|
options.registry.unsubscribeSocket(socket.id);
|
|
94
|
-
|
|
115
|
+
// Suspended seats included, and the presence rooms read BEFORE the seats are given back: the
|
|
116
|
+
// hub forgets a topic's declaration with its last member.
|
|
117
|
+
const topics = options.hub.topicsOf(socket);
|
|
118
|
+
const rooms = topics.filter(hasRoster);
|
|
95
119
|
for (const name of topics) options.hub.unsubscribe(socket, name);
|
|
96
120
|
sockets.remove(socket.id);
|
|
97
121
|
grants.delete(socket.id);
|
|
@@ -101,7 +125,9 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
101
125
|
// bus and the close callback is synchronous, so it cannot be awaited here.
|
|
102
126
|
const leaves: Promise<unknown>[] = [];
|
|
103
127
|
if (presence) {
|
|
104
|
-
|
|
128
|
+
// Only a channel declared `events: true` has a roster: a leave on any other was a KV read,
|
|
129
|
+
// a KV write and a presence event on a channel that never carried one.
|
|
130
|
+
for (const name of rooms) {
|
|
105
131
|
const leave = presence.leave(name, socket.id);
|
|
106
132
|
// Detached as well as returned: `detach` attaches the reporting catch, so a caller that
|
|
107
133
|
// awaits this later is awaiting a promise whose rejection is already handled.
|
|
@@ -130,6 +156,11 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
130
156
|
lastSocketLeft = done;
|
|
131
157
|
});
|
|
132
158
|
|
|
159
|
+
/** Whether this topic's channel carries presence — asked while the hub still knows the topic. */
|
|
160
|
+
function hasRoster(name: Topic): boolean {
|
|
161
|
+
return options.hub.channelOf(name)?.channel.events === true;
|
|
162
|
+
}
|
|
163
|
+
|
|
133
164
|
const evict = (socket: SyncSocket, code: number, reason: string): readonly Promise<unknown>[] => {
|
|
134
165
|
socket.close(code, reason);
|
|
135
166
|
return teardown(socket);
|
|
@@ -141,17 +172,47 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
141
172
|
* was authorized for as long as it stayed open — and an active client's socket never idles out,
|
|
142
173
|
* because every inbound frame touches it.
|
|
143
174
|
*/
|
|
144
|
-
const reauthenticate =
|
|
175
|
+
const reauthenticate = (): Promise<void> => {
|
|
176
|
+
// One pass at a time. The interval does not wait for the pass it started: after a deploy every
|
|
177
|
+
// grant expires in one window, a pass over them can outlast the interval, and each overlapping
|
|
178
|
+
// tick then refreshed the same grants again and re-snapshotted the same subscriptions.
|
|
179
|
+
reauthPass ??= reauthPassOnce().finally(() => {
|
|
180
|
+
reauthPass = null;
|
|
181
|
+
});
|
|
182
|
+
return reauthPass;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
const reauthPassOnce = async (): Promise<void> => {
|
|
145
186
|
await sweepGrants({
|
|
146
187
|
grants,
|
|
147
188
|
clock,
|
|
189
|
+
refreshDeadlineMs: options.grantRefreshDeadlineMs,
|
|
148
190
|
onActor: async (socketId, actor) => {
|
|
149
191
|
const socket = sockets.get(socketId);
|
|
150
192
|
if (!socket) return;
|
|
193
|
+
const rooms = new Set(options.hub.topicsOf(socket).filter(hasRoster));
|
|
151
194
|
// The hub sets `socket.actor` and drops the topics this actor may no longer read; the
|
|
152
|
-
// registry re-decides every live subscription
|
|
153
|
-
// delivery re-snapshots them under the new
|
|
154
|
-
|
|
195
|
+
// registry re-decides every live subscription (refusing each one it drops under its sid)
|
|
196
|
+
// and desyncs the survivors, so the next delivery re-snapshots them under the new
|
|
197
|
+
// authority rather than the old window.
|
|
198
|
+
for (const name of await options.hub.onActorChange(socket, actor)) {
|
|
199
|
+
// A dropped seat is SAID: unsaid, the client kept rendering the channel as live.
|
|
200
|
+
const sid = channelSids.sidOf(socket, name);
|
|
201
|
+
channelSids.delete(socket, name);
|
|
202
|
+
if (sid !== undefined) {
|
|
203
|
+
refuseSubscription(
|
|
204
|
+
socket,
|
|
205
|
+
sid,
|
|
206
|
+
new TopicForbiddenError({
|
|
207
|
+
topic: name,
|
|
208
|
+
actorId: socket.actorId,
|
|
209
|
+
reason: 'the session changed and the channel policy no longer admits it',
|
|
210
|
+
}),
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
if (presence && rooms.has(name))
|
|
214
|
+
detach(presence.leave(name, socket.id), 'presence.leave', name);
|
|
215
|
+
}
|
|
155
216
|
await options.registry.reauthorize(socket);
|
|
156
217
|
},
|
|
157
218
|
onRevoked: (socketId) => {
|
|
@@ -180,6 +241,8 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
180
241
|
registry: options.registry,
|
|
181
242
|
buildId: options.buildId,
|
|
182
243
|
presence,
|
|
244
|
+
channelSids,
|
|
245
|
+
heartbeatMs: clientHeartbeatMs(presence?.heartbeatMs, sockets.idleTimeoutMs),
|
|
183
246
|
});
|
|
184
247
|
|
|
185
248
|
return {
|
|
@@ -200,7 +263,14 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
200
263
|
// subscribers are re-served on the next change to each query.
|
|
201
264
|
if (gaps.observe(envelope)) {
|
|
202
265
|
const marked = options.registry.invalidate();
|
|
203
|
-
|
|
266
|
+
// Channels too: their seq is minted from what this node sees, so the hole is invisible
|
|
267
|
+
// to a ring that would otherwise replay across it as complete history.
|
|
268
|
+
const channelGaps = options.hub.invalidate();
|
|
269
|
+
logger.warn('live.change_gap', {
|
|
270
|
+
entity: envelope.change.entity,
|
|
271
|
+
desynced: marked,
|
|
272
|
+
channelGaps,
|
|
273
|
+
});
|
|
204
274
|
}
|
|
205
275
|
// Not awaited: the bus handler must return before the next change, and ordering is the
|
|
206
276
|
// registry's — one serial lane per query id. What this call site owes is the failure. An
|
|
@@ -210,6 +280,15 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
210
280
|
// hub turns this change into `records` frames on every channel it touches.
|
|
211
281
|
options.hub.deliverChange(envelope.change);
|
|
212
282
|
});
|
|
283
|
+
// A bus that reconnected is changes this node never saw, and the gap detector only notices
|
|
284
|
+
// on the NEXT message: with no later write, every window and cursor here stayed behind and
|
|
285
|
+
// every new subscriber joined the stale window. Said by the transport, repaired now.
|
|
286
|
+
reconnects = options.transport.onReconnect(() => {
|
|
287
|
+
gaps.forget();
|
|
288
|
+
const marked = options.registry.invalidate();
|
|
289
|
+
const channelGaps = options.hub.invalidate();
|
|
290
|
+
logger.warn('live.bus_reconnected', { desynced: marked, channelGaps });
|
|
291
|
+
});
|
|
213
292
|
// One pass per heartbeat window: a member is swept only once it has actually missed its
|
|
214
293
|
// window, and the interval never holds the process open — shutdown is the drain's job.
|
|
215
294
|
if (presence) {
|
|
@@ -270,6 +349,8 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
270
349
|
newSocketId: () => uuid(),
|
|
271
350
|
authenticate: options.authenticate,
|
|
272
351
|
allowedOrigins: options.allowedOrigins,
|
|
352
|
+
admitReachedOrigin: options.admitReachedOrigin,
|
|
353
|
+
healthDetailPeers: options.healthDetailPeers ?? DEFAULT_HEALTH_DETAIL_PEERS,
|
|
273
354
|
onGranted: (socketId, grant) => grants.set(socketId, grant),
|
|
274
355
|
// The other half of recording the grant before the upgrade: an upgrade that never took
|
|
275
356
|
// gets no `close` callback, so this is the only thing that can free its entry.
|
|
@@ -403,9 +484,3 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
403
484
|
},
|
|
404
485
|
};
|
|
405
486
|
}
|
|
406
|
-
|
|
407
|
-
/**
|
|
408
|
-
* An upgrade refused before a socket exists, rendered as the error contract rather than as a word.
|
|
409
|
-
* There is no frame to carry it — the client never got a connection — so the body is the only
|
|
410
|
-
* channel, and `--json` on every error means this one too.
|
|
411
|
-
*/
|
package/src/sync-origin.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Single responsibility: whether a websocket upgrade may carry this app's ambient credential. No
|
|
2
|
-
// CORS applies to a websocket and the session cookie rides it, so without this a page
|
|
3
|
-
//
|
|
4
|
-
//
|
|
2
|
+
// CORS applies to a websocket and the session cookie rides it, so without this a page that is not
|
|
3
|
+
// this app opened a socket as its visitor. The rule is core's `proveSameOrigin`, the one
|
|
4
|
+
// `@ultimat3/http`'s CSRF check asks too.
|
|
5
5
|
|
|
6
6
|
import { type OriginVerdict, proveSameOrigin } from '@ultimat3/core';
|
|
7
7
|
|
|
@@ -9,25 +9,40 @@ import { type OriginVerdict, proveSameOrigin } from '@ultimat3/core';
|
|
|
9
9
|
* Admitted, in order:
|
|
10
10
|
* - **no `Origin`** — RFC 6455 has a browser send one on every handshake, so a dial without it is
|
|
11
11
|
* not a browser and no visitor's cookie can be riding it;
|
|
12
|
-
* -
|
|
13
|
-
* the
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
12
|
+
* - **`allowedOrigins`, exactly, when any is declared** (`APP_URL`'s origin, from the CLI). A
|
|
13
|
+
* declaration IS the app's origin, so it is the whole list: the origin the node sees is derived
|
|
14
|
+
* from the `Host` header, which names whatever host the request was aimed at — a sibling
|
|
15
|
+
* subdomain pointed at this node is "its own origin" by that reading, and a `Domain=`-wide
|
|
16
|
+
* session cookie rides its socket;
|
|
17
|
+
* - **the node's own ORIGIN, only when nothing is declared** — host and port exactly as the
|
|
18
|
+
* request reached it (`x dev`, a combined-role container). A host name alone is not this app:
|
|
19
|
+
* cookies are not isolated by port. `admitReached` keeps it beside a declared list (`x dev`).
|
|
17
20
|
*/
|
|
18
21
|
export function upgradeOrigin(
|
|
19
22
|
request: Request,
|
|
20
23
|
url: URL,
|
|
21
24
|
allowedOrigins: readonly string[],
|
|
25
|
+
admitReached = false,
|
|
22
26
|
): OriginVerdict {
|
|
23
27
|
const origin = request.headers.get('origin');
|
|
24
28
|
if (origin === null) return { ok: true };
|
|
25
|
-
|
|
29
|
+
// Normalised once (`https://x/` and `https://x` are one origin), then matched as whole strings.
|
|
30
|
+
const admitted = allowedOrigins.map((allowed) => URL.parse(allowed)?.origin);
|
|
26
31
|
return proveSameOrigin({
|
|
27
|
-
selfOrigins:
|
|
32
|
+
selfOrigins: admitted.length > 0 && !admitReached ? [] : selfOrigins(url),
|
|
28
33
|
origin,
|
|
29
34
|
secFetchSite: request.headers.get('sec-fetch-site'),
|
|
30
|
-
listed: () =>
|
|
35
|
+
listed: (asked) => admitted.includes(asked),
|
|
31
36
|
listName: 'APP_URL or createSyncNode({ allowedOrigins })',
|
|
32
37
|
});
|
|
33
38
|
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The origins an UNDECLARED node is reached on. TLS usually ends at a proxy, so a node reached over
|
|
42
|
+
* plain http cannot know its pages' scheme: the https spelling of the same host and port is this
|
|
43
|
+
* app too. That also admits the plain-http page, which is why a deployment behind TLS declares
|
|
44
|
+
* `APP_URL` — the chart sets it from the ingress host.
|
|
45
|
+
*/
|
|
46
|
+
function selfOrigins(url: URL): readonly string[] {
|
|
47
|
+
return url.protocol === 'http:' ? [url.origin, `https://${url.host}`] : [url.origin];
|
|
48
|
+
}
|