@ultimat3/realtime 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +591 -0
- package/README.md +320 -19
- package/package.json +6 -3
- package/src/apply-patches.ts +60 -0
- package/src/change-buffer.ts +77 -11
- package/src/channel.ts +174 -19
- package/src/client-contract.ts +81 -0
- package/src/client-frames.ts +175 -0
- package/src/client-heartbeat.ts +77 -0
- package/src/client-mutations.ts +114 -0
- package/src/client-topics.ts +54 -0
- package/src/client.ts +307 -273
- package/src/cursor.ts +7 -1
- package/src/errors.ts +193 -4
- package/src/frame-lanes.ts +58 -0
- package/src/hooks.ts +19 -5
- package/src/identity-map.ts +141 -0
- package/src/index.ts +96 -28
- package/src/json.ts +38 -1
- package/src/live-contract.ts +67 -0
- package/src/live-definition.ts +16 -11
- package/src/live-fanout.ts +150 -0
- package/src/live-query.ts +215 -268
- package/src/live-rows.ts +143 -0
- package/src/local-store.ts +86 -43
- package/src/nats-client.ts +132 -0
- package/src/nats-fake.ts +389 -344
- package/src/nats-jetstream.ts +21 -20
- package/src/nats-kv.ts +7 -7
- package/src/nats-lib-client.ts +210 -0
- package/src/nats-transport.ts +109 -138
- package/src/offline-queue.ts +146 -30
- package/src/pg-entity-row.ts +99 -31
- package/src/pg-replication.ts +84 -27
- package/src/pg-socket.ts +4 -1
- package/src/policy-gate.ts +13 -5
- package/src/presence.ts +76 -6
- package/src/query-hook.ts +56 -0
- package/src/query-window.ts +151 -0
- package/src/rebase.ts +68 -8
- package/src/replicator.ts +84 -11
- package/src/socket.ts +170 -14
- package/src/subscriber-gate.ts +209 -0
- package/src/subscription-book.ts +237 -0
- package/src/sync-auth.ts +124 -0
- package/src/sync-frames.ts +185 -0
- package/src/sync-listen.ts +73 -0
- package/src/sync-node.ts +284 -243
- package/src/sync-protocol.ts +115 -24
- package/src/sync-upgrade.ts +124 -0
- package/src/thundering-herd.ts +21 -0
- package/src/transport-env.ts +3 -3
- package/src/type-pins.ts +72 -0
- package/src/window-lock.ts +21 -0
- package/src/nats-commands.ts +0 -97
- package/src/nats-connection-fixture.ts +0 -105
- package/src/nats-connection.ts +0 -464
- package/src/nats-protocol.ts +0 -222
- package/src/nats-socket.ts +0 -236
- package/src/pg-connection-fixture.ts +0 -215
- package/src/pg-replication-fixture.ts +0 -261
package/src/sync-node.ts
CHANGED
|
@@ -4,44 +4,54 @@
|
|
|
4
4
|
// client may reconnect to any node and resume from its cursor, which is why drain is allowed to
|
|
5
5
|
// redistribute connections at all.
|
|
6
6
|
|
|
7
|
-
import {
|
|
8
|
-
type Clock,
|
|
9
|
-
healthzPayload,
|
|
10
|
-
logger,
|
|
11
|
-
markListening,
|
|
12
|
-
markReady,
|
|
13
|
-
onShutdown,
|
|
14
|
-
readyzPayload,
|
|
15
|
-
systemClock,
|
|
16
|
-
uuid,
|
|
17
|
-
} from '@ultimat3/core';
|
|
7
|
+
import { type Clock, logger, markReady, reportError, systemClock, uuid } from '@ultimat3/core';
|
|
18
8
|
import type { ChannelHub, Topic } from './channel';
|
|
19
|
-
import {
|
|
9
|
+
import { isClientFault } from './errors';
|
|
20
10
|
import type { Transport, TransportSubscription } from './fanout';
|
|
21
|
-
import type { JsonValue, Row } from './json';
|
|
22
11
|
import type { LiveQueryRegistry } from './live-query';
|
|
23
|
-
import {
|
|
24
|
-
import { CHANGE_SUBJECT_PREFIX,
|
|
25
|
-
import {
|
|
12
|
+
import type { PresenceRegistry } from './presence';
|
|
13
|
+
import { CHANGE_SUBJECT_PREFIX, parseEnvelope, SeqGapDetector } from './replicator';
|
|
14
|
+
import {
|
|
15
|
+
CLOSE,
|
|
16
|
+
DEFAULT_MAX_BUFFERED_BYTES,
|
|
17
|
+
SocketRegistry,
|
|
18
|
+
SyncSocket,
|
|
19
|
+
type WsLike,
|
|
20
|
+
} from './socket';
|
|
21
|
+
import { GrantBook, type SyncAuthenticator, sweepGrants } from './sync-auth';
|
|
22
|
+
import { ackRefOf, createFrameRouter, type MutationHandler } from './sync-frames';
|
|
26
23
|
import { decode, type Frame, PROTOCOL_VERSION, toWireError } from './sync-protocol';
|
|
24
|
+
import { handleUpgrade, type UpgradeTarget, type WsData } from './sync-upgrade';
|
|
27
25
|
import { AcceptBudget, drainPlan, type Rng, reconnectFrame } from './thundering-herd';
|
|
28
26
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
readonly clientBuildId: string;
|
|
32
|
-
readonly actorId: string | null;
|
|
33
|
-
}
|
|
27
|
+
/** Declared with the upgrade that builds it — this file only ever reads one. */
|
|
28
|
+
export type { UpgradeTarget, WsData } from './sync-upgrade';
|
|
34
29
|
|
|
35
30
|
export type SyncWs = WsLike & { readonly data: WsData };
|
|
36
31
|
|
|
37
|
-
/**
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
32
|
+
/**
|
|
33
|
+
* How often an expired grant is re-decided. A third of the shortest TTL worth issuing: a grant is
|
|
34
|
+
* re-checked on the pass after it expires, so the window a revoked actor keeps its socket is this
|
|
35
|
+
* interval and not its token's lifetime.
|
|
36
|
+
*/
|
|
37
|
+
export const DEFAULT_REAUTH_INTERVAL_MS = 30_000;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Concurrent sockets one node will hold. The accept budget bounds the accept RATE and nothing
|
|
41
|
+
* bounded the COUNT: at the 500/s that budget permits, an attacker holding each socket open with
|
|
42
|
+
* one keepalive frame a minute reaches 1.8M sockets an hour, each carrying a `GrantBook` entry.
|
|
43
|
+
*
|
|
44
|
+
* The number clears the 50,000 real clients this repo has measured on one node
|
|
45
|
+
* (`scripts/bench/restart-bench.ts`) with room to spare, because a ceiling that refuses a proven
|
|
46
|
+
* workload is an outage the framework caused.
|
|
47
|
+
*/
|
|
48
|
+
export const DEFAULT_MAX_CONNECTIONS = 250_000;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Inbound bytes one frame may carry. Bun's own default is 16 MiB, which one authenticated socket
|
|
52
|
+
* can push continuously; a `subscribe` frame carrying a full 512-id cursor is under 32 KiB.
|
|
53
|
+
*/
|
|
54
|
+
export const DEFAULT_MAX_FRAME_BYTES = 256 * 1024;
|
|
45
55
|
|
|
46
56
|
export interface SyncNodeOptions {
|
|
47
57
|
readonly hub: ChannelHub;
|
|
@@ -51,7 +61,34 @@ export interface SyncNodeOptions {
|
|
|
51
61
|
readonly presence?: PresenceRegistry;
|
|
52
62
|
readonly sockets?: SocketRegistry;
|
|
53
63
|
readonly accept?: AcceptBudget;
|
|
64
|
+
/** Concurrent sockets this node will hold. The count the accept budget does not bound. */
|
|
65
|
+
readonly maxConnections?: number;
|
|
66
|
+
/** Inbound bytes one frame may carry, handed to whatever server mounts `websocket`. */
|
|
67
|
+
readonly maxFrameBytes?: number;
|
|
68
|
+
/** Sustained inbound frames one socket may have routed per second. */
|
|
69
|
+
readonly maxFramesPerSecond?: number;
|
|
70
|
+
/** Burst allowance on that rate, per socket. */
|
|
71
|
+
readonly frameBurst?: number;
|
|
72
|
+
/**
|
|
73
|
+
* When a socket starts dropping frames, and how many drops close it. On `SyncSocket` too, but
|
|
74
|
+
* this node builds every socket it holds — so unforwarded they were reachable only by abandoning
|
|
75
|
+
* `createSyncNode`, and a dropped channel frame is the one loss nothing replays.
|
|
76
|
+
*/
|
|
77
|
+
readonly maxBufferedBytes?: number;
|
|
78
|
+
readonly maxDroppedFrames?: number;
|
|
54
79
|
readonly onMutate?: MutationHandler;
|
|
80
|
+
/**
|
|
81
|
+
* Who is dialling. Injected for the same reason `onMutate` is: `sync` owns no business logic and
|
|
82
|
+
* imports no authenticator, so an app supplies the one function that turns an upgrade request
|
|
83
|
+
* into an actor — from `@ultimat3/auth` or from anywhere else.
|
|
84
|
+
*
|
|
85
|
+
* **Omitted, every socket on this node is anonymous** and every policy downstream — the topic
|
|
86
|
+
* guard, `authorize`, `visible`, the per-tenant subscription cap — decides against `null`. That
|
|
87
|
+
* is a single-tenant node, and `start()` says so in the log.
|
|
88
|
+
*/
|
|
89
|
+
readonly authenticate?: SyncAuthenticator;
|
|
90
|
+
/** How often an expired grant is re-decided. The clock a socket's authority runs on. */
|
|
91
|
+
readonly reauthenticateIntervalMs?: number;
|
|
55
92
|
readonly clock?: Clock;
|
|
56
93
|
readonly rng?: Rng;
|
|
57
94
|
/** WS endpoint. One path, no negotiation — the protocol version lives in the frames. */
|
|
@@ -59,21 +96,30 @@ export interface SyncNodeOptions {
|
|
|
59
96
|
readonly drainSpreadMs?: number;
|
|
60
97
|
}
|
|
61
98
|
|
|
62
|
-
/** Structural view of `Bun.serve`'s server object; keeps this module free of a Bun import. */
|
|
63
|
-
export interface UpgradeTarget {
|
|
64
|
-
upgrade(request: Request, options: { data: WsData }): boolean;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
99
|
export interface SyncNode {
|
|
68
100
|
readonly sockets: SocketRegistry;
|
|
69
101
|
readonly ready: boolean;
|
|
70
102
|
start(): Promise<void>;
|
|
103
|
+
/**
|
|
104
|
+
* Refuse new connections, keep every one this node holds. The SIGTERM `accept` phase calls it —
|
|
105
|
+
* `/readyz` answers 503 so the load balancer stops routing here, and an upgrade arriving in the
|
|
106
|
+
* meantime is shed with a retry delay instead of landing on a process that is going away. It is
|
|
107
|
+
* NOT `stop()`: a draining node still owes its clients their patches, and `stop()` releases the
|
|
108
|
+
* change subscription that carries them.
|
|
109
|
+
*/
|
|
110
|
+
stopAccepting(): void;
|
|
71
111
|
stop(): Promise<void>;
|
|
72
|
-
|
|
112
|
+
/**
|
|
113
|
+
* Async because `authenticate` is: the credential is decided *before* `server.upgrade`, so a
|
|
114
|
+
* refused one never costs a websocket. Bun's `fetch` may return a promise, and an upgrade that
|
|
115
|
+
* awaits first is still an upgrade.
|
|
116
|
+
*/
|
|
117
|
+
fetch(request: Request, server: UpgradeTarget): Promise<Response | undefined>;
|
|
73
118
|
readonly websocket: {
|
|
74
119
|
idleTimeout: number;
|
|
75
120
|
backpressureLimit: number;
|
|
76
|
-
|
|
121
|
+
/** Inbound ceiling. Declared here so every host that mounts this handler inherits it. */
|
|
122
|
+
maxPayloadLength: number;
|
|
77
123
|
sendPings: boolean;
|
|
78
124
|
open(ws: SyncWs): void;
|
|
79
125
|
message(ws: SyncWs, message: string | Uint8Array): void;
|
|
@@ -88,129 +134,116 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
88
134
|
options.sockets ?? new SocketRegistry({ ...(options.clock ? { clock: options.clock } : {}) });
|
|
89
135
|
const clock = options.clock ?? systemClock;
|
|
90
136
|
const accept = options.accept ?? new AcceptBudget({ perSecond: 500, burst: 2000, clock });
|
|
137
|
+
const maxConnections = options.maxConnections ?? DEFAULT_MAX_CONNECTIONS;
|
|
91
138
|
const path = options.path ?? '/_x/sync';
|
|
92
139
|
const presence = options.presence;
|
|
140
|
+
const grants = new GrantBook();
|
|
141
|
+
const gaps = new SeqGapDetector();
|
|
93
142
|
let ready = false;
|
|
94
143
|
let changes: TransportSubscription | null = null;
|
|
95
144
|
let sweeping: ReturnType<typeof setInterval> | null = null;
|
|
145
|
+
let reauthing: ReturnType<typeof setInterval> | null = null;
|
|
96
146
|
|
|
97
147
|
/**
|
|
98
|
-
*
|
|
99
|
-
* It reaches the bus, so it can fail; failing must not take
|
|
100
|
-
* and must not be silent either, or "the room still shows someone
|
|
148
|
+
* Work nobody is waiting on — a presence leave from a synchronous close, a sweep on a timer, a
|
|
149
|
+
* fanout off the change bus. It reaches the bus or a policy, so it can fail; failing must not take
|
|
150
|
+
* a socket or the process with it, and must not be silent either, or "the room still shows someone
|
|
151
|
+
* who left" and "that change reached nobody" have nothing to read. `operation` stays low
|
|
152
|
+
* cardinality so the monitor can group on it; the topic or entity goes in `at`.
|
|
101
153
|
*/
|
|
102
|
-
const detach = (work: Promise<unknown>,
|
|
154
|
+
const detach = (work: Promise<unknown>, operation: string, at?: string): void => {
|
|
103
155
|
void work.catch((error: unknown) => {
|
|
104
|
-
logger.error(
|
|
105
|
-
at,
|
|
156
|
+
logger.error(`${operation} failed`, {
|
|
157
|
+
...(at === undefined ? {} : { at }),
|
|
106
158
|
error: error instanceof Error ? error.message : String(error),
|
|
107
159
|
});
|
|
160
|
+
// Nobody is awaiting this, so the log is the only trace it leaves — and a log is not a
|
|
161
|
+
// signal anyone is paged on. The bus is this node's dependency, never the client's.
|
|
162
|
+
reportError(error, { source: 'realtime', scope: { operation } });
|
|
108
163
|
});
|
|
109
164
|
};
|
|
110
165
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
return;
|
|
146
|
-
}
|
|
147
|
-
if (frame.op === 'drop') {
|
|
148
|
-
options.registry.unsubscribe(frame.sid);
|
|
149
|
-
return;
|
|
150
|
-
}
|
|
151
|
-
const { frame: reply } = await options.registry.subscribe({
|
|
152
|
-
socket,
|
|
153
|
-
name: frame.target.qid,
|
|
154
|
-
input: frame.target.input,
|
|
155
|
-
sid: frame.sid,
|
|
156
|
-
cursor: frame.target.cursor,
|
|
157
|
-
});
|
|
158
|
-
socket.send(reply);
|
|
159
|
-
return;
|
|
160
|
-
}
|
|
161
|
-
case 'mutate': {
|
|
162
|
-
if (!options.onMutate) {
|
|
163
|
-
socket.send({
|
|
164
|
-
type: 'ack',
|
|
165
|
-
v: PROTOCOL_VERSION,
|
|
166
|
-
ref: frame.key,
|
|
167
|
-
lsn: null,
|
|
168
|
-
error: toWireError({
|
|
169
|
-
code: 'X_NOT_IMPLEMENTED',
|
|
170
|
-
cause: 'this sync node was started without a mutation handler',
|
|
171
|
-
fix: 'pass onMutate to createSyncNode({ onMutate })',
|
|
172
|
-
}),
|
|
173
|
-
});
|
|
174
|
-
return;
|
|
175
|
-
}
|
|
176
|
-
const result = await options.onMutate({
|
|
177
|
-
socket,
|
|
178
|
-
name: frame.name,
|
|
179
|
-
key: frame.key,
|
|
180
|
-
seq: frame.seq,
|
|
181
|
-
input: frame.input,
|
|
182
|
-
});
|
|
183
|
-
socket.send({
|
|
184
|
-
type: 'ack',
|
|
185
|
-
v: PROTOCOL_VERSION,
|
|
186
|
-
ref: frame.key,
|
|
187
|
-
lsn: result.lsn ?? null,
|
|
188
|
-
error: null,
|
|
189
|
-
});
|
|
190
|
-
if (result.entity !== undefined) {
|
|
191
|
-
socket.send({
|
|
192
|
-
type: 'rebase',
|
|
193
|
-
v: PROTOCOL_VERSION,
|
|
194
|
-
key: frame.key,
|
|
195
|
-
entity: result.entity,
|
|
196
|
-
strategy: 'server-wins',
|
|
197
|
-
row: result.row ?? null,
|
|
198
|
-
});
|
|
199
|
-
}
|
|
200
|
-
return;
|
|
201
|
-
}
|
|
202
|
-
// Server-authored frames are never received from a client.
|
|
203
|
-
case 'snapshot':
|
|
204
|
-
case 'patch':
|
|
205
|
-
case 'ack':
|
|
206
|
-
case 'rebase':
|
|
207
|
-
case 'presence':
|
|
208
|
-
case 'reconnect':
|
|
209
|
-
case 'update-available':
|
|
210
|
-
return;
|
|
166
|
+
/**
|
|
167
|
+
* Everything `start()` acquired that is not a socket: the change subscription and the presence
|
|
168
|
+
* sweep. Both `drain()` and `stop()` run it, because a `drain()` is terminal on its own — it
|
|
169
|
+
* closes the hub — and `listenSyncNode` is the only caller that follows one with the other. A
|
|
170
|
+
* node that drained and kept its subscription goes on pulling changes off the bus and sweeping
|
|
171
|
+
* presence for a fleet it has already left, with no socket to deliver either to. Idempotent:
|
|
172
|
+
* running it twice is the normal case.
|
|
173
|
+
*/
|
|
174
|
+
const release = (): void => {
|
|
175
|
+
changes?.unsubscribe();
|
|
176
|
+
changes = null;
|
|
177
|
+
if (sweeping !== null) clearInterval(sweeping);
|
|
178
|
+
sweeping = null;
|
|
179
|
+
if (reauthing !== null) clearInterval(reauthing);
|
|
180
|
+
reauthing = null;
|
|
181
|
+
gaps.forget();
|
|
182
|
+
};
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Everything one socket held, released once. Bun's `close` callback runs it, and so does a
|
|
186
|
+
* revoked grant — a socket this node closes itself gets no callback in a unit test, and in
|
|
187
|
+
* production the second run is the no-op every step here already is.
|
|
188
|
+
*/
|
|
189
|
+
const teardown = (socket: SyncSocket): void => {
|
|
190
|
+
options.registry.unsubscribeSocket(socket.id);
|
|
191
|
+
const topics = [...socket.topics] as Topic[];
|
|
192
|
+
for (const name of topics) options.hub.unsubscribe(socket, name);
|
|
193
|
+
sockets.remove(socket.id);
|
|
194
|
+
grants.delete(socket.id);
|
|
195
|
+
// A closed socket is a leave, said now rather than left to TTL: everyone else would otherwise
|
|
196
|
+
// keep rendering a member who is provably gone for the rest of its window. The write is on the
|
|
197
|
+
// bus and the close callback is synchronous, so it cannot be awaited here.
|
|
198
|
+
if (presence) {
|
|
199
|
+
for (const name of topics) detach(presence.leave(name, socket.id), 'presence.leave', name);
|
|
211
200
|
}
|
|
212
201
|
};
|
|
213
202
|
|
|
203
|
+
/**
|
|
204
|
+
* One pass over the grants whose window has closed. This is the half R2 was missing: `reauthorize`
|
|
205
|
+
* and `onActorChange` were both written and neither had a caller, so a socket that was accepted
|
|
206
|
+
* was authorized for as long as it stayed open — and an active client's socket never idles out,
|
|
207
|
+
* because every inbound frame touches it.
|
|
208
|
+
*/
|
|
209
|
+
const reauthenticate = async (): Promise<void> => {
|
|
210
|
+
await sweepGrants({
|
|
211
|
+
grants,
|
|
212
|
+
clock,
|
|
213
|
+
onActor: async (socketId, actor) => {
|
|
214
|
+
const socket = sockets.get(socketId);
|
|
215
|
+
if (!socket) return;
|
|
216
|
+
// The hub sets `socket.actor` and drops the topics this actor may no longer read; the
|
|
217
|
+
// registry re-decides every live subscription and desyncs the survivors, so the next
|
|
218
|
+
// delivery re-snapshots them under the new authority rather than the old window.
|
|
219
|
+
await options.hub.onActorChange(socket, actor);
|
|
220
|
+
await options.registry.reauthorize(socket);
|
|
221
|
+
},
|
|
222
|
+
onRevoked: (socketId) => {
|
|
223
|
+
const socket = sockets.get(socketId);
|
|
224
|
+
if (!socket) return;
|
|
225
|
+
teardown(socket);
|
|
226
|
+
socket.close(CLOSE.policy, 'grant expired');
|
|
227
|
+
},
|
|
228
|
+
onRefreshFailed: (socketId, error) => {
|
|
229
|
+
// Not a denial: the grant is kept and retried next pass. Reported because a socket nobody
|
|
230
|
+
// can re-decide is not something to discover from a connection graph.
|
|
231
|
+
reportError(error, {
|
|
232
|
+
source: 'realtime',
|
|
233
|
+
scope: { operation: 'sync.reauthenticate', extra: { socketId } },
|
|
234
|
+
});
|
|
235
|
+
},
|
|
236
|
+
});
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
const routeFrame = createFrameRouter({
|
|
240
|
+
hub: options.hub,
|
|
241
|
+
registry: options.registry,
|
|
242
|
+
buildId: options.buildId,
|
|
243
|
+
presence,
|
|
244
|
+
onMutate: options.onMutate,
|
|
245
|
+
});
|
|
246
|
+
|
|
214
247
|
return {
|
|
215
248
|
sockets,
|
|
216
249
|
|
|
@@ -220,68 +253,113 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
220
253
|
|
|
221
254
|
async start(): Promise<void> {
|
|
222
255
|
changes = await options.transport.subscribe(`${CHANGE_SUBJECT_PREFIX}.>`, (payload) => {
|
|
223
|
-
const
|
|
224
|
-
if (
|
|
256
|
+
const envelope = parseEnvelope(payload);
|
|
257
|
+
if (!envelope) return;
|
|
258
|
+
// Fanout is at-most-once over core NATS, so a reconnect is changes this node never saw.
|
|
259
|
+
// Nothing downstream could notice: no window's lsn moved, so no cursor moved, so nothing
|
|
260
|
+
// ever asked for a re-snapshot. A gap invalidates every window here instead, and the
|
|
261
|
+
// subscribers are re-served on the next change to each query.
|
|
262
|
+
if (gaps.observe(envelope)) {
|
|
263
|
+
const marked = options.registry.invalidate();
|
|
264
|
+
logger.warn('live.change_gap', { entity: envelope.change.entity, desynced: marked });
|
|
265
|
+
}
|
|
266
|
+
// Not awaited: the bus handler must return before the next change, and ordering is the
|
|
267
|
+
// registry's — one serial lane per query id. What this call site owes is the failure. An
|
|
268
|
+
// unhandled rejection here is a fanout that reached nobody, reported as a dead process.
|
|
269
|
+
detach(options.registry.deliver(envelope.change), 'live.deliver', envelope.change.entity);
|
|
225
270
|
});
|
|
226
271
|
// One pass per heartbeat window: a member is swept only once it has actually missed its
|
|
227
272
|
// window, and the interval never holds the process open — shutdown is the drain's job.
|
|
228
273
|
if (presence) {
|
|
229
|
-
sweeping = setInterval(
|
|
274
|
+
sweeping = setInterval(
|
|
275
|
+
() => detach(presence.sweepAll(), 'presence.sweep'),
|
|
276
|
+
presence.heartbeatMs,
|
|
277
|
+
);
|
|
230
278
|
sweeping.unref();
|
|
231
279
|
}
|
|
280
|
+
if (options.authenticate) {
|
|
281
|
+
reauthing = setInterval(
|
|
282
|
+
() => detach(reauthenticate(), 'sync.reauthenticate'),
|
|
283
|
+
options.reauthenticateIntervalMs ?? DEFAULT_REAUTH_INTERVAL_MS,
|
|
284
|
+
);
|
|
285
|
+
reauthing.unref();
|
|
286
|
+
} else {
|
|
287
|
+
// Enforced where it can be: nothing here can invent a credential, so the one honest signal
|
|
288
|
+
// is that every policy on this node is about to be asked about `null`.
|
|
289
|
+
logger.warn('sync node has no authenticator: every socket is anonymous', {
|
|
290
|
+
buildId: options.buildId,
|
|
291
|
+
fix: 'pass authenticate to createSyncNode({ authenticate })',
|
|
292
|
+
});
|
|
293
|
+
}
|
|
232
294
|
ready = true;
|
|
233
295
|
markReady();
|
|
234
296
|
logger.info('sync node ready', { buildId: options.buildId, path });
|
|
235
297
|
},
|
|
236
298
|
|
|
299
|
+
stopAccepting(): void {
|
|
300
|
+
ready = false;
|
|
301
|
+
},
|
|
302
|
+
|
|
237
303
|
async stop(): Promise<void> {
|
|
238
304
|
ready = false;
|
|
239
|
-
|
|
240
|
-
changes = null;
|
|
241
|
-
if (sweeping !== null) clearInterval(sweeping);
|
|
242
|
-
sweeping = null;
|
|
305
|
+
release();
|
|
243
306
|
},
|
|
244
307
|
|
|
245
|
-
fetch(request: Request, server: UpgradeTarget): Response | undefined {
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
clientBuildId: url.searchParams.get('build') ?? options.buildId,
|
|
265
|
-
actorId: null,
|
|
266
|
-
};
|
|
267
|
-
return server.upgrade(request, { data })
|
|
268
|
-
? undefined
|
|
269
|
-
: new Response('expected websocket', { status: 426 });
|
|
308
|
+
async fetch(request: Request, server: UpgradeTarget): Promise<Response | undefined> {
|
|
309
|
+
return await handleUpgrade(
|
|
310
|
+
{
|
|
311
|
+
path,
|
|
312
|
+
buildId: options.buildId,
|
|
313
|
+
maxConnections,
|
|
314
|
+
accept,
|
|
315
|
+
rng: options.rng ?? Math.random,
|
|
316
|
+
// Read per call, never captured: `ready` and the socket count both move while a request
|
|
317
|
+
// is parked inside `authenticate`, which is the whole reason they are functions.
|
|
318
|
+
ready: () => ready,
|
|
319
|
+
socketCount: () => sockets.count,
|
|
320
|
+
newSocketId: () => uuid(),
|
|
321
|
+
authenticate: options.authenticate,
|
|
322
|
+
onGranted: (socketId, grant) => grants.set(socketId, grant),
|
|
323
|
+
},
|
|
324
|
+
request,
|
|
325
|
+
server,
|
|
326
|
+
);
|
|
270
327
|
},
|
|
271
328
|
|
|
272
329
|
websocket: {
|
|
273
330
|
idleTimeout: 120,
|
|
274
|
-
|
|
275
|
-
|
|
331
|
+
// The same number `SyncSocket` refuses to add past, never a second spelling of it. Bun's
|
|
332
|
+
// limit set lower and our own check never fires: the runtime drops the frame with nothing
|
|
333
|
+
// marked desynced, which is the silent divergence the mark exists to prevent.
|
|
334
|
+
backpressureLimit: DEFAULT_MAX_BUFFERED_BYTES,
|
|
335
|
+
// No `publishToSelf`: this node never publishes to a native topic. Every channel frame is
|
|
336
|
+
// one filtered `send` per socket through `SocketRegistry.deliver`, which is the only path
|
|
337
|
+
// that can count the frame it dropped — a flag configuring a mechanism nothing uses reads
|
|
338
|
+
// as a live one to the next person who has to decide how delivery works.
|
|
339
|
+
maxPayloadLength: options.maxFrameBytes ?? DEFAULT_MAX_FRAME_BYTES,
|
|
276
340
|
sendPings: true,
|
|
277
341
|
|
|
278
342
|
open(ws: SyncWs): void {
|
|
343
|
+
// The actor the upgrade resolved, carried into the socket the whole pipeline decides
|
|
344
|
+
// against — the topic guard, `authorize`, `visible`, the per-tenant cap. It was hardcoded
|
|
345
|
+
// `null` here, which made every one of those a decision about nobody.
|
|
279
346
|
const socket = new SyncSocket({
|
|
280
347
|
ws,
|
|
281
348
|
id: ws.data.socketId,
|
|
282
349
|
clientBuildId: ws.data.clientBuildId,
|
|
283
350
|
serverBuildId: options.buildId,
|
|
351
|
+
actor: grants.get(ws.data.socketId)?.actor ?? null,
|
|
284
352
|
clock,
|
|
353
|
+
...(options.maxFramesPerSecond === undefined
|
|
354
|
+
? {}
|
|
355
|
+
: { maxFramesPerSecond: options.maxFramesPerSecond }),
|
|
356
|
+
...(options.frameBurst === undefined ? {} : { frameBurst: options.frameBurst }),
|
|
357
|
+
...(options.maxBufferedBytes === undefined
|
|
358
|
+
? {}
|
|
359
|
+
: { maxBufferedBytes: options.maxBufferedBytes }),
|
|
360
|
+
...(options.maxDroppedFrames === undefined
|
|
361
|
+
? {}
|
|
362
|
+
: { maxDroppedFrames: options.maxDroppedFrames }),
|
|
285
363
|
});
|
|
286
364
|
sockets.add(socket);
|
|
287
365
|
},
|
|
@@ -290,13 +368,26 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
290
368
|
const socket = sockets.get(ws.data.socketId);
|
|
291
369
|
if (!socket) return;
|
|
292
370
|
void (async () => {
|
|
371
|
+
// Decoded into a binding the failure path can read: an ack has to name the thing that
|
|
372
|
+
// failed — the mutation key the client's queue holds, the sid its subscription holds —
|
|
373
|
+
// and a frame that could not be decoded is the one case where there is nothing to name.
|
|
374
|
+
let frame: Frame | null = null;
|
|
293
375
|
try {
|
|
294
|
-
|
|
376
|
+
frame = decode(message);
|
|
377
|
+
await routeFrame(socket, frame);
|
|
295
378
|
} catch (error) {
|
|
379
|
+
// The ack frame tells the client what it did wrong; the monitor only hears about what
|
|
380
|
+
// this node did wrong. Same rule the HTTP pipeline applies at `status >= 500`.
|
|
381
|
+
if (!isClientFault(error)) {
|
|
382
|
+
reportError(error, {
|
|
383
|
+
source: 'realtime',
|
|
384
|
+
scope: { operation: 'sync.frame', extra: { socketId: socket.id } },
|
|
385
|
+
});
|
|
386
|
+
}
|
|
296
387
|
socket.send({
|
|
297
388
|
type: 'ack',
|
|
298
389
|
v: PROTOCOL_VERSION,
|
|
299
|
-
ref: ws.data.socketId,
|
|
390
|
+
ref: ackRefOf(frame, ws.data.socketId),
|
|
300
391
|
lsn: null,
|
|
301
392
|
error: toWireError(error),
|
|
302
393
|
});
|
|
@@ -306,15 +397,13 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
306
397
|
|
|
307
398
|
close(ws: SyncWs): void {
|
|
308
399
|
const socket = sockets.get(ws.data.socketId);
|
|
309
|
-
if (!socket)
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
// write is on the bus, and this callback is synchronous, so it cannot be awaited here.
|
|
317
|
-
if (presence) for (const name of topics) detach(presence.leave(name, socket.id), name);
|
|
400
|
+
if (!socket) {
|
|
401
|
+
// The socket is already gone, but a grant recorded for an upgrade whose `open` never ran
|
|
402
|
+
// is not — and nothing else would ever reach it.
|
|
403
|
+
grants.delete(ws.data.socketId);
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
teardown(socket);
|
|
318
407
|
},
|
|
319
408
|
},
|
|
320
409
|
|
|
@@ -333,68 +422,20 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
333
422
|
for (const socket of [...sockets.all()]) {
|
|
334
423
|
socket.close(CLOSE.goingAway, 'drain');
|
|
335
424
|
sockets.remove(socket.id);
|
|
425
|
+
grants.delete(socket.id);
|
|
336
426
|
}
|
|
427
|
+
// Released once the sockets are gone rather than at the top: a client is entitled to its
|
|
428
|
+
// patches for the whole grace window, and it is entitled to them *before* the hub the
|
|
429
|
+
// fanout writes through is closed.
|
|
430
|
+
release();
|
|
337
431
|
await options.hub.close();
|
|
338
432
|
return plan;
|
|
339
433
|
},
|
|
340
434
|
};
|
|
341
435
|
}
|
|
342
436
|
|
|
343
|
-
export interface ListenOptions {
|
|
344
|
-
readonly port?: number;
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
export interface SyncListener {
|
|
348
|
-
/** The bound websocket origin, e.g. `ws://localhost:3001`. With `port: 0` only the OS knows it. */
|
|
349
|
-
readonly url: string;
|
|
350
|
-
stop(): void;
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
/**
|
|
354
|
-
* Binds the node to `Bun.serve` and wires SIGTERM to `drain()`. Kept tiny so the node itself stays
|
|
355
|
-
* testable without a server.
|
|
356
|
-
*/
|
|
357
|
-
export function listenSyncNode(node: SyncNode, options: ListenOptions = {}): SyncListener {
|
|
358
|
-
const server = Bun.serve({
|
|
359
|
-
port: options.port ?? 3001,
|
|
360
|
-
fetch: node.fetch,
|
|
361
|
-
websocket: node.websocket,
|
|
362
|
-
});
|
|
363
|
-
// Same rule as @ultimat3/http: every socket the framework opens announces itself, so a request
|
|
364
|
-
// back to it is recognisably this process calling itself rather than egress.
|
|
365
|
-
const stopListening = markListening(server.url.origin);
|
|
366
|
-
// Unregistered by `stop()`: a hook left behind after the listener is gone drains a node that is
|
|
367
|
-
// already stopped, and the next process-wide shutdown hangs on it.
|
|
368
|
-
const unregister = onShutdown('realtime:sync', async () => {
|
|
369
|
-
await node.drain();
|
|
370
|
-
await node.stop();
|
|
371
|
-
server.stop();
|
|
372
|
-
stopListening();
|
|
373
|
-
});
|
|
374
|
-
return {
|
|
375
|
-
url: websocketOrigin(server.url),
|
|
376
|
-
stop: () => {
|
|
377
|
-
unregister();
|
|
378
|
-
server.stop();
|
|
379
|
-
stopListening();
|
|
380
|
-
},
|
|
381
|
-
};
|
|
382
|
-
}
|
|
383
|
-
|
|
384
437
|
/**
|
|
385
|
-
*
|
|
386
|
-
*
|
|
387
|
-
*
|
|
438
|
+
* An upgrade refused before a socket exists, rendered as the error contract rather than as a word.
|
|
439
|
+
* There is no frame to carry it — the client never got a connection — so the body is the only
|
|
440
|
+
* channel, and `--json` on every error means this one too.
|
|
388
441
|
*/
|
|
389
|
-
function websocketOrigin(url: URL): string {
|
|
390
|
-
const ws = new URL(url);
|
|
391
|
-
ws.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
|
|
392
|
-
return ws.origin;
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
function json(payload: { status: number; body: unknown }): Response {
|
|
396
|
-
return new Response(JSON.stringify(payload.body), {
|
|
397
|
-
status: payload.status,
|
|
398
|
-
headers: { 'content-type': 'application/json' },
|
|
399
|
-
});
|
|
400
|
-
}
|