@ultimat3/realtime 22.15.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +89 -102
- package/README.md +140 -20
- package/package.json +4 -4
- package/src/advisory-lock.ts +57 -0
- package/src/boot.ts +12 -7
- package/src/change-buffer.ts +96 -41
- package/src/changefeed-env.ts +2 -2
- package/src/changefeed.ts +39 -1
- package/src/channel-authz.ts +43 -12
- package/src/channel-latch.ts +52 -0
- package/src/channel-logs.ts +34 -4
- package/src/channel-records.ts +33 -2
- package/src/channel-sids.ts +37 -0
- package/src/channel.ts +143 -44
- package/src/client-channels.ts +10 -1
- package/src/client-contract.ts +4 -1
- package/src/client-frames.ts +34 -4
- package/src/client-heartbeat.ts +39 -9
- package/src/client.ts +1 -0
- package/src/cursor.ts +11 -0
- package/src/errors.ts +70 -19
- package/src/fanout.ts +13 -0
- package/src/index.ts +1 -1
- package/src/live-contract.ts +6 -2
- package/src/live-definition.ts +46 -11
- package/src/live-fanout.ts +70 -11
- package/src/live-floor.ts +35 -0
- package/src/live-query-options.ts +38 -0
- package/src/live-query.ts +119 -91
- package/src/live-refusal.ts +21 -0
- package/src/live-replicator.ts +6 -6
- package/src/live-resume.ts +74 -0
- package/src/live-tenant.ts +25 -0
- package/src/nats-client.ts +11 -2
- package/src/nats-open.ts +9 -0
- package/src/nats-transport.ts +23 -1
- package/src/offline-queue.ts +48 -5
- package/src/page-errors.ts +17 -0
- package/src/page-outbox.ts +121 -50
- package/src/pg-advisory-lock.ts +45 -1
- package/src/pg-auth.ts +9 -4
- package/src/pg-bytes.ts +4 -1
- package/src/pg-connection.ts +119 -21
- package/src/pg-entity-row.ts +27 -2
- package/src/pg-identity-tables.ts +25 -0
- package/src/pg-preflight.ts +50 -4
- package/src/pg-replication-row.ts +82 -0
- package/src/pg-replication.ts +44 -45
- package/src/pg-socket.ts +6 -4
- package/src/pg-tls.ts +18 -0
- package/src/pg-wire.ts +5 -3
- package/src/pgoutput.ts +19 -9
- package/src/presence.ts +85 -8
- package/src/query-window.ts +28 -7
- package/src/record-persister.ts +7 -2
- package/src/replication-errors.ts +8 -4
- package/src/replicator-envelope.ts +110 -0
- package/src/replicator-row.ts +75 -0
- package/src/replicator.ts +319 -186
- package/src/server.ts +11 -7
- package/src/socket-engine.ts +14 -2
- package/src/socket-idle.ts +3 -1
- package/src/socket-routes.ts +8 -3
- package/src/subscriber-gate.ts +11 -1
- package/src/sync-auth.ts +47 -2
- package/src/sync-frames.ts +73 -20
- package/src/sync-node-bounds.ts +15 -0
- package/src/sync-node-contract.ts +20 -2
- package/src/sync-node.ts +92 -17
- package/src/sync-origin.ts +26 -11
- package/src/sync-protocol.ts +13 -3
- package/src/sync-upgrade.ts +76 -8
- package/src/use-mutation.ts +33 -5
- package/src/wire-channel.ts +13 -4
package/src/channel-records.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// socket, no seq, no policy — `ChannelHub` does those per node and per socket. The only derivation
|
|
3
3
|
// of "which channel carries this row" lives here, so a write never names a channel (axiom 2).
|
|
4
4
|
|
|
5
|
+
import { type Counter, counter, logger } from '@ultimat3/core';
|
|
5
6
|
import type { Row } from '@ultimat3/core/page';
|
|
6
7
|
import type { RecordProjection } from '@ultimat3/entity/record';
|
|
7
8
|
import type { ChangeEvent } from './changefeed';
|
|
@@ -16,6 +17,23 @@ export interface TopicUpdate {
|
|
|
16
17
|
readonly adopt: readonly RecordPart[];
|
|
17
18
|
readonly remove: readonly { readonly type: string; readonly key: string }[];
|
|
18
19
|
readonly row: Row;
|
|
20
|
+
/**
|
|
21
|
+
* The change could not carry this row whole (`ChangeEvent.omitted` names a column the record
|
|
22
|
+
* projects), so there is nothing sound to adopt: the topic's members are told to re-read.
|
|
23
|
+
*/
|
|
24
|
+
readonly gap?: true;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const removalsSkipped: Counter = counter('channel_removals_skipped_total', {
|
|
28
|
+
unit: '{removal}',
|
|
29
|
+
description:
|
|
30
|
+
'Channel record removals that could not be routed: the old row image lacked the channel params (the table is not REPLICA IDENTITY FULL), so members keep the removed record.',
|
|
31
|
+
});
|
|
32
|
+
let skipped = 0;
|
|
33
|
+
|
|
34
|
+
/** Removals this process could not route since boot — the in-process read of the counter above. */
|
|
35
|
+
export function skippedRemovals(): number {
|
|
36
|
+
return skipped;
|
|
19
37
|
}
|
|
20
38
|
|
|
21
39
|
/**
|
|
@@ -25,7 +43,9 @@ export interface TopicUpdate {
|
|
|
25
43
|
* - insert/update: the new row is adopted on the topic its params name.
|
|
26
44
|
* - an update that moved the row to other params (`orgId` changed): a remove on the OLD topic too.
|
|
27
45
|
* - delete: a remove on the topic the old image names — which needs the params columns in the
|
|
28
|
-
* `before` image, i.e. `REPLICA IDENTITY FULL`; a key-only image names no topic and is skipped
|
|
46
|
+
* `before` image, i.e. `REPLICA IDENTITY FULL`; a key-only image names no topic and is skipped
|
|
47
|
+
* — COUNTED and logged (`channel.removal_skipped`), because the members of that topic keep a
|
|
48
|
+
* record the database no longer has and nothing else would ever say so.
|
|
29
49
|
*/
|
|
30
50
|
export function updatesFor(channels: Iterable<Channel>, change: ChangeEvent): TopicUpdate[] {
|
|
31
51
|
const updates: TopicUpdate[] = [];
|
|
@@ -39,17 +59,28 @@ export function updatesFor(channels: Iterable<Channel>, change: ChangeEvent): To
|
|
|
39
59
|
const before = change.before;
|
|
40
60
|
const afterParams = after === null ? null : channel.paramsOf(after);
|
|
41
61
|
const beforeParams = before === null ? null : channel.paramsOf(before);
|
|
62
|
+
if (before !== null && beforeParams === null && channel.params.length > 0) {
|
|
63
|
+
skipped += 1;
|
|
64
|
+
removalsSkipped.add(1);
|
|
65
|
+
logger.warn('channel.removal_skipped', { channel: channel.name, table: change.entity });
|
|
66
|
+
}
|
|
42
67
|
const afterTopic = afterParams === null ? null : channel.topic(afterParams);
|
|
43
68
|
const beforeTopic = beforeParams === null ? null : channel.topic(beforeParams);
|
|
44
69
|
|
|
45
70
|
if (after !== null && afterParams !== null && afterTopic !== null) {
|
|
71
|
+
// An unchanged out-of-line value Postgres logged no bytes for: adopted, the partial row
|
|
72
|
+
// would be retained in the ring as the whole record and handed to every later joiner.
|
|
73
|
+
const partial = (change.omitted ?? []).some((column) =>
|
|
74
|
+
Object.hasOwn(projection.schema.properties ?? {}, column),
|
|
75
|
+
);
|
|
46
76
|
updates.push({
|
|
47
77
|
channel,
|
|
48
78
|
topic: afterTopic,
|
|
49
79
|
params: afterParams,
|
|
50
|
-
adopt: [{ type: projection.type, key: projection.key(after), row: after }],
|
|
80
|
+
adopt: partial ? [] : [{ type: projection.type, key: projection.key(after), row: after }],
|
|
51
81
|
remove: [],
|
|
52
82
|
row: after,
|
|
83
|
+
...(partial ? { gap: true as const } : {}),
|
|
53
84
|
});
|
|
54
85
|
}
|
|
55
86
|
if (
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// The sid each of a socket's channel seats was answered under, keyed by TOPIC. A sid is client
|
|
2
|
+
// data: keyed by it, one socket minted an entry per fresh sid without bound, and two sids naming
|
|
3
|
+
// one topic aliased one membership — dropping either unsubscribed the other.
|
|
4
|
+
|
|
5
|
+
import type { Topic } from './channel-decl';
|
|
6
|
+
import type { SyncSocket } from './socket';
|
|
7
|
+
|
|
8
|
+
/** Weakly keyed by socket, so a socket's entries die with it; bounded by the hub's topic cap. */
|
|
9
|
+
export class ChannelSids {
|
|
10
|
+
readonly #bySocket = new WeakMap<SyncSocket, Map<Topic, string>>();
|
|
11
|
+
|
|
12
|
+
/** The latest sid wins: a repeated `add` under a new sid re-names the one seat, never adds one. */
|
|
13
|
+
set(socket: SyncSocket, topic: Topic, sid: string): void {
|
|
14
|
+
const topics = this.#bySocket.get(socket) ?? new Map<Topic, string>();
|
|
15
|
+
this.#bySocket.set(socket, topics.set(topic, sid));
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** The sid a refusal for this seat must name — `undefined` when the socket holds no such seat. */
|
|
19
|
+
sidOf(socket: SyncSocket, topic: Topic): string | undefined {
|
|
20
|
+
return this.#bySocket.get(socket)?.get(topic);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** The topic currently held under `sid`. A sid an earlier `add` was re-named away from holds none. */
|
|
24
|
+
topicOf(socket: SyncSocket, sid: string): Topic | undefined {
|
|
25
|
+
for (const [topic, held] of this.#bySocket.get(socket) ?? []) if (held === sid) return topic;
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
delete(socket: SyncSocket, topic: Topic): void {
|
|
30
|
+
this.#bySocket.get(socket)?.delete(topic);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Seats recorded for one socket — what a test bounds. */
|
|
34
|
+
count(socket: SyncSocket): number {
|
|
35
|
+
return this.#bySocket.get(socket)?.size ?? 0;
|
|
36
|
+
}
|
|
37
|
+
}
|
package/src/channel.ts
CHANGED
|
@@ -16,7 +16,8 @@ import type { ChangeEvent } from './changefeed';
|
|
|
16
16
|
import { authorizeChannel } from './channel-authz';
|
|
17
17
|
import { type Bridge, unsubscribeWhenOpen } from './channel-bridge';
|
|
18
18
|
import type { Channel, Topic } from './channel-decl';
|
|
19
|
-
import {
|
|
19
|
+
import { DenialLatch } from './channel-latch';
|
|
20
|
+
import { ChannelLogs, type ChannelTopic } from './channel-logs';
|
|
20
21
|
import { getChannel, registeredChannels } from './channel-registry';
|
|
21
22
|
import type { ChannelEventsFrame, ChannelSubscribeTarget } from './channel-wire';
|
|
22
23
|
import {
|
|
@@ -52,6 +53,12 @@ export interface ChannelHubOptions {
|
|
|
52
53
|
readonly ringSize?: number;
|
|
53
54
|
}
|
|
54
55
|
|
|
56
|
+
/** The longest value one channel param may carry — a uuid is 36. */
|
|
57
|
+
export const MAX_CHANNEL_PARAM_LENGTH = 128;
|
|
58
|
+
|
|
59
|
+
/** `#settle`'s "nobody has decided this seat yet": distinct from `null`, which is an actor. */
|
|
60
|
+
const UNDECIDED = Symbol('undecided');
|
|
61
|
+
|
|
55
62
|
/** Distinct topics one node bridges before `X_SUBSCRIPTION_LIMIT`. */
|
|
56
63
|
export const DEFAULT_MAX_TOPICS_PER_NODE = 10_000;
|
|
57
64
|
|
|
@@ -80,12 +87,13 @@ export class ChannelHub {
|
|
|
80
87
|
readonly #only: ReadonlyMap<string, Channel> | null;
|
|
81
88
|
readonly #logs: ChannelLogs;
|
|
82
89
|
readonly #ctx: Ctx;
|
|
90
|
+
/** Topics a socket's policy refused, until its actor changes (`channel-latch.ts`). */
|
|
91
|
+
readonly #latched: DenialLatch;
|
|
83
92
|
/**
|
|
84
|
-
* Topics
|
|
85
|
-
*
|
|
86
|
-
* THAT channel is refused, every other one on the socket keeps flowing.
|
|
93
|
+
* Topics whose guard could not DECIDE on a re-auth. Not a denial, so the seat and its bridge
|
|
94
|
+
* reference are kept; not a pass either, so nothing is delivered until one succeeds.
|
|
87
95
|
*/
|
|
88
|
-
readonly #
|
|
96
|
+
readonly #suspended = new WeakMap<SyncSocket, Set<Topic>>();
|
|
89
97
|
|
|
90
98
|
constructor(options: ChannelHubOptions) {
|
|
91
99
|
this.#transport = options.transport;
|
|
@@ -100,6 +108,7 @@ export class ChannelHub {
|
|
|
100
108
|
'maxTopicsPerNode',
|
|
101
109
|
options.maxTopicsPerNode ?? DEFAULT_MAX_TOPICS_PER_NODE,
|
|
102
110
|
);
|
|
111
|
+
this.#latched = new DenialLatch(this.#maxTopicsPerSocket, 'maxTopicsPerSocket');
|
|
103
112
|
this.#ctx = options.ctx ?? createContext();
|
|
104
113
|
this.#logs = new ChannelLogs(options.sockets, options.ringSize);
|
|
105
114
|
this.#only =
|
|
@@ -123,31 +132,93 @@ export class ChannelHub {
|
|
|
123
132
|
}
|
|
124
133
|
const params: Record<string, string> = {};
|
|
125
134
|
for (const param of declared.params) {
|
|
126
|
-
|
|
135
|
+
const value = Object.hasOwn(target.params, param) ? (target.params[param] ?? '') : '';
|
|
136
|
+
// A topic is retained (the bridge table, the latch, the client's sid), and `topic()` checks
|
|
137
|
+
// a segment's alphabet, never its size — so a param is the client's to grow only this far.
|
|
138
|
+
if (value.length > MAX_CHANNEL_PARAM_LENGTH) {
|
|
139
|
+
throw new TopicForbiddenError({
|
|
140
|
+
topic: target.channel,
|
|
141
|
+
actorId: socket.actorId,
|
|
142
|
+
reason: `param "${param}" is ${value.length} characters; the limit is ${MAX_CHANNEL_PARAM_LENGTH}`,
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
params[param] = value;
|
|
127
146
|
}
|
|
128
147
|
const name = declared.topic(params);
|
|
129
|
-
if (this.#latched.
|
|
148
|
+
if (this.#latched.has(socket, name)) {
|
|
130
149
|
throw new TopicForbiddenError({
|
|
131
150
|
topic: name,
|
|
132
151
|
actorId: socket.actorId,
|
|
133
152
|
reason: 'denied earlier on this connection; it is re-decided when the session changes',
|
|
134
153
|
});
|
|
135
154
|
}
|
|
155
|
+
this.#latched.assertRoom(socket);
|
|
156
|
+
const seat: ChannelTopic = { channel: declared, params };
|
|
157
|
+
// A suspended seat asked for again (the client's beat) is the next pass, never a second seat.
|
|
158
|
+
if (this.#suspended.get(socket)?.has(name) === true) {
|
|
159
|
+
await this.#settle(socket, name, seat, UNDECIDED);
|
|
160
|
+
return name;
|
|
161
|
+
}
|
|
136
162
|
// Asked before the join seats it: a repeated `add` (the presence beat) is not a fresh seat.
|
|
137
163
|
const fresh = !socket.topics.has(name);
|
|
164
|
+
// Captured, because the guard awaits and a re-auth may land under it: the verdict below is
|
|
165
|
+
// about THIS actor, and `onActorChange` cannot re-decide a seat that does not exist yet.
|
|
166
|
+
const actor = socket.actor;
|
|
138
167
|
await this.#join(socket, name, async () => {
|
|
139
168
|
try {
|
|
140
|
-
await authorizeChannel(declared, this.#ctx,
|
|
169
|
+
await authorizeChannel(declared, this.#ctx, actor, name, params);
|
|
141
170
|
} catch (error) {
|
|
142
|
-
if (error instanceof TopicForbiddenError
|
|
171
|
+
if (error instanceof TopicForbiddenError && socket.actor === actor) {
|
|
172
|
+
this.#latched.add(socket, name);
|
|
173
|
+
}
|
|
143
174
|
throw error;
|
|
144
175
|
}
|
|
145
176
|
});
|
|
146
|
-
|
|
177
|
+
// The socket died while the guard or the bridge was answering: `#join` gave the seat back, so
|
|
178
|
+
// there is no member to open a ring for or to resume.
|
|
179
|
+
if (!socket.topics.has(name)) return name;
|
|
180
|
+
this.#logs.open(name, seat);
|
|
181
|
+
await this.#settle(socket, name, seat, actor);
|
|
182
|
+
// Dropped is a throw; suspended is a seat nothing is delivered on, the resume included.
|
|
183
|
+
if (!socket.topics.has(name)) return name;
|
|
147
184
|
this.#logs.resume(socket, name, target.since, fresh);
|
|
148
185
|
return name;
|
|
149
186
|
}
|
|
150
187
|
|
|
188
|
+
/**
|
|
189
|
+
* Decide a seat under the socket's CURRENT actor until the actor it was decided for is still the
|
|
190
|
+
* one on the socket. A seat granted to the actor a re-auth has since replaced was kept until the
|
|
191
|
+
* next grant expiry; a re-ask on a suspended seat ignored a denial and answered success.
|
|
192
|
+
*/
|
|
193
|
+
async #settle(
|
|
194
|
+
socket: SyncSocket,
|
|
195
|
+
name: Topic,
|
|
196
|
+
seat: ChannelTopic,
|
|
197
|
+
decidedFor: Actor | null | typeof UNDECIDED,
|
|
198
|
+
): Promise<void> {
|
|
199
|
+
let actor = decidedFor;
|
|
200
|
+
while (actor === UNDECIDED || socket.actor !== actor) {
|
|
201
|
+
actor = socket.actor;
|
|
202
|
+
if ((await this.#decide(socket, name, seat, actor)) !== 'denied') continue;
|
|
203
|
+
if (socket.actor === actor) this.#latched.add(socket, name);
|
|
204
|
+
throw new TopicForbiddenError({
|
|
205
|
+
topic: name,
|
|
206
|
+
actorId: socket.actorId,
|
|
207
|
+
reason: `channel "${seat.channel.name}" policy denied the subscribe`,
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Every open records topic is presumed to have missed a change: a new epoch, and every member
|
|
214
|
+
* told `replay-gap`. The `sync` node calls it beside `LiveQueryRegistry.invalidate()` — seq is
|
|
215
|
+
* minted here from the changes this node SEES, so a change the bus dropped leaves a hole the
|
|
216
|
+
* ring cannot detect and would replay as complete history.
|
|
217
|
+
*/
|
|
218
|
+
invalidate(): number {
|
|
219
|
+
return this.#logs.invalidate();
|
|
220
|
+
}
|
|
221
|
+
|
|
151
222
|
/**
|
|
152
223
|
* One committed change from the feed, turned into `records` frames on every declared channel it
|
|
153
224
|
* touches and delivered on THIS node. Called for every change the node receives — the same
|
|
@@ -178,6 +249,9 @@ export class ChannelHub {
|
|
|
178
249
|
* a topic a caller spelled; every `Topic` comes from a declaration.
|
|
179
250
|
*/
|
|
180
251
|
async emit(name: Topic, event: JsonObject): Promise<void> {
|
|
252
|
+
// A topic this node holds open on a channel declared without `events` carries none: a
|
|
253
|
+
// presence leave for it was an event on a channel whose members were promised records only.
|
|
254
|
+
if (this.#logs.target(name)?.channel.events === false) return;
|
|
181
255
|
const frame: ChannelEventsFrame = { type: 'events', v: PROTOCOL_VERSION, channel: name, event };
|
|
182
256
|
await this.#transport.publish(`${CHANNEL_SUBJECT_PREFIX}.${name}`, JSON.stringify(frame));
|
|
183
257
|
}
|
|
@@ -189,12 +263,6 @@ export class ChannelHub {
|
|
|
189
263
|
return this.#logs.target(name);
|
|
190
264
|
}
|
|
191
265
|
|
|
192
|
-
#latch(socket: SyncSocket, name: string): void {
|
|
193
|
-
const latched = this.#latched.get(socket) ?? new Set<string>();
|
|
194
|
-
latched.add(name);
|
|
195
|
-
this.#latched.set(socket, latched);
|
|
196
|
-
}
|
|
197
|
-
|
|
198
266
|
/** Sockets this node will deliver `name` to. The metric the fanout reads. */
|
|
199
267
|
subscriberCount(name: Topic): number {
|
|
200
268
|
return this.#sockets.subscriberCount(name);
|
|
@@ -222,7 +290,8 @@ export class ChannelHub {
|
|
|
222
290
|
async #join(socket: SyncSocket, name: Topic, authorize: () => Promise<void>): Promise<void> {
|
|
223
291
|
if (socket.topics.has(name)) return;
|
|
224
292
|
const claimed = this.#claimed.get(socket) ?? 0;
|
|
225
|
-
|
|
293
|
+
const held = socket.topics.size + (this.#suspended.get(socket)?.size ?? 0);
|
|
294
|
+
if (held + claimed >= this.#maxTopicsPerSocket) {
|
|
226
295
|
throw new SubscriptionLimitError({
|
|
227
296
|
scope: 'socket',
|
|
228
297
|
id: socket.id,
|
|
@@ -263,51 +332,81 @@ export class ChannelHub {
|
|
|
263
332
|
// fanout reads are one fact, and two call sites for one fact is the drift that makes an index
|
|
264
333
|
// wrong. The registry owns it because it is the only thing that sees a socket die.
|
|
265
334
|
this.#sockets.joinTopic(socket, name);
|
|
335
|
+
// The socket closed while this subscribe was parked: its teardown has already walked its
|
|
336
|
+
// topics, so a seat taken now is a bridge pinned with no member and nothing left to release it.
|
|
337
|
+
if (socket.closed) this.unsubscribe(socket, name);
|
|
266
338
|
}
|
|
267
339
|
|
|
268
340
|
unsubscribe(socket: SyncSocket, name: Topic): void {
|
|
269
|
-
|
|
270
|
-
|
|
341
|
+
const suspended = this.#suspended.get(socket)?.delete(name) === true;
|
|
342
|
+
if (!suspended && !socket.topics.has(name)) return;
|
|
343
|
+
if (!suspended) this.#sockets.leaveTopic(socket, name);
|
|
271
344
|
this.#release(name);
|
|
272
345
|
}
|
|
273
346
|
|
|
347
|
+
/** Every topic this socket holds a seat on — the delivered ones and the suspended ones. */
|
|
348
|
+
topicsOf(socket: SyncSocket): readonly Topic[] {
|
|
349
|
+
return [...socket.topics, ...(this.#suspended.get(socket) ?? [])] as Topic[];
|
|
350
|
+
}
|
|
351
|
+
|
|
274
352
|
/**
|
|
275
353
|
* Called when a socket's session changes (login, logout, role change, token refresh).
|
|
276
354
|
*
|
|
277
|
-
* A denial drops the topic
|
|
278
|
-
*
|
|
279
|
-
* outage
|
|
280
|
-
*
|
|
281
|
-
*
|
|
355
|
+
* A denial drops the topic. Anything else is not a decision — a guard is app code and may reach
|
|
356
|
+
* a database, and reading a store that timed out as a revoked grant dropped every topic on every
|
|
357
|
+
* re-authenticated socket during one outage. It is not a pass either: the seat is kept and
|
|
358
|
+
* SUSPENDED, so nothing is delivered on it until a later pass (the next re-auth, or the client's
|
|
359
|
+
* own beat) decides. Returns the topics dropped — denials and nothing else.
|
|
282
360
|
*/
|
|
283
361
|
async onActorChange(socket: SyncSocket, actor: Actor | null): Promise<readonly Topic[]> {
|
|
284
362
|
socket.actor = actor;
|
|
285
363
|
// A new session re-decides everything, the latched denials included.
|
|
286
|
-
this.#latched.
|
|
364
|
+
this.#latched.clear(socket);
|
|
287
365
|
const dropped: Topic[] = [];
|
|
288
|
-
for (const name of
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
await authorizeChannel(target.channel, this.#ctx, actor, name, target.params);
|
|
293
|
-
}
|
|
294
|
-
} catch (error) {
|
|
295
|
-
if (isPolicyDenial(error) || error instanceof TopicForbiddenError) {
|
|
296
|
-
this.unsubscribe(socket, name);
|
|
297
|
-
dropped.push(name);
|
|
298
|
-
continue;
|
|
299
|
-
}
|
|
300
|
-
this.#guardFailures += 1;
|
|
301
|
-
logger.warn('channel.guard_failed', {
|
|
302
|
-
topic: name,
|
|
303
|
-
socketId: socket.id,
|
|
304
|
-
error: renderThrowable(error),
|
|
305
|
-
});
|
|
306
|
-
}
|
|
366
|
+
for (const name of this.topicsOf(socket)) {
|
|
367
|
+
const target = this.#logs.target(name);
|
|
368
|
+
if (target === undefined) continue;
|
|
369
|
+
if ((await this.#decide(socket, name, target, actor)) === 'denied') dropped.push(name);
|
|
307
370
|
}
|
|
308
371
|
return dropped;
|
|
309
372
|
}
|
|
310
373
|
|
|
374
|
+
/** One seat, re-decided: dropped on a denial, suspended on a failure, delivered on a pass. */
|
|
375
|
+
async #decide(
|
|
376
|
+
socket: SyncSocket,
|
|
377
|
+
name: Topic,
|
|
378
|
+
target: ChannelTopic,
|
|
379
|
+
actor: Actor | null,
|
|
380
|
+
): Promise<'allowed' | 'denied' | 'undecided'> {
|
|
381
|
+
try {
|
|
382
|
+
await authorizeChannel(target.channel, this.#ctx, actor, name, target.params);
|
|
383
|
+
} catch (error) {
|
|
384
|
+
if (isPolicyDenial(error) || error instanceof TopicForbiddenError) {
|
|
385
|
+
this.unsubscribe(socket, name);
|
|
386
|
+
return 'denied';
|
|
387
|
+
}
|
|
388
|
+
this.#guardFailures += 1;
|
|
389
|
+
logger.warn('channel.guard_failed', {
|
|
390
|
+
topic: name,
|
|
391
|
+
socketId: socket.id,
|
|
392
|
+
error: renderThrowable(error),
|
|
393
|
+
});
|
|
394
|
+
if (socket.topics.has(name)) {
|
|
395
|
+
this.#sockets.leaveTopic(socket, name);
|
|
396
|
+
const suspended = this.#suspended.get(socket) ?? new Set<Topic>();
|
|
397
|
+
this.#suspended.set(socket, suspended.add(name));
|
|
398
|
+
}
|
|
399
|
+
return 'undecided';
|
|
400
|
+
}
|
|
401
|
+
// Still suspended, and still open: a socket that closed meanwhile gave the seat back already.
|
|
402
|
+
if (this.#suspended.get(socket)?.delete(name) === true) {
|
|
403
|
+
this.#sockets.joinTopic(socket, name);
|
|
404
|
+
// What the suspension withheld is unknown to the client, so a records channel is re-read.
|
|
405
|
+
this.#logs.resume(socket, name, undefined, true);
|
|
406
|
+
}
|
|
407
|
+
return 'allowed';
|
|
408
|
+
}
|
|
409
|
+
|
|
311
410
|
async close(): Promise<void> {
|
|
312
411
|
// Set BEFORE the table is walked, because the table is not the whole story: a reservation an
|
|
313
412
|
// in-flight `subscribe` has not opened yet is `sub === null`, so `unsubscribeWhenOpen` does
|
package/src/client-channels.ts
CHANGED
|
@@ -208,6 +208,10 @@ export class ChannelBook {
|
|
|
208
208
|
event(frame: ChannelEventsFrame): void {
|
|
209
209
|
const entry = this.#byTopic.get(frame.channel);
|
|
210
210
|
const presence = readPresence(frame.event);
|
|
211
|
+
// The node delivers an event only to a member it has seated, and for an events-only channel
|
|
212
|
+
// (no rows, so no `records` and no `replay-gap`) the roster is the one answer a subscribe
|
|
213
|
+
// gets. Only from `joining`: a read in flight, a refusal and a dead socket are not undone.
|
|
214
|
+
if (entry?.state === 'joining') this.#set(entry, 'live');
|
|
211
215
|
for (const holder of entry?.holders ?? []) {
|
|
212
216
|
if (presence !== null) holder.onPresence?.(presence);
|
|
213
217
|
else holder.onEvent?.(frame.event);
|
|
@@ -230,7 +234,12 @@ export class ChannelBook {
|
|
|
230
234
|
const contiguous = entry.contiguous ?? frame.seq - 1;
|
|
231
235
|
// A duplicate is dropped; a replayed seq filling a hole is new. A numeric hole on its own is
|
|
232
236
|
// never a gap — a row this socket may not see is skipped for it; only `replay-gap` is.
|
|
233
|
-
if (frame.seq <= contiguous || entry.above.has(frame.seq))
|
|
237
|
+
if (frame.seq <= contiguous || entry.above.has(frame.seq)) {
|
|
238
|
+
// Still a frame of THIS epoch on THIS seat — and the whole of the node's answer to a resume
|
|
239
|
+
// that was already current (`ChannelLogs.resume`): nothing to apply, and no longer joining.
|
|
240
|
+
if (entry.state !== 'live') this.#set(entry, 'live');
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
234
243
|
const store = this.#deps.store;
|
|
235
244
|
store.batch(() => {
|
|
236
245
|
for (const [type, rows] of Object.entries(frame.adopt ?? {})) store.adopt(type, rows);
|
package/src/client-contract.ts
CHANGED
|
@@ -63,7 +63,10 @@ export interface LiveClientOptions {
|
|
|
63
63
|
readonly clock?: Clock;
|
|
64
64
|
/** How a pending reconnect is armed. Defaults to `setTimeout`; tests fire theirs by hand. */
|
|
65
65
|
readonly scheduler?: Scheduler;
|
|
66
|
-
/**
|
|
66
|
+
/**
|
|
67
|
+
* How often a live socket re-announces itself, in ms, until the node names its own beat on its
|
|
68
|
+
* `hello` (`hello.heartbeatMs`). `0` disables the pass, whatever the node says. Default 10s.
|
|
69
|
+
*/
|
|
67
70
|
readonly heartbeatMs?: number;
|
|
68
71
|
/** Where a failure nobody awaits is reported. Defaults to `console.error`. */
|
|
69
72
|
readonly onError?: (error: unknown) => void;
|
package/src/client-frames.ts
CHANGED
|
@@ -3,11 +3,12 @@
|
|
|
3
3
|
// every piece of the client a frame may touch, so the blast radius of a new frame kind is a
|
|
4
4
|
// reviewable list rather than "whatever the router could reach through `this`".
|
|
5
5
|
|
|
6
|
+
import { UltimateError } from '@ultimat3/core/page';
|
|
6
7
|
import type { ChannelBook } from './client-channels';
|
|
7
8
|
import { CLOSE } from './close-codes';
|
|
8
9
|
import { advance } from './cursor';
|
|
9
10
|
import type { Registration, RowWindows } from './live-rows';
|
|
10
|
-
import type { Frame } from './sync-protocol';
|
|
11
|
+
import type { Frame, WireError } from './sync-protocol';
|
|
11
12
|
|
|
12
13
|
/** Declared with the window it projects; re-exported here because the router is what writes it. */
|
|
13
14
|
export type { LiveState, Registration } from './live-rows';
|
|
@@ -35,6 +36,8 @@ export interface ClientFrameTarget {
|
|
|
35
36
|
now(): number;
|
|
36
37
|
/** A newer build is live; the app decides when to reload. */
|
|
37
38
|
setUpdate(buildId: string | null): void;
|
|
39
|
+
/** The node named the beat this socket must keep (`hello.heartbeatMs`). */
|
|
40
|
+
followHeartbeat(intervalMs: number): void;
|
|
38
41
|
/** The node assigned this socket its own delay before closing it. */
|
|
39
42
|
scheduleReconnect(afterMs: number | null): void;
|
|
40
43
|
closeSocket(code: number, reason: string): void;
|
|
@@ -42,6 +45,28 @@ export interface ClientFrameTarget {
|
|
|
42
45
|
report(error: unknown): void;
|
|
43
46
|
}
|
|
44
47
|
|
|
48
|
+
/** An absolute HTTP(S) link, or nothing: a node's `docs` is data and may be `javascript:`. */
|
|
49
|
+
const HTTP_URL = /^https?:\/\/[^\s]+$/;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* A refusal off the wire, as the error the node raised: the same code, cause and fix, BRANDED —
|
|
53
|
+
* the shape `@ultimat3/core`'s `problemError` gives an HTTP failure, marked `origin: 'remote'` the
|
|
54
|
+
* same way. Handed over as the bare `WireError` it failed `isUltimateError`, so every reader that
|
|
55
|
+
* asks that question — `<ErrorState>`, a boundary, a reporter — fell through to "an unknown value
|
|
56
|
+
* was thrown": `X_INTERNAL`, the real error JSON-stringified into its cause, and a fix telling the
|
|
57
|
+
* reader to throw the `UltimateError` it already was.
|
|
58
|
+
*/
|
|
59
|
+
export function refusalError(wire: WireError): UltimateError {
|
|
60
|
+
const docs = wire.docs !== undefined && HTTP_URL.test(wire.docs) ? wire.docs : undefined;
|
|
61
|
+
return new UltimateError({
|
|
62
|
+
code: wire.code,
|
|
63
|
+
cause: wire.cause,
|
|
64
|
+
fix: wire.fix,
|
|
65
|
+
meta: { origin: 'remote' },
|
|
66
|
+
...(docs === undefined ? {} : { docs }),
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
|
|
45
70
|
export function applyFrame(frame: Frame, target: ClientFrameTarget): void {
|
|
46
71
|
switch (frame.type) {
|
|
47
72
|
case 'snapshot': {
|
|
@@ -78,13 +103,14 @@ export function applyFrame(frame: Frame, target: ClientFrameTarget): void {
|
|
|
78
103
|
// The socket carries no writes, so an `ack` is only ever a refusal: of a subscription (its
|
|
79
104
|
// `ref` is the sid) or of a frame the node could not read at all (its `ref` is the socket).
|
|
80
105
|
if (frame.error === null) return;
|
|
106
|
+
const refusal = refusalError(frame.error);
|
|
81
107
|
const registration = target.registration(frame.ref);
|
|
82
108
|
if (registration === undefined) {
|
|
83
|
-
if (!target.channels.refused(frame.ref,
|
|
109
|
+
if (!target.channels.refused(frame.ref, refusal)) target.report(refusal);
|
|
84
110
|
return;
|
|
85
111
|
}
|
|
86
112
|
registration.state = 'failed';
|
|
87
|
-
registration.error =
|
|
113
|
+
registration.error = refusal;
|
|
88
114
|
registration.notify();
|
|
89
115
|
return;
|
|
90
116
|
}
|
|
@@ -111,8 +137,12 @@ export function applyFrame(frame: Frame, target: ClientFrameTarget): void {
|
|
|
111
137
|
target.channels.gap(frame);
|
|
112
138
|
return;
|
|
113
139
|
case 'hello':
|
|
140
|
+
// The node's reply. It carries one instruction — how often to beat — and a node that names
|
|
141
|
+
// none (an older build) leaves the client on its own default.
|
|
142
|
+
if (frame.heartbeatMs !== undefined) target.followHeartbeat(frame.heartbeatMs);
|
|
143
|
+
return;
|
|
114
144
|
case 'subscribe':
|
|
115
|
-
//
|
|
145
|
+
// A client-authored frame: never received. Ignored rather than thrown, so a future
|
|
116
146
|
// bidirectional use of the same kind cannot break an old client.
|
|
117
147
|
return;
|
|
118
148
|
}
|
package/src/client-heartbeat.ts
CHANGED
|
@@ -5,15 +5,22 @@
|
|
|
5
5
|
import type { Scheduler } from './thundering-herd';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* A third of the node's default presence ttl (`DEFAULT_PRESENCE_TTL_MS`, 30 s), which is what the
|
|
9
|
+
* node derives for itself (`PresenceRegistry.heartbeatMs` is `ttlMs / 3`): two beats land inside
|
|
10
|
+
* every ttl, so ONE lost beat is never a false leave. It was 15 s — half the ttl, where the beat
|
|
11
|
+
* after a lost one lands on the expiry itself. `client-heartbeat.test.ts` holds the two equal;
|
|
12
|
+
* this file cannot import the server's constant (`transport-env.ts` links `nats`).
|
|
12
13
|
*/
|
|
13
|
-
export const DEFAULT_HEARTBEAT_MS =
|
|
14
|
+
export const DEFAULT_HEARTBEAT_MS = 10_000;
|
|
15
|
+
|
|
16
|
+
/** The node floors the beat it names at 1 s (`clientHeartbeatMs`); less than that is not its word. */
|
|
17
|
+
export const MIN_HEARTBEAT_MS = 1_000;
|
|
14
18
|
|
|
15
19
|
export interface HeartbeatOptions {
|
|
16
|
-
/**
|
|
20
|
+
/**
|
|
21
|
+
* Until the node names its own (`follow`). `0` (or less) disables the pass entirely, whatever
|
|
22
|
+
* the node says — the shape a test that owns the clock wants.
|
|
23
|
+
*/
|
|
17
24
|
readonly intervalMs: number;
|
|
18
25
|
readonly schedule: Scheduler;
|
|
19
26
|
readonly now: () => number;
|
|
@@ -32,19 +39,42 @@ export class Heartbeat {
|
|
|
32
39
|
readonly #options: HeartbeatOptions;
|
|
33
40
|
#cancel: (() => void) | null = null;
|
|
34
41
|
#lastSeen = 0;
|
|
42
|
+
/** The beat in force on THIS socket: the option's, until the node's `hello` names another. */
|
|
43
|
+
#intervalMs: number;
|
|
35
44
|
|
|
36
45
|
constructor(options: HeartbeatOptions) {
|
|
37
46
|
this.#options = options;
|
|
47
|
+
this.#intervalMs = options.intervalMs;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
get intervalMs(): number {
|
|
51
|
+
return this.#intervalMs;
|
|
38
52
|
}
|
|
39
53
|
|
|
40
54
|
/** The socket is up. `now` seeds the silence window, so the first tick judges this connection. */
|
|
41
55
|
start(now: number): void {
|
|
42
56
|
this.stop();
|
|
43
|
-
|
|
57
|
+
// A new socket may be another node's: its beat is not known until its `hello` says so.
|
|
58
|
+
this.#intervalMs = this.#options.intervalMs;
|
|
59
|
+
if (this.#intervalMs <= 0) return;
|
|
44
60
|
this.#lastSeen = now;
|
|
45
61
|
this.#arm();
|
|
46
62
|
}
|
|
47
63
|
|
|
64
|
+
/**
|
|
65
|
+
* The node named the beat (`hello.heartbeatMs`): a third of ITS presence ttl, bounded by ITS idle
|
|
66
|
+
* budget — two numbers a browser cannot read, and the reason a fixed client beat was wrong for
|
|
67
|
+
* every app that set either. Re-armed at once, so a shorter beat does not wait out the long one.
|
|
68
|
+
*/
|
|
69
|
+
follow(intervalMs: number): void {
|
|
70
|
+
if (this.#options.intervalMs <= 0) return;
|
|
71
|
+
if (intervalMs < MIN_HEARTBEAT_MS || intervalMs === this.#intervalMs) return;
|
|
72
|
+
this.#intervalMs = intervalMs;
|
|
73
|
+
if (this.#cancel === null) return;
|
|
74
|
+
this.stop();
|
|
75
|
+
this.#arm();
|
|
76
|
+
}
|
|
77
|
+
|
|
48
78
|
/** A frame arrived. Anything counts: the point is that bytes still cross in this direction. */
|
|
49
79
|
saw(now: number): void {
|
|
50
80
|
this.#lastSeen = now;
|
|
@@ -60,7 +90,7 @@ export class Heartbeat {
|
|
|
60
90
|
this.#cancel = this.#options.schedule(() => {
|
|
61
91
|
this.#cancel = null;
|
|
62
92
|
this.#tick();
|
|
63
|
-
}, this.#
|
|
93
|
+
}, this.#intervalMs);
|
|
64
94
|
}
|
|
65
95
|
|
|
66
96
|
#tick(): void {
|
|
@@ -68,7 +98,7 @@ export class Heartbeat {
|
|
|
68
98
|
// Two windows, not one: a beat and the answer to it share the window they were sent in, so a
|
|
69
99
|
// single quiet interval is a slow round trip and not a dead socket. Nothing is re-armed after
|
|
70
100
|
// a silence — `onSilence` drops the socket, and the next `start()` is the next connection's.
|
|
71
|
-
if (now - this.#lastSeen > this.#
|
|
101
|
+
if (now - this.#lastSeen > this.#intervalMs * 2) {
|
|
72
102
|
this.#options.onSilence();
|
|
73
103
|
return;
|
|
74
104
|
}
|
package/src/client.ts
CHANGED
|
@@ -321,6 +321,7 @@ export class LiveClient {
|
|
|
321
321
|
// against a re-snapshot, so the frame path reads the same clock every other path does.
|
|
322
322
|
now: () => this.#clock.now().getTime(),
|
|
323
323
|
setUpdate: (buildId) => this.#setStatus({ update: buildId }),
|
|
324
|
+
followHeartbeat: (intervalMs) => this.#heartbeat.follow(intervalMs),
|
|
324
325
|
scheduleReconnect: (afterMs) => this.#scheduleReconnect(afterMs),
|
|
325
326
|
closeSocket: (code, reason) => this.#socket?.close(code, reason),
|
|
326
327
|
report: (error) => this.#onError(error),
|
package/src/cursor.ts
CHANGED
|
@@ -76,6 +76,17 @@ export interface ResumeSource {
|
|
|
76
76
|
* the entry, which is the only moment anything knows the window is unreachable.
|
|
77
77
|
*/
|
|
78
78
|
forget?(qid: string): void;
|
|
79
|
+
/**
|
|
80
|
+
* The window behind `qid` was read at `lsn`, so what is retained for it is complete from there
|
|
81
|
+
* and from nowhere earlier; `exclusive` refuses a cursor exactly at it (a forced re-read), `sole`
|
|
82
|
+
* answers that cursor and no other below the first patch (a read with no position behind it).
|
|
83
|
+
* Optional for `forget`'s reason. When it is called, and why, is one rule: `change-buffer.ts`.
|
|
84
|
+
*/
|
|
85
|
+
floorAt?(
|
|
86
|
+
qid: string,
|
|
87
|
+
lsn: string,
|
|
88
|
+
options?: { readonly exclusive?: boolean; readonly sole?: boolean },
|
|
89
|
+
): void;
|
|
79
90
|
}
|
|
80
91
|
|
|
81
92
|
export type ResumeResult<R extends Row = Row> =
|