@ultimat3/realtime 23.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +118 -126
- package/README.md +229 -54
- package/package.json +9 -6
- package/src/advisory-lock.ts +62 -0
- package/src/boot.ts +25 -15
- package/src/change-buffer.ts +96 -41
- package/src/changefeed-env.ts +9 -9
- package/src/changefeed.ts +81 -19
- package/src/channel-authz.ts +48 -17
- package/src/channel-describe.ts +1 -1
- package/src/channel-latch.ts +52 -0
- package/src/channel-logs.ts +36 -15
- package/src/channel-presence.ts +3 -3
- package/src/channel-records.ts +35 -4
- package/src/channel-ref.ts +2 -2
- package/src/channel-registry.ts +2 -2
- package/src/channel-sids.ts +37 -0
- package/src/channel.ts +148 -48
- package/src/client-channels.ts +12 -1
- package/src/client-contract.ts +4 -1
- package/src/client-frames.ts +16 -3
- package/src/client-heartbeat.ts +39 -9
- package/src/client.ts +30 -2
- package/src/cursor.ts +11 -0
- package/src/error-titles.ts +91 -0
- package/src/errors.ts +113 -105
- package/src/fanout.ts +13 -0
- package/src/first-paint-hold.ts +49 -0
- package/src/idb-types.ts +2 -2
- package/src/index.ts +25 -7
- package/src/json.ts +16 -3
- package/src/live-contract.ts +14 -0
- package/src/live-definition.ts +34 -9
- package/src/live-deliver.ts +55 -0
- package/src/live-fanout.ts +122 -29
- package/src/live-floor.ts +35 -0
- package/src/live-query-options.ts +44 -0
- package/src/live-query.ts +96 -175
- package/src/live-reauth.ts +127 -0
- package/src/live-replicator.ts +28 -8
- package/src/live-resume.ts +44 -8
- package/src/live-rows.ts +32 -3
- package/src/live-spend.ts +50 -0
- package/src/local-store-idb.ts +5 -0
- package/src/matcher-bridge.ts +11 -5
- package/src/nats-client.ts +37 -6
- package/src/nats-lib-client.ts +36 -7
- package/src/nats-subscriptions.ts +63 -0
- package/src/nats-transport.ts +149 -15
- package/src/offline-queue.ts +53 -5
- package/src/page-errors.ts +25 -9
- package/src/page-outbox.ts +125 -54
- package/src/page-runtime-wait.ts +72 -0
- package/src/page-runtime.ts +32 -0
- package/src/page-socket.ts +7 -16
- package/src/page-store.ts +83 -16
- package/src/pg-advisory-lock.ts +54 -5
- package/src/pg-auth.ts +9 -4
- package/src/pg-bytes.ts +4 -1
- package/src/pg-connection.ts +126 -22
- package/src/pg-entity-row.ts +16 -0
- package/src/pg-identity-tables.ts +25 -0
- package/src/pg-preflight.ts +50 -4
- package/src/pg-replication-row.ts +82 -0
- package/src/pg-replication.ts +50 -51
- package/src/pg-socket.ts +26 -7
- package/src/pg-tls.ts +18 -0
- package/src/pg-values.ts +29 -14
- package/src/pg-wire.ts +5 -3
- package/src/pgoutput.ts +19 -9
- package/src/policy-gate.ts +7 -7
- package/src/presence.ts +87 -10
- package/src/principal-sockets.ts +75 -0
- package/src/query-window.ts +17 -5
- package/src/record-persister.ts +7 -2
- package/src/record-publisher.ts +178 -0
- package/src/record-store.ts +25 -4
- package/src/record-synced.ts +43 -1
- package/src/replication-errors.ts +11 -7
- package/src/replicator-envelope.ts +160 -0
- package/src/replicator-row.ts +75 -0
- package/src/replicator.ts +313 -191
- package/src/server.ts +45 -23
- package/src/socket-defaults.ts +20 -0
- package/src/socket-engine.ts +14 -2
- package/src/socket-idle.ts +3 -1
- package/src/socket-routes.ts +8 -3
- package/src/socket.ts +14 -19
- package/src/subscriber-gate.ts +11 -1
- package/src/subscription-book.ts +129 -71
- package/src/subscription-count.ts +151 -0
- package/src/sync-actor-change.ts +56 -0
- package/src/sync-auth.ts +47 -2
- package/src/sync-bus-handlers.ts +131 -0
- package/src/sync-frames.ts +74 -21
- package/src/sync-node-bounds.ts +16 -1
- package/src/sync-node-contract.ts +36 -4
- package/src/sync-node.ts +163 -82
- package/src/sync-origin.ts +27 -12
- package/src/sync-protocol.ts +86 -16
- package/src/sync-upgrade.ts +115 -8
- package/src/type-pins.ts +9 -1
- package/src/use-channel.ts +19 -14
- package/src/use-connection.ts +3 -3
- package/src/use-mutation.ts +41 -20
- package/src/use-query.ts +11 -11
- package/src/use-record.ts +89 -10
- package/src/wire-channel.ts +18 -9
- package/src/wire-read.ts +2 -2
- package/src/idb-fake.ts +0 -133
- package/src/policy-fake.ts +0 -14
package/src/live-query.ts
CHANGED
|
@@ -6,54 +6,28 @@
|
|
|
6
6
|
// policy is never sent to that actor — it arrives as a `delete` if they hold it, and is dropped
|
|
7
7
|
// otherwise.
|
|
8
8
|
|
|
9
|
-
import { type
|
|
9
|
+
import { type Clock, finiteOption, systemClock, uuidV7 } from '@ultimat3/core';
|
|
10
10
|
import { queryHash } from '@ultimat3/query';
|
|
11
11
|
import type { ChangeEvent } from './changefeed';
|
|
12
|
-
import { type LiveCursor, makeCursor
|
|
13
|
-
import {
|
|
12
|
+
import { type LiveCursor, makeCursor } from './cursor';
|
|
13
|
+
import { LiveQueryUnknownError, SubscriptionLimitError } from './errors';
|
|
14
14
|
import type { JsonValue } from './json';
|
|
15
15
|
import type { LiveQueryDefinition, LiveSubscription, SnapshotResult } from './live-contract';
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
16
|
+
import { deliverChange } from './live-deliver';
|
|
17
|
+
import { type FanoutDeps, snapshotFrame } from './live-fanout';
|
|
18
|
+
import { floorAfterRead } from './live-floor';
|
|
19
|
+
import { DEFAULT_MAX_ENTRIES, type LiveQueryRegistryOptions } from './live-query-options';
|
|
20
|
+
import { reauthorizeSocket } from './live-reauth';
|
|
18
21
|
import { resumeOnto } from './live-resume';
|
|
22
|
+
import { type Charge, type SubscribeArgs, spendOnce } from './live-spend';
|
|
19
23
|
import { liveTenantOf, windowId } from './live-tenant';
|
|
20
|
-
import {
|
|
24
|
+
import { fillWindow, type QueryEntry, queryEntry } from './query-window';
|
|
21
25
|
import type { SyncSocket } from './socket';
|
|
22
|
-
import { type Subscriber, SubscriberGate
|
|
26
|
+
import { type Subscriber, SubscriberGate } from './subscriber-gate';
|
|
23
27
|
import { SubscriptionBook, subscriptionKey } from './subscription-book';
|
|
24
28
|
import type { Frame } from './sync-protocol';
|
|
25
|
-
import type { Scheduler } from './thundering-herd';
|
|
26
29
|
|
|
27
|
-
export
|
|
28
|
-
readonly source: ResumeSource;
|
|
29
|
-
readonly budget?: ReconnectBudget;
|
|
30
|
-
readonly clock?: Clock;
|
|
31
|
-
readonly maxPerSocket?: number;
|
|
32
|
-
readonly maxPerTenant?: number;
|
|
33
|
-
readonly tenantOf?: (actor: Actor | null) => string | null;
|
|
34
|
-
/**
|
|
35
|
-
* Distinct `(query, input)` pairs this node will hold at once. A `qid` derives from
|
|
36
|
-
* client-chosen input, so without a ceiling one socket mints entries — a matcher, a row window,
|
|
37
|
-
* a `WindowLock` and a fanout target each — until the process dies.
|
|
38
|
-
*/
|
|
39
|
-
readonly maxEntries?: number;
|
|
40
|
-
/**
|
|
41
|
-
* How long one entry's SHARED snapshot read may hold its slot. Defaults to
|
|
42
|
-
* `DEFAULT_READ_DEADLINE_MS`. Without it a `definition.snapshot` that never settles pinned the
|
|
43
|
-
* slot for the life of the process and every later cold subscriber joined a promise nothing
|
|
44
|
-
* would resolve — one wedged read taking every future subscriber of that query id with it.
|
|
45
|
-
*/
|
|
46
|
-
readonly readDeadlineMs?: number;
|
|
47
|
-
/** Injected so that deadline is provable without waiting for one. Production uses `setTimeout`. */
|
|
48
|
-
readonly schedule?: Scheduler;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
/**
|
|
52
|
-
* Live `(query, input)` pairs one node holds. Reached, the next NEW pair is refused with
|
|
53
|
-
* `X_SUBSCRIPTION_LIMIT`; subscribing to a pair that already exists keeps working, because the
|
|
54
|
-
* cost this bounds is the entry, not the subscriber.
|
|
55
|
-
*/
|
|
56
|
-
export const DEFAULT_MAX_ENTRIES = 10_000;
|
|
30
|
+
export { DEFAULT_MAX_ENTRIES, type LiveQueryRegistryOptions } from './live-query-options';
|
|
57
31
|
|
|
58
32
|
export class LiveQueryRegistry {
|
|
59
33
|
readonly #definitions = new Map<string, LiveQueryDefinition>();
|
|
@@ -67,6 +41,11 @@ export class LiveQueryRegistry {
|
|
|
67
41
|
/** What one lane needs, and nothing this class holds beyond it. */
|
|
68
42
|
readonly #fanout: FanoutDeps;
|
|
69
43
|
#lastLsn = '';
|
|
44
|
+
/**
|
|
45
|
+
* This node's mark for a window read before it held any position: unique to the process, and
|
|
46
|
+
* `!` sorts below every hex digit, so every real change is above it.
|
|
47
|
+
*/
|
|
48
|
+
readonly #origin = `!${uuidV7()}`;
|
|
70
49
|
#staleChanges = 0;
|
|
71
50
|
|
|
72
51
|
constructor(options: LiveQueryRegistryOptions) {
|
|
@@ -108,9 +87,9 @@ export class LiveQueryRegistry {
|
|
|
108
87
|
* ever ask for a re-snapshot. The repair lands on the next change to each query — which is the
|
|
109
88
|
* event that proves the query is moving at all.
|
|
110
89
|
*
|
|
111
|
-
* `entity` narrows it to the windows that read
|
|
112
|
-
* (`updateWhere`/`deleteWhere`) names its
|
|
113
|
-
* for it re-read every live query in the process on each bulk write
|
|
90
|
+
* `entity` — the RELATION, as on `ChangeEvent.table` — narrows it to the windows that read it:
|
|
91
|
+
* a filtered write (`updateWhere`/`deleteWhere`) names its table but no row, and staling every
|
|
92
|
+
* window on the node for it re-read every live query in the process on each bulk write.
|
|
114
93
|
*/
|
|
115
94
|
invalidate(entity?: string): number {
|
|
116
95
|
let marked = 0;
|
|
@@ -147,26 +126,36 @@ export class LiveQueryRegistry {
|
|
|
147
126
|
* Subscribe or resume. Returns the frame this subscriber needs: a `snapshot` on a cold start or a
|
|
148
127
|
* blown reconnect budget, a `patch` when the cursor is inside the retained window.
|
|
149
128
|
*/
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
129
|
+
subscribe(args: SubscribeArgs): Promise<{ subscription: LiveSubscription; frame: Frame }> {
|
|
130
|
+
return this.#subscribe(args, { due: true });
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** `charge.due` false: a re-seat, whose subscription was paid for when it was first made. */
|
|
134
|
+
async #subscribe(
|
|
135
|
+
args: SubscribeArgs,
|
|
136
|
+
charge: Charge,
|
|
137
|
+
): Promise<{ subscription: LiveSubscription; frame: Frame }> {
|
|
157
138
|
const definition = this.#definitions.get(args.name);
|
|
158
139
|
// A name this node never registered, and not a protocol skew: the frame parsed, the version
|
|
159
140
|
// matched, and one string in it names nothing. Reporting it as `X_PROTOCOL_VERSION` handed the
|
|
160
141
|
// client "x build && redeploy the client" for a typo no rebuild changes.
|
|
161
142
|
if (!definition) throw new LiveQueryUnknownError({ name: args.name });
|
|
162
|
-
const sid = args.sid ??
|
|
143
|
+
const sid = args.sid ?? uuidV7();
|
|
163
144
|
// Everything this subscribe can be refused for, decided in one synchronous step BEFORE the
|
|
164
145
|
// first await — the caps and the sid both. Read at the top and acted on three awaits later,
|
|
165
146
|
// they were bypassed by the ordinary case: one WebSocket write carrying N subscribe frames,
|
|
166
147
|
// dispatched concurrently, N of them reading a count nothing had grown yet.
|
|
167
148
|
const slot = this.#book.reserve(args.socket, sid);
|
|
168
149
|
try {
|
|
169
|
-
|
|
150
|
+
for (;;) {
|
|
151
|
+
// `authorize`, the read and the row pass all await, and a re-auth that lands under them
|
|
152
|
+
// re-decides only what is ATTACHED. So a subscription seated for an actor the socket no
|
|
153
|
+
// longer carries is taken back and served again under the one it does.
|
|
154
|
+
const actor = args.socket.actor;
|
|
155
|
+
const served = await this.#subscribeReserved(definition, sid, args, charge);
|
|
156
|
+
if (args.socket.actor === actor) return served;
|
|
157
|
+
this.unsubscribe(args.socket.id, sid);
|
|
158
|
+
}
|
|
170
159
|
} finally {
|
|
171
160
|
// After the attach on every path, so the slot is only ever given back to a count that has
|
|
172
161
|
// already grown — or, on a failure, to one that never will.
|
|
@@ -183,19 +172,22 @@ export class LiveQueryRegistry {
|
|
|
183
172
|
input: JsonValue;
|
|
184
173
|
cursor?: LiveCursor | null;
|
|
185
174
|
},
|
|
175
|
+
charge: Charge,
|
|
186
176
|
): Promise<{ subscription: LiveSubscription; frame: Frame }> {
|
|
187
177
|
await definition.authorize?.({ actor: args.socket.actor, input: args.input });
|
|
188
|
-
// After this subscriber's own decision, never before it: resolving a shape for a caller who
|
|
189
|
-
// may not subscribe is work an unauthorized client gets to schedule.
|
|
190
|
-
await definition.prepare?.(args.input);
|
|
191
|
-
|
|
192
178
|
// The window is the subscriber's TENANT's: two orgs on one `(query, input)` are two entries,
|
|
193
179
|
// two reads and two retained rings, so no shared row ever stands between them and a policy.
|
|
194
180
|
const tenant = liveTenantOf(args.socket.actor);
|
|
195
181
|
const qid = windowId(queryHash(args.name, args.input), tenant);
|
|
182
|
+
// After this subscriber's own decision, never before it: resolving a shape for a caller who
|
|
183
|
+
// may not subscribe is work an unauthorized client gets to schedule.
|
|
184
|
+
await definition.prepare?.(args.input);
|
|
185
|
+
|
|
196
186
|
const entry = this.#entryFor(qid, definition, args.input, tenant);
|
|
197
187
|
try {
|
|
198
|
-
|
|
188
|
+
// Charged where a read happens, never from the cursor's say-so (`live-spend.ts`).
|
|
189
|
+
const pay = () => spendOnce(definition, args.socket, charge, this.#clock);
|
|
190
|
+
return await this.#serve(entry, sid, args, pay);
|
|
199
191
|
} catch (error) {
|
|
200
192
|
// The entry was born above, before anything could fill it. Everything below can throw — the
|
|
201
193
|
// snapshot, the resume, the window's own read deadline — and `unsubscribe` is the only other
|
|
@@ -215,6 +207,7 @@ export class LiveQueryRegistry {
|
|
|
215
207
|
input: JsonValue;
|
|
216
208
|
cursor?: LiveCursor | null;
|
|
217
209
|
},
|
|
210
|
+
pay: () => Promise<void>,
|
|
218
211
|
): Promise<{ subscription: LiveSubscription; frame: Frame }> {
|
|
219
212
|
const qid = entry.qid;
|
|
220
213
|
const now = this.#clock.now().getTime();
|
|
@@ -223,7 +216,10 @@ export class LiveQueryRegistry {
|
|
|
223
216
|
// replays. One that names another window — the same browser, signed in to another org, or a
|
|
224
217
|
// forged frame — is not a position in this one: it is a cold start, never a replay of a ring
|
|
225
218
|
// this subscriber's window does not own.
|
|
226
|
-
|
|
219
|
+
//
|
|
220
|
+
// Nor is ANY cursor a position in a window known to have missed a change: the ring still
|
|
221
|
+
// answers a delta for it, and that delta is the hole. A stale entry serves cold starts only.
|
|
222
|
+
if (args.cursor && args.cursor.qid === qid && !entry.stale) {
|
|
227
223
|
const who = { sid, actor: args.socket.actor };
|
|
228
224
|
const resumed = await resumeOnto(
|
|
229
225
|
{
|
|
@@ -231,7 +227,11 @@ export class LiveQueryRegistry {
|
|
|
231
227
|
budget: this.#options.budget,
|
|
232
228
|
clock: this.#clock,
|
|
233
229
|
gate: this.#gate,
|
|
234
|
-
read: () =>
|
|
230
|
+
read: async () => {
|
|
231
|
+
await pay();
|
|
232
|
+
return await this.#read(entry, who);
|
|
233
|
+
},
|
|
234
|
+
beforeFill: pay,
|
|
235
235
|
},
|
|
236
236
|
entry,
|
|
237
237
|
who,
|
|
@@ -242,6 +242,7 @@ export class LiveQueryRegistry {
|
|
|
242
242
|
return { subscription, frame: resumed.frame };
|
|
243
243
|
}
|
|
244
244
|
|
|
245
|
+
await pay();
|
|
245
246
|
const fresh = await this.#read(entry, { sid, actor: args.socket.actor });
|
|
246
247
|
const cursor = makeCursor(qid, fresh.lsn, fresh.rows, now);
|
|
247
248
|
const subscription = this.#attachUnlessGone(entry, args.socket, sid, cursor);
|
|
@@ -290,103 +291,34 @@ export class LiveQueryRegistry {
|
|
|
290
291
|
for (const subscription of this.#book.ofSocket(socketId)) {
|
|
291
292
|
this.unsubscribe(socketId, subscription.sid);
|
|
292
293
|
}
|
|
294
|
+
// Held or not: a socket re-authed while holding nothing is remembered by nothing else.
|
|
295
|
+
this.#book.forgetSocket(socketId);
|
|
293
296
|
}
|
|
294
297
|
|
|
295
|
-
/**
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
* under the new actor's row policy. Returns the sids that were dropped — a denial and nothing
|
|
299
|
-
* else, so a caller may tell the client "you may no longer see this" and be right.
|
|
300
|
-
*
|
|
301
|
-
* A survivor whose actor now belongs to ANOTHER tenant cannot stay where it is: its window is the
|
|
302
|
-
* old org's. It is re-seated — dropped, subscribed again under the new tenant, and sent that
|
|
303
|
-
* window's snapshot under the same sid — or, when that cannot complete, refused on the socket
|
|
304
|
-
* under the sid (`refuseSubscription`), never kept on the old org's window.
|
|
305
|
-
*/
|
|
306
|
-
async reauthorize(socket: SyncSocket): Promise<readonly string[]> {
|
|
307
|
-
const dropped: string[] = [];
|
|
308
|
-
// The actor changed, so what tenant this socket's subscriptions count against may have too.
|
|
309
|
-
// Told here rather than derived per lookup: the per-tenant cap is an index now, and an index
|
|
310
|
-
// nobody updates is a count that drifts from the book for the rest of the process.
|
|
311
|
-
this.#book.retenant(socket);
|
|
312
|
-
for (const subscription of this.#book.ofSocket(socket.id)) {
|
|
313
|
-
let failed: { readonly error: unknown } | undefined;
|
|
314
|
-
try {
|
|
315
|
-
await subscription.definition.authorize?.({
|
|
316
|
-
actor: socket.actor,
|
|
317
|
-
input: subscription.input,
|
|
318
|
-
});
|
|
319
|
-
} catch (error) {
|
|
320
|
-
if (isPolicyDenial(error)) {
|
|
321
|
-
this.unsubscribe(socket.id, subscription.sid);
|
|
322
|
-
dropped.push(subscription.sid);
|
|
323
|
-
continue;
|
|
324
|
-
}
|
|
325
|
-
// Not a decision — the gate never reached one. Destroying the subscription would report a
|
|
326
|
-
// database timeout as a revoked grant, and a client does not resubscribe to a denial. It
|
|
327
|
-
// survives, desynced: nothing is delivered from the window built under the old actor, and
|
|
328
|
-
// the row gate still decides every row under the new one, from the same policy `authorize`
|
|
329
|
-
// consults. The failure is counted and reported rather than silently absorbed.
|
|
330
|
-
this.#gate.failedAuthorize(
|
|
331
|
-
subscription.qid,
|
|
332
|
-
{ sid: subscription.sid, actor: socket.actor },
|
|
333
|
-
error,
|
|
334
|
-
);
|
|
335
|
-
failed = { error };
|
|
336
|
-
}
|
|
337
|
-
if (await this.#reseat(socket, subscription, failed)) continue;
|
|
338
|
-
socket.markDesynced(subscription.sid);
|
|
339
|
-
}
|
|
340
|
-
return dropped;
|
|
298
|
+
/** Sockets the spanning caps still remember a key for. A leak is this number not falling. */
|
|
299
|
+
get trackedSockets(): number {
|
|
300
|
+
return this.#book.trackedSockets;
|
|
341
301
|
}
|
|
342
302
|
|
|
343
|
-
/**
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
this.unsubscribe(socket.id, sid);
|
|
361
|
-
if (failed !== undefined) {
|
|
362
|
-
refuseSubscription(socket, sid, failed.error);
|
|
363
|
-
return true;
|
|
364
|
-
}
|
|
365
|
-
try {
|
|
366
|
-
const { frame } = await this.subscribe({ socket, name: definition.name, input, sid });
|
|
367
|
-
if (!socket.send(frame)) socket.markDesynced(sid);
|
|
368
|
-
} catch (error) {
|
|
369
|
-
this.#gate.failedAuthorize(wanted, { sid, actor: socket.actor }, error);
|
|
370
|
-
refuseSubscription(socket, sid, error);
|
|
371
|
-
}
|
|
372
|
-
return true;
|
|
303
|
+
/** Live subscriptions one principal (`actor:<id>` / `address:<ip>`) holds on this node. */
|
|
304
|
+
actorCount(principal: string): number {
|
|
305
|
+
return this.#book.actorCount(principal);
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/** Actor changed mid-connection — re-decided per subscription; `live-reauth.ts` owns how. */
|
|
309
|
+
reauthorize(socket: SyncSocket): Promise<readonly string[]> {
|
|
310
|
+
return reauthorizeSocket(
|
|
311
|
+
{
|
|
312
|
+
book: this.#book,
|
|
313
|
+
gate: this.#gate,
|
|
314
|
+
unsubscribe: (socketId, sid) => this.unsubscribe(socketId, sid),
|
|
315
|
+
// A re-seat is the subscription it already was, so it is never charged again.
|
|
316
|
+
resubscribe: (args) => this.#subscribe(args, { due: false }),
|
|
317
|
+
},
|
|
318
|
+
socket,
|
|
319
|
+
);
|
|
373
320
|
}
|
|
374
321
|
|
|
375
|
-
/**
|
|
376
|
-
* Fan one change out. Matched once per query id, authorized once per subscriber. Returns the
|
|
377
|
-
* number of frames sent — the metric the reconnect benchmark watches.
|
|
378
|
-
*
|
|
379
|
-
* Each entry's turn is taken in that entry's lane. Nothing upstream orders this: `sync` fires
|
|
380
|
-
* `void registry.deliver(change)` straight off the bus subscription, so two changes arriving back
|
|
381
|
-
* to back would otherwise interleave inside one query id — lsn 2 delivered before lsn 1, the
|
|
382
|
-
* subscriber's cursor rewound to 1, and every gate deciding against whichever window won.
|
|
383
|
-
*
|
|
384
|
-
* Every lane is *entered* before any of them is awaited, and nothing inside a fanout takes a
|
|
385
|
-
* second lane, so holding all of them at once cannot be a cycle. That is what makes the ordering
|
|
386
|
-
* claim true: two deliveries queue onto each query id in call order, serialized per query id and
|
|
387
|
-
* never per node — awaiting one entry before entering the next made one slow policy pass the
|
|
388
|
-
* whole node's pace, and let a lane that threw skip every entry behind it with nobody desynced.
|
|
389
|
-
*/
|
|
390
322
|
/**
|
|
391
323
|
* The newest change position this registry has been handed. What a node's snapshot may claim
|
|
392
324
|
* (`liveQueryDefinition`'s `lsn`): a read begun after it holds at least that change, and every
|
|
@@ -396,34 +328,12 @@ export class LiveQueryRegistry {
|
|
|
396
328
|
return this.#lastLsn;
|
|
397
329
|
}
|
|
398
330
|
|
|
331
|
+
/** Fan one change out, every query id in its own lane — `live-deliver.ts` owns why. */
|
|
399
332
|
async deliver(change: ChangeEvent): Promise<number> {
|
|
400
333
|
if (change.lsn > this.#lastLsn) this.#lastLsn = change.lsn;
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
const result = await entry.lock.run(() => fanoutChange(this.#fanout, entry, change));
|
|
404
|
-
this.#staleChanges += result.stale;
|
|
405
|
-
return result.sent;
|
|
406
|
-
} catch (error) {
|
|
407
|
-
// The window advanced under a fanout that did not finish, so every subscriber of this one
|
|
408
|
-
// query id now holds a cursor below the change and no later flush would correct them:
|
|
409
|
-
// desynced here, re-snapshotted on the next one. Silent divergence is the whole reason
|
|
410
|
-
// `markDesynced` exists, and skipping this is how a failure became one.
|
|
411
|
-
for (const subscription of entry.subscribers.values()) {
|
|
412
|
-
subscription.socket.markDesynced(subscription.sid);
|
|
413
|
-
}
|
|
414
|
-
throw error;
|
|
415
|
-
}
|
|
334
|
+
return await deliverChange(this.#entries.values(), this.#fanout, change, (stale) => {
|
|
335
|
+
this.#staleChanges += stale;
|
|
416
336
|
});
|
|
417
|
-
// `allSettled`, so one lane's rejection neither cancels the others nor goes unhandled. The
|
|
418
|
-
// first failure still reaches the caller — `sync` logs it — but it costs one query id.
|
|
419
|
-
let sent = 0;
|
|
420
|
-
let failure: { readonly error: unknown } | null = null;
|
|
421
|
-
for (const lane of await Promise.allSettled(lanes)) {
|
|
422
|
-
if (lane.status === 'fulfilled') sent += lane.value;
|
|
423
|
-
else failure ??= { error: lane.reason };
|
|
424
|
-
}
|
|
425
|
-
if (failure !== null) throw failure.error;
|
|
426
|
-
return sent;
|
|
427
337
|
}
|
|
428
338
|
|
|
429
339
|
#attachUnlessGone(
|
|
@@ -433,6 +343,10 @@ export class LiveQueryRegistry {
|
|
|
433
343
|
cursor: LiveCursor,
|
|
434
344
|
): LiveSubscription {
|
|
435
345
|
const subscription = this.#attach(entry, socket, sid, cursor);
|
|
346
|
+
// The policy pass ran outside the lane, so a change may have been folded into the window
|
|
347
|
+
// between the rows this subscriber was served and this attach: it is in neither their frame
|
|
348
|
+
// nor their patch stream. Marked, so the next delivery re-snapshots them out of the window.
|
|
349
|
+
if (entry.lsn !== cursor.lsn) socket.markDesynced(sid);
|
|
436
350
|
if (socket.closed) this.unsubscribe(socket.id, sid);
|
|
437
351
|
return subscription;
|
|
438
352
|
}
|
|
@@ -443,6 +357,9 @@ export class LiveQueryRegistry {
|
|
|
443
357
|
sid: string,
|
|
444
358
|
cursor: LiveCursor,
|
|
445
359
|
): LiveSubscription {
|
|
360
|
+
// Before anything is written: a re-auth during the read may have moved this socket to a
|
|
361
|
+
// principal that never had room for it.
|
|
362
|
+
this.#book.assertAttachable(socket, sid);
|
|
446
363
|
const subscription: LiveSubscription = {
|
|
447
364
|
sid,
|
|
448
365
|
qid: entry.qid,
|
|
@@ -479,7 +396,7 @@ export class LiveQueryRegistry {
|
|
|
479
396
|
knob: 'maxEntries',
|
|
480
397
|
});
|
|
481
398
|
}
|
|
482
|
-
const created =
|
|
399
|
+
const created = queryEntry(qid, definition, input, definition.matcher(input), {
|
|
483
400
|
readDeadlineMs: this.#options.readDeadlineMs,
|
|
484
401
|
schedule: this.#options.schedule,
|
|
485
402
|
tenant,
|
|
@@ -495,6 +412,10 @@ export class LiveQueryRegistry {
|
|
|
495
412
|
*/
|
|
496
413
|
async #read(entry: QueryEntry, who: Subscriber): Promise<SnapshotResult> {
|
|
497
414
|
const window = await fillWindow(entry);
|
|
498
|
-
|
|
415
|
+
const lsn = floorAfterRead(this.#options.source, entry, window, {
|
|
416
|
+
lastLsn: this.#lastLsn,
|
|
417
|
+
origin: this.#origin,
|
|
418
|
+
});
|
|
419
|
+
return { rows: await this.#gate.filterRows(entry, who, window.rows), lsn };
|
|
499
420
|
}
|
|
500
421
|
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// A socket whose actor changed mid-connection: every subscription it holds re-decided under the
|
|
2
|
+
// new actor, re-seated onto the new tenant's window, and — new principal's cap first — refused
|
|
3
|
+
// rather than carried past a ceiling. Split from `live-query.ts` at its size ceiling: the registry
|
|
4
|
+
// owns the entry table, this owns what a re-authorization does to one socket's subscriptions.
|
|
5
|
+
|
|
6
|
+
import { queryHash } from '@ultimat3/query';
|
|
7
|
+
import { isPolicyDenial } from './errors';
|
|
8
|
+
import type { LiveSubscription } from './live-contract';
|
|
9
|
+
import { refuseSubscription } from './live-refusal';
|
|
10
|
+
import type { SubscribeArgs } from './live-spend';
|
|
11
|
+
import { liveTenantOf, windowId } from './live-tenant';
|
|
12
|
+
import type { SyncSocket } from './socket';
|
|
13
|
+
import type { SubscriberGate } from './subscriber-gate';
|
|
14
|
+
import type { SubscriptionBook } from './subscription-book';
|
|
15
|
+
import type { Frame } from './sync-protocol';
|
|
16
|
+
|
|
17
|
+
/** What a re-authorization reaches on the registry, and nothing else. */
|
|
18
|
+
export interface ReauthDeps {
|
|
19
|
+
readonly book: SubscriptionBook;
|
|
20
|
+
readonly gate: SubscriberGate;
|
|
21
|
+
unsubscribe(socketId: string, sid: string): void;
|
|
22
|
+
/** Subscribe again under the same sid, uncharged — a re-seat is not a new subscription. */
|
|
23
|
+
resubscribe(args: SubscribeArgs): Promise<{ readonly frame: Frame }>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Actor changed mid-connection (login, logout, role change): re-run subscribe-time authz and drop
|
|
28
|
+
* what is no longer allowed. Survivors are marked desynced so the next flush re-snapshots them
|
|
29
|
+
* under the new actor's row policy. Returns the sids that were dropped — a denial and nothing
|
|
30
|
+
* else, so a caller may tell the client "you may no longer see this" and be right.
|
|
31
|
+
*
|
|
32
|
+
* A survivor whose actor now belongs to ANOTHER tenant cannot stay where it is: its window is the
|
|
33
|
+
* old org's. It is re-seated — dropped, subscribed again under the new tenant, and sent that
|
|
34
|
+
* window's snapshot under the same sid — or, when that cannot complete, refused on the socket
|
|
35
|
+
* under the sid (`refuseSubscription`), never kept on the old org's window.
|
|
36
|
+
*/
|
|
37
|
+
export async function reauthorizeSocket(
|
|
38
|
+
deps: ReauthDeps,
|
|
39
|
+
socket: SyncSocket,
|
|
40
|
+
): Promise<readonly string[]> {
|
|
41
|
+
const dropped: string[] = [];
|
|
42
|
+
// The actor changed, so what tenant this socket's subscriptions count against may have too.
|
|
43
|
+
// Told here rather than derived per lookup: the per-tenant cap is an index now, and an index
|
|
44
|
+
// nobody updates is a count that drifts from the book for the rest of the process.
|
|
45
|
+
refuseOverflow(deps, socket);
|
|
46
|
+
deps.book.retenant(socket);
|
|
47
|
+
for (const subscription of deps.book.ofSocket(socket.id)) {
|
|
48
|
+
let failed: { readonly error: unknown } | undefined;
|
|
49
|
+
try {
|
|
50
|
+
await subscription.definition.authorize?.({
|
|
51
|
+
actor: socket.actor,
|
|
52
|
+
input: subscription.input,
|
|
53
|
+
});
|
|
54
|
+
} catch (error) {
|
|
55
|
+
if (isPolicyDenial(error)) {
|
|
56
|
+
deps.unsubscribe(socket.id, subscription.sid);
|
|
57
|
+
// Said under the sid, as a refused subscribe is: unsaid, the client kept the rows on
|
|
58
|
+
// screen in state `live` for a subscription this node no longer serves.
|
|
59
|
+
refuseSubscription(socket, subscription.sid, error);
|
|
60
|
+
dropped.push(subscription.sid);
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
// Not a decision — the gate never reached one. Destroying the subscription would report a
|
|
64
|
+
// database timeout as a revoked grant, and a client does not resubscribe to a denial. It
|
|
65
|
+
// survives, desynced: nothing is delivered from the window built under the old actor, and
|
|
66
|
+
// the row gate still decides every row under the new one, from the same policy `authorize`
|
|
67
|
+
// consults. The failure is counted and reported rather than silently absorbed.
|
|
68
|
+
deps.gate.failedAuthorize(
|
|
69
|
+
subscription.qid,
|
|
70
|
+
{ sid: subscription.sid, actor: socket.actor },
|
|
71
|
+
error,
|
|
72
|
+
);
|
|
73
|
+
failed = { error };
|
|
74
|
+
}
|
|
75
|
+
if (await reseat(deps, socket, subscription, failed)) continue;
|
|
76
|
+
socket.markDesynced(subscription.sid);
|
|
77
|
+
}
|
|
78
|
+
return dropped;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Move one subscription onto its actor's CURRENT tenant's window. `false` when it is already
|
|
83
|
+
* there. A re-seat that cannot complete leaves the subscription dropped — never attached to the
|
|
84
|
+
* window of an org its actor has left — and REFUSED on the socket under its sid, the frame a
|
|
85
|
+
* refused subscribe gets, so the client's window renders `failed` instead of going quiet.
|
|
86
|
+
*
|
|
87
|
+
* `failed` is the `authorize` this pass already saw fail: it is not asked again — the store that
|
|
88
|
+
* just timed out is the one `subscribe` would ask — and it is not counted twice.
|
|
89
|
+
*/
|
|
90
|
+
async function reseat(
|
|
91
|
+
deps: ReauthDeps,
|
|
92
|
+
socket: SyncSocket,
|
|
93
|
+
subscription: LiveSubscription,
|
|
94
|
+
failed: { readonly error: unknown } | undefined,
|
|
95
|
+
): Promise<boolean> {
|
|
96
|
+
const { sid, input, definition } = subscription;
|
|
97
|
+
const wanted = windowId(queryHash(definition.name, input), liveTenantOf(socket.actor));
|
|
98
|
+
if (wanted === subscription.qid) return false;
|
|
99
|
+
deps.unsubscribe(socket.id, sid);
|
|
100
|
+
if (failed !== undefined) {
|
|
101
|
+
refuseSubscription(socket, sid, failed.error);
|
|
102
|
+
return true;
|
|
103
|
+
}
|
|
104
|
+
try {
|
|
105
|
+
const { frame } = await deps.resubscribe({ socket, name: definition.name, input, sid });
|
|
106
|
+
if (!socket.send(frame)) socket.markDesynced(sid);
|
|
107
|
+
} catch (error) {
|
|
108
|
+
deps.gate.failedAuthorize(wanted, { sid, actor: socket.actor }, error);
|
|
109
|
+
refuseSubscription(socket, sid, error);
|
|
110
|
+
}
|
|
111
|
+
return true;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The subscriptions the socket's NEW tenant or actor has no room for, refused under their sids
|
|
116
|
+
* before the move — so the move never carries a key past its cap, and filling an address then
|
|
117
|
+
* signing in cannot be repeated to stack one actor past it. The newest go first: the client's
|
|
118
|
+
* oldest windows are the ones it has been rendering longest. Not a denial, so not in `dropped`.
|
|
119
|
+
*/
|
|
120
|
+
function refuseOverflow(deps: ReauthDeps, socket: SyncSocket): void {
|
|
121
|
+
const overflow = deps.book.overflowOnRekey(socket);
|
|
122
|
+
if (overflow === null) return;
|
|
123
|
+
for (const subscription of deps.book.ofSocket(socket.id).slice(-overflow.excess)) {
|
|
124
|
+
deps.unsubscribe(socket.id, subscription.sid);
|
|
125
|
+
refuseSubscription(socket, subscription.sid, overflow.error);
|
|
126
|
+
}
|
|
127
|
+
}
|
package/src/live-replicator.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// WHAT IS REAL HERE, and it is everything downstream of the decoder: the matcher, the shared window,
|
|
8
8
|
// the per-subscriber `visible` gate, the cursor, the frames. This substitutes for the WAL DECODER
|
|
9
9
|
// and for nothing else — `@ultimat3/entity`'s `setRowObserver` reports what a repository wrote, in
|
|
10
|
-
// this process, and the events are shaped exactly as `
|
|
10
|
+
// this process, and the events are shaped exactly as `PostgresChangeFeed` shapes them.
|
|
11
11
|
//
|
|
12
12
|
// WHAT IS NOT: a write another process made is invisible, because nothing here reads a log. That is
|
|
13
13
|
// the honest bound, and it is why `selectChangeFeed` never picks it — the boot that installs it
|
|
@@ -27,6 +27,7 @@ import type { RowBulkChange, RowChange, RowObserver } from '@ultimat3/entity';
|
|
|
27
27
|
import type { ChangeEvent, ChangeOp } from './changefeed';
|
|
28
28
|
import type { Row } from './json';
|
|
29
29
|
import type { LiveQueryRegistry } from './live-query';
|
|
30
|
+
import { isPublishedTable } from './record-publisher';
|
|
30
31
|
|
|
31
32
|
/** What a caller does with a change nobody could deliver. */
|
|
32
33
|
export interface LiveReplicatorOptions {
|
|
@@ -35,8 +36,13 @@ export interface LiveReplicatorOptions {
|
|
|
35
36
|
* The node's declared channels, fed the same `ChangeEvent` — what a real node's change
|
|
36
37
|
* subscription does beside `registry.deliver` (`sync-node.ts`). Without it a channel's `records`
|
|
37
38
|
* frames never carried a write made under `x dev`, and every other tab stayed on the old row.
|
|
39
|
+
* `invalidate` is the other half, called beside `registry.invalidate` as the node pairs them: a
|
|
40
|
+
* bulk write reaches no `records` frame, so the topics carrying its table must re-read instead.
|
|
38
41
|
*/
|
|
39
|
-
readonly channels?: {
|
|
42
|
+
readonly channels?: {
|
|
43
|
+
deliverChange(change: ChangeEvent): unknown;
|
|
44
|
+
invalidate(entity?: string): unknown;
|
|
45
|
+
};
|
|
40
46
|
/** Tenant column, hoisted out of the row so fanout filters without parsing it. */
|
|
41
47
|
readonly tenantColumn?: string;
|
|
42
48
|
readonly onError?: (error: unknown) => void;
|
|
@@ -80,10 +86,19 @@ export async function startLiveReplicator(options: LiveReplicatorOptions): Promi
|
|
|
80
86
|
let position = 0;
|
|
81
87
|
let delivered = 0;
|
|
82
88
|
// One promise chain, because ORDERING is the guarantee the whole pipeline is built on — the same
|
|
83
|
-
// reason `
|
|
89
|
+
// reason `MemoryChangeFeed` serializes its deliveries rather than firing them concurrently.
|
|
84
90
|
let tail: Promise<void> = Promise.resolve();
|
|
85
91
|
let stopped = false;
|
|
86
92
|
|
|
93
|
+
/**
|
|
94
|
+
* The RELATION an entity's rows live in — what `ChangeEvent.table` carries on every producer:
|
|
95
|
+
* the WAL decoder reads it off the Relation message, a channel matches `projection.table`, and a
|
|
96
|
+
* live shape is `from('<table>', …)`. The row observer reports the entity's declared NAME, which
|
|
97
|
+
* is the table only until an entity adopts one (`entity('ledger', { table: 'gl_entries' })`);
|
|
98
|
+
* passed through, such an entity reached no channel, no window and no bulk re-read here.
|
|
99
|
+
*/
|
|
100
|
+
const tableOf = (name: string): string => entity.getEntity(name)?.tableName ?? name;
|
|
101
|
+
|
|
87
102
|
const enqueue = (work: () => Promise<void>): void => {
|
|
88
103
|
tail = tail.then(work).catch((error: unknown) => {
|
|
89
104
|
// Never rethrown into the chain: one failed fanout must not silence every change behind it,
|
|
@@ -100,7 +115,7 @@ export async function startLiveReplicator(options: LiveReplicatorOptions): Promi
|
|
|
100
115
|
const row = change.after ?? change.before;
|
|
101
116
|
const orgId = typeof row?.[tenant] === 'string' ? (row[tenant] as string) : null;
|
|
102
117
|
const event: ChangeEvent = {
|
|
103
|
-
|
|
118
|
+
table: tableOf(change.entity),
|
|
104
119
|
// A repository's three row ops are three of the feed's four; a truncate never comes this way.
|
|
105
120
|
op: change.op satisfies ChangeOp,
|
|
106
121
|
before: asRow(change.before),
|
|
@@ -113,12 +128,14 @@ export async function startLiveReplicator(options: LiveReplicatorOptions): Promi
|
|
|
113
128
|
at,
|
|
114
129
|
// The keyed write it belongs to, read off the request scope — what the WAL decoder reads
|
|
115
130
|
// off the transaction's opening message — so a channel frame names it here as it would there.
|
|
116
|
-
|
|
131
|
+
write: change.write ?? null,
|
|
117
132
|
};
|
|
118
133
|
enqueue(async () => {
|
|
119
134
|
// Channels first, as the node does: a live query's fanout that throws must not also cost
|
|
120
|
-
// every declared channel the change.
|
|
121
|
-
|
|
135
|
+
// every declared channel the change. Not an entity a `recordPublisher` claims: the app's
|
|
136
|
+
// own publish is its one channel delivery, as in production with no replicator, and both
|
|
137
|
+
// was every insert twice (#682). Asked per change, so a publisher built later counts.
|
|
138
|
+
if (!isPublishedTable(event.table)) options.channels?.deliverChange(event);
|
|
122
139
|
delivered += await registry.deliver(event);
|
|
123
140
|
});
|
|
124
141
|
},
|
|
@@ -132,7 +149,10 @@ export async function startLiveReplicator(options: LiveReplicatorOptions): Promi
|
|
|
132
149
|
*/
|
|
133
150
|
onBulk(change: RowBulkChange): void {
|
|
134
151
|
if (stopped) return;
|
|
135
|
-
|
|
152
|
+
const table = tableOf(change.entity);
|
|
153
|
+
registry.invalidate(table);
|
|
154
|
+
// Channels too, or a tab seated on one held the pre-update rows with nothing to re-read.
|
|
155
|
+
options.channels?.invalidate(table);
|
|
136
156
|
},
|
|
137
157
|
};
|
|
138
158
|
|