@ultimat3/realtime 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +591 -0
- package/README.md +320 -19
- package/package.json +6 -3
- package/src/apply-patches.ts +60 -0
- package/src/change-buffer.ts +77 -11
- package/src/channel.ts +174 -19
- package/src/client-contract.ts +81 -0
- package/src/client-frames.ts +175 -0
- package/src/client-heartbeat.ts +77 -0
- package/src/client-mutations.ts +114 -0
- package/src/client-topics.ts +54 -0
- package/src/client.ts +307 -273
- package/src/cursor.ts +7 -1
- package/src/errors.ts +193 -4
- package/src/frame-lanes.ts +58 -0
- package/src/hooks.ts +19 -5
- package/src/identity-map.ts +141 -0
- package/src/index.ts +96 -28
- package/src/json.ts +38 -1
- package/src/live-contract.ts +67 -0
- package/src/live-definition.ts +16 -11
- package/src/live-fanout.ts +150 -0
- package/src/live-query.ts +215 -268
- package/src/live-rows.ts +143 -0
- package/src/local-store.ts +86 -43
- package/src/nats-client.ts +132 -0
- package/src/nats-fake.ts +389 -344
- package/src/nats-jetstream.ts +21 -20
- package/src/nats-kv.ts +7 -7
- package/src/nats-lib-client.ts +210 -0
- package/src/nats-transport.ts +109 -138
- package/src/offline-queue.ts +146 -30
- package/src/pg-entity-row.ts +99 -31
- package/src/pg-replication.ts +84 -27
- package/src/pg-socket.ts +4 -1
- package/src/policy-gate.ts +13 -5
- package/src/presence.ts +76 -6
- package/src/query-hook.ts +56 -0
- package/src/query-window.ts +151 -0
- package/src/rebase.ts +68 -8
- package/src/replicator.ts +84 -11
- package/src/socket.ts +170 -14
- package/src/subscriber-gate.ts +209 -0
- package/src/subscription-book.ts +237 -0
- package/src/sync-auth.ts +124 -0
- package/src/sync-frames.ts +185 -0
- package/src/sync-listen.ts +73 -0
- package/src/sync-node.ts +284 -243
- package/src/sync-protocol.ts +115 -24
- package/src/sync-upgrade.ts +124 -0
- package/src/thundering-herd.ts +21 -0
- package/src/transport-env.ts +3 -3
- package/src/type-pins.ts +72 -0
- package/src/window-lock.ts +21 -0
- package/src/nats-commands.ts +0 -97
- package/src/nats-connection-fixture.ts +0 -105
- package/src/nats-connection.ts +0 -464
- package/src/nats-protocol.ts +0 -222
- package/src/nats-socket.ts +0 -236
- package/src/pg-connection-fixture.ts +0 -215
- package/src/pg-replication-fixture.ts +0 -261
package/src/live-query.ts
CHANGED
|
@@ -9,67 +9,28 @@
|
|
|
9
9
|
import { type Actor, type Clock, systemClock, uuid } from '@ultimat3/core';
|
|
10
10
|
import type { ChangeEvent } from './changefeed';
|
|
11
11
|
import {
|
|
12
|
-
advance,
|
|
13
12
|
type LiveCursor,
|
|
14
13
|
makeCursor,
|
|
15
14
|
type ReconnectBudget,
|
|
16
15
|
type ResumeSource,
|
|
17
16
|
resumeFrom,
|
|
18
17
|
} from './cursor';
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
18
|
+
import { isPolicyDenial, LiveQueryUnknownError, SubscriptionLimitError } from './errors';
|
|
19
|
+
import type { JsonValue } from './json';
|
|
21
20
|
import {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
type
|
|
26
|
-
} from './
|
|
21
|
+
type LiveQueryDefinition,
|
|
22
|
+
type LiveSubscription,
|
|
23
|
+
qidOf,
|
|
24
|
+
type SnapshotResult,
|
|
25
|
+
} from './live-contract';
|
|
26
|
+
import { type FanoutDeps, fanoutChange, snapshotFrame } from './live-fanout';
|
|
27
|
+
import { createEntry, fillWindow, type QueryEntry } from './query-window';
|
|
27
28
|
import type { SyncSocket } from './socket';
|
|
29
|
+
import { type Subscriber, SubscriberGate, type SubscriberGateOptions } from './subscriber-gate';
|
|
30
|
+
import { SubscriptionBook, subscriptionKey } from './subscription-book';
|
|
28
31
|
import { type Frame, PROTOCOL_VERSION } from './sync-protocol';
|
|
29
32
|
|
|
30
|
-
|
|
31
|
-
export function qidOf(name: string, input: JsonValue): string {
|
|
32
|
-
return `${name}:${fnv1a(canonicalJson(input))}`;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
export interface SnapshotResult<R extends Row = Row> {
|
|
36
|
-
readonly rows: readonly R[];
|
|
37
|
-
readonly lsn: string;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export interface LiveQueryDefinition<R extends Row = Row> {
|
|
41
|
-
readonly name: string;
|
|
42
|
-
/** Dependency set for the pre-filter. `x verify` rejects a `live: true` query without one. */
|
|
43
|
-
readonly entities: readonly string[];
|
|
44
|
-
/** Read set. Lets the pre-filter skip updates that touch no column this query reads. */
|
|
45
|
-
readonly columns?: readonly string[];
|
|
46
|
-
/** Bounded read (`orderBy` + `limit`, enforced by `x verify`), unfiltered by policy. */
|
|
47
|
-
snapshot(args: { input: JsonValue }): Promise<SnapshotResult<R>>;
|
|
48
|
-
/** Subscribe-time gate. Throws to deny — the same `policy` used by HTTP, jobs, and MCP. */
|
|
49
|
-
authorize?(args: { actor: Actor | null; input: JsonValue }): void | Promise<void>;
|
|
50
|
-
/** Row-level gate, evaluated per subscriber. The only row filter in the pipeline. */
|
|
51
|
-
visible(args: { actor: Actor | null; row: R; input: JsonValue }): boolean | Promise<boolean>;
|
|
52
|
-
/** Built once per `qid`, since a qid pins both the query and its input. */
|
|
53
|
-
matcher(input: JsonValue): IncrementalMatcher;
|
|
54
|
-
/**
|
|
55
|
-
* Resolve whatever this input needs before an entry is built. `matcher` is synchronous by
|
|
56
|
-
* design — a change event must not await anything — so a definition that has to compile a
|
|
57
|
-
* source or a shape does it here, after `authorize` allowed this subscriber and before the
|
|
58
|
-
* shared window exists.
|
|
59
|
-
*/
|
|
60
|
-
prepare?(input: JsonValue): Promise<void>;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
export interface LiveSubscription {
|
|
64
|
-
readonly sid: string;
|
|
65
|
-
readonly qid: string;
|
|
66
|
-
readonly socket: SyncSocket;
|
|
67
|
-
readonly input: JsonValue;
|
|
68
|
-
readonly definition: LiveQueryDefinition;
|
|
69
|
-
cursor: LiveCursor;
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
export interface LiveQueryRegistryOptions {
|
|
33
|
+
export interface LiveQueryRegistryOptions extends SubscriberGateOptions {
|
|
73
34
|
readonly source: ResumeSource;
|
|
74
35
|
readonly budget?: ReconnectBudget;
|
|
75
36
|
readonly clock?: Clock;
|
|
@@ -77,61 +38,78 @@ export interface LiveQueryRegistryOptions {
|
|
|
77
38
|
readonly maxPerTenant?: number;
|
|
78
39
|
readonly tenantOf?: (actor: Actor | null) => string | null;
|
|
79
40
|
/**
|
|
80
|
-
* `
|
|
81
|
-
*
|
|
82
|
-
*
|
|
41
|
+
* Distinct `(query, input)` pairs this node will hold at once. A `qid` derives from
|
|
42
|
+
* client-chosen input, so without a ceiling one socket mints entries — a matcher, a row window,
|
|
43
|
+
* a `WindowLock` and a fanout target each — until the process dies.
|
|
83
44
|
*/
|
|
84
|
-
readonly
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
/** One row withheld from one subscriber. Carries no row payload: the ids are the whole point. */
|
|
88
|
-
export interface RowDenied {
|
|
89
|
-
readonly qid: string;
|
|
90
|
-
readonly sid: string;
|
|
91
|
-
readonly actorId: string | null;
|
|
92
|
-
readonly rowId: string;
|
|
45
|
+
readonly maxEntries?: number;
|
|
93
46
|
}
|
|
94
47
|
|
|
95
48
|
/**
|
|
96
|
-
*
|
|
97
|
-
*
|
|
49
|
+
* Live `(query, input)` pairs one node holds. Reached, the next NEW pair is refused with
|
|
50
|
+
* `X_SUBSCRIPTION_LIMIT`; subscribing to a pair that already exists keeps working, because the
|
|
51
|
+
* cost this bounds is the entry, not the subscriber.
|
|
98
52
|
*/
|
|
99
|
-
|
|
100
|
-
readonly sid: string;
|
|
101
|
-
readonly actor: Actor | null;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
interface QueryEntry {
|
|
105
|
-
readonly qid: string;
|
|
106
|
-
readonly definition: LiveQueryDefinition;
|
|
107
|
-
readonly input: JsonValue;
|
|
108
|
-
readonly shape: SubscriptionShape;
|
|
109
|
-
readonly matcher: IncrementalMatcher;
|
|
110
|
-
readonly subscribers: Map<string, LiveSubscription>;
|
|
111
|
-
/**
|
|
112
|
-
* The shared, *pre-policy* result window. One per query id, bounded by the query's `limit`, and
|
|
113
|
-
* the reason the matcher can run once for N subscribers: the read is shared, the authz is not.
|
|
114
|
-
*/
|
|
115
|
-
rows: readonly Row[];
|
|
116
|
-
lsn: string;
|
|
117
|
-
}
|
|
53
|
+
export const DEFAULT_MAX_ENTRIES = 10_000;
|
|
118
54
|
|
|
119
55
|
export class LiveQueryRegistry {
|
|
120
56
|
readonly #definitions = new Map<string, LiveQueryDefinition>();
|
|
121
57
|
readonly #entries = new Map<string, QueryEntry>();
|
|
122
|
-
|
|
58
|
+
/** Keyed by `(socket, sid)`, never by `sid` alone — `subscription-book.ts` owns why. */
|
|
59
|
+
readonly #book: SubscriptionBook;
|
|
123
60
|
readonly #options: LiveQueryRegistryOptions;
|
|
124
61
|
readonly #clock: Clock;
|
|
125
|
-
#
|
|
62
|
+
readonly #gate: SubscriberGate;
|
|
63
|
+
readonly #maxEntries: number;
|
|
64
|
+
/** What one lane needs, and nothing this class holds beyond it. */
|
|
65
|
+
readonly #fanout: FanoutDeps;
|
|
66
|
+
#staleChanges = 0;
|
|
126
67
|
|
|
127
68
|
constructor(options: LiveQueryRegistryOptions) {
|
|
128
69
|
this.#options = options;
|
|
129
70
|
this.#clock = options.clock ?? systemClock;
|
|
71
|
+
this.#gate = new SubscriberGate(options);
|
|
72
|
+
this.#maxEntries = options.maxEntries ?? DEFAULT_MAX_ENTRIES;
|
|
73
|
+
// The book owns the caps because it is the only thing that can answer them in O(1).
|
|
74
|
+
this.#book = new SubscriptionBook(options);
|
|
75
|
+
this.#fanout = { gate: this.#gate, source: options.source, clock: this.#clock };
|
|
130
76
|
}
|
|
131
77
|
|
|
132
78
|
/** `live.rows_denied` for this node: rows a subscriber's policy refused since boot. */
|
|
133
79
|
get rowsDenied(): number {
|
|
134
|
-
return this.#rowsDenied;
|
|
80
|
+
return this.#gate.rowsDenied;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** `live.gate_failed` for this node: gates that raised instead of deciding. Never a denial. */
|
|
84
|
+
get gateFailures(): number {
|
|
85
|
+
return this.#gate.gateFailures;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** `live.changes_stale`: changes at or below a window's own lsn, refused rather than folded. */
|
|
89
|
+
get staleChanges(): number {
|
|
90
|
+
return this.#staleChanges;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Every window on this node is presumed to have missed a change, so nothing may be patched or
|
|
95
|
+
* served out of one until it has been re-read, and every subscriber is re-snapshotted.
|
|
96
|
+
*
|
|
97
|
+
* The `sync` node calls this when the change stream skips a sequence: over core NATS a fanout is
|
|
98
|
+
* at-most-once, and a node that missed eleven changes during a reconnect otherwise holds a window
|
|
99
|
+
* whose lsn never moved, subscribers whose cursors never moved, and therefore nothing that would
|
|
100
|
+
* ever ask for a re-snapshot. The repair lands on the next change to each query — which is the
|
|
101
|
+
* event that proves the query is moving at all.
|
|
102
|
+
*/
|
|
103
|
+
invalidate(): number {
|
|
104
|
+
let marked = 0;
|
|
105
|
+
for (const entry of this.#entries.values()) {
|
|
106
|
+
entry.stale = true;
|
|
107
|
+
for (const subscription of entry.subscribers.values()) {
|
|
108
|
+
subscription.socket.markDesynced(subscription.sid);
|
|
109
|
+
marked += 1;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return marked;
|
|
135
113
|
}
|
|
136
114
|
|
|
137
115
|
register(definition: LiveQueryDefinition): this {
|
|
@@ -147,8 +125,9 @@ export class LiveQueryRegistry {
|
|
|
147
125
|
return this.#entries.get(qid)?.subscribers.size ?? 0;
|
|
148
126
|
}
|
|
149
127
|
|
|
150
|
-
subscription
|
|
151
|
-
|
|
128
|
+
/** One socket's subscription. A sid alone does not identify one — see `subscription-book.ts`. */
|
|
129
|
+
subscription(socketId: string, sid: string): LiveSubscription | undefined {
|
|
130
|
+
return this.#book.get(socketId, sid);
|
|
152
131
|
}
|
|
153
132
|
|
|
154
133
|
/**
|
|
@@ -163,14 +142,35 @@ export class LiveQueryRegistry {
|
|
|
163
142
|
cursor?: LiveCursor | null;
|
|
164
143
|
}): Promise<{ subscription: LiveSubscription; frame: Frame }> {
|
|
165
144
|
const definition = this.#definitions.get(args.name);
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
145
|
+
// A name this node never registered, and not a protocol skew: the frame parsed, the version
|
|
146
|
+
// matched, and one string in it names nothing. Reporting it as `X_PROTOCOL_VERSION` handed the
|
|
147
|
+
// client "x build && redeploy the client" for a typo no rebuild changes.
|
|
148
|
+
if (!definition) throw new LiveQueryUnknownError({ name: args.name });
|
|
149
|
+
const sid = args.sid ?? uuid();
|
|
150
|
+
// Everything this subscribe can be refused for, decided in one synchronous step BEFORE the
|
|
151
|
+
// first await — the caps and the sid both. Read at the top and acted on three awaits later,
|
|
152
|
+
// they were bypassed by the ordinary case: one WebSocket write carrying N subscribe frames,
|
|
153
|
+
// dispatched concurrently, N of them reading a count nothing had grown yet.
|
|
154
|
+
const slot = this.#book.reserve(args.socket, sid);
|
|
155
|
+
try {
|
|
156
|
+
return await this.#subscribeReserved(definition, sid, args);
|
|
157
|
+
} finally {
|
|
158
|
+
// After the attach on every path, so the slot is only ever given back to a count that has
|
|
159
|
+
// already grown — or, on a failure, to one that never will.
|
|
160
|
+
slot.release();
|
|
172
161
|
}
|
|
173
|
-
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
async #subscribeReserved(
|
|
165
|
+
definition: LiveQueryDefinition,
|
|
166
|
+
sid: string,
|
|
167
|
+
args: {
|
|
168
|
+
socket: SyncSocket;
|
|
169
|
+
name: string;
|
|
170
|
+
input: JsonValue;
|
|
171
|
+
cursor?: LiveCursor | null;
|
|
172
|
+
},
|
|
173
|
+
): Promise<{ subscription: LiveSubscription; frame: Frame }> {
|
|
174
174
|
await definition.authorize?.({ actor: args.socket.actor, input: args.input });
|
|
175
175
|
// After this subscriber's own decision, never before it: resolving a shape for a caller who
|
|
176
176
|
// may not subscribe is work an unauthorized client gets to schedule.
|
|
@@ -178,89 +178,111 @@ export class LiveQueryRegistry {
|
|
|
178
178
|
|
|
179
179
|
const qid = qidOf(args.name, args.input);
|
|
180
180
|
const entry = this.#entryFor(qid, definition, args.input);
|
|
181
|
-
const sid = args.sid ?? uuid();
|
|
182
181
|
const now = this.#clock.now().getTime();
|
|
183
182
|
|
|
184
183
|
if (args.cursor) {
|
|
185
|
-
const
|
|
184
|
+
const cursor = args.cursor;
|
|
185
|
+
const resumed = await resumeFrom(cursor, {
|
|
186
186
|
source: this.#options.source,
|
|
187
187
|
...(this.#options.budget ? { budget: this.#options.budget } : {}),
|
|
188
188
|
clock: this.#clock,
|
|
189
189
|
snapshot: async () => await this.#read(entry, { sid, actor: args.socket.actor }),
|
|
190
190
|
});
|
|
191
191
|
if (resumed.kind === 'delta') {
|
|
192
|
-
|
|
192
|
+
// The gate decides about whole rows out of the shared window, and an entry nothing has read
|
|
193
|
+
// yet has none — every patch would meet an empty window and be withheld. Filling is
|
|
194
|
+
// conditional on purpose: a restart storm resumes onto entries that already hold a live
|
|
195
|
+
// window, and re-reading per resuming subscriber is the cost a delta resume exists to skip.
|
|
196
|
+
if (entry.lsn === '') await fillWindow(entry);
|
|
197
|
+
// The live entry on purpose: a resume runs outside the lane, so the window under it may
|
|
198
|
+
// have moved on — always forwards, and a row whose grant was revoked in the meantime is
|
|
199
|
+
// one this pass must refuse rather than replay from the state it had at the cursor's lsn.
|
|
200
|
+
const patches = await this.#gate.filterPatches(
|
|
193
201
|
entry,
|
|
194
202
|
{ sid, actor: args.socket.actor },
|
|
195
203
|
resumed.patches,
|
|
196
|
-
|
|
204
|
+
new Set(cursor.ids),
|
|
197
205
|
);
|
|
198
|
-
const subscription = this.#
|
|
206
|
+
const subscription = this.#attachUnlessGone(entry, args.socket, sid, resumed.cursor);
|
|
199
207
|
return {
|
|
200
208
|
subscription,
|
|
201
209
|
frame: { type: 'patch', v: PROTOCOL_VERSION, sid, patches, lsn: resumed.cursor.lsn },
|
|
202
210
|
};
|
|
203
211
|
}
|
|
204
|
-
const subscription = this.#
|
|
212
|
+
const subscription = this.#attachUnlessGone(entry, args.socket, sid, resumed.cursor);
|
|
205
213
|
return {
|
|
206
214
|
subscription,
|
|
207
|
-
frame:
|
|
208
|
-
type: 'snapshot',
|
|
209
|
-
v: PROTOCOL_VERSION,
|
|
210
|
-
sid,
|
|
211
|
-
rows: resumed.rows,
|
|
212
|
-
cursor: resumed.cursor,
|
|
213
|
-
},
|
|
215
|
+
frame: snapshotFrame(entry, sid, resumed.rows, resumed.cursor),
|
|
214
216
|
};
|
|
215
217
|
}
|
|
216
218
|
|
|
217
219
|
const fresh = await this.#read(entry, { sid, actor: args.socket.actor });
|
|
218
220
|
const cursor = makeCursor(qid, fresh.lsn, fresh.rows, now);
|
|
219
|
-
const subscription = this.#
|
|
220
|
-
return {
|
|
221
|
-
subscription,
|
|
222
|
-
frame: { type: 'snapshot', v: PROTOCOL_VERSION, sid, rows: fresh.rows, cursor },
|
|
223
|
-
};
|
|
221
|
+
const subscription = this.#attachUnlessGone(entry, args.socket, sid, cursor);
|
|
222
|
+
return { subscription, frame: snapshotFrame(entry, sid, fresh.rows, cursor) };
|
|
224
223
|
}
|
|
225
224
|
|
|
226
|
-
|
|
227
|
-
|
|
225
|
+
/** Scoped to the socket that asked: a client may only drop its own subscription. */
|
|
226
|
+
unsubscribe(socketId: string, sid: string): void {
|
|
227
|
+
const subscription = this.#book.get(socketId, sid);
|
|
228
228
|
if (!subscription) return;
|
|
229
|
-
this.#
|
|
229
|
+
this.#book.delete(socketId, sid);
|
|
230
230
|
subscription.socket.queries.delete(sid);
|
|
231
231
|
subscription.socket.clearDesynced(sid);
|
|
232
232
|
const entry = this.#entries.get(subscription.qid);
|
|
233
233
|
if (!entry) return;
|
|
234
|
-
entry.subscribers.delete(sid);
|
|
234
|
+
entry.subscribers.delete(subscriptionKey(socketId, sid));
|
|
235
235
|
// An entry with no subscribers stops costing a matcher and a change window.
|
|
236
|
-
if (entry.subscribers.size
|
|
236
|
+
if (entry.subscribers.size !== 0) return;
|
|
237
|
+
this.#entries.delete(subscription.qid);
|
|
238
|
+
// And the retained patches go with it. `forget` had no caller: the entry was dropped here and
|
|
239
|
+
// the `ResumeSource` was never told, so its ring for that qid sat at full capacity until the
|
|
240
|
+
// LRU happened to evict it — a client-chosen input's memory outliving the last subscriber.
|
|
241
|
+
this.#options.source.forget?.(subscription.qid);
|
|
237
242
|
}
|
|
238
243
|
|
|
239
244
|
unsubscribeSocket(socketId: string): void {
|
|
240
|
-
for (const subscription of
|
|
241
|
-
|
|
245
|
+
for (const subscription of this.#book.ofSocket(socketId)) {
|
|
246
|
+
this.unsubscribe(socketId, subscription.sid);
|
|
242
247
|
}
|
|
243
248
|
}
|
|
244
249
|
|
|
245
250
|
/**
|
|
246
251
|
* Actor changed mid-connection (login, logout, role change): re-run subscribe-time authz and drop
|
|
247
252
|
* what is no longer allowed. Survivors are marked desynced so the next flush re-snapshots them
|
|
248
|
-
* under the new actor's row policy.
|
|
253
|
+
* under the new actor's row policy. Returns the sids that were dropped — a denial and nothing
|
|
254
|
+
* else, so a caller may tell the client "you may no longer see this" and be right.
|
|
249
255
|
*/
|
|
250
256
|
async reauthorize(socket: SyncSocket): Promise<readonly string[]> {
|
|
251
257
|
const dropped: string[] = [];
|
|
252
|
-
|
|
253
|
-
|
|
258
|
+
// The actor changed, so what tenant this socket's subscriptions count against may have too.
|
|
259
|
+
// Told here rather than derived per lookup: the per-tenant cap is an index now, and an index
|
|
260
|
+
// nobody updates is a count that drifts from the book for the rest of the process.
|
|
261
|
+
this.#book.retenant(socket);
|
|
262
|
+
for (const subscription of this.#book.ofSocket(socket.id)) {
|
|
254
263
|
try {
|
|
255
264
|
await subscription.definition.authorize?.({
|
|
256
265
|
actor: socket.actor,
|
|
257
266
|
input: subscription.input,
|
|
258
267
|
});
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
268
|
+
} catch (error) {
|
|
269
|
+
if (isPolicyDenial(error)) {
|
|
270
|
+
this.unsubscribe(socket.id, subscription.sid);
|
|
271
|
+
dropped.push(subscription.sid);
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
// Not a decision — the gate never reached one. Destroying the subscription would report a
|
|
275
|
+
// database timeout as a revoked grant, and a client does not resubscribe to a denial. It
|
|
276
|
+
// survives, desynced: nothing is delivered from the window built under the old actor, and
|
|
277
|
+
// the row gate still decides every row under the new one, from the same policy `authorize`
|
|
278
|
+
// consults. The failure is counted and reported rather than silently absorbed.
|
|
279
|
+
this.#gate.failedAuthorize(
|
|
280
|
+
subscription.qid,
|
|
281
|
+
{ sid: subscription.sid, actor: socket.actor },
|
|
282
|
+
error,
|
|
283
|
+
);
|
|
263
284
|
}
|
|
285
|
+
socket.markDesynced(subscription.sid);
|
|
264
286
|
}
|
|
265
287
|
return dropped;
|
|
266
288
|
}
|
|
@@ -268,53 +290,58 @@ export class LiveQueryRegistry {
|
|
|
268
290
|
/**
|
|
269
291
|
* Fan one change out. Matched once per query id, authorized once per subscriber. Returns the
|
|
270
292
|
* number of frames sent — the metric the reconnect benchmark watches.
|
|
293
|
+
*
|
|
294
|
+
* Each entry's turn is taken in that entry's lane. Nothing upstream orders this: `sync` fires
|
|
295
|
+
* `void registry.deliver(change)` straight off the bus subscription, so two changes arriving back
|
|
296
|
+
* to back would otherwise interleave inside one query id — lsn 2 delivered before lsn 1, the
|
|
297
|
+
* subscriber's cursor rewound to 1, and every gate deciding against whichever window won.
|
|
298
|
+
*
|
|
299
|
+
* Every lane is *entered* before any of them is awaited, and nothing inside a fanout takes a
|
|
300
|
+
* second lane, so holding all of them at once cannot be a cycle. That is what makes the ordering
|
|
301
|
+
* claim true: two deliveries queue onto each query id in call order, serialized per query id and
|
|
302
|
+
* never per node — awaiting one entry before entering the next made one slow policy pass the
|
|
303
|
+
* whole node's pace, and let a lane that threw skip every entry behind it with nobody desynced.
|
|
271
304
|
*/
|
|
272
305
|
async deliver(change: ChangeEvent): Promise<number> {
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
// The window lost its tail: guessing is how a sync engine silently diverges.
|
|
285
|
-
subscription.socket.markDesynced(subscription.sid);
|
|
286
|
-
continue;
|
|
287
|
-
}
|
|
288
|
-
const who: Subscriber = { sid: subscription.sid, actor: subscription.socket.actor };
|
|
289
|
-
const allowed: RowPatch[] = [];
|
|
290
|
-
for (const patch of result.patches) {
|
|
291
|
-
const gated = await this.#gate(
|
|
292
|
-
entry,
|
|
293
|
-
who,
|
|
294
|
-
patch,
|
|
295
|
-
subscription.cursor.ids.includes(patch.id),
|
|
296
|
-
);
|
|
297
|
-
if (gated) allowed.push(gated);
|
|
298
|
-
}
|
|
299
|
-
if (allowed.length === 0) continue;
|
|
300
|
-
const frame: Frame = {
|
|
301
|
-
type: 'patch',
|
|
302
|
-
v: PROTOCOL_VERSION,
|
|
303
|
-
sid: subscription.sid,
|
|
304
|
-
patches: allowed,
|
|
305
|
-
lsn: change.lsn,
|
|
306
|
-
};
|
|
307
|
-
if (subscription.socket.send(frame)) {
|
|
308
|
-
subscription.cursor = advance(subscription.cursor, allowed, change.lsn, change.at);
|
|
309
|
-
sent += 1;
|
|
310
|
-
} else {
|
|
306
|
+
const lanes = [...this.#entries.values()].map(async (entry) => {
|
|
307
|
+
try {
|
|
308
|
+
const result = await entry.lock.run(() => fanoutChange(this.#fanout, entry, change));
|
|
309
|
+
this.#staleChanges += result.stale;
|
|
310
|
+
return result.sent;
|
|
311
|
+
} catch (error) {
|
|
312
|
+
// The window advanced under a fanout that did not finish, so every subscriber of this one
|
|
313
|
+
// query id now holds a cursor below the change and no later flush would correct them:
|
|
314
|
+
// desynced here, re-snapshotted on the next one. Silent divergence is the whole reason
|
|
315
|
+
// `markDesynced` exists, and skipping this is how a failure became one.
|
|
316
|
+
for (const subscription of entry.subscribers.values()) {
|
|
311
317
|
subscription.socket.markDesynced(subscription.sid);
|
|
312
318
|
}
|
|
319
|
+
throw error;
|
|
313
320
|
}
|
|
321
|
+
});
|
|
322
|
+
// `allSettled`, so one lane's rejection neither cancels the others nor goes unhandled. The
|
|
323
|
+
// first failure still reaches the caller — `sync` logs it — but it costs one query id.
|
|
324
|
+
let sent = 0;
|
|
325
|
+
let failure: { readonly error: unknown } | null = null;
|
|
326
|
+
for (const lane of await Promise.allSettled(lanes)) {
|
|
327
|
+
if (lane.status === 'fulfilled') sent += lane.value;
|
|
328
|
+
else failure ??= { error: lane.reason };
|
|
314
329
|
}
|
|
330
|
+
if (failure !== null) throw failure.error;
|
|
315
331
|
return sent;
|
|
316
332
|
}
|
|
317
333
|
|
|
334
|
+
#attachUnlessGone(
|
|
335
|
+
entry: QueryEntry,
|
|
336
|
+
socket: SyncSocket,
|
|
337
|
+
sid: string,
|
|
338
|
+
cursor: LiveCursor,
|
|
339
|
+
): LiveSubscription {
|
|
340
|
+
const subscription = this.#attach(entry, socket, sid, cursor);
|
|
341
|
+
if (socket.closed) this.unsubscribe(socket.id, sid);
|
|
342
|
+
return subscription;
|
|
343
|
+
}
|
|
344
|
+
|
|
318
345
|
#attach(
|
|
319
346
|
entry: QueryEntry,
|
|
320
347
|
socket: SyncSocket,
|
|
@@ -329,8 +356,10 @@ export class LiveQueryRegistry {
|
|
|
329
356
|
definition: entry.definition,
|
|
330
357
|
cursor,
|
|
331
358
|
};
|
|
332
|
-
entry
|
|
333
|
-
|
|
359
|
+
// The entry's own map takes the SAME composite key: a sid alone would collide across sockets
|
|
360
|
+
// here exactly as it did in the book, and `unsubscribe` deletes from both by one identity.
|
|
361
|
+
entry.subscribers.set(subscriptionKey(socket.id, sid), subscription);
|
|
362
|
+
this.#book.add(subscription);
|
|
334
363
|
socket.queries.set(sid, entry.qid);
|
|
335
364
|
socket.clearDesynced(sid);
|
|
336
365
|
return subscription;
|
|
@@ -339,111 +368,29 @@ export class LiveQueryRegistry {
|
|
|
339
368
|
#entryFor(qid: string, definition: LiveQueryDefinition, input: JsonValue): QueryEntry {
|
|
340
369
|
const existing = this.#entries.get(qid);
|
|
341
370
|
if (existing) return existing;
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
...(definition.columns ? { columns: definition.columns } : {}),
|
|
355
|
-
},
|
|
356
|
-
matcher,
|
|
357
|
-
subscribers: new Map(),
|
|
358
|
-
rows: [],
|
|
359
|
-
lsn: '',
|
|
360
|
-
};
|
|
371
|
+
// The node-wide ceiling, refused where the entry would be born. `qid` derives from
|
|
372
|
+
// client-chosen input, so one socket varying it mints a matcher, a row window and a
|
|
373
|
+
// `WindowLock` per value, and every change then fans out over all of them.
|
|
374
|
+
if (this.#entries.size >= this.#maxEntries) {
|
|
375
|
+
throw new SubscriptionLimitError({
|
|
376
|
+
scope: 'node',
|
|
377
|
+
id: definition.name,
|
|
378
|
+
limit: this.#maxEntries,
|
|
379
|
+
knob: 'maxEntries',
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
const created = createEntry(qid, definition, input, definition.matcher(input));
|
|
361
383
|
this.#entries.set(qid, created);
|
|
362
384
|
return created;
|
|
363
385
|
}
|
|
364
386
|
|
|
365
|
-
/** One read, then one policy pass per subscriber. Never one read per subscriber. */
|
|
366
|
-
async #read(entry: QueryEntry, who: Subscriber): Promise<SnapshotResult> {
|
|
367
|
-
const result = await entry.definition.snapshot({ input: entry.input });
|
|
368
|
-
entry.rows = result.rows;
|
|
369
|
-
entry.lsn = result.lsn;
|
|
370
|
-
const rows: Row[] = [];
|
|
371
|
-
for (const row of result.rows) {
|
|
372
|
-
if (await entry.definition.visible({ actor: who.actor, row, input: entry.input })) {
|
|
373
|
-
rows.push(row);
|
|
374
|
-
} else {
|
|
375
|
-
this.#denied(entry, who, row.id);
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
return { rows, lsn: result.lsn };
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
async #filterPatches(
|
|
382
|
-
entry: QueryEntry,
|
|
383
|
-
who: Subscriber,
|
|
384
|
-
patches: readonly RowPatch[],
|
|
385
|
-
cursor: LiveCursor,
|
|
386
|
-
): Promise<RowPatch[]> {
|
|
387
|
-
const held = new Set(cursor.ids);
|
|
388
|
-
const out: RowPatch[] = [];
|
|
389
|
-
for (const patch of patches) {
|
|
390
|
-
const allowed = await this.#gate(entry, who, patch, held.has(patch.id));
|
|
391
|
-
if (allowed) out.push(allowed);
|
|
392
|
-
}
|
|
393
|
-
return out;
|
|
394
|
-
}
|
|
395
|
-
|
|
396
387
|
/**
|
|
397
|
-
*
|
|
398
|
-
*
|
|
388
|
+
* One read, then one policy pass per subscriber. Never one read per subscriber — and never a
|
|
389
|
+
* partial pass: a gate that fails raises out of `subscribe`, because a snapshot missing the rows
|
|
390
|
+
* nobody could decide about is a short result set this subscriber would render as the whole one.
|
|
399
391
|
*/
|
|
400
|
-
async #
|
|
401
|
-
entry
|
|
402
|
-
who:
|
|
403
|
-
patch: RowPatch,
|
|
404
|
-
holds: boolean,
|
|
405
|
-
): Promise<RowPatch | null> {
|
|
406
|
-
if (patch.op === 'delete' || patch.row === null) return patch;
|
|
407
|
-
// The policy always sees the whole row from the shared window — a patch carries changed
|
|
408
|
-
// columns only, and authorizing a partial row is how a row policy silently starts failing.
|
|
409
|
-
const full = entry.rows.find((row) => row.id === patch.id);
|
|
410
|
-
const row: Row = { ...(full ?? {}), ...patch.row, id: patch.id };
|
|
411
|
-
if (await entry.definition.visible({ actor: who.actor, row, input: entry.input })) return patch;
|
|
412
|
-
this.#denied(entry, who, patch.id);
|
|
413
|
-
return holds ? { op: 'delete', id: patch.id, row: null, lsn: patch.lsn } : null;
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
/** `live.rows_denied`. Counted here and nowhere else, so every drop is one increment. */
|
|
417
|
-
#denied(entry: QueryEntry, who: Subscriber, rowId: string): void {
|
|
418
|
-
this.#rowsDenied += 1;
|
|
419
|
-
this.#options.onRowDenied?.({
|
|
420
|
-
qid: entry.qid,
|
|
421
|
-
sid: who.sid,
|
|
422
|
-
actorId: who.actor === null ? null : who.actor.id,
|
|
423
|
-
rowId,
|
|
424
|
-
});
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
#assertLimits(socket: SyncSocket): void {
|
|
428
|
-
const perSocket = this.#options.maxPerSocket ?? 128;
|
|
429
|
-
if (socket.queries.size >= perSocket) {
|
|
430
|
-
throw new SubscriptionLimitError({ scope: 'socket', id: socket.id, limit: perSocket });
|
|
431
|
-
}
|
|
432
|
-
const perTenant = this.#options.maxPerTenant;
|
|
433
|
-
const tenant = this.#options.tenantOf?.(socket.actor) ?? null;
|
|
434
|
-
if (perTenant === undefined || tenant === null) return;
|
|
435
|
-
let count = 0;
|
|
436
|
-
for (const subscription of this.#bySid.values()) {
|
|
437
|
-
if ((this.#options.tenantOf?.(subscription.socket.actor) ?? null) === tenant) count += 1;
|
|
438
|
-
}
|
|
439
|
-
if (count >= perTenant) {
|
|
440
|
-
throw new SubscriptionLimitError({ scope: 'tenant', id: tenant, limit: perTenant });
|
|
441
|
-
}
|
|
392
|
+
async #read(entry: QueryEntry, who: Subscriber): Promise<SnapshotResult> {
|
|
393
|
+
const window = await fillWindow(entry);
|
|
394
|
+
return { rows: await this.#gate.filterRows(entry, who, window.rows), lsn: window.lsn };
|
|
442
395
|
}
|
|
443
396
|
}
|
|
444
|
-
|
|
445
|
-
function orgIdOf(input: JsonValue): string | null {
|
|
446
|
-
if (typeof input !== 'object' || input === null || Array.isArray(input)) return null;
|
|
447
|
-
const value = input['orgId'];
|
|
448
|
-
return typeof value === 'string' ? value : null;
|
|
449
|
-
}
|