@ultimat3/realtime 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +591 -0
- package/README.md +320 -19
- package/package.json +6 -3
- package/src/apply-patches.ts +60 -0
- package/src/change-buffer.ts +77 -11
- package/src/channel.ts +174 -19
- package/src/client-contract.ts +81 -0
- package/src/client-frames.ts +175 -0
- package/src/client-heartbeat.ts +77 -0
- package/src/client-mutations.ts +114 -0
- package/src/client-topics.ts +54 -0
- package/src/client.ts +307 -273
- package/src/cursor.ts +7 -1
- package/src/errors.ts +193 -4
- package/src/frame-lanes.ts +58 -0
- package/src/hooks.ts +19 -5
- package/src/identity-map.ts +141 -0
- package/src/index.ts +96 -28
- package/src/json.ts +38 -1
- package/src/live-contract.ts +67 -0
- package/src/live-definition.ts +16 -11
- package/src/live-fanout.ts +150 -0
- package/src/live-query.ts +215 -268
- package/src/live-rows.ts +143 -0
- package/src/local-store.ts +86 -43
- package/src/nats-client.ts +132 -0
- package/src/nats-fake.ts +389 -344
- package/src/nats-jetstream.ts +21 -20
- package/src/nats-kv.ts +7 -7
- package/src/nats-lib-client.ts +210 -0
- package/src/nats-transport.ts +109 -138
- package/src/offline-queue.ts +146 -30
- package/src/pg-entity-row.ts +99 -31
- package/src/pg-replication.ts +84 -27
- package/src/pg-socket.ts +4 -1
- package/src/policy-gate.ts +13 -5
- package/src/presence.ts +76 -6
- package/src/query-hook.ts +56 -0
- package/src/query-window.ts +151 -0
- package/src/rebase.ts +68 -8
- package/src/replicator.ts +84 -11
- package/src/socket.ts +170 -14
- package/src/subscriber-gate.ts +209 -0
- package/src/subscription-book.ts +237 -0
- package/src/sync-auth.ts +124 -0
- package/src/sync-frames.ts +185 -0
- package/src/sync-listen.ts +73 -0
- package/src/sync-node.ts +284 -243
- package/src/sync-protocol.ts +115 -24
- package/src/sync-upgrade.ts +124 -0
- package/src/thundering-herd.ts +21 -0
- package/src/transport-env.ts +3 -3
- package/src/type-pins.ts +72 -0
- package/src/window-lock.ts +21 -0
- package/src/nats-commands.ts +0 -97
- package/src/nats-connection-fixture.ts +0 -105
- package/src/nats-connection.ts +0 -464
- package/src/nats-protocol.ts +0 -222
- package/src/nats-socket.ts +0 -236
- package/src/pg-connection-fixture.ts +0 -215
- package/src/pg-replication-fixture.ts +0 -261
package/src/index.ts
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
// Public API. Explicit, tier by tier: channels, live queries, local-first sync, plus the wire and
|
|
2
2
|
// the server/client halves that carry all three.
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
// ---- the client's one stateless piece, reusable against an app's own store ----------------------
|
|
5
|
+
export { applyPatches, orderAfterPatches } from './apply-patches';
|
|
6
|
+
export {
|
|
7
|
+
type ChangeBufferOptions,
|
|
8
|
+
DEFAULT_MAX_BUFFER_BYTES,
|
|
9
|
+
DEFAULT_MAX_BUFFER_BYTES_PER_QUERY,
|
|
10
|
+
RingChangeBuffer,
|
|
11
|
+
} from './change-buffer';
|
|
5
12
|
// ---- tier 2: live queries ----------------------------------------------------------------------
|
|
6
13
|
export {
|
|
7
14
|
type ChangeEvent,
|
|
@@ -25,12 +32,12 @@ export {
|
|
|
25
32
|
type SelectChangeFeedOptions,
|
|
26
33
|
selectChangeFeed,
|
|
27
34
|
} from './changefeed-env';
|
|
28
|
-
|
|
29
35
|
// ---- tier 1: channels + presence ---------------------------------------------------------------
|
|
30
36
|
export {
|
|
31
37
|
ChannelHub,
|
|
32
38
|
type ChannelHubOptions,
|
|
33
39
|
channelFrame,
|
|
40
|
+
DEFAULT_MAX_TOPICS_PER_NODE,
|
|
34
41
|
type Topic,
|
|
35
42
|
type TopicGuard,
|
|
36
43
|
type TopicGuardArgs,
|
|
@@ -39,7 +46,6 @@ export {
|
|
|
39
46
|
} from './channel';
|
|
40
47
|
// ---- server + client halves -------------------------------------------------------------------
|
|
41
48
|
export {
|
|
42
|
-
applyPatches,
|
|
43
49
|
type ClientSocket,
|
|
44
50
|
LiveClient,
|
|
45
51
|
type LiveClientOptions,
|
|
@@ -48,6 +54,7 @@ export {
|
|
|
48
54
|
type LiveState,
|
|
49
55
|
type MutatorRef,
|
|
50
56
|
type SignalFactory,
|
|
57
|
+
type Unsubscribe,
|
|
51
58
|
} from './client';
|
|
52
59
|
// ---- reconnect ----------------------------------------------------------------------------------
|
|
53
60
|
export {
|
|
@@ -71,10 +78,13 @@ export {
|
|
|
71
78
|
// ---- errors ----------------------------------------------------------------------------------
|
|
72
79
|
export {
|
|
73
80
|
CursorStaleError,
|
|
81
|
+
FrameRateLimitError,
|
|
74
82
|
LiveClientMissingError,
|
|
83
|
+
LiveQueryUnknownError,
|
|
75
84
|
LiveRowUnidentifiedError,
|
|
76
85
|
NotImplementedError,
|
|
77
86
|
ProtocolVersionError,
|
|
87
|
+
QueryNotSubscribableError,
|
|
78
88
|
REALTIME_ERROR_CODES,
|
|
79
89
|
REALTIME_ERROR_TITLES,
|
|
80
90
|
RealtimeError,
|
|
@@ -115,6 +125,15 @@ export {
|
|
|
115
125
|
useMutation,
|
|
116
126
|
useMutationQueue,
|
|
117
127
|
} from './hooks';
|
|
128
|
+
// ---- the client's single source of truth: one row per (entity, id) ------------------------------
|
|
129
|
+
export {
|
|
130
|
+
type IdentityListener,
|
|
131
|
+
IdentityMap,
|
|
132
|
+
privateScope,
|
|
133
|
+
type RowKey,
|
|
134
|
+
type RowScope,
|
|
135
|
+
rowKey,
|
|
136
|
+
} from './identity-map';
|
|
118
137
|
// ---- shared value domain ---------------------------------------------------------------------
|
|
119
138
|
export {
|
|
120
139
|
canonicalJson,
|
|
@@ -128,16 +147,19 @@ export {
|
|
|
128
147
|
type RowOp,
|
|
129
148
|
type RowPatch,
|
|
130
149
|
} from './json';
|
|
131
|
-
export { type LiveDefinitionOptions, liveQueryDefinition } from './live-definition';
|
|
132
150
|
export {
|
|
133
151
|
type LiveQueryDefinition,
|
|
134
|
-
LiveQueryRegistry,
|
|
135
|
-
type LiveQueryRegistryOptions,
|
|
136
152
|
type LiveSubscription,
|
|
137
153
|
qidOf,
|
|
138
|
-
type RowDenied,
|
|
139
154
|
type SnapshotResult,
|
|
155
|
+
} from './live-contract';
|
|
156
|
+
export { type LiveDefinitionOptions, liveQueryDefinition } from './live-definition';
|
|
157
|
+
export {
|
|
158
|
+
DEFAULT_MAX_ENTRIES,
|
|
159
|
+
LiveQueryRegistry,
|
|
160
|
+
type LiveQueryRegistryOptions,
|
|
140
161
|
} from './live-query';
|
|
162
|
+
export { type Registration, RowWindows } from './live-rows';
|
|
141
163
|
// ---- tier 3: local-first ------------------------------------------------------------------------
|
|
142
164
|
export {
|
|
143
165
|
createOpfsLocalStore,
|
|
@@ -161,15 +183,23 @@ export {
|
|
|
161
183
|
type SubscriptionShape,
|
|
162
184
|
toBridgeResult,
|
|
163
185
|
} from './matcher-bridge';
|
|
164
|
-
export type { NatsConnectOptions } from './nats-commands';
|
|
165
186
|
// ---- the production bus -------------------------------------------------------------------------
|
|
166
187
|
export {
|
|
167
|
-
|
|
168
|
-
|
|
188
|
+
DEFAULT_NATS_PORT,
|
|
189
|
+
DEFAULT_REQUEST_TIMEOUT_MS,
|
|
190
|
+
type NatsClient,
|
|
191
|
+
type NatsClientOptions,
|
|
192
|
+
type NatsConnect,
|
|
193
|
+
type NatsHeaders,
|
|
194
|
+
type NatsMessage,
|
|
169
195
|
type NatsMessageHandler,
|
|
196
|
+
type NatsRequestManyOptions,
|
|
197
|
+
type NatsRequestOptions,
|
|
170
198
|
type NatsSubscription,
|
|
171
|
-
|
|
172
|
-
|
|
199
|
+
type NatsTarget,
|
|
200
|
+
parseNatsUrl,
|
|
201
|
+
} from './nats-client';
|
|
202
|
+
export { FakeNatsBroker, type FakeNatsOptions, fakeNatsConnect } from './nats-fake';
|
|
173
203
|
export {
|
|
174
204
|
assertBucket,
|
|
175
205
|
assertServerVersion,
|
|
@@ -178,24 +208,12 @@ export {
|
|
|
178
208
|
type KvRecord,
|
|
179
209
|
kvGet,
|
|
180
210
|
kvLast,
|
|
211
|
+
kvStream,
|
|
181
212
|
kvSubject,
|
|
182
213
|
kvWrite,
|
|
183
214
|
} from './nats-jetstream';
|
|
184
215
|
export { decodeToken, encodeToken, NatsKvSet, type NatsKvSetOptions } from './nats-kv';
|
|
185
|
-
export {
|
|
186
|
-
type NatsHeaders,
|
|
187
|
-
type NatsMessage,
|
|
188
|
-
type NatsOperation,
|
|
189
|
-
NatsProtocolParser,
|
|
190
|
-
type NatsServerInfo,
|
|
191
|
-
} from './nats-protocol';
|
|
192
|
-
export {
|
|
193
|
-
bunNatsStream,
|
|
194
|
-
type NatsStream,
|
|
195
|
-
type NatsTarget,
|
|
196
|
-
natsStreamOver,
|
|
197
|
-
parseNatsUrl,
|
|
198
|
-
} from './nats-socket';
|
|
216
|
+
export { openNatsClient } from './nats-lib-client';
|
|
199
217
|
export { NatsTransport, type NatsTransportOptions } from './nats-transport';
|
|
200
218
|
export {
|
|
201
219
|
type DrainReport,
|
|
@@ -226,12 +244,28 @@ export {
|
|
|
226
244
|
} from './pgoutput';
|
|
227
245
|
export { authorizeWithPolicy, type GateOptions, visibleWithPolicy } from './policy-gate';
|
|
228
246
|
export {
|
|
247
|
+
DEFAULT_MAX_PRESENCE_MEMBERS,
|
|
229
248
|
PRESENCE_KEY_PREFIX,
|
|
249
|
+
PRESENCE_SWEEP_PREFIX,
|
|
230
250
|
type PresenceInput,
|
|
231
251
|
type PresenceOptions,
|
|
232
252
|
PresenceRegistry,
|
|
253
|
+
type PresenceRoster,
|
|
233
254
|
presenceFrame,
|
|
234
255
|
} from './presence';
|
|
256
|
+
/** The typed projection: one query bound to one named hook, `useLiveFeed({ orgId })`. */
|
|
257
|
+
export {
|
|
258
|
+
type LiveQueryHook,
|
|
259
|
+
type LiveQuerySource,
|
|
260
|
+
liveHookFor,
|
|
261
|
+
} from './query-hook';
|
|
262
|
+
export {
|
|
263
|
+
createEntry,
|
|
264
|
+
fillWindow,
|
|
265
|
+
orgIdOf,
|
|
266
|
+
type QueryEntry,
|
|
267
|
+
refillWindowInLane,
|
|
268
|
+
} from './query-window';
|
|
235
269
|
export {
|
|
236
270
|
type ConflictStrategy,
|
|
237
271
|
type CustomMerge,
|
|
@@ -249,30 +283,61 @@ export {
|
|
|
249
283
|
export {
|
|
250
284
|
type AdvisoryLock,
|
|
251
285
|
CHANGE_SUBJECT_PREFIX,
|
|
286
|
+
type ChangeEnvelope,
|
|
252
287
|
changeSubject,
|
|
253
288
|
createReplicator,
|
|
254
289
|
InMemoryAdvisoryLock,
|
|
255
290
|
normalize,
|
|
256
291
|
parseChange,
|
|
292
|
+
parseEnvelope,
|
|
257
293
|
type Replicator,
|
|
258
294
|
type ReplicatorOptions,
|
|
259
295
|
type ReplicatorStats,
|
|
296
|
+
SeqGapDetector,
|
|
260
297
|
} from './replicator';
|
|
261
298
|
export {
|
|
262
299
|
actorIdOf,
|
|
263
300
|
CLOSE,
|
|
301
|
+
DEFAULT_FRAME_BURST,
|
|
302
|
+
DEFAULT_MAX_BUFFERED_BYTES,
|
|
303
|
+
DEFAULT_MAX_FRAMES_PER_SECOND,
|
|
264
304
|
SocketRegistry,
|
|
265
305
|
type SocketRegistryOptions,
|
|
266
306
|
SyncSocket,
|
|
267
307
|
type SyncSocketOptions,
|
|
268
308
|
type WsLike,
|
|
269
309
|
} from './socket';
|
|
310
|
+
export type {
|
|
311
|
+
GateFailed,
|
|
312
|
+
GateStage,
|
|
313
|
+
RowDenied,
|
|
314
|
+
Subscriber,
|
|
315
|
+
SubscriberGateOptions,
|
|
316
|
+
} from './subscriber-gate';
|
|
317
|
+
export {
|
|
318
|
+
GrantBook,
|
|
319
|
+
type GrantSweepDeps,
|
|
320
|
+
type GrantSweepResult,
|
|
321
|
+
type SyncAuthenticator,
|
|
322
|
+
type SyncGrant,
|
|
323
|
+
sweepGrants,
|
|
324
|
+
} from './sync-auth';
|
|
325
|
+
export {
|
|
326
|
+
createFrameRouter,
|
|
327
|
+
type FrameRouter,
|
|
328
|
+
type FrameRouterOptions,
|
|
329
|
+
type MutationHandler,
|
|
330
|
+
} from './sync-frames';
|
|
270
331
|
export {
|
|
271
|
-
createSyncNode,
|
|
272
332
|
type ListenOptions,
|
|
273
333
|
listenSyncNode,
|
|
274
|
-
type MutationHandler,
|
|
275
334
|
type SyncListener,
|
|
335
|
+
} from './sync-listen';
|
|
336
|
+
export {
|
|
337
|
+
createSyncNode,
|
|
338
|
+
DEFAULT_MAX_CONNECTIONS,
|
|
339
|
+
DEFAULT_MAX_FRAME_BYTES,
|
|
340
|
+
DEFAULT_REAUTH_INTERVAL_MS,
|
|
276
341
|
type SyncNode,
|
|
277
342
|
type SyncNodeOptions,
|
|
278
343
|
type SyncWs,
|
|
@@ -286,6 +351,7 @@ export {
|
|
|
286
351
|
decode,
|
|
287
352
|
encode,
|
|
288
353
|
FRAME_KINDS,
|
|
354
|
+
FRAME_LIMITS,
|
|
289
355
|
type Frame,
|
|
290
356
|
type FrameKind,
|
|
291
357
|
type HelloFrame,
|
|
@@ -316,6 +382,8 @@ export {
|
|
|
316
382
|
type ReconnectReason,
|
|
317
383
|
type Rng,
|
|
318
384
|
reconnectFrame,
|
|
385
|
+
type Scheduler,
|
|
386
|
+
timeoutScheduler,
|
|
319
387
|
} from './thundering-herd';
|
|
320
388
|
export {
|
|
321
389
|
DEFAULT_PRESENCE_BUCKET,
|
package/src/json.ts
CHANGED
|
@@ -49,8 +49,28 @@ export function changedColumns(before: JsonObject | null, after: JsonObject): Js
|
|
|
49
49
|
return out;
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Key-sorted JSON so a query id derived from input is stable across property order — and
|
|
54
|
+
* INJECTIVE, because that id decides who shares a window.
|
|
55
|
+
*
|
|
56
|
+
* `qidOf` is `stableDigest(canonicalJson(input))` and a qid HIT hands the joiner the existing
|
|
57
|
+
* entry: the first subscriber's compiled source, its matcher and its seated rows. Two inputs that
|
|
58
|
+
* canonicalise to one string are therefore two clients served out of one window. `JSON.stringify`
|
|
59
|
+
* is not injective over numbers — `NaN` and `±Infinity` are both `"null"`, which also collides
|
|
60
|
+
* with JSON `null` itself, and `-0` is `"0"` — so the number branch is spelled out here.
|
|
61
|
+
*
|
|
62
|
+
* `-0` is the one of those a client can put on the wire (`JSON.parse('{"a":-0}')` answers `-0`);
|
|
63
|
+
* the non-finite three have no JSON spelling and arrive only from a caller building `input` in JS,
|
|
64
|
+
* such as `useLive(feed, () => ({ limit: Number.parseInt(raw) }))` on an unparseable `raw`.
|
|
65
|
+
* The tokens are bare, never quoted: this output is only ever hashed, and the `string` branch
|
|
66
|
+
* always quotes, so an unquoted word cannot collide with the text that spells it.
|
|
67
|
+
*
|
|
68
|
+
* The twin of `@ultimat3/query`'s and `@ultimat3/action`'s rules in their own `stable.ts`. All
|
|
69
|
+
* three are tier 3, so no two of them can import each other; the shared home is `@ultimat3/core`
|
|
70
|
+
* if one is ever made.
|
|
71
|
+
*/
|
|
53
72
|
export function canonicalJson(value: JsonValue): string {
|
|
73
|
+
if (typeof value === 'number') return canonicalNumber(value);
|
|
54
74
|
if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null';
|
|
55
75
|
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
|
|
56
76
|
const keys = Object.keys(value).sort();
|
|
@@ -58,6 +78,23 @@ export function canonicalJson(value: JsonValue): string {
|
|
|
58
78
|
return `{${parts.join(',')}}`;
|
|
59
79
|
}
|
|
60
80
|
|
|
81
|
+
/** Ordinary numbers are `String(n)`, byte-identical to what this emitted before. */
|
|
82
|
+
function canonicalNumber(value: number): string {
|
|
83
|
+
if (Number.isNaN(value)) return 'NaN';
|
|
84
|
+
if (!Number.isFinite(value)) return value > 0 ? 'Infinity' : '-Infinity';
|
|
85
|
+
return Object.is(value, -0) ? '-0' : String(value);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* SHA-256, first 16 hex characters. For the hashes that are also SHARING keys — a `qid` decides
|
|
90
|
+
* which subscribers are served from one window, and it is derived from input a client chooses, so
|
|
91
|
+
* the 32 bits `fnv1a` answers are a collision anyone can find offline in seconds. Same primitive
|
|
92
|
+
* and same width `@ultimat3/entity`'s `planScope` already chose for a cursor's scope.
|
|
93
|
+
*/
|
|
94
|
+
export function stableDigest(text: string): string {
|
|
95
|
+
return new Bun.CryptoHasher('sha256').update(text).digest('hex').slice(0, 16);
|
|
96
|
+
}
|
|
97
|
+
|
|
61
98
|
/** FNV-1a, 32-bit, hex. Not cryptographic — it identifies and detects drift, it does not protect. */
|
|
62
99
|
export function fnv1a(text: string): string {
|
|
63
100
|
let hash = 0x811c9dc5;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// What a live query IS: the id one is keyed by, the contract a definition satisfies, and the
|
|
2
|
+
// subscription one socket holds. Split from `live-query.ts` because four modules need the shape
|
|
3
|
+
// and none of them needs the registry that runs it — and because one file runs one job.
|
|
4
|
+
|
|
5
|
+
import type { Actor } from '@ultimat3/core';
|
|
6
|
+
import type { LiveCursor } from './cursor';
|
|
7
|
+
import { canonicalJson, type JsonValue, type Row, stableDigest } from './json';
|
|
8
|
+
import type { IncrementalMatcher } from './matcher-bridge';
|
|
9
|
+
import type { SyncSocket } from './socket';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* `qid` = hash(query name, input). Fanout subjects and change windows are keyed by it.
|
|
13
|
+
*
|
|
14
|
+
* The hash is a **sharing** key, which is why it is `stableDigest` and not `fnv1a`: `#entryFor`
|
|
15
|
+
* answers a hit with the EXISTING entry and `liveQueryDefinition` answers with the seated
|
|
16
|
+
* `SharedWindow`, both carrying the first subscriber's input, compiled source and rows. Input is
|
|
17
|
+
* client-chosen, so a second input colliding with the first passes `authorize` against its own
|
|
18
|
+
* arguments and is then served out of somebody else's window — and 32 bits is a collision found
|
|
19
|
+
* offline in seconds.
|
|
20
|
+
*/
|
|
21
|
+
export function qidOf(name: string, input: JsonValue): string {
|
|
22
|
+
return `${name}:${stableDigest(canonicalJson(input))}`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface SnapshotResult<R extends Row = Row> {
|
|
26
|
+
readonly rows: readonly R[];
|
|
27
|
+
readonly lsn: string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface LiveQueryDefinition<R extends Row = Row> {
|
|
31
|
+
readonly name: string;
|
|
32
|
+
/** Dependency set for the pre-filter. `x verify` rejects a `live: true` query without one. */
|
|
33
|
+
readonly entities: readonly string[];
|
|
34
|
+
/** Read set. Lets the pre-filter skip updates that touch no column this query reads. */
|
|
35
|
+
readonly columns?: readonly string[];
|
|
36
|
+
/** Bounded read (`orderBy` + `limit`, enforced by `x verify`), unfiltered by policy. */
|
|
37
|
+
snapshot(args: { input: JsonValue }): Promise<SnapshotResult<R>>;
|
|
38
|
+
/** Subscribe-time gate. Throws to deny — the same `policy` used by HTTP, jobs, and MCP. */
|
|
39
|
+
authorize?(args: { actor: Actor | null; input: JsonValue }): void | Promise<void>;
|
|
40
|
+
/** Row-level gate, evaluated per subscriber. The only row filter in the pipeline. */
|
|
41
|
+
visible(args: { actor: Actor | null; row: R; input: JsonValue }): boolean | Promise<boolean>;
|
|
42
|
+
/** Built once per `qid`, since a qid pins both the query and its input. */
|
|
43
|
+
matcher(input: JsonValue): IncrementalMatcher;
|
|
44
|
+
/**
|
|
45
|
+
* The entity every row of this result set belongs to, resolved per input exactly as `matcher`
|
|
46
|
+
* is. It is the client's identity scope, and it can only come from here: a browser cannot
|
|
47
|
+
* compile the shape a `sql` produces. `null` means "not stated", and the client then keeps the
|
|
48
|
+
* rows private to that one subscription rather than guessing.
|
|
49
|
+
*/
|
|
50
|
+
rowEntity?(input: JsonValue): string | null;
|
|
51
|
+
/**
|
|
52
|
+
* Resolve whatever this input needs before an entry is built. `matcher` is synchronous by
|
|
53
|
+
* design — a change event must not await anything — so a definition that has to compile a
|
|
54
|
+
* source or a shape does it here, after `authorize` allowed this subscriber and before the
|
|
55
|
+
* shared window exists.
|
|
56
|
+
*/
|
|
57
|
+
prepare?(input: JsonValue): Promise<void>;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export interface LiveSubscription {
|
|
61
|
+
readonly sid: string;
|
|
62
|
+
readonly qid: string;
|
|
63
|
+
readonly socket: SyncSocket;
|
|
64
|
+
readonly input: JsonValue;
|
|
65
|
+
readonly definition: LiveQueryDefinition;
|
|
66
|
+
cursor: LiveCursor;
|
|
67
|
+
}
|
package/src/live-definition.ts
CHANGED
|
@@ -9,10 +9,10 @@
|
|
|
9
9
|
// every time. Collapsing the second onto the first is privilege escalation with a cache hit rate.
|
|
10
10
|
|
|
11
11
|
import type { Ctx } from '@ultimat3/core';
|
|
12
|
-
import { type AnyQuery, queryName
|
|
12
|
+
import { type AnyQuery, queryName } from '@ultimat3/query';
|
|
13
13
|
import { LiveRowUnidentifiedError } from './errors';
|
|
14
14
|
import { isRow, type JsonValue, type Row } from './json';
|
|
15
|
-
import { type LiveQueryDefinition, qidOf, type SnapshotResult } from './live-
|
|
15
|
+
import { type LiveQueryDefinition, qidOf, type SnapshotResult } from './live-contract';
|
|
16
16
|
import { type IncrementalMatcher, matcherFor } from './matcher-bridge';
|
|
17
17
|
import { authorizeWithPolicy, visibleWithPolicy } from './policy-gate';
|
|
18
18
|
|
|
@@ -44,6 +44,8 @@ export interface LiveDefinitionOptions {
|
|
|
44
44
|
*/
|
|
45
45
|
interface SharedWindow {
|
|
46
46
|
readonly matcher: IncrementalMatcher;
|
|
47
|
+
/** The compiled shape's root entity — the client's identity scope for every row of this read. */
|
|
48
|
+
readonly rowEntity: string;
|
|
47
49
|
read(): Promise<readonly Row[]>;
|
|
48
50
|
}
|
|
49
51
|
|
|
@@ -81,17 +83,17 @@ export function liveQueryDefinition(
|
|
|
81
83
|
enforce: false,
|
|
82
84
|
...(options.epoch === undefined ? {} : { epoch: options.epoch }),
|
|
83
85
|
});
|
|
84
|
-
//
|
|
85
|
-
//
|
|
86
|
-
//
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
enforce: false,
|
|
90
|
-
surface: 'live',
|
|
91
|
-
});
|
|
86
|
+
// One build, and the window reads through it. `live.execute()` runs the source the shape and
|
|
87
|
+
// the dependency set above were taken from, so the rows a subscriber is served and the matcher
|
|
88
|
+
// that patches them describe the same `(query, input)` by construction. Asking `sourceFor` for
|
|
89
|
+
// a second subject-less copy — which is what this did — paid for the parse and the `sql()`
|
|
90
|
+
// twice per query id and left two descriptions of one read that agreed only by luck.
|
|
92
91
|
const built: SharedWindow = {
|
|
93
92
|
matcher: matcherFor(live),
|
|
94
|
-
|
|
93
|
+
// `assertMatchable` already refused a shape without one, so this is the entity the matcher
|
|
94
|
+
// patches rows of — the same name `ChangeEvent.entity` and `tx.<table>` use.
|
|
95
|
+
rowEntity: live.shape.entity,
|
|
96
|
+
read: async () => rowsOf(name, await live.execute()),
|
|
95
97
|
};
|
|
96
98
|
windows.set(qid, built);
|
|
97
99
|
evictOldest(windows, options.maxWindows ?? 256);
|
|
@@ -112,6 +114,9 @@ export function liveQueryDefinition(
|
|
|
112
114
|
return { rows: await window.read(), lsn: options.lsn?.() ?? '' };
|
|
113
115
|
},
|
|
114
116
|
matcher: (input) => windows.get(qidOf(name, input))?.matcher ?? UNRESOLVED,
|
|
117
|
+
// Read off the same resolved window as the matcher, so the scope the client keys rows under and
|
|
118
|
+
// the entity the matcher patches them from can never be two different names.
|
|
119
|
+
rowEntity: (input) => windows.get(qidOf(name, input))?.rowEntity ?? null,
|
|
115
120
|
// The two per-subscriber gates, both through the package's one authz seam. Neither result is
|
|
116
121
|
// memoised anywhere: `authorize` runs on every subscribe, `visible` on every row of every
|
|
117
122
|
// delivery, and there is no key here an actor could share with another actor.
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// One change, one query id, inside that query's lane: match it, fold it into the shared window,
|
|
2
|
+
// then one policy pass per subscriber over what came out. Split from `live-query.ts` because the
|
|
3
|
+
// registry owns the lanes and the entry table while this owns what happens inside one of them —
|
|
4
|
+
// and because one file runs one job. The lane itself is never taken here: the caller is holding it.
|
|
5
|
+
|
|
6
|
+
import type { Clock } from '@ultimat3/core';
|
|
7
|
+
import type { ChangeEvent } from './changefeed';
|
|
8
|
+
import { advance, type LiveCursor, makeCursor, type ResumeSource } from './cursor';
|
|
9
|
+
import type { Row, RowPatch } from './json';
|
|
10
|
+
import type { LiveSubscription } from './live-contract';
|
|
11
|
+
import { applyToWindow, bridgeChange } from './matcher-bridge';
|
|
12
|
+
import { type QueryEntry, refillWindowInLane } from './query-window';
|
|
13
|
+
import type { Subscriber, SubscriberGate } from './subscriber-gate';
|
|
14
|
+
import { type Frame, PROTOCOL_VERSION } from './sync-protocol';
|
|
15
|
+
|
|
16
|
+
export interface FanoutDeps {
|
|
17
|
+
readonly gate: SubscriberGate;
|
|
18
|
+
readonly source: ResumeSource;
|
|
19
|
+
readonly clock: Clock;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface FanoutResult {
|
|
23
|
+
/** Frames that left this node for this query id. */
|
|
24
|
+
readonly sent: number;
|
|
25
|
+
/** `1` when the change was at or below the window's own lsn — `live.changes_stale`. */
|
|
26
|
+
readonly stale: number;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* The fanout for one entry, run inside that entry's lane — so the window this mutates at the top is
|
|
31
|
+
* still the window every subscriber's gate reads at the bottom, and the patches reach the retained
|
|
32
|
+
* buffer in the order the client will be asked to fold them.
|
|
33
|
+
*/
|
|
34
|
+
export async function fanoutChange(
|
|
35
|
+
deps: FanoutDeps,
|
|
36
|
+
entry: QueryEntry,
|
|
37
|
+
change: ChangeEvent,
|
|
38
|
+
): Promise<FanoutResult> {
|
|
39
|
+
// A window that missed a change must be replaced before it is patched again, and it can only be
|
|
40
|
+
// replaced here — a fanout holds this entry's lane, and `fillWindow` takes the same one.
|
|
41
|
+
if (entry.stale) await refillWindowInLane(entry);
|
|
42
|
+
// The consume-side twin of the replicator's own duplicate guard, which had none. `entry.lsn =
|
|
43
|
+
// change.lsn` was unconditional, so a change the window already holds — a redelivery, or one
|
|
44
|
+
// that arrived behind the snapshot that already included it — rewound every subscriber's cursor
|
|
45
|
+
// to it and asked them to fold state they had already folded over newer rows.
|
|
46
|
+
if (entry.lsn !== '' && change.lsn <= entry.lsn) return { sent: 0, stale: 1 };
|
|
47
|
+
const result = bridgeChange(entry.shape, entry.matcher, change, entry.rows);
|
|
48
|
+
if (!result) return { sent: 0, stale: 0 };
|
|
49
|
+
entry.lsn = change.lsn;
|
|
50
|
+
entry.rows = applyToWindow(entry.rows, result.patches);
|
|
51
|
+
// The window lost its tail, so what it holds is a guess — the next delivery re-reads it rather
|
|
52
|
+
// than patching a guess, and every subscriber below is re-snapshotted out of what that returns.
|
|
53
|
+
if (result.refill) entry.stale = true;
|
|
54
|
+
// The retained window holds the pre-policy patch; resume re-filters it per subscriber.
|
|
55
|
+
for (const patch of result.patches) deps.source.append(entry.qid, patch);
|
|
56
|
+
|
|
57
|
+
let sent = 0;
|
|
58
|
+
for (const subscription of entry.subscribers.values()) {
|
|
59
|
+
if (result.refill) {
|
|
60
|
+
// The window lost its tail: guessing is how a sync engine silently diverges. Checked BEFORE
|
|
61
|
+
// the mark, because a repair reads `entry.rows` — which this fanout has just declared a
|
|
62
|
+
// guess — and then CLEARS the mark. A subscriber already diverged would be recorded as
|
|
63
|
+
// repaired against rows nothing trusts, and the next change, having refilled the window,
|
|
64
|
+
// sends it a patch instead of the snapshot it is still owed. A lost tail degrades every
|
|
65
|
+
// subscriber the same way, whatever each was holding.
|
|
66
|
+
subscription.socket.markDesynced(subscription.sid);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
// `desynced` had four writers and no reader: a subscriber whose patch was dropped by
|
|
70
|
+
// backpressure, whose gate failed, or whose window lost its tail was recorded as diverged and
|
|
71
|
+
// then served the next patch as if nothing had happened — permanently and silently stale on a
|
|
72
|
+
// healthy socket. A marked subscriber is re-snapshotted out of the shared window instead, at
|
|
73
|
+
// the cost of one frame and no DB read, and only then is the mark cleared.
|
|
74
|
+
if (subscription.socket.desynced.has(subscription.sid)) {
|
|
75
|
+
if (await resnapshot(deps, entry, subscription)) sent += 1;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
const who: Subscriber = { sid: subscription.sid, actor: subscription.socket.actor };
|
|
79
|
+
let allowed: readonly RowPatch[];
|
|
80
|
+
try {
|
|
81
|
+
allowed = await deps.gate.filterPatches(
|
|
82
|
+
entry,
|
|
83
|
+
who,
|
|
84
|
+
result.patches,
|
|
85
|
+
new Set(subscription.cursor.ids),
|
|
86
|
+
);
|
|
87
|
+
} catch {
|
|
88
|
+
// Already counted and reported as a gate failure. Degrade this one subscriber the way a
|
|
89
|
+
// lost window tail degrades them — desynced, re-snapshotted on the next flush — because
|
|
90
|
+
// rejecting here would abandon the fanout to every other subscriber over one actor's
|
|
91
|
+
// broken rule, and delivering the patches anyway would be the leak.
|
|
92
|
+
subscription.socket.markDesynced(subscription.sid);
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
if (allowed.length === 0) continue;
|
|
96
|
+
const frame: Frame = {
|
|
97
|
+
type: 'patch',
|
|
98
|
+
v: PROTOCOL_VERSION,
|
|
99
|
+
sid: subscription.sid,
|
|
100
|
+
patches: allowed,
|
|
101
|
+
lsn: change.lsn,
|
|
102
|
+
};
|
|
103
|
+
if (subscription.socket.send(frame)) {
|
|
104
|
+
subscription.cursor = advance(subscription.cursor, allowed, change.lsn, change.at);
|
|
105
|
+
sent += 1;
|
|
106
|
+
} else {
|
|
107
|
+
subscription.socket.markDesynced(subscription.sid);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return { sent, stale: 0 };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* The repair for one diverged subscriber, out of the window the lane is already holding — no DB
|
|
115
|
+
* read, one frame. Its cursor is rebuilt from what this subscriber may actually see, exactly as
|
|
116
|
+
* `subscribe` does, because a cursor over the pre-policy window would claim ids the client was
|
|
117
|
+
* never sent. The mark is cleared only on a frame that left: a send refused by backpressure keeps
|
|
118
|
+
* the subscriber diverged, which is the state it is actually in.
|
|
119
|
+
*/
|
|
120
|
+
async function resnapshot(
|
|
121
|
+
deps: FanoutDeps,
|
|
122
|
+
entry: QueryEntry,
|
|
123
|
+
subscription: LiveSubscription,
|
|
124
|
+
): Promise<boolean> {
|
|
125
|
+
const who: Subscriber = { sid: subscription.sid, actor: subscription.socket.actor };
|
|
126
|
+
let rows: readonly Row[];
|
|
127
|
+
try {
|
|
128
|
+
rows = await deps.gate.filterRows(entry, who, entry.rows);
|
|
129
|
+
} catch {
|
|
130
|
+
// Counted and reported as a gate failure already. It stays desynced: a subscriber whose rule
|
|
131
|
+
// cannot decide is not one to serve rows to, and the next change tries again.
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
const cursor = makeCursor(entry.qid, entry.lsn, rows, deps.clock.now().getTime());
|
|
135
|
+
if (!subscription.socket.send(snapshotFrame(entry, subscription.sid, rows, cursor))) return false;
|
|
136
|
+
subscription.cursor = cursor;
|
|
137
|
+
subscription.socket.clearDesynced(subscription.sid);
|
|
138
|
+
return true;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** The one place a snapshot frame is built, so the identity scope cannot be told to one caller only. */
|
|
142
|
+
export function snapshotFrame(
|
|
143
|
+
entry: QueryEntry,
|
|
144
|
+
sid: string,
|
|
145
|
+
rows: readonly Row[],
|
|
146
|
+
cursor: LiveCursor,
|
|
147
|
+
): Frame {
|
|
148
|
+
const base = { type: 'snapshot', v: PROTOCOL_VERSION, sid, rows, cursor } as const;
|
|
149
|
+
return entry.rowEntity === null ? base : { ...base, entity: entry.rowEntity };
|
|
150
|
+
}
|