@ultimat3/realtime 22.15.0 → 24.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/CLAUDE.md +89 -102
  2. package/README.md +140 -20
  3. package/package.json +4 -4
  4. package/src/advisory-lock.ts +57 -0
  5. package/src/boot.ts +12 -7
  6. package/src/change-buffer.ts +96 -41
  7. package/src/changefeed-env.ts +2 -2
  8. package/src/changefeed.ts +39 -1
  9. package/src/channel-authz.ts +43 -12
  10. package/src/channel-latch.ts +52 -0
  11. package/src/channel-logs.ts +34 -4
  12. package/src/channel-records.ts +33 -2
  13. package/src/channel-sids.ts +37 -0
  14. package/src/channel.ts +143 -44
  15. package/src/client-channels.ts +10 -1
  16. package/src/client-contract.ts +4 -1
  17. package/src/client-frames.ts +34 -4
  18. package/src/client-heartbeat.ts +39 -9
  19. package/src/client.ts +1 -0
  20. package/src/cursor.ts +11 -0
  21. package/src/errors.ts +70 -19
  22. package/src/fanout.ts +13 -0
  23. package/src/index.ts +1 -1
  24. package/src/live-contract.ts +6 -2
  25. package/src/live-definition.ts +46 -11
  26. package/src/live-fanout.ts +70 -11
  27. package/src/live-floor.ts +35 -0
  28. package/src/live-query-options.ts +38 -0
  29. package/src/live-query.ts +119 -91
  30. package/src/live-refusal.ts +21 -0
  31. package/src/live-replicator.ts +6 -6
  32. package/src/live-resume.ts +74 -0
  33. package/src/live-tenant.ts +25 -0
  34. package/src/nats-client.ts +11 -2
  35. package/src/nats-open.ts +9 -0
  36. package/src/nats-transport.ts +23 -1
  37. package/src/offline-queue.ts +48 -5
  38. package/src/page-errors.ts +17 -0
  39. package/src/page-outbox.ts +121 -50
  40. package/src/pg-advisory-lock.ts +45 -1
  41. package/src/pg-auth.ts +9 -4
  42. package/src/pg-bytes.ts +4 -1
  43. package/src/pg-connection.ts +119 -21
  44. package/src/pg-entity-row.ts +27 -2
  45. package/src/pg-identity-tables.ts +25 -0
  46. package/src/pg-preflight.ts +50 -4
  47. package/src/pg-replication-row.ts +82 -0
  48. package/src/pg-replication.ts +44 -45
  49. package/src/pg-socket.ts +6 -4
  50. package/src/pg-tls.ts +18 -0
  51. package/src/pg-wire.ts +5 -3
  52. package/src/pgoutput.ts +19 -9
  53. package/src/presence.ts +85 -8
  54. package/src/query-window.ts +28 -7
  55. package/src/record-persister.ts +7 -2
  56. package/src/replication-errors.ts +8 -4
  57. package/src/replicator-envelope.ts +110 -0
  58. package/src/replicator-row.ts +75 -0
  59. package/src/replicator.ts +319 -186
  60. package/src/server.ts +11 -7
  61. package/src/socket-engine.ts +14 -2
  62. package/src/socket-idle.ts +3 -1
  63. package/src/socket-routes.ts +8 -3
  64. package/src/subscriber-gate.ts +11 -1
  65. package/src/sync-auth.ts +47 -2
  66. package/src/sync-frames.ts +73 -20
  67. package/src/sync-node-bounds.ts +15 -0
  68. package/src/sync-node-contract.ts +20 -2
  69. package/src/sync-node.ts +92 -17
  70. package/src/sync-origin.ts +26 -11
  71. package/src/sync-protocol.ts +13 -3
  72. package/src/sync-upgrade.ts +76 -8
  73. package/src/use-mutation.ts +33 -5
  74. package/src/wire-channel.ts +13 -4
package/src/live-query.ts CHANGED
@@ -1,63 +1,31 @@
1
1
  // Tier 2: live queries. Registration, per-subscriber authz, snapshot, patch stream.
2
2
  //
3
3
  // The rule this file exists to enforce: **policy is evaluated once per subscriber, never once per
4
- // query**. The DB read is shared across subscribers of the same query id; the authz decision is
5
- // not. Two actors on one live query see two different result sets, and a row that fails an actor's
4
+ // query**. The DB read is shared across subscribers of the same query id IN ONE TENANT
5
+ // (`live-tenant.ts`); the authz decision is not shared at all. Two actors on one live query see two different result sets, and a row that fails an actor's
6
6
  // policy is never sent to that actor — it arrives as a `delete` if they hold it, and is dropped
7
7
  // otherwise.
8
8
 
9
- import { type Actor, type Clock, finiteOption, systemClock, uuid } from '@ultimat3/core';
9
+ import { type Clock, finiteOption, systemClock, uuid } from '@ultimat3/core';
10
10
  import { queryHash } from '@ultimat3/query';
11
11
  import type { ChangeEvent } from './changefeed';
12
- import {
13
- advance,
14
- type LiveCursor,
15
- makeCursor,
16
- type ReconnectBudget,
17
- type ResumeSource,
18
- resumeFrom,
19
- } from './cursor';
12
+ import { type LiveCursor, makeCursor } from './cursor';
20
13
  import { isPolicyDenial, LiveQueryUnknownError, SubscriptionLimitError } from './errors';
21
14
  import type { JsonValue } from './json';
22
15
  import type { LiveQueryDefinition, LiveSubscription, SnapshotResult } from './live-contract';
23
16
  import { type FanoutDeps, fanoutChange, snapshotFrame } from './live-fanout';
17
+ import { floorAfterRead } from './live-floor';
18
+ import { DEFAULT_MAX_ENTRIES, type LiveQueryRegistryOptions } from './live-query-options';
19
+ import { refuseSubscription } from './live-refusal';
20
+ import { resumeOnto } from './live-resume';
21
+ import { liveTenantOf, windowId } from './live-tenant';
24
22
  import { createEntry, fillWindow, type QueryEntry } from './query-window';
25
23
  import type { SyncSocket } from './socket';
26
- import { type Subscriber, SubscriberGate, type SubscriberGateOptions } from './subscriber-gate';
24
+ import { type Subscriber, SubscriberGate } from './subscriber-gate';
27
25
  import { SubscriptionBook, subscriptionKey } from './subscription-book';
28
- import { type Frame, PROTOCOL_VERSION } from './sync-protocol';
29
- import type { Scheduler } from './thundering-herd';
26
+ import type { Frame } from './sync-protocol';
30
27
 
31
- export interface LiveQueryRegistryOptions extends SubscriberGateOptions {
32
- readonly source: ResumeSource;
33
- readonly budget?: ReconnectBudget;
34
- readonly clock?: Clock;
35
- readonly maxPerSocket?: number;
36
- readonly maxPerTenant?: number;
37
- readonly tenantOf?: (actor: Actor | null) => string | null;
38
- /**
39
- * Distinct `(query, input)` pairs this node will hold at once. A `qid` derives from
40
- * client-chosen input, so without a ceiling one socket mints entries — a matcher, a row window,
41
- * a `WindowLock` and a fanout target each — until the process dies.
42
- */
43
- readonly maxEntries?: number;
44
- /**
45
- * How long one entry's SHARED snapshot read may hold its slot. Defaults to
46
- * `DEFAULT_READ_DEADLINE_MS`. Without it a `definition.snapshot` that never settles pinned the
47
- * slot for the life of the process and every later cold subscriber joined a promise nothing
48
- * would resolve — one wedged read taking every future subscriber of that query id with it.
49
- */
50
- readonly readDeadlineMs?: number;
51
- /** Injected so that deadline is provable without waiting for one. Production uses `setTimeout`. */
52
- readonly schedule?: Scheduler;
53
- }
54
-
55
- /**
56
- * Live `(query, input)` pairs one node holds. Reached, the next NEW pair is refused with
57
- * `X_SUBSCRIPTION_LIMIT`; subscribing to a pair that already exists keeps working, because the
58
- * cost this bounds is the entry, not the subscriber.
59
- */
60
- export const DEFAULT_MAX_ENTRIES = 10_000;
28
+ export { DEFAULT_MAX_ENTRIES, type LiveQueryRegistryOptions } from './live-query-options';
61
29
 
62
30
  export class LiveQueryRegistry {
63
31
  readonly #definitions = new Map<string, LiveQueryDefinition>();
@@ -71,6 +39,11 @@ export class LiveQueryRegistry {
71
39
  /** What one lane needs, and nothing this class holds beyond it. */
72
40
  readonly #fanout: FanoutDeps;
73
41
  #lastLsn = '';
42
+ /**
43
+ * This node's mark for a window read before it held any position: unique to the process, and
44
+ * `!` sorts below every hex digit, so every real change is above it.
45
+ */
46
+ readonly #origin = `!${uuid()}`;
74
47
  #staleChanges = 0;
75
48
 
76
49
  constructor(options: LiveQueryRegistryOptions) {
@@ -111,10 +84,15 @@ export class LiveQueryRegistry {
111
84
  * whose lsn never moved, subscribers whose cursors never moved, and therefore nothing that would
112
85
  * ever ask for a re-snapshot. The repair lands on the next change to each query — which is the
113
86
  * event that proves the query is moving at all.
87
+ *
88
+ * `entity` narrows it to the windows that read that entity: a filtered write
89
+ * (`updateWhere`/`deleteWhere`) names its entity but no row, and staling every window on the node
90
+ * for it re-read every live query in the process on each bulk write to any table.
114
91
  */
115
- invalidate(): number {
92
+ invalidate(entity?: string): number {
116
93
  let marked = 0;
117
94
  for (const entry of this.#entries.values()) {
95
+ if (entity !== undefined && !entry.shape.entities.includes(entity)) continue;
118
96
  entry.stale = true;
119
97
  for (const subscription of entry.subscribers.values()) {
120
98
  subscription.socket.markDesynced(subscription.sid);
@@ -165,7 +143,15 @@ export class LiveQueryRegistry {
165
143
  // dispatched concurrently, N of them reading a count nothing had grown yet.
166
144
  const slot = this.#book.reserve(args.socket, sid);
167
145
  try {
168
- return await this.#subscribeReserved(definition, sid, args);
146
+ for (;;) {
147
+ // `authorize`, the read and the row pass all await, and a re-auth that lands under them
148
+ // re-decides only what is ATTACHED. So a subscription seated for an actor the socket no
149
+ // longer carries is taken back and served again under the one it does.
150
+ const actor = args.socket.actor;
151
+ const served = await this.#subscribeReserved(definition, sid, args);
152
+ if (args.socket.actor === actor) return served;
153
+ this.unsubscribe(args.socket.id, sid);
154
+ }
169
155
  } finally {
170
156
  // After the attach on every path, so the slot is only ever given back to a count that has
171
157
  // already grown — or, on a failure, to one that never will.
@@ -188,8 +174,11 @@ export class LiveQueryRegistry {
188
174
  // may not subscribe is work an unauthorized client gets to schedule.
189
175
  await definition.prepare?.(args.input);
190
176
 
191
- const qid = queryHash(args.name, args.input);
192
- const entry = this.#entryFor(qid, definition, args.input);
177
+ // The window is the subscriber's TENANT's: two orgs on one `(query, input)` are two entries,
178
+ // two reads and two retained rings, so no shared row ever stands between them and a policy.
179
+ const tenant = liveTenantOf(args.socket.actor);
180
+ const qid = windowId(queryHash(args.name, args.input), tenant);
181
+ const entry = this.#entryFor(qid, definition, args.input, tenant);
193
182
  try {
194
183
  return await this.#serve(entry, sid, args);
195
184
  } catch (error) {
@@ -215,48 +204,30 @@ export class LiveQueryRegistry {
215
204
  const qid = entry.qid;
216
205
  const now = this.#clock.now().getTime();
217
206
 
218
- if (args.cursor) {
219
- const cursor = args.cursor;
220
- const resumed = await resumeFrom(cursor, {
221
- source: this.#options.source,
222
- ...(this.#options.budget ? { budget: this.#options.budget } : {}),
223
- clock: this.#clock,
224
- snapshot: async () => await this.#read(entry, { sid, actor: args.socket.actor }),
225
- });
226
- if (resumed.kind === 'delta') {
227
- // The gate decides about whole rows out of the shared window, and an entry nothing has read
228
- // yet has none — every patch would meet an empty window and be withheld. Filling is
229
- // conditional on purpose: a restart storm resumes onto entries that already hold a live
230
- // window, and re-reading per resuming subscriber is the cost a delta resume exists to skip.
231
- if (entry.lsn === '') await fillWindow(entry);
232
- // The live entry on purpose: a resume runs outside the lane, so the window under it may
233
- // have moved on — always forwards, and a row whose grant was revoked in the meantime is
234
- // one this pass must refuse rather than replay from the state it had at the cursor's lsn.
235
- const patches = await this.#gate.filterPatches(
236
- entry,
237
- { sid, actor: args.socket.actor },
238
- resumed.patches,
239
- new Set(cursor.ids),
240
- );
241
- // Advanced over the FILTERED list, never over `resumed.cursor` — which `resumeFrom` built
242
- // by advancing across the retained window, and that window is PRE-POLICY. Seated as it
243
- // came back, this subscriber's `cursor.ids` gained the id of every row inserted for every
244
- // OTHER actor while it was away; `subscriber-gate` then reads `held.has(patch.id)` off it,
245
- // takes the "the subscriber holds this row" branch, and delivers a `delete` frame carrying
246
- // another tenant's row id and the instant it went. The leak that branch closes, re-opened
247
- // one layer up. `live-fanout.ts` advances over `allowed` for exactly this reason.
248
- const seated = advance(cursor, patches, resumed.cursor.lsn, now);
249
- const subscription = this.#attachUnlessGone(entry, args.socket, sid, seated);
250
- return {
251
- subscription,
252
- frame: { type: 'patch', v: PROTOCOL_VERSION, sid, patches, lsn: seated.lsn },
253
- };
254
- }
207
+ // A cursor is CLIENT data, and its `qid` names the window whose retained patches a resume
208
+ // replays. One that names another window — the same browser, signed in to another org, or a
209
+ // forged frame — is not a position in this one: it is a cold start, never a replay of a ring
210
+ // this subscriber's window does not own.
211
+ //
212
+ // Nor is ANY cursor a position in a window known to have missed a change: the ring still
213
+ // answers a delta for it, and that delta is the hole. A stale entry serves cold starts only.
214
+ if (args.cursor && args.cursor.qid === qid && !entry.stale) {
215
+ const who = { sid, actor: args.socket.actor };
216
+ const resumed = await resumeOnto(
217
+ {
218
+ source: this.#options.source,
219
+ budget: this.#options.budget,
220
+ clock: this.#clock,
221
+ gate: this.#gate,
222
+ read: () => this.#read(entry, who),
223
+ },
224
+ entry,
225
+ who,
226
+ args.cursor,
227
+ now,
228
+ );
255
229
  const subscription = this.#attachUnlessGone(entry, args.socket, sid, resumed.cursor);
256
- return {
257
- subscription,
258
- frame: snapshotFrame(entry, sid, resumed.rows, resumed.cursor),
259
- };
230
+ return { subscription, frame: resumed.frame };
260
231
  }
261
232
 
262
233
  const fresh = await this.#read(entry, { sid, actor: args.socket.actor });
@@ -314,6 +285,11 @@ export class LiveQueryRegistry {
314
285
  * what is no longer allowed. Survivors are marked desynced so the next flush re-snapshots them
315
286
  * under the new actor's row policy. Returns the sids that were dropped — a denial and nothing
316
287
  * else, so a caller may tell the client "you may no longer see this" and be right.
288
+ *
289
+ * A survivor whose actor now belongs to ANOTHER tenant cannot stay where it is: its window is the
290
+ * old org's. It is re-seated — dropped, subscribed again under the new tenant, and sent that
291
+ * window's snapshot under the same sid — or, when that cannot complete, refused on the socket
292
+ * under the sid (`refuseSubscription`), never kept on the old org's window.
317
293
  */
318
294
  async reauthorize(socket: SyncSocket): Promise<readonly string[]> {
319
295
  const dropped: string[] = [];
@@ -322,6 +298,7 @@ export class LiveQueryRegistry {
322
298
  // nobody updates is a count that drifts from the book for the rest of the process.
323
299
  this.#book.retenant(socket);
324
300
  for (const subscription of this.#book.ofSocket(socket.id)) {
301
+ let failed: { readonly error: unknown } | undefined;
325
302
  try {
326
303
  await subscription.definition.authorize?.({
327
304
  actor: socket.actor,
@@ -330,6 +307,9 @@ export class LiveQueryRegistry {
330
307
  } catch (error) {
331
308
  if (isPolicyDenial(error)) {
332
309
  this.unsubscribe(socket.id, subscription.sid);
310
+ // Said under the sid, as a refused subscribe is: unsaid, the client kept the rows on
311
+ // screen in state `live` for a subscription this node no longer serves.
312
+ refuseSubscription(socket, subscription.sid, error);
333
313
  dropped.push(subscription.sid);
334
314
  continue;
335
315
  }
@@ -343,12 +323,46 @@ export class LiveQueryRegistry {
343
323
  { sid: subscription.sid, actor: socket.actor },
344
324
  error,
345
325
  );
326
+ failed = { error };
346
327
  }
328
+ if (await this.#reseat(socket, subscription, failed)) continue;
347
329
  socket.markDesynced(subscription.sid);
348
330
  }
349
331
  return dropped;
350
332
  }
351
333
 
334
+ /**
335
+ * Move one subscription onto its actor's CURRENT tenant's window. `false` when it is already
336
+ * there. A re-seat that cannot complete leaves the subscription dropped — never attached to the
337
+ * window of an org its actor has left — and REFUSED on the socket under its sid, the frame a
338
+ * refused subscribe gets, so the client's window renders `failed` instead of going quiet.
339
+ *
340
+ * `failed` is the `authorize` this pass already saw fail: it is not asked again — the store that
341
+ * just timed out is the one `subscribe` would ask — and it is not counted twice.
342
+ */
343
+ async #reseat(
344
+ socket: SyncSocket,
345
+ subscription: LiveSubscription,
346
+ failed: { readonly error: unknown } | undefined,
347
+ ): Promise<boolean> {
348
+ const { sid, input, definition } = subscription;
349
+ const wanted = windowId(queryHash(definition.name, input), liveTenantOf(socket.actor));
350
+ if (wanted === subscription.qid) return false;
351
+ this.unsubscribe(socket.id, sid);
352
+ if (failed !== undefined) {
353
+ refuseSubscription(socket, sid, failed.error);
354
+ return true;
355
+ }
356
+ try {
357
+ const { frame } = await this.subscribe({ socket, name: definition.name, input, sid });
358
+ if (!socket.send(frame)) socket.markDesynced(sid);
359
+ } catch (error) {
360
+ this.#gate.failedAuthorize(wanted, { sid, actor: socket.actor }, error);
361
+ refuseSubscription(socket, sid, error);
362
+ }
363
+ return true;
364
+ }
365
+
352
366
  /**
353
367
  * Fan one change out. Matched once per query id, authorized once per subscriber. Returns the
354
368
  * number of frames sent — the metric the reconnect benchmark watches.
@@ -410,6 +424,10 @@ export class LiveQueryRegistry {
410
424
  cursor: LiveCursor,
411
425
  ): LiveSubscription {
412
426
  const subscription = this.#attach(entry, socket, sid, cursor);
427
+ // The policy pass ran outside the lane, so a change may have been folded into the window
428
+ // between the rows this subscriber was served and this attach: it is in neither their frame
429
+ // nor their patch stream. Marked, so the next delivery re-snapshots them out of the window.
430
+ if (entry.lsn !== cursor.lsn) socket.markDesynced(sid);
413
431
  if (socket.closed) this.unsubscribe(socket.id, sid);
414
432
  return subscription;
415
433
  }
@@ -437,7 +455,12 @@ export class LiveQueryRegistry {
437
455
  return subscription;
438
456
  }
439
457
 
440
- #entryFor(qid: string, definition: LiveQueryDefinition, input: JsonValue): QueryEntry {
458
+ #entryFor(
459
+ qid: string,
460
+ definition: LiveQueryDefinition,
461
+ input: JsonValue,
462
+ tenant: string | null,
463
+ ): QueryEntry {
441
464
  const existing = this.#entries.get(qid);
442
465
  if (existing) return existing;
443
466
  // The node-wide ceiling, refused where the entry would be born. `qid` derives from
@@ -454,6 +477,7 @@ export class LiveQueryRegistry {
454
477
  const created = createEntry(qid, definition, input, definition.matcher(input), {
455
478
  readDeadlineMs: this.#options.readDeadlineMs,
456
479
  schedule: this.#options.schedule,
480
+ tenant,
457
481
  });
458
482
  this.#entries.set(qid, created);
459
483
  return created;
@@ -466,6 +490,10 @@ export class LiveQueryRegistry {
466
490
  */
467
491
  async #read(entry: QueryEntry, who: Subscriber): Promise<SnapshotResult> {
468
492
  const window = await fillWindow(entry);
469
- return { rows: await this.#gate.filterRows(entry, who, window.rows), lsn: window.lsn };
493
+ const lsn = floorAfterRead(this.#options.source, entry, window, {
494
+ lastLsn: this.#lastLsn,
495
+ origin: this.#origin,
496
+ });
497
+ return { rows: await this.#gate.filterRows(entry, who, window.rows), lsn };
470
498
  }
471
499
  }
@@ -0,0 +1,21 @@
1
+ // A live subscription refused AFTER it was served: the frame a refused `subscribe` gets — an `ack`
2
+ // whose `ref` is the sid — sent by the node itself, so the client's window renders `failed` with
3
+ // the code, cause and fix instead of going quiet. `sync-node.ts` sends the same frame when a
4
+ // subscribe frame throws.
5
+
6
+ import { logger } from '@ultimat3/core';
7
+ import type { SyncSocket } from './socket';
8
+ import { PROTOCOL_VERSION, toWireError } from './sync-protocol';
9
+
10
+ export function refuseSubscription(socket: SyncSocket, sid: string, error: unknown): void {
11
+ const sent = socket.send({
12
+ type: 'ack',
13
+ v: PROTOCOL_VERSION,
14
+ ref: sid,
15
+ lsn: null,
16
+ error: toWireError(error),
17
+ });
18
+ // Read, never assumed: the subscription is already gone, so a dropped refusal leaves a client
19
+ // waiting on a window nothing serves — the log is the only trace it would leave.
20
+ if (!sent) logger.warn('sync.refusal_dropped', { socketId: socket.id, sid });
21
+ }
@@ -125,14 +125,14 @@ export async function startLiveReplicator(options: LiveReplicatorOptions): Promi
125
125
 
126
126
  /**
127
127
  * A filtered write names rows this seam never saw, so there is no event to shape. Every window
128
- * on the node is marked stale instead and re-read on the next change — `invalidate()` is the
129
- * node's own answer to "the change stream skipped something", used here for the one write that
130
- * genuinely does. Silence would be the alternative, and a subscriber told nothing happened
131
- * diverges with nobody ever asking again.
128
+ * over THAT entity is marked stale instead and re-read, its subscribers re-snapshotted, on the
129
+ * next change — `invalidate()` is the node's own answer to "the change stream skipped
130
+ * something", used here for the one write that genuinely does. Silence would be the
131
+ * alternative, and a subscriber told nothing happened diverges with nobody ever asking again.
132
132
  */
133
- onBulk(_change: RowBulkChange): void {
133
+ onBulk(change: RowBulkChange): void {
134
134
  if (stopped) return;
135
- registry.invalidate();
135
+ registry.invalidate(change.entity);
136
136
  },
137
137
  };
138
138
 
@@ -0,0 +1,74 @@
1
+ // One subscriber resuming onto its window from a cursor: replay the retained patches when the
2
+ // cursor is inside them, serve one bounded snapshot when it is not. Split from `live-query.ts` at
3
+ // that file's size ceiling — the registry owns the entry table and who is attached to what; this
4
+ // owns what a returning subscriber is SENT. It attaches nothing: the caller seats the cursor.
5
+
6
+ import type { Clock } from '@ultimat3/core';
7
+ import {
8
+ advance,
9
+ type LiveCursor,
10
+ type ReconnectBudget,
11
+ type ResumeSource,
12
+ resumeFrom,
13
+ } from './cursor';
14
+ import type { SnapshotResult } from './live-contract';
15
+ import { snapshotFrame } from './live-fanout';
16
+ import { fillWindow, type QueryEntry } from './query-window';
17
+ import type { Subscriber, SubscriberGate } from './subscriber-gate';
18
+ import { type Frame, PROTOCOL_VERSION } from './sync-protocol';
19
+
20
+ export interface ResumeOntoDeps {
21
+ readonly source: ResumeSource;
22
+ readonly budget?: ReconnectBudget | undefined;
23
+ readonly clock: Clock;
24
+ readonly gate: SubscriberGate;
25
+ /** The window, read once for the entry and filtered for THIS subscriber. */
26
+ read(): Promise<SnapshotResult>;
27
+ }
28
+
29
+ /**
30
+ * The frame a resuming subscriber needs and the cursor it is seated under. The caller has already
31
+ * established that `cursor` names THIS window: a cursor is client data, and one minted for another
32
+ * window is a cold start, never a replay of a ring this window does not own.
33
+ */
34
+ export async function resumeOnto(
35
+ deps: ResumeOntoDeps,
36
+ entry: QueryEntry,
37
+ who: Subscriber,
38
+ cursor: LiveCursor,
39
+ now: number,
40
+ ): Promise<{ readonly cursor: LiveCursor; readonly frame: Frame }> {
41
+ const resumed = await resumeFrom(cursor, {
42
+ source: deps.source,
43
+ ...(deps.budget ? { budget: deps.budget } : {}),
44
+ clock: deps.clock,
45
+ snapshot: () => deps.read(),
46
+ });
47
+ if (resumed.kind !== 'delta') {
48
+ return {
49
+ cursor: resumed.cursor,
50
+ frame: snapshotFrame(entry, who.sid, resumed.rows, resumed.cursor),
51
+ };
52
+ }
53
+ // The gate decides about whole rows out of the shared window, and an entry nothing has read
54
+ // yet has none — every patch would meet an empty window and be withheld. Filling is
55
+ // conditional on purpose: a restart storm resumes onto entries that already hold a live
56
+ // window, and re-reading per resuming subscriber is the cost a delta resume exists to skip.
57
+ if (entry.lsn === '') await fillWindow(entry);
58
+ // The live entry on purpose: a resume runs outside the lane, so the window under it may
59
+ // have moved on — always forwards, and a row whose grant was revoked in the meantime is
60
+ // one this pass must refuse rather than replay from the state it had at the cursor's lsn.
61
+ const patches = await deps.gate.filterPatches(entry, who, resumed.patches, new Set(cursor.ids));
62
+ // Advanced over the FILTERED list, never over `resumed.cursor` — which `resumeFrom` built
63
+ // by advancing across the retained window, and that window is PRE-POLICY. Seated as it
64
+ // came back, this subscriber's `cursor.ids` gained the id of every row inserted for every
65
+ // OTHER actor while it was away; `subscriber-gate` then reads `held.has(patch.id)` off it,
66
+ // takes the "the subscriber holds this row" branch, and delivers a `delete` frame carrying
67
+ // another tenant's row id and the instant it went. The leak that branch closes, re-opened
68
+ // one layer up. `live-fanout.ts` advances over `allowed` for exactly this reason.
69
+ const seated = advance(cursor, patches, resumed.cursor.lsn, now);
70
+ return {
71
+ cursor: seated,
72
+ frame: { type: 'patch', v: PROTOCOL_VERSION, sid: who.sid, patches, lsn: seated.lsn },
73
+ };
74
+ }
@@ -0,0 +1,25 @@
1
+ // Which tenant a live window belongs to, and the id that keeps two tenants' windows apart.
2
+ //
3
+ // A window is the SHARED, pre-policy read of one `(query, input)`. Shared across every subscriber
4
+ // of one org it is the fan-out the pipeline is built on; shared across two orgs it is one org's
5
+ // rows in a window the other is served from, with a row policy as the only thing in between. So
6
+ // the tenant is part of the window's identity — the same place `@ultimat3/query`'s read cache puts
7
+ // its authority — and the read under it runs in a context that carries that tenant
8
+ // (`live-definition.ts`), which is where `@ultimat3/entity`'s guard takes it from on every surface.
9
+
10
+ import type { Actor } from '@ultimat3/core';
11
+
12
+ /** The org a subscriber's windows are read for. `null`: anonymous, or an actor in no org. */
13
+ export function liveTenantOf(actor: Actor | null): string | null {
14
+ const orgId = actor?.orgId;
15
+ return typeof orgId === 'string' && orgId !== '' ? orgId : null;
16
+ }
17
+
18
+ /**
19
+ * One window's id: the query id, qualified by the tenant it is read for. JSON for the reason
20
+ * `readAuthority` gives — an org id is app data and may carry any separator, and a value that can
21
+ * spell a boundary can spell someone else's. An org-less window keeps the bare query id.
22
+ */
23
+ export function windowId(queryId: string, tenant: string | null): string {
24
+ return tenant === null ? queryId : JSON.stringify([queryId, tenant]);
25
+ }
@@ -81,6 +81,8 @@ export interface NatsClientOptions {
81
81
  export type NatsConnect = (options: NatsClientOptions) => Promise<NatsClient>;
82
82
 
83
83
  export const DEFAULT_NATS_PORT = 4222;
84
+
85
+ const NATS_URL_FIX = 'export NATS_URL=nats://user:password@host:4222';
84
86
  export const DEFAULT_REQUEST_TIMEOUT_MS = 5_000;
85
87
 
86
88
  /**
@@ -92,9 +94,12 @@ export function parseNatsUrl(url: string): NatsTarget {
92
94
  try {
93
95
  parsed = new URL(url);
94
96
  } catch {
97
+ // The variable, never its value: a bus URL carries the credential, and an error is the one
98
+ // thing that reaches a log, `--json`, an agent transcript and a ticket. `parsePgUrl`'s rule.
95
99
  throw new TransportUnavailableError({
96
100
  transport: 'nats',
97
- reason: `"${url}" is not a connection URL`,
101
+ reason: 'NATS_URL is not a connection URL',
102
+ fix: NATS_URL_FIX,
98
103
  });
99
104
  }
100
105
  if (parsed.protocol !== 'nats:' && parsed.protocol !== 'tls:') {
@@ -104,7 +109,11 @@ export function parseNatsUrl(url: string): NatsTarget {
104
109
  });
105
110
  }
106
111
  if (parsed.hostname === '') {
107
- throw new TransportUnavailableError({ transport: 'nats', reason: `"${url}" has no host` });
112
+ throw new TransportUnavailableError({
113
+ transport: 'nats',
114
+ reason: 'NATS_URL has no host',
115
+ fix: NATS_URL_FIX,
116
+ });
108
117
  }
109
118
  const hasUser = parsed.username !== '';
110
119
  const hasPass = parsed.password !== '';
@@ -0,0 +1,9 @@
1
+ // The production `NatsConnect`, as a door: the adapter and the `nats` library behind it — 65
2
+ // vendor modules — are imported when a client is OPENED, never when this package is. Every role
3
+ // of every app imports `@ultimat3/realtime/server`; only one configured for the NATS transport
4
+ // ever opens a client.
5
+
6
+ import type { NatsClient, NatsClientOptions } from './nats-client';
7
+
8
+ export const openNatsClient = async (options: NatsClientOptions): Promise<NatsClient> =>
9
+ (await import('./nats-lib-client')).openNatsClient(options);
@@ -17,7 +17,7 @@ import type { NatsClient, NatsConnect } from './nats-client';
17
17
  import { parseNatsUrl } from './nats-client';
18
18
  import { ensureKvBucket } from './nats-jetstream';
19
19
  import { NatsKvSet } from './nats-kv';
20
- import { openNatsClient } from './nats-lib-client';
20
+ import { openNatsClient } from './nats-open';
21
21
  import { type BackoffPolicy, defaultBackoff, policyDelay, type Rng } from './thundering-herd';
22
22
 
23
23
  const encoder = new TextEncoder();
@@ -55,6 +55,7 @@ export class NatsTransport implements Transport {
55
55
  #dialing: Promise<NatsClient> | undefined;
56
56
  #retries = 0;
57
57
  #closed = false;
58
+ readonly #reconnectListeners = new Set<() => void>();
58
59
 
59
60
  constructor(options: NatsTransportOptions) {
60
61
  // Parsed here rather than at the first publish: a malformed NATS_URL is a boot-time fault, and
@@ -116,6 +117,19 @@ export class NatsTransport implements Transport {
116
117
  return { subject, unsubscribe: () => live.unsubscribe() };
117
118
  }
118
119
 
120
+ /**
121
+ * Told every time the client comes back from a drop. Fanout is at-most-once, so the gap between
122
+ * the drop and this call is changes the subscriber never saw — and with no later message there
123
+ * is no sequence number to notice it by. A `sync` node invalidates its windows on it. Returns
124
+ * the unsubscribe.
125
+ */
126
+ onReconnect(listener: () => void): () => void {
127
+ this.#reconnectListeners.add(listener);
128
+ return () => {
129
+ this.#reconnectListeners.delete(listener);
130
+ };
131
+ }
132
+
119
133
  async close(): Promise<void> {
120
134
  this.#closed = true;
121
135
  const client = this.#client;
@@ -200,7 +214,15 @@ export class NatsTransport implements Transport {
200
214
  this.#retries = 0;
201
215
  const client = this.#client;
202
216
  if (client === undefined) return;
217
+ // Not awaited: the library calls this from its status loop. The failure has one place to go.
203
218
  void this.#ensureBucket(client).catch((error: unknown) => this.#report(error, this.name));
219
+ for (const listener of [...this.#reconnectListeners]) {
220
+ try {
221
+ listener();
222
+ } catch (error) {
223
+ this.#report(error, this.name);
224
+ }
225
+ }
204
226
  }
205
227
 
206
228
  /**