@ultimat3/realtime 23.0.0 → 24.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CLAUDE.md +75 -81
  2. package/README.md +107 -17
  3. package/package.json +4 -4
  4. package/src/advisory-lock.ts +57 -0
  5. package/src/boot.ts +12 -7
  6. package/src/change-buffer.ts +96 -41
  7. package/src/changefeed-env.ts +2 -2
  8. package/src/changefeed.ts +39 -1
  9. package/src/channel-authz.ts +43 -12
  10. package/src/channel-latch.ts +52 -0
  11. package/src/channel-logs.ts +34 -4
  12. package/src/channel-records.ts +33 -2
  13. package/src/channel-sids.ts +37 -0
  14. package/src/channel.ts +143 -44
  15. package/src/client-channels.ts +10 -1
  16. package/src/client-contract.ts +4 -1
  17. package/src/client-frames.ts +7 -1
  18. package/src/client-heartbeat.ts +39 -9
  19. package/src/client.ts +1 -0
  20. package/src/cursor.ts +11 -0
  21. package/src/errors.ts +70 -19
  22. package/src/fanout.ts +13 -0
  23. package/src/index.ts +1 -1
  24. package/src/live-definition.ts +22 -7
  25. package/src/live-fanout.ts +43 -7
  26. package/src/live-floor.ts +35 -0
  27. package/src/live-query-options.ts +38 -0
  28. package/src/live-query.ts +36 -37
  29. package/src/nats-client.ts +11 -2
  30. package/src/nats-transport.ts +22 -0
  31. package/src/offline-queue.ts +48 -5
  32. package/src/page-errors.ts +17 -0
  33. package/src/page-outbox.ts +121 -50
  34. package/src/pg-advisory-lock.ts +45 -1
  35. package/src/pg-auth.ts +9 -4
  36. package/src/pg-bytes.ts +4 -1
  37. package/src/pg-connection.ts +119 -21
  38. package/src/pg-entity-row.ts +16 -0
  39. package/src/pg-identity-tables.ts +25 -0
  40. package/src/pg-preflight.ts +50 -4
  41. package/src/pg-replication-row.ts +82 -0
  42. package/src/pg-replication.ts +44 -45
  43. package/src/pg-socket.ts +6 -4
  44. package/src/pg-tls.ts +18 -0
  45. package/src/pg-wire.ts +5 -3
  46. package/src/pgoutput.ts +19 -9
  47. package/src/presence.ts +85 -8
  48. package/src/query-window.ts +16 -4
  49. package/src/record-persister.ts +7 -2
  50. package/src/replication-errors.ts +8 -4
  51. package/src/replicator-envelope.ts +110 -0
  52. package/src/replicator-row.ts +75 -0
  53. package/src/replicator.ts +319 -186
  54. package/src/server.ts +10 -6
  55. package/src/socket-engine.ts +14 -2
  56. package/src/socket-idle.ts +3 -1
  57. package/src/socket-routes.ts +8 -3
  58. package/src/subscriber-gate.ts +11 -1
  59. package/src/sync-auth.ts +47 -2
  60. package/src/sync-frames.ts +73 -20
  61. package/src/sync-node-bounds.ts +15 -0
  62. package/src/sync-node-contract.ts +20 -2
  63. package/src/sync-node.ts +92 -17
  64. package/src/sync-origin.ts +26 -11
  65. package/src/sync-protocol.ts +13 -3
  66. package/src/sync-upgrade.ts +76 -8
  67. package/src/use-mutation.ts +33 -5
  68. package/src/wire-channel.ts +13 -4
@@ -27,8 +27,10 @@ export interface LiveDefinitionOptions {
27
27
  */
28
28
  readonly ctx: Ctx;
29
29
  /**
30
- * Where the shared window sits in the change stream, asked at snapshot time. Without a feed
31
- * position a reconnect can only re-snapshot, so a node with a replicator should pass its lsn.
30
+ * Where the shared window sits in the change stream, asked at snapshot time — the node's newest
31
+ * received change (`registry.lastLsn`). Omitted, or answering `''` before the first change, the
32
+ * snapshot claims no position and the retained ring is floored by its first patch instead: a
33
+ * cursor minted before that patch re-snapshots, one that received it resumes as a delta.
32
34
  */
33
35
  readonly lsn?: () => string;
34
36
  /** Pins the reconnect epoch in tests; the server derives it from the build. */
@@ -165,6 +167,9 @@ export function liveQueryDefinition(
165
167
  // The position is taken BEFORE the rows: the rows are then at least that new, so the claim
166
168
  // is true. Taken after, a commit landing mid-read was claimed and missing — and every change
167
169
  // up to it is dropped downstream as already folded.
170
+ // `''` when the node holds no position — no `lsn` option, or no change received yet. Never
171
+ // a made-up one: a position the node never held becomes a ring floor every foreign cursor
172
+ // is "inside", and the client that resumed there kept rows this node never patched.
168
173
  const lsn = options.lsn?.() ?? '';
169
174
  return { rows: await readFor(options.ctx, tenant, () => window.read()), lsn };
170
175
  },
@@ -191,15 +196,25 @@ function evictOldest(windows: Map<string, SharedWindow>, max: number): void {
191
196
  }
192
197
 
193
198
  /**
194
- * Rows crossing the wire are addressed by `id` — patches, cursors and the local store all key on
195
- * it. A projection without one is refused here rather than delivered as a row nobody can patch:
196
- * the alternative is a subscription that appears to work until the first update.
199
+ * Rows crossing the wire are addressed by a TEXT `id` — patches, cursors and the local store all
200
+ * key on it. A safe-integer id is stringified, the one normalisation the change feed applies to the
201
+ * same row (`pg-replication.ts`'s `toRow`), so a snapshot row and its later patch name one identity.
202
+ * Anything else is refused here rather than delivered as a row nobody can patch.
197
203
  */
198
204
  function rowsOf(query: string, rows: readonly object[]): readonly Row[] {
199
205
  const out: Row[] = [];
200
206
  for (const row of rows) {
201
- if (!isRow(row)) throw new LiveRowUnidentifiedError({ query, keys: Object.keys(row) });
202
- out.push(row);
207
+ const id: unknown = (row as { id?: unknown }).id;
208
+ const candidate =
209
+ typeof id === 'number' && Number.isSafeInteger(id) ? { ...row, id: String(id) } : row;
210
+ if (!isRow(candidate)) {
211
+ throw new LiveRowUnidentifiedError({
212
+ query,
213
+ keys: Object.keys(row),
214
+ ...(id === undefined || id === null ? {} : { idType: typeof id }),
215
+ });
216
+ }
217
+ out.push(candidate);
203
218
  }
204
219
  return out;
205
220
  }
@@ -8,7 +8,7 @@ import type { ChangeEvent } from './changefeed';
8
8
  import { advance, type LiveCursor, makeCursor, type ResumeSource } from './cursor';
9
9
  import type { Row, RowPatch } from './json';
10
10
  import type { LiveSubscription } from './live-contract';
11
- import { applyToWindow, bridgeChange } from './matcher-bridge';
11
+ import { applyToWindow, bridgeChange, type Projection, projectionOf } from './matcher-bridge';
12
12
  import { type QueryEntry, refillWindowInLane } from './query-window';
13
13
  import { type Subscriber, type SubscriberGate, windowIndex } from './subscriber-gate';
14
14
  import { type Frame, PROTOCOL_VERSION } from './sync-protocol';
@@ -50,6 +50,8 @@ export async function fanoutChange(
50
50
  const reread = entry.stale;
51
51
  if (reread) {
52
52
  await refillWindowInLane(entry);
53
+ // What was retained before this read has a hole exactly where the staleness came from.
54
+ deps.source.floorAt?.(entry.qid, entry.lsn, { exclusive: true });
53
55
  // Every subscriber was fed from the rows this read just replaced, so each is owed a snapshot —
54
56
  // not only the ones `invalidate()` marked: a read that failed or timed out stales the window
55
57
  // and marks nobody.
@@ -72,19 +74,26 @@ export async function fanoutChange(
72
74
  if (!bridged) return { sent: reread ? await resnapshotAll(deps, entry) : 0, stale: 0 };
73
75
  // Keyed ONCE, here, before the retained window stores them: a resume replays the same key.
74
76
  const result = { ...bridged, patches: keyPatches(entry, change, bridged.patches) };
77
+ // Asked of the window as it stood BEFORE this change: the partial row must not teach it a shape.
78
+ const partial = lacksOmitted(change, result.patches, projectionOf(entry.rows));
75
79
  entry.lsn = change.lsn;
76
80
  entry.rows = applyToWindow(entry.rows, result.patches);
77
- // The window lost its tail, so what it holds is a guess — the next delivery re-reads it rather
78
- // than patching a guess, and every subscriber below is re-snapshotted out of what that returns.
79
- if (result.refill) entry.stale = true;
80
- // The retained window holds the pre-policy patch; resume re-filters it per subscriber.
81
- for (const patch of result.patches) deps.source.append(entry.qid, patch);
81
+ // The window lost its tail — or adopted a row the change could not carry whole — so what it
82
+ // holds is a guess: the next delivery re-reads it rather than patching a guess, and every
83
+ // subscriber below is re-snapshotted out of what that returns.
84
+ const guessed = result.refill || partial;
85
+ if (guessed) entry.stale = true;
86
+ // The retained window holds the pre-policy patch; resume re-filters it per subscriber. A partial
87
+ // row is never retained: replayed, it is the same missing column on a resuming client.
88
+ if (!partial) for (const patch of result.patches) deps.source.append(entry.qid, patch);
89
+ // And the ring says so now, not at the re-read: a resume in between got a delta lacking it.
90
+ else deps.source.floorAt?.(entry.qid, change.lsn, { exclusive: true });
82
91
 
83
92
  let sent = 0;
84
93
  // Indexed once for every subscriber below, never searched per subscriber per patch.
85
94
  const index = windowIndex(entry.rows);
86
95
  for (const subscription of entry.subscribers.values()) {
87
- if (result.refill) {
96
+ if (guessed) {
88
97
  // The window lost its tail: guessing is how a sync engine silently diverges. Checked BEFORE
89
98
  // the mark, because a repair reads `entry.rows` — which this fanout has just declared a
90
99
  // guess — and then CLEARS the mark. A subscriber already diverged would be recorded as
@@ -139,6 +148,32 @@ export async function fanoutChange(
139
148
  return { sent, stale: 0 };
140
149
  }
141
150
 
151
+ /**
152
+ * Whether a row ENTERS the window without a column the change did not carry
153
+ * (`ChangeEvent.omitted`: an unchanged out-of-line value Postgres logged no bytes for). An update
154
+ * patch is safe — it merges onto a window row that already holds the column — but an add adopts
155
+ * the row as the whole row, into the window, every later snapshot and the retained ring. An
156
+ * unread projection (an empty window) cannot rule a column out, so every omitted one counts.
157
+ */
158
+ function lacksOmitted(
159
+ change: ChangeEvent,
160
+ patches: readonly RowPatch[],
161
+ projection: Projection,
162
+ ): boolean {
163
+ const omitted = change.omitted;
164
+ if (omitted === undefined || omitted.length === 0) return false;
165
+ return patches.some(
166
+ (patch) =>
167
+ patch.op === 'insert' &&
168
+ patch.row !== null &&
169
+ omitted.some(
170
+ (column) =>
171
+ (projection === undefined || projection.has(column)) &&
172
+ !Object.hasOwn(patch.row ?? {}, column),
173
+ ),
174
+ );
175
+ }
176
+
142
177
  /**
143
178
  * Every row of a relation this window reads is gone. There is nothing to patch from — a truncate
144
179
  * names no row — so the window is re-read now, in the lane, and every subscriber is re-snapshotted
@@ -153,6 +188,7 @@ async function truncated(
153
188
  if (!entry.shape.entities.includes(change.entity)) return { sent: 0, stale: 0 };
154
189
  await refillWindowInLane(entry);
155
190
  if (change.lsn > entry.lsn) entry.lsn = change.lsn;
191
+ deps.source.floorAt?.(entry.qid, entry.lsn, { exclusive: true });
156
192
  for (const subscription of entry.subscribers.values()) {
157
193
  subscription.socket.markDesynced(subscription.sid);
158
194
  }
@@ -0,0 +1,35 @@
1
+ // What a window read that just returned owes the retained ring, and the position the subscriber it
2
+ // serves is told. The rule itself is `change-buffer.ts`'s (`Ring`); this is its one caller for a
3
+ // read, split from `live-query.ts` at that file's size ceiling.
4
+
5
+ import type { ResumeSource } from './cursor';
6
+ import type { FilledWindow, QueryEntry } from './query-window';
7
+
8
+ export interface NodePosition {
9
+ /** The newest change this node has received; `''` before the first. */
10
+ readonly lastLsn: string;
11
+ /** This node's own mark for a read with no position behind it. Sorts below every real lsn. */
12
+ readonly origin: string;
13
+ }
14
+
15
+ /**
16
+ * Synchronous on purpose: the caller awaits a policy pass next, and the position has to be the
17
+ * one these ROWS were read at — read after that await, a change folded in meanwhile would be
18
+ * claimed by a snapshot that does not hold it.
19
+ */
20
+ export function floorAfterRead(
21
+ source: ResumeSource,
22
+ entry: QueryEntry,
23
+ window: FilledWindow,
24
+ node: NodePosition,
25
+ ): string {
26
+ // Served out of a window that was already filled: its patch history is still true.
27
+ if (window.landed === null) return window.lsn;
28
+ // A read with no position behind it is marked with this node's own origin, so the cursor it
29
+ // mints is one no other node could have issued.
30
+ const lsn = window.lsn === '' ? node.origin : window.lsn;
31
+ if (entry.lsn === '') entry.lsn = lsn;
32
+ if (window.landed === 'forced') source.floorAt?.(entry.qid, lsn, { exclusive: true });
33
+ else source.floorAt?.(entry.qid, lsn, { sole: node.lastLsn === '' || window.lsn === '' });
34
+ return lsn;
35
+ }
@@ -0,0 +1,38 @@
1
+ // What a `LiveQueryRegistry` is built from, and the one ceiling it defaults. Split from
2
+ // `live-query.ts` at that file's size ceiling: the registry owns behaviour, this owns its knobs.
3
+
4
+ import type { Actor, Clock } from '@ultimat3/core';
5
+ import type { ReconnectBudget, ResumeSource } from './cursor';
6
+ import type { SubscriberGateOptions } from './subscriber-gate';
7
+ import type { Scheduler } from './thundering-herd';
8
+
9
+ export interface LiveQueryRegistryOptions extends SubscriberGateOptions {
10
+ readonly source: ResumeSource;
11
+ readonly budget?: ReconnectBudget;
12
+ readonly clock?: Clock;
13
+ readonly maxPerSocket?: number;
14
+ readonly maxPerTenant?: number;
15
+ readonly tenantOf?: (actor: Actor | null) => string | null;
16
+ /**
17
+ * Distinct `(query, input)` pairs this node will hold at once. A `qid` derives from
18
+ * client-chosen input, so without a ceiling one socket mints entries — a matcher, a row window,
19
+ * a `WindowLock` and a fanout target each — until the process dies.
20
+ */
21
+ readonly maxEntries?: number;
22
+ /**
23
+ * How long one entry's SHARED snapshot read may hold its slot. Defaults to
24
+ * `DEFAULT_READ_DEADLINE_MS`. Without it a `definition.snapshot` that never settles pinned the
25
+ * slot for the life of the process and every later cold subscriber joined a promise nothing
26
+ * would resolve — one wedged read taking every future subscriber of that query id with it.
27
+ */
28
+ readonly readDeadlineMs?: number;
29
+ /** Injected so that deadline is provable without waiting for one. Production uses `setTimeout`. */
30
+ readonly schedule?: Scheduler;
31
+ }
32
+
33
+ /**
34
+ * Live `(query, input)` pairs one node holds. Reached, the next NEW pair is refused with
35
+ * `X_SUBSCRIPTION_LIMIT`; subscribing to a pair that already exists keeps working, because the
36
+ * cost this bounds is the entry, not the subscriber.
37
+ */
38
+ export const DEFAULT_MAX_ENTRIES = 10_000;
package/src/live-query.ts CHANGED
@@ -6,54 +6,26 @@
6
6
  // policy is never sent to that actor — it arrives as a `delete` if they hold it, and is dropped
7
7
  // otherwise.
8
8
 
9
- import { type Actor, type Clock, finiteOption, systemClock, uuid } from '@ultimat3/core';
9
+ import { type Clock, finiteOption, systemClock, uuid } from '@ultimat3/core';
10
10
  import { queryHash } from '@ultimat3/query';
11
11
  import type { ChangeEvent } from './changefeed';
12
- import { type LiveCursor, makeCursor, type ReconnectBudget, type ResumeSource } from './cursor';
12
+ import { type LiveCursor, makeCursor } from './cursor';
13
13
  import { isPolicyDenial, LiveQueryUnknownError, SubscriptionLimitError } from './errors';
14
14
  import type { JsonValue } from './json';
15
15
  import type { LiveQueryDefinition, LiveSubscription, SnapshotResult } from './live-contract';
16
16
  import { type FanoutDeps, fanoutChange, snapshotFrame } from './live-fanout';
17
+ import { floorAfterRead } from './live-floor';
18
+ import { DEFAULT_MAX_ENTRIES, type LiveQueryRegistryOptions } from './live-query-options';
17
19
  import { refuseSubscription } from './live-refusal';
18
20
  import { resumeOnto } from './live-resume';
19
21
  import { liveTenantOf, windowId } from './live-tenant';
20
22
  import { createEntry, fillWindow, type QueryEntry } from './query-window';
21
23
  import type { SyncSocket } from './socket';
22
- import { type Subscriber, SubscriberGate, type SubscriberGateOptions } from './subscriber-gate';
24
+ import { type Subscriber, SubscriberGate } from './subscriber-gate';
23
25
  import { SubscriptionBook, subscriptionKey } from './subscription-book';
24
26
  import type { Frame } from './sync-protocol';
25
- import type { Scheduler } from './thundering-herd';
26
27
 
27
- export interface LiveQueryRegistryOptions extends SubscriberGateOptions {
28
- readonly source: ResumeSource;
29
- readonly budget?: ReconnectBudget;
30
- readonly clock?: Clock;
31
- readonly maxPerSocket?: number;
32
- readonly maxPerTenant?: number;
33
- readonly tenantOf?: (actor: Actor | null) => string | null;
34
- /**
35
- * Distinct `(query, input)` pairs this node will hold at once. A `qid` derives from
36
- * client-chosen input, so without a ceiling one socket mints entries — a matcher, a row window,
37
- * a `WindowLock` and a fanout target each — until the process dies.
38
- */
39
- readonly maxEntries?: number;
40
- /**
41
- * How long one entry's SHARED snapshot read may hold its slot. Defaults to
42
- * `DEFAULT_READ_DEADLINE_MS`. Without it a `definition.snapshot` that never settles pinned the
43
- * slot for the life of the process and every later cold subscriber joined a promise nothing
44
- * would resolve — one wedged read taking every future subscriber of that query id with it.
45
- */
46
- readonly readDeadlineMs?: number;
47
- /** Injected so that deadline is provable without waiting for one. Production uses `setTimeout`. */
48
- readonly schedule?: Scheduler;
49
- }
50
-
51
- /**
52
- * Live `(query, input)` pairs one node holds. Reached, the next NEW pair is refused with
53
- * `X_SUBSCRIPTION_LIMIT`; subscribing to a pair that already exists keeps working, because the
54
- * cost this bounds is the entry, not the subscriber.
55
- */
56
- export const DEFAULT_MAX_ENTRIES = 10_000;
28
+ export { DEFAULT_MAX_ENTRIES, type LiveQueryRegistryOptions } from './live-query-options';
57
29
 
58
30
  export class LiveQueryRegistry {
59
31
  readonly #definitions = new Map<string, LiveQueryDefinition>();
@@ -67,6 +39,11 @@ export class LiveQueryRegistry {
67
39
  /** What one lane needs, and nothing this class holds beyond it. */
68
40
  readonly #fanout: FanoutDeps;
69
41
  #lastLsn = '';
42
+ /**
43
+ * This node's mark for a window read before it held any position: unique to the process, and
44
+ * `!` sorts below every hex digit, so every real change is above it.
45
+ */
46
+ readonly #origin = `!${uuid()}`;
70
47
  #staleChanges = 0;
71
48
 
72
49
  constructor(options: LiveQueryRegistryOptions) {
@@ -166,7 +143,15 @@ export class LiveQueryRegistry {
166
143
  // dispatched concurrently, N of them reading a count nothing had grown yet.
167
144
  const slot = this.#book.reserve(args.socket, sid);
168
145
  try {
169
- return await this.#subscribeReserved(definition, sid, args);
146
+ for (;;) {
147
+ // `authorize`, the read and the row pass all await, and a re-auth that lands under them
148
+ // re-decides only what is ATTACHED. So a subscription seated for an actor the socket no
149
+ // longer carries is taken back and served again under the one it does.
150
+ const actor = args.socket.actor;
151
+ const served = await this.#subscribeReserved(definition, sid, args);
152
+ if (args.socket.actor === actor) return served;
153
+ this.unsubscribe(args.socket.id, sid);
154
+ }
170
155
  } finally {
171
156
  // After the attach on every path, so the slot is only ever given back to a count that has
172
157
  // already grown — or, on a failure, to one that never will.
@@ -223,7 +208,10 @@ export class LiveQueryRegistry {
223
208
  // replays. One that names another window — the same browser, signed in to another org, or a
224
209
  // forged frame — is not a position in this one: it is a cold start, never a replay of a ring
225
210
  // this subscriber's window does not own.
226
- if (args.cursor && args.cursor.qid === qid) {
211
+ //
212
+ // Nor is ANY cursor a position in a window known to have missed a change: the ring still
213
+ // answers a delta for it, and that delta is the hole. A stale entry serves cold starts only.
214
+ if (args.cursor && args.cursor.qid === qid && !entry.stale) {
227
215
  const who = { sid, actor: args.socket.actor };
228
216
  const resumed = await resumeOnto(
229
217
  {
@@ -319,6 +307,9 @@ export class LiveQueryRegistry {
319
307
  } catch (error) {
320
308
  if (isPolicyDenial(error)) {
321
309
  this.unsubscribe(socket.id, subscription.sid);
310
+ // Said under the sid, as a refused subscribe is: unsaid, the client kept the rows on
311
+ // screen in state `live` for a subscription this node no longer serves.
312
+ refuseSubscription(socket, subscription.sid, error);
322
313
  dropped.push(subscription.sid);
323
314
  continue;
324
315
  }
@@ -433,6 +424,10 @@ export class LiveQueryRegistry {
433
424
  cursor: LiveCursor,
434
425
  ): LiveSubscription {
435
426
  const subscription = this.#attach(entry, socket, sid, cursor);
427
+ // The policy pass ran outside the lane, so a change may have been folded into the window
428
+ // between the rows this subscriber was served and this attach: it is in neither their frame
429
+ // nor their patch stream. Marked, so the next delivery re-snapshots them out of the window.
430
+ if (entry.lsn !== cursor.lsn) socket.markDesynced(sid);
436
431
  if (socket.closed) this.unsubscribe(socket.id, sid);
437
432
  return subscription;
438
433
  }
@@ -495,6 +490,10 @@ export class LiveQueryRegistry {
495
490
  */
496
491
  async #read(entry: QueryEntry, who: Subscriber): Promise<SnapshotResult> {
497
492
  const window = await fillWindow(entry);
498
- return { rows: await this.#gate.filterRows(entry, who, window.rows), lsn: window.lsn };
493
+ const lsn = floorAfterRead(this.#options.source, entry, window, {
494
+ lastLsn: this.#lastLsn,
495
+ origin: this.#origin,
496
+ });
497
+ return { rows: await this.#gate.filterRows(entry, who, window.rows), lsn };
499
498
  }
500
499
  }
@@ -81,6 +81,8 @@ export interface NatsClientOptions {
81
81
  export type NatsConnect = (options: NatsClientOptions) => Promise<NatsClient>;
82
82
 
83
83
  export const DEFAULT_NATS_PORT = 4222;
84
+
85
+ const NATS_URL_FIX = 'export NATS_URL=nats://user:password@host:4222';
84
86
  export const DEFAULT_REQUEST_TIMEOUT_MS = 5_000;
85
87
 
86
88
  /**
@@ -92,9 +94,12 @@ export function parseNatsUrl(url: string): NatsTarget {
92
94
  try {
93
95
  parsed = new URL(url);
94
96
  } catch {
97
+ // The variable, never its value: a bus URL carries the credential, and an error is the one
98
+ // thing that reaches a log, `--json`, an agent transcript and a ticket. `parsePgUrl`'s rule.
95
99
  throw new TransportUnavailableError({
96
100
  transport: 'nats',
97
- reason: `"${url}" is not a connection URL`,
101
+ reason: 'NATS_URL is not a connection URL',
102
+ fix: NATS_URL_FIX,
98
103
  });
99
104
  }
100
105
  if (parsed.protocol !== 'nats:' && parsed.protocol !== 'tls:') {
@@ -104,7 +109,11 @@ export function parseNatsUrl(url: string): NatsTarget {
104
109
  });
105
110
  }
106
111
  if (parsed.hostname === '') {
107
- throw new TransportUnavailableError({ transport: 'nats', reason: `"${url}" has no host` });
112
+ throw new TransportUnavailableError({
113
+ transport: 'nats',
114
+ reason: 'NATS_URL has no host',
115
+ fix: NATS_URL_FIX,
116
+ });
108
117
  }
109
118
  const hasUser = parsed.username !== '';
110
119
  const hasPass = parsed.password !== '';
@@ -55,6 +55,7 @@ export class NatsTransport implements Transport {
55
55
  #dialing: Promise<NatsClient> | undefined;
56
56
  #retries = 0;
57
57
  #closed = false;
58
+ readonly #reconnectListeners = new Set<() => void>();
58
59
 
59
60
  constructor(options: NatsTransportOptions) {
60
61
  // Parsed here rather than at the first publish: a malformed NATS_URL is a boot-time fault, and
@@ -116,6 +117,19 @@ export class NatsTransport implements Transport {
116
117
  return { subject, unsubscribe: () => live.unsubscribe() };
117
118
  }
118
119
 
120
+ /**
121
+ * Told every time the client comes back from a drop. Fanout is at-most-once, so the gap between
122
+ * the drop and this call is changes the subscriber never saw — and with no later message there
123
+ * is no sequence number to notice it by. A `sync` node invalidates its windows on it. Returns
124
+ * the unsubscribe.
125
+ */
126
+ onReconnect(listener: () => void): () => void {
127
+ this.#reconnectListeners.add(listener);
128
+ return () => {
129
+ this.#reconnectListeners.delete(listener);
130
+ };
131
+ }
132
+
119
133
  async close(): Promise<void> {
120
134
  this.#closed = true;
121
135
  const client = this.#client;
@@ -200,7 +214,15 @@ export class NatsTransport implements Transport {
200
214
  this.#retries = 0;
201
215
  const client = this.#client;
202
216
  if (client === undefined) return;
217
+ // Not awaited: the library calls this from its status loop. The failure has one place to go.
203
218
  void this.#ensureBucket(client).catch((error: unknown) => this.#report(error, this.name));
219
+ for (const listener of [...this.#reconnectListeners]) {
220
+ try {
221
+ listener();
222
+ } catch (error) {
223
+ this.#report(error, this.name);
224
+ }
225
+ }
204
226
  }
205
227
 
206
228
  /**
@@ -11,6 +11,7 @@
11
11
 
12
12
  import { renderThrowable, stringField } from '@ultimat3/core/page';
13
13
  import type { JsonValue } from './json';
14
+ import { OfflineQueueAbandonedError } from './page-errors';
14
15
  import type { WireError } from './sync-protocol';
15
16
 
16
17
  export type MutationStatus = 'pending' | 'inflight' | 'acked' | 'failed';
@@ -91,6 +92,8 @@ export class OfflineQueue {
91
92
  * is parked inside `send`. Bumped by every loss so a pass that resumes afterwards claims nothing.
92
93
  */
93
94
  #epoch = 0;
95
+ /** Set by `abandon`, never cleared: this queue's principal is gone. */
96
+ #abandoned = false;
94
97
 
95
98
  private constructor(store: QueueStore, state: QueueState) {
96
99
  this.#store = store;
@@ -103,8 +106,10 @@ export class OfflineQueue {
103
106
  *
104
107
  * An `inflight` entry on disk belonged to a page that is gone, so it goes back to `pending`. Left
105
108
  * as it was, `#sendable` skipped it forever — no ack was coming to a page that no longer exists —
106
- * and every later write overtook it. The replay carries its idempotency key, so a write the old
107
- * page did get through is answered from the action's idempotency store, never applied twice.
109
+ * and every later write overtook it. The replay carries its idempotency key, and a `mutator()`
110
+ * is REQUIRED to declare `idempotent: true` (`X_MUTATOR_NOT_IDEMPOTENT` at declaration), so a
111
+ * write the old page did get through is answered from the action's idempotency store and is
112
+ * never applied twice. That requirement is what makes this resend safe, not the key alone.
108
113
  */
109
114
  static async open(store: QueueStore): Promise<OfflineQueue> {
110
115
  const queue = new OfflineQueue(store, await store.load());
@@ -119,6 +124,7 @@ export class OfflineQueue {
119
124
  * other pass can have one on the wire.
120
125
  */
121
126
  async reload(): Promise<void> {
127
+ if (this.#abandoned) return;
122
128
  const state = await this.#store.load();
123
129
  this.#mutations = state.mutations.map((mutation) => ({ ...mutation }));
124
130
  this.#nextSeq = Math.max(this.#nextSeq, state.nextSeq);
@@ -167,6 +173,17 @@ export class OfflineQueue {
167
173
  input: JsonValue;
168
174
  at?: number;
169
175
  }): Promise<QueuedMutation> {
176
+ // Read BEFORE the key is looked up, never between the lookup and the push: with the await in
177
+ // between, two enqueues of one key both found nothing and both pushed — one intent, two
178
+ // entries under one sequence number. Another tab of the same user may have taken sequence
179
+ // numbers since this one loaded, which is what the read is for.
180
+ //
181
+ // An abandoned queue REFUSES, before the read and after it (the principal can leave during
182
+ // it): a write accepted here would sit in memory, shown as queued, with nothing to send it.
183
+ if (this.#abandoned) throw new OfflineQueueAbandonedError({ name: args.name });
184
+ const floor = (await this.#store.load()).nextSeq;
185
+ if (this.#abandoned) throw new OfflineQueueAbandonedError({ name: args.name });
186
+ this.#nextSeq = Math.max(this.#nextSeq, floor);
170
187
  const existing = this.find(args.key);
171
188
  if (existing && existing.status !== 'failed') {
172
189
  this.#collapsed += 1;
@@ -178,8 +195,6 @@ export class OfflineQueue {
178
195
  // entry is dropped and this one takes a new sequence at the back of the queue.
179
196
  // By identity: `existing` IS the entry for this key, found above — no second key comparison.
180
197
  if (existing) this.#mutations = this.#mutations.filter((entry) => entry !== existing);
181
- // Another tab of the same user may have taken sequence numbers since this one loaded.
182
- this.#nextSeq = Math.max(this.#nextSeq, (await this.#store.load()).nextSeq);
183
198
  const mutation: QueuedMutation = {
184
199
  key: args.key,
185
200
  seq: this.#nextSeq,
@@ -248,6 +263,22 @@ export class OfflineQueue {
248
263
  return returned;
249
264
  }
250
265
 
266
+ /**
267
+ * The principal this queue belongs to is gone (`page-outbox.ts` calls it on a `rescope`), and it
268
+ * is final: nothing more is SENT, WRITTEN or ACCEPTED (`enqueue` refuses). The epoch bump stops a pass parked
269
+ * inside `send` before its next entry — which would otherwise leave under the next principal's
270
+ * session — and the flag stops a pass that starts later, and every write-back: the store was
271
+ * wiped for this principal, and a pass that then saved what it touched put the writes back.
272
+ */
273
+ abandon(): void {
274
+ this.#epoch += 1;
275
+ this.#abandoned = true;
276
+ }
277
+
278
+ get abandoned(): boolean {
279
+ return this.#abandoned;
280
+ }
281
+
251
282
  /** Server acknowledged: the mutation leaves the queue and its rebase entry can be committed. */
252
283
  async ack(key: string): Promise<void> {
253
284
  const mutation = this.find(key);
@@ -294,7 +325,7 @@ export class OfflineQueue {
294
325
  /** One drain pass. Never called concurrently with itself — `drain` owns that. */
295
326
  async #pass(send: MutationSender): Promise<DrainReport> {
296
327
  const epoch = this.#epoch;
297
- const sendable = this.#sendable();
328
+ const sendable = this.#abandoned ? [] : this.#sendable();
298
329
  // Nothing to do: a pass chained behind one that already sent everything must not rewrite the
299
330
  // durable state for the privilege of reporting zero.
300
331
  if (sendable.length === 0) {
@@ -326,6 +357,12 @@ export class OfflineQueue {
326
357
  // `WebSocket.send` on a CLOSING socket discards it and returns normally — so calling that
327
358
  // an ack drops the mutation on exactly the socket death this queue exists to survive.
328
359
  // Only `ack`/`fail` (the server) or `requeueInflight` (a lost connection) moves it on.
360
+ //
361
+ // A sender may settle the write itself before it returns — the page outbox does, over
362
+ // HTTP. One it REFUSED keeps the error `fail` just recorded and is not counted as sent:
363
+ // clearing it here put `{ status: 'failed', error: null }` on disk and reported a refused
364
+ // write as a sent one.
365
+ if (statusOf(mutation) === 'failed') continue;
329
366
  mutation.error = null;
330
367
  sent += 1;
331
368
  } catch (error) {
@@ -364,6 +401,7 @@ export class OfflineQueue {
364
401
  * queue: a second tab's entries are not this tab's to overwrite or delete.
365
402
  */
366
403
  async #persist(puts: readonly QueuedMutation[], deletes: readonly string[] = []): Promise<void> {
404
+ if (this.#abandoned) return;
367
405
  await this.#store.write({
368
406
  puts: puts.map((mutation) => ({ ...mutation })),
369
407
  deletes,
@@ -372,6 +410,11 @@ export class OfflineQueue {
372
410
  }
373
411
  }
374
412
 
413
+ /** Read through a call: `send` awaited in between, and the compiler still narrows to `inflight`. */
414
+ function statusOf(mutation: QueuedMutation): MutationStatus {
415
+ return mutation.status;
416
+ }
417
+
375
418
  /** A thrown value as the queue records it — `code`, `cause`, `fix`, never a throw of its own. */
376
419
  export function toQueueError(error: unknown): WireError {
377
420
  return {
@@ -93,6 +93,23 @@ export class LocalStoreUnavailableError extends RealtimeError {
93
93
  }
94
94
  }
95
95
 
96
+ /**
97
+ * A write reached a queue whose principal is gone: the page signed out, or in as someone else,
98
+ * while the write was being queued. It is not stored and never sent — a write issued under one
99
+ * session must not leave under the next — and it is RAISED, because a user's write dropped in
100
+ * silence is a loss nobody is told about. `name` is the mutator's registered name; the write's
101
+ * idempotency key is deliberately not rendered.
102
+ */
103
+ export class OfflineQueueAbandonedError extends RealtimeError {
104
+ constructor(args: { name: string }) {
105
+ super({
106
+ code: 'X_OFFLINE_QUEUE_ABANDONED',
107
+ cause: `the write "${args.name}" was queued after its page changed principal, so it was not stored and will not be sent — call the mutation again under the current session`,
108
+ fix: 'x errors explain X_OFFLINE_QUEUE_ABANDONED --json',
109
+ });
110
+ }
111
+ }
112
+
96
113
  /**
97
114
  * Something that can only mean "talk to the socket" ran on the server client — a mutation, a
98
115
  * publish, a topic subscription, a dial. There is no socket during a server render and there never