@ultimat3/realtime 1.2.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CLAUDE.md +591 -0
  2. package/README.md +320 -19
  3. package/package.json +6 -3
  4. package/src/apply-patches.ts +60 -0
  5. package/src/change-buffer.ts +77 -11
  6. package/src/channel.ts +174 -19
  7. package/src/client-contract.ts +81 -0
  8. package/src/client-frames.ts +175 -0
  9. package/src/client-heartbeat.ts +77 -0
  10. package/src/client-mutations.ts +114 -0
  11. package/src/client-topics.ts +54 -0
  12. package/src/client.ts +307 -273
  13. package/src/cursor.ts +7 -1
  14. package/src/errors.ts +193 -4
  15. package/src/frame-lanes.ts +58 -0
  16. package/src/hooks.ts +19 -5
  17. package/src/identity-map.ts +141 -0
  18. package/src/index.ts +96 -28
  19. package/src/json.ts +38 -1
  20. package/src/live-contract.ts +67 -0
  21. package/src/live-definition.ts +16 -11
  22. package/src/live-fanout.ts +150 -0
  23. package/src/live-query.ts +215 -268
  24. package/src/live-rows.ts +143 -0
  25. package/src/local-store.ts +86 -43
  26. package/src/nats-client.ts +132 -0
  27. package/src/nats-fake.ts +389 -344
  28. package/src/nats-jetstream.ts +21 -20
  29. package/src/nats-kv.ts +7 -7
  30. package/src/nats-lib-client.ts +210 -0
  31. package/src/nats-transport.ts +109 -138
  32. package/src/offline-queue.ts +146 -30
  33. package/src/pg-entity-row.ts +99 -31
  34. package/src/pg-replication.ts +84 -27
  35. package/src/pg-socket.ts +4 -1
  36. package/src/policy-gate.ts +13 -5
  37. package/src/presence.ts +76 -6
  38. package/src/query-hook.ts +56 -0
  39. package/src/query-window.ts +151 -0
  40. package/src/rebase.ts +68 -8
  41. package/src/replicator.ts +84 -11
  42. package/src/socket.ts +170 -14
  43. package/src/subscriber-gate.ts +209 -0
  44. package/src/subscription-book.ts +237 -0
  45. package/src/sync-auth.ts +124 -0
  46. package/src/sync-frames.ts +185 -0
  47. package/src/sync-listen.ts +73 -0
  48. package/src/sync-node.ts +284 -243
  49. package/src/sync-protocol.ts +115 -24
  50. package/src/sync-upgrade.ts +124 -0
  51. package/src/thundering-herd.ts +21 -0
  52. package/src/transport-env.ts +3 -3
  53. package/src/type-pins.ts +72 -0
  54. package/src/window-lock.ts +21 -0
  55. package/src/nats-commands.ts +0 -97
  56. package/src/nats-connection-fixture.ts +0 -105
  57. package/src/nats-connection.ts +0 -464
  58. package/src/nats-protocol.ts +0 -222
  59. package/src/nats-socket.ts +0 -236
  60. package/src/pg-connection-fixture.ts +0 -215
  61. package/src/pg-replication-fixture.ts +0 -261
@@ -1,15 +1,16 @@
1
1
  // Single responsibility: the production `Transport` — core NATS for fanout, JetStream KV for the
2
- // shared presence sets. Subscriptions are held as intent rather than as socket state, so a lost
3
- // connection is re-established and re-subscribed underneath the caller: that is what makes a `sync`
4
- // node stateless and lets any client resubscribe to any node.
2
+ // shared presence sets. The client underneath owns the wire and the reconnect, including
3
+ // re-establishing subscriptions, which is what makes a `sync` node stateless: a lost connection is
4
+ // re-dialled and re-subscribed underneath the caller, and this file keeps no socket state at all.
5
5
 
6
- import { type Clock, isUltimateError, logger, systemClock } from '@ultimat3/core';
6
+ import { type Clock, isUltimateError, logger, renderThrowable, systemClock } from '@ultimat3/core';
7
7
  import { TransportUnavailableError } from './errors';
8
8
  import type { Transport, TransportHandler, TransportSet, TransportSubscription } from './fanout';
9
- import { NatsConnection, type NatsSubscription } from './nats-connection';
9
+ import type { NatsClient, NatsConnect } from './nats-client';
10
+ import { parseNatsUrl } from './nats-client';
10
11
  import { ensureKvBucket } from './nats-jetstream';
11
12
  import { NatsKvSet } from './nats-kv';
12
- import { bunNatsStream, type NatsStream, type NatsTarget, parseNatsUrl } from './nats-socket';
13
+ import { openNatsClient } from './nats-lib-client';
13
14
  import { type BackoffPolicy, backoffDelay, defaultBackoff, type Rng } from './thundering-herd';
14
15
 
15
16
  const encoder = new TextEncoder();
@@ -28,44 +29,37 @@ export interface NatsTransportOptions {
28
29
  readonly onError?: (error: unknown, subject: string) => void;
29
30
  readonly rng?: Rng;
30
31
  /** Injected so the whole transport — reconnect included — runs in a test with no network. */
31
- readonly open?: (target: NatsTarget) => Promise<NatsStream>;
32
- readonly sleep?: (ms: number) => Promise<void>;
33
- }
34
-
35
- interface Wanted {
36
- readonly subject: string;
37
- readonly handler: TransportHandler;
38
- live: NatsSubscription | undefined;
32
+ readonly connect?: NatsConnect;
39
33
  }
40
34
 
41
35
  const DEFAULT_ATTEMPTS = 10;
36
+ const DEFAULT_PRESENCE_TTL_MS = 30_000;
42
37
 
43
38
  /** The production bus: NATS subjects for fanout, a JetStream KV bucket for presence. */
44
39
  export class NatsTransport implements Transport {
45
40
  readonly name = 'nats';
46
41
  readonly shared: TransportSet;
47
- readonly #target: NatsTarget;
48
42
  readonly #options: NatsTransportOptions;
49
- readonly #wanted = new Map<number, Wanted>();
43
+ readonly #connect: NatsConnect;
50
44
  readonly #backoff: BackoffPolicy;
51
45
  readonly #attempts: number;
52
46
  readonly #rng: Rng;
53
- readonly #sleep: (ms: number) => Promise<void>;
54
- #connection: NatsConnection | undefined;
55
- #dialing: Promise<NatsConnection> | undefined;
56
- #next = 0;
57
- #losses = 0;
47
+ #client: NatsClient | undefined;
48
+ #dialing: Promise<NatsClient> | undefined;
49
+ #retries = 0;
58
50
  #closed = false;
59
51
 
60
52
  constructor(options: NatsTransportOptions) {
61
- this.#target = parseNatsUrl(options.url);
53
+ // Parsed here rather than at the first publish: a malformed NATS_URL is a boot-time fault, and
54
+ // a container that reports itself healthy on one is a container nothing will ever page about.
55
+ parseNatsUrl(options.url);
62
56
  this.#options = options;
57
+ this.#connect = options.connect ?? openNatsClient;
63
58
  this.#backoff = options.backoff ?? defaultBackoff;
64
59
  this.#attempts = options.maxReconnectAttempts ?? DEFAULT_ATTEMPTS;
65
60
  this.#rng = options.rng ?? Math.random;
66
- this.#sleep = options.sleep ?? ((ms) => Bun.sleep(ms));
67
61
  this.shared = new NatsKvSet({
68
- connection: () => this.#ensure(),
62
+ client: () => this.#ensure(),
69
63
  bucket: options.bucket,
70
64
  clock: options.clock ?? systemClock,
71
65
  });
@@ -77,107 +71,88 @@ export class NatsTransport implements Transport {
77
71
  }
78
72
 
79
73
  get connected(): boolean {
80
- return this.#connection !== undefined && !this.#connection.closed;
74
+ return this.#client?.connected === true;
81
75
  }
82
76
 
83
77
  async publish(subject: string, payload: string): Promise<void> {
84
- const connection = await this.#ensure();
85
- await connection.publish(subject, encoder.encode(payload));
78
+ const client = await this.#ensure();
79
+ // `client.publish` is synchronous and refuses locally: a bad subject, a payload over the
80
+ // server's `max_payload`, a connection torn down between the `#ensure` and this line. Those
81
+ // are the LIBRARY's errors — or an app-supplied `connect`'s — so they arrive uncoded, and
82
+ // `ChannelHub`'s bridge, `SocketRegistry` and the replicator all await this call.
83
+ this.#translating(`publish to ${subject}`, () =>
84
+ client.publish(subject, encoder.encode(payload)),
85
+ );
86
86
  }
87
87
 
88
+ /**
89
+ * The subscription is the client's to keep: it survives a drop and comes back with the reconnect,
90
+ * so there is no intent map here to re-bind from — and therefore no way for a re-bind to run
91
+ * twice and double every change on the subject.
92
+ */
88
93
  async subscribe(subject: string, handler: TransportHandler): Promise<TransportSubscription> {
89
- this.#next += 1;
90
- const id = this.#next;
91
- const wanted: Wanted = { subject, handler, live: undefined };
92
- this.#wanted.set(id, wanted);
93
- try {
94
- const connection = await this.#ensure();
95
- // The dial this may have triggered already re-bound everything it found registered, this
96
- // one included — binding again here would double every delivery on the subject.
97
- wanted.live ??= await this.#bind(connection, wanted);
98
- } catch (error) {
99
- this.#wanted.delete(id);
100
- throw error;
101
- }
102
- return {
103
- subject,
104
- unsubscribe: () => {
105
- this.#wanted.delete(id);
106
- void wanted.live?.unsubscribe().catch((error: unknown) => this.#report(error, subject));
107
- wanted.live = undefined;
108
- },
109
- };
94
+ const client = await this.#ensure();
95
+ // Same seam as `publish`: a permissions violation on the subject is refused here, not later.
96
+ const live = this.#translating(`subscribe to ${subject}`, () =>
97
+ client.subscribe(subject, (message) => {
98
+ try {
99
+ handler(decoder.decode(message.payload), message.subject);
100
+ } catch (error) {
101
+ this.#report(error, message.subject);
102
+ }
103
+ }),
104
+ );
105
+ return { subject, unsubscribe: () => live.unsubscribe() };
110
106
  }
111
107
 
112
108
  async close(): Promise<void> {
113
109
  this.#closed = true;
114
- this.#wanted.clear();
115
- const connection = this.#connection;
116
- this.#connection = undefined;
117
- await connection?.close();
118
- }
119
-
120
- #bind(connection: NatsConnection, wanted: Wanted): Promise<NatsSubscription> {
121
- return connection.subscribe(wanted.subject, (message) => {
122
- try {
123
- wanted.handler(decoder.decode(message.payload), message.subject);
124
- } catch (error) {
125
- this.#report(error, message.subject);
126
- }
127
- });
110
+ const client = this.#client;
111
+ this.#client = undefined;
112
+ await client?.close();
128
113
  }
129
114
 
130
- #ensure(): Promise<NatsConnection> {
115
+ /**
116
+ * One dial, shared by every caller that races it. A client that has already been handed out is
117
+ * reused whatever its state: while it is reconnecting the library is re-establishing that same
118
+ * connection and its subscriptions, and a second dial alongside it would double every delivery.
119
+ * A budget that ran out is a readiness failure, not a reason to start an unbounded retry here.
120
+ */
121
+ #ensure(): Promise<NatsClient> {
131
122
  if (this.#closed) {
132
123
  return Promise.reject(
133
124
  new TransportUnavailableError({ transport: this.name, reason: 'transport is closed' }),
134
125
  );
135
126
  }
136
- const current = this.#connection;
137
- if (current !== undefined && !current.closed) return Promise.resolve(current);
127
+ const current = this.#client;
128
+ if (current !== undefined) return Promise.resolve(current);
138
129
  this.#dialing ??= this.#dial().finally(() => {
139
130
  this.#dialing = undefined;
140
131
  });
141
132
  return this.#dialing;
142
133
  }
143
134
 
144
- /** Retry is bounded: a bus that is down for longer than the budget is a readiness failure. */
145
- async #dial(): Promise<NatsConnection> {
146
- for (let attempt = 0; ; attempt += 1) {
147
- try {
148
- return await this.#establish();
149
- } catch (error) {
150
- // A protocol mismatch answers the same on every attempt — a server too old for per-message
151
- // TTL stays too old — so retrying only delays the one report that names the fix.
152
- const fatal = isUltimateError(error) && error.code === 'X_TRANSPORT_PROTOCOL';
153
- if (fatal || this.#closed || attempt >= this.#attempts) {
154
- throw isUltimateError(error)
155
- ? error
156
- : new TransportUnavailableError({
157
- transport: this.name,
158
- reason: `${this.#target.host}:${this.#target.port} — ${String(error)}`,
159
- });
160
- }
161
- await this.#sleep(backoffDelay(attempt, this.#backoff, this.#rng));
162
- }
163
- }
164
- }
165
-
166
135
  /**
167
- * One attempt, published only once it is whole. A connection parked in `#connection` before its
168
- * bucket and its subscriptions are up answers `connected` for a dial that rejected, keeps its
169
- * `onClose` which then clears the connection that replaced it and leaves half a rebind
170
- * behind, so the next dial binds the same subject a second time and every change arrives twice.
171
- * A failed attempt therefore takes its own socket down with it rather than leaking one per retry.
136
+ * One attempt, published only once it is whole. A client parked in `#client` before its bucket is
137
+ * up answers `connected` for a dial that rejected, and the next caller then writes presence into
138
+ * a bucket that does not exist. A failed attempt therefore closes its own connection rather than
139
+ * leaking one per retry.
172
140
  */
173
- async #establish(): Promise<NatsConnection> {
174
- const connection = await this.#open();
141
+ async #dial(): Promise<NatsClient> {
142
+ const client = await this.#connect({
143
+ url: this.#options.url,
144
+ name: 'ultimate',
145
+ maxReconnectAttempts: this.#attempts,
146
+ // The library retries; the spread is ours, so a cluster restart does not bring every node
147
+ // back on the same millisecond.
148
+ reconnectDelay: () => backoffDelay(this.#retries++, this.#backoff, this.#rng),
149
+ onError: (error) => this.#report(error, this.name),
150
+ onReconnect: () => this.#recovered(),
151
+ });
175
152
  try {
176
- await ensureKvBucket(connection, this.#options.bucket, this.#options.presenceTtlMs ?? 30_000);
177
- for (const wanted of this.#wanted.values())
178
- wanted.live = await this.#bind(connection, wanted);
179
- // `close()` can land while an attempt is in flight, and it only closes what it can see:
180
- // publishing now would leave a socket open that nothing will ever close again.
153
+ await this.#ensureBucket(client);
154
+ // `close()` can land while a dial is in flight, and it only closes what it can see:
155
+ // publishing now would leave a connection open that nothing will ever close again.
181
156
  if (this.#closed) {
182
157
  throw new TransportUnavailableError({
183
158
  transport: this.name,
@@ -185,55 +160,51 @@ export class NatsTransport implements Transport {
185
160
  });
186
161
  }
187
162
  } catch (error) {
188
- for (const wanted of this.#wanted.values()) wanted.live = undefined;
189
- await connection.close();
163
+ await client.close();
190
164
  throw error;
191
165
  }
192
- this.#connection = connection;
193
- this.#losses = 0;
194
- return connection;
166
+ this.#client = client;
167
+ this.#retries = 0;
168
+ return client;
195
169
  }
196
170
 
197
- async #open(): Promise<NatsConnection> {
198
- const stream = await (this.#options.open ?? bunNatsStream)(this.#target);
199
- // The connection names itself in its own `onClose` so a drop can be matched against the live
200
- // one. It goes through a holder rather than a `const`, because a buffered `-ERR` closes the
201
- // session from inside `open()`, while a `const` binding would still be in its dead zone.
202
- const held: { connection: NatsConnection | undefined } = { connection: undefined };
203
- held.connection = await NatsConnection.open({
204
- stream,
205
- target: this.#target,
206
- name: 'ultimate',
207
- rng: this.#options.rng,
208
- onClose: (error) => this.#lost(error, held.connection),
209
- onError: (error) => this.#report(error, this.name),
210
- });
211
- return held.connection;
171
+ #ensureBucket(client: NatsClient): Promise<void> {
172
+ return ensureKvBucket(
173
+ client,
174
+ this.#options.bucket,
175
+ this.#options.presenceTtlMs ?? DEFAULT_PRESENCE_TTL_MS,
176
+ );
212
177
  }
213
178
 
214
179
  /**
215
- * A lost connection re-dials on its own rather than waiting for the next publish: a `sync` node
216
- * whose subscriptions are down is silently delivering nothing, which is worse than an error.
180
+ * A reconnect may have landed on a different cluster a restarted single node, or a failover to
181
+ * one that never held this bucket. The subscriptions came back with the client; the bucket is the
182
+ * one thing the library knows nothing about, so it is re-asserted here. It is idempotent.
217
183
  */
218
- #lost(error: unknown, connection: NatsConnection | undefined): void {
219
- // Only the live connection may declare a loss. One abandoned mid-dial drops on its own clock,
220
- // and without this it would clear the subscriptions — and the reconnect budget — of whichever
221
- // connection replaced it, then report a failure the transport had already recovered from.
222
- if (connection === undefined || this.#connection !== connection) return;
223
- this.#connection = undefined;
224
- for (const wanted of this.#wanted.values()) wanted.live = undefined;
225
- this.#report(error, this.name);
226
- if (this.#closed || this.#wanted.size === 0) return;
227
- this.#losses += 1;
228
- if (this.#losses > this.#attempts) return;
229
- void this.#recover();
184
+ #recovered(): void {
185
+ this.#retries = 0;
186
+ const client = this.#client;
187
+ if (client === undefined) return;
188
+ void this.#ensureBucket(client).catch((error: unknown) => this.#report(error, this.name));
230
189
  }
231
190
 
232
- async #recover(): Promise<void> {
233
- // Backoff first: a server that accepts and immediately drops must not become a hot loop.
234
- await this.#sleep(backoffDelay(this.#losses - 1, this.#backoff, this.#rng));
235
- if (this.#closed || this.connected) return;
236
- await this.#ensure().catch((error: unknown) => this.#report(error, this.name));
191
+ /**
192
+ * One call into the client, with its refusal translated. An `UltimateError` passes through the
193
+ * port raises its own for a closed client, and re-wrapping would bury the code a caller branches
194
+ * on while anything else becomes `X_TRANSPORT_UNAVAILABLE` carrying the library's own words as
195
+ * evidence. Never a bare `Error` out of this file: a raw `NatsError` has no code, no `fix:` and
196
+ * nothing an operator can act on, which is the whole reason the port is here.
197
+ */
198
+ #translating<T>(what: string, call: () => T): T {
199
+ try {
200
+ return call();
201
+ } catch (error) {
202
+ if (isUltimateError(error)) throw error;
203
+ throw new TransportUnavailableError({
204
+ transport: this.name,
205
+ reason: `${what} was refused: ${renderThrowable(error)}`,
206
+ });
207
+ }
237
208
  }
238
209
 
239
210
  /**
@@ -1,11 +1,15 @@
1
- // Tier 3: the durable mutation queue. Two invariants, both enforced here rather than documented:
1
+ // Tier 3: the durable mutation queue. Three invariants, all enforced here rather than documented:
2
2
  //
3
3
  // 1. **Order.** Mutations drain in client sequence order and stop at the first failure. A mutator
4
4
  // that assumed `like` ran before `unlike` must never see them swapped.
5
5
  // 2. **Dedupe.** The idempotency key is the identity of the intent. Re-enqueueing a key that is
6
6
  // already queued (double click, replay after a crash) collapses onto the existing entry and
7
7
  // never gets a new sequence number.
8
+ // 3. **Only the server removes a mutation.** A `send` that returned proves the frame was handed
9
+ // to a socket and nothing more, so a drained mutation is `inflight` — not `acked` — until an
10
+ // `ack`/`fail` frame settles it, or a lost connection returns it to the queue.
8
11
 
12
+ import { renderThrowable, stringField } from '@ultimat3/core';
9
13
  import type { JsonValue } from './json';
10
14
  import { type Frame, PROTOCOL_VERSION, type WireError } from './sync-protocol';
11
15
 
@@ -50,6 +54,7 @@ export class MemoryQueueStore implements QueueStore {
50
54
  export interface DrainReport {
51
55
  readonly sent: number;
52
56
  readonly collapsed: number;
57
+ /** Still to send. Not the queue depth: a sent mutation is unacknowledged, never unsent. */
53
58
  readonly remaining: number;
54
59
  readonly stoppedAt: string | null;
55
60
  }
@@ -61,6 +66,14 @@ export class OfflineQueue {
61
66
  #mutations: QueuedMutation[] = [];
62
67
  #nextSeq = 1;
63
68
  #collapsed = 0;
69
+ /** The drain lane: one pass at a time, in call order. See `drain`. */
70
+ #draining: Promise<DrainReport> | null = null;
71
+ /**
72
+ * Which connection the current pass is draining into. The lane orders passes against each other
73
+ * but `requeueInflight` is not a pass — it is a socket death, and it cannot reach into one that
74
+ * is parked inside `send`. Bumped by every loss so a pass that resumes afterwards claims nothing.
75
+ */
76
+ #epoch = 0;
64
77
 
65
78
  private constructor(store: QueueStore, state: QueueState) {
66
79
  this.#store = store;
@@ -89,7 +102,10 @@ export class OfflineQueue {
89
102
  return this.#mutations.find((mutation) => mutation.key === key);
90
103
  }
91
104
 
92
- /** Sorted by sequence. This is the only order anything downstream is allowed to use. */
105
+ /**
106
+ * Everything the server has not settled yet, sorted by sequence — this is the only order
107
+ * anything downstream is allowed to use, and the count a UI renders as "unsynced".
108
+ */
93
109
  pending(): readonly QueuedMutation[] {
94
110
  return this.#mutations
95
111
  .filter((mutation) => mutation.status === 'pending' || mutation.status === 'inflight')
@@ -107,10 +123,16 @@ export class OfflineQueue {
107
123
  at?: number;
108
124
  }): Promise<QueuedMutation> {
109
125
  const existing = this.find(args.key);
110
- if (existing) {
126
+ if (existing && existing.status !== 'failed') {
111
127
  this.#collapsed += 1;
112
128
  return existing;
113
129
  }
130
+ // A terminally failed entry is a decision the server already made about this key, kept for the
131
+ // UI — collapsing onto it makes an explicit idempotency key unusable for the rest of the
132
+ // session, because nothing ever retries a denial. Re-issuing one is a NEW intent, so the old
133
+ // entry is dropped and this one takes a new sequence at the back of the queue.
134
+ if (existing)
135
+ this.#mutations = this.#mutations.filter((candidate) => candidate.key !== args.key);
114
136
  const mutation: QueuedMutation = {
115
137
  key: args.key,
116
138
  seq: this.#nextSeq,
@@ -130,31 +152,51 @@ export class OfflineQueue {
130
152
  /**
131
153
  * Drains in sequence order and stops at the first failure. Continuing past a failure is how a
132
154
  * sync engine reorders a user's intent — so it does not continue.
155
+ *
156
+ * **One pass at a time, chained rather than joined.** Two passes overlapping read the same entry
157
+ * as sendable and put the same key on the wire twice — with the same seq, and the node dedupes
158
+ * nothing — and a pass that started later could pass a mutation the pass in front of it has not
159
+ * reached yet, which is the ordering guarantee above, gone. Chained rather than joined because a
160
+ * caller that enqueued after the running pass began must still see its own mutation sent: it
161
+ * gets a pass BEHIND that one, not that one's promise. The chain hangs off a settled shadow, so
162
+ * one pass that rejected does not reject every pass behind it.
133
163
  */
134
164
  async drain(send: MutationSender): Promise<DrainReport> {
135
- let sent = 0;
136
- for (const mutation of this.pending()) {
137
- mutation.status = 'inflight';
138
- mutation.attempts += 1;
139
- try {
140
- await send(mutation);
141
- mutation.status = 'acked';
142
- mutation.error = null;
143
- sent += 1;
144
- } catch (error) {
145
- mutation.status = 'pending';
146
- mutation.error = toQueueError(error);
147
- await this.#persist();
148
- return {
149
- sent,
150
- collapsed: this.#collapsed,
151
- remaining: this.pending().length,
152
- stoppedAt: mutation.key,
153
- };
154
- }
165
+ const ahead = this.#draining?.then(
166
+ () => undefined,
167
+ () => undefined,
168
+ );
169
+ const pass = (ahead ?? Promise.resolve()).then(() => this.#pass(send));
170
+ this.#draining = pass;
171
+ try {
172
+ return await pass;
173
+ } finally {
174
+ // Cleared only by the last pass in the chain, so the next drain starts fresh instead of
175
+ // queueing behind a promise that settled a lifetime ago.
176
+ if (this.#draining === pass) this.#draining = null;
155
177
  }
156
- await this.#persist();
157
- return { sent, collapsed: this.#collapsed, remaining: this.pending().length, stoppedAt: null };
178
+ }
179
+
180
+ /**
181
+ * A lost connection: everything handed to the dead socket goes back to `pending`, because a
182
+ * `send` that returned is not an acknowledgement and those frames may never have left the tab.
183
+ * At least once by construction — the idempotency key is what makes the resend safe.
184
+ *
185
+ * The epoch is bumped BEFORE the scan, not after: a pass parked at `await send(p1)` on the socket
186
+ * that just died resumes into this same turn and would otherwise mark p2 and p3 `inflight` for a
187
+ * connection that is gone. `#sendable` excludes `inflight`, so the next drain skips them, no ack
188
+ * will ever arrive, and the writes are lost — which is exactly what invariant 3 forbids.
189
+ */
190
+ async requeueInflight(): Promise<number> {
191
+ this.#epoch += 1;
192
+ let returned = 0;
193
+ for (const mutation of this.#mutations) {
194
+ if (mutation.status !== 'inflight') continue;
195
+ mutation.status = 'pending';
196
+ returned += 1;
197
+ }
198
+ if (returned > 0) await this.#persist();
199
+ return returned;
158
200
  }
159
201
 
160
202
  /** Server acknowledged: the mutation leaves the queue and its rebase entry can be committed. */
@@ -175,13 +217,82 @@ export class OfflineQueue {
175
217
  await this.#persist();
176
218
  }
177
219
 
220
+ /** Never sent on this connection. `inflight` is excluded: it is already on a socket. */
221
+ #sendable(): readonly QueuedMutation[] {
222
+ return this.#mutations
223
+ .filter((mutation) => mutation.status === 'pending')
224
+ .sort((a, b) => a.seq - b.seq);
225
+ }
226
+
227
+ /** One drain pass. Never called concurrently with itself — `drain` owns that. */
228
+ async #pass(send: MutationSender): Promise<DrainReport> {
229
+ const epoch = this.#epoch;
230
+ const sendable = this.#sendable();
231
+ // Nothing to do: a pass chained behind one that already sent everything must not rewrite the
232
+ // durable state for the privilege of reporting zero.
233
+ if (sendable.length === 0) {
234
+ return { sent: 0, collapsed: this.#collapsed, remaining: 0, stoppedAt: null };
235
+ }
236
+ let sent = 0;
237
+ for (const mutation of sendable) {
238
+ // The connection this pass was draining into is gone, and `requeueInflight` has already
239
+ // handed back what was on it. Everything left stays `pending` for the pass the next
240
+ // connection arms — claiming it here would strand it on a socket that cannot answer.
241
+ if (epoch !== this.#epoch) {
242
+ return {
243
+ sent,
244
+ collapsed: this.#collapsed,
245
+ remaining: this.#sendable().length,
246
+ stoppedAt: mutation.key,
247
+ };
248
+ }
249
+ mutation.status = 'inflight';
250
+ mutation.attempts += 1;
251
+ try {
252
+ await send(mutation);
253
+ // Stays `inflight`. `send` resolving means the frame reached a socket — a browser
254
+ // `WebSocket.send` on a CLOSING socket discards it and returns normally — so calling that
255
+ // an ack drops the mutation on exactly the socket death this queue exists to survive.
256
+ // Only `ack`/`fail` (the server) or `requeueInflight` (a lost connection) moves it on.
257
+ mutation.error = null;
258
+ sent += 1;
259
+ } catch (error) {
260
+ mutation.status = 'pending';
261
+ mutation.error = toQueueError(error);
262
+ await this.#persist();
263
+ return {
264
+ sent,
265
+ collapsed: this.#collapsed,
266
+ remaining: this.#sendable().length,
267
+ stoppedAt: mutation.key,
268
+ };
269
+ }
270
+ }
271
+ await this.#persist();
272
+ return {
273
+ sent,
274
+ collapsed: this.#collapsed,
275
+ remaining: this.#sendable().length,
276
+ stoppedAt: null,
277
+ };
278
+ }
279
+
178
280
  async clear(): Promise<void> {
179
281
  this.#mutations = [];
180
282
  await this.#persist();
181
283
  }
182
284
 
285
+ /**
286
+ * A snapshot, never the live entries. `save` is a durable write — OPFS, IndexedDB — and it is
287
+ * allowed to await before it reads. Handed the array itself, a store that resolves after the next
288
+ * pass has moved on persists a status that was never true when it was called; `inflight` is the
289
+ * one a reload cannot recover from, because `#sendable` skips it and no ack is coming.
290
+ */
183
291
  async #persist(): Promise<void> {
184
- await this.#store.save({ mutations: this.#mutations, nextSeq: this.#nextSeq });
292
+ await this.#store.save({
293
+ mutations: this.#mutations.map((mutation) => ({ ...mutation })),
294
+ nextSeq: this.#nextSeq,
295
+ });
185
296
  }
186
297
  }
187
298
 
@@ -197,10 +308,15 @@ export function mutateFrame(mutation: QueuedMutation): Frame {
197
308
  }
198
309
 
199
310
  function toQueueError(error: unknown): WireError {
200
- const shape = error as { code?: unknown; cause?: unknown; fix?: unknown } | null;
201
311
  return {
202
- code: typeof shape?.code === 'string' ? shape.code : 'X_TRANSPORT_UNAVAILABLE',
203
- cause: typeof shape?.cause === 'string' ? shape.cause : String(error),
204
- fix: typeof shape?.fix === 'string' ? shape.fix : 'the queue retries on the next reconnect',
312
+ // `stringField`, not `shape?.code`: the sender is a transport the app supplied, so the probe
313
+ // for "did it throw a coded error" is itself a property read on an app value. A getter that
314
+ // throws escaped `drain`'s catch through the probe rather than the render — the same contract
315
+ // break one line earlier than the one the comment below records.
316
+ code: stringField(error, 'code') ?? 'X_TRANSPORT_UNAVAILABLE',
317
+ // Whatever the sender threw. `String()` here escaped `drain`'s own catch, so the queue's
318
+ // stop-at-the-first-failure contract broke on the failure it exists to record.
319
+ cause: stringField(error, 'cause') ?? renderThrowable(error),
320
+ fix: stringField(error, 'fix') ?? 'the queue retries on the next reconnect',
205
321
  };
206
322
  }