@ultimat3/realtime 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +591 -0
- package/README.md +320 -19
- package/package.json +6 -3
- package/src/apply-patches.ts +60 -0
- package/src/change-buffer.ts +77 -11
- package/src/channel.ts +174 -19
- package/src/client-contract.ts +81 -0
- package/src/client-frames.ts +175 -0
- package/src/client-heartbeat.ts +77 -0
- package/src/client-mutations.ts +114 -0
- package/src/client-topics.ts +54 -0
- package/src/client.ts +307 -273
- package/src/cursor.ts +7 -1
- package/src/errors.ts +193 -4
- package/src/frame-lanes.ts +58 -0
- package/src/hooks.ts +19 -5
- package/src/identity-map.ts +141 -0
- package/src/index.ts +96 -28
- package/src/json.ts +38 -1
- package/src/live-contract.ts +67 -0
- package/src/live-definition.ts +16 -11
- package/src/live-fanout.ts +150 -0
- package/src/live-query.ts +215 -268
- package/src/live-rows.ts +143 -0
- package/src/local-store.ts +86 -43
- package/src/nats-client.ts +132 -0
- package/src/nats-fake.ts +389 -344
- package/src/nats-jetstream.ts +21 -20
- package/src/nats-kv.ts +7 -7
- package/src/nats-lib-client.ts +210 -0
- package/src/nats-transport.ts +109 -138
- package/src/offline-queue.ts +146 -30
- package/src/pg-entity-row.ts +99 -31
- package/src/pg-replication.ts +84 -27
- package/src/pg-socket.ts +4 -1
- package/src/policy-gate.ts +13 -5
- package/src/presence.ts +76 -6
- package/src/query-hook.ts +56 -0
- package/src/query-window.ts +151 -0
- package/src/rebase.ts +68 -8
- package/src/replicator.ts +84 -11
- package/src/socket.ts +170 -14
- package/src/subscriber-gate.ts +209 -0
- package/src/subscription-book.ts +237 -0
- package/src/sync-auth.ts +124 -0
- package/src/sync-frames.ts +185 -0
- package/src/sync-listen.ts +73 -0
- package/src/sync-node.ts +284 -243
- package/src/sync-protocol.ts +115 -24
- package/src/sync-upgrade.ts +124 -0
- package/src/thundering-herd.ts +21 -0
- package/src/transport-env.ts +3 -3
- package/src/type-pins.ts +72 -0
- package/src/window-lock.ts +21 -0
- package/src/nats-commands.ts +0 -97
- package/src/nats-connection-fixture.ts +0 -105
- package/src/nats-connection.ts +0 -464
- package/src/nats-protocol.ts +0 -222
- package/src/nats-socket.ts +0 -236
- package/src/pg-connection-fixture.ts +0 -215
- package/src/pg-replication-fixture.ts +0 -261
package/src/nats-transport.ts
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
// Single responsibility: the production `Transport` — core NATS for fanout, JetStream KV for the
|
|
2
|
-
// shared presence sets.
|
|
3
|
-
//
|
|
4
|
-
//
|
|
2
|
+
// shared presence sets. The client underneath owns the wire and the reconnect, including
|
|
3
|
+
// re-establishing subscriptions, which is what makes a `sync` node stateless: a lost connection is
|
|
4
|
+
// re-dialled and re-subscribed underneath the caller, and this file keeps no socket state at all.
|
|
5
5
|
|
|
6
|
-
import { type Clock, isUltimateError, logger, systemClock } from '@ultimat3/core';
|
|
6
|
+
import { type Clock, isUltimateError, logger, renderThrowable, systemClock } from '@ultimat3/core';
|
|
7
7
|
import { TransportUnavailableError } from './errors';
|
|
8
8
|
import type { Transport, TransportHandler, TransportSet, TransportSubscription } from './fanout';
|
|
9
|
-
import {
|
|
9
|
+
import type { NatsClient, NatsConnect } from './nats-client';
|
|
10
|
+
import { parseNatsUrl } from './nats-client';
|
|
10
11
|
import { ensureKvBucket } from './nats-jetstream';
|
|
11
12
|
import { NatsKvSet } from './nats-kv';
|
|
12
|
-
import {
|
|
13
|
+
import { openNatsClient } from './nats-lib-client';
|
|
13
14
|
import { type BackoffPolicy, backoffDelay, defaultBackoff, type Rng } from './thundering-herd';
|
|
14
15
|
|
|
15
16
|
const encoder = new TextEncoder();
|
|
@@ -28,44 +29,37 @@ export interface NatsTransportOptions {
|
|
|
28
29
|
readonly onError?: (error: unknown, subject: string) => void;
|
|
29
30
|
readonly rng?: Rng;
|
|
30
31
|
/** Injected so the whole transport — reconnect included — runs in a test with no network. */
|
|
31
|
-
readonly
|
|
32
|
-
readonly sleep?: (ms: number) => Promise<void>;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
interface Wanted {
|
|
36
|
-
readonly subject: string;
|
|
37
|
-
readonly handler: TransportHandler;
|
|
38
|
-
live: NatsSubscription | undefined;
|
|
32
|
+
readonly connect?: NatsConnect;
|
|
39
33
|
}
|
|
40
34
|
|
|
41
35
|
const DEFAULT_ATTEMPTS = 10;
|
|
36
|
+
const DEFAULT_PRESENCE_TTL_MS = 30_000;
|
|
42
37
|
|
|
43
38
|
/** The production bus: NATS subjects for fanout, a JetStream KV bucket for presence. */
|
|
44
39
|
export class NatsTransport implements Transport {
|
|
45
40
|
readonly name = 'nats';
|
|
46
41
|
readonly shared: TransportSet;
|
|
47
|
-
readonly #target: NatsTarget;
|
|
48
42
|
readonly #options: NatsTransportOptions;
|
|
49
|
-
readonly #
|
|
43
|
+
readonly #connect: NatsConnect;
|
|
50
44
|
readonly #backoff: BackoffPolicy;
|
|
51
45
|
readonly #attempts: number;
|
|
52
46
|
readonly #rng: Rng;
|
|
53
|
-
|
|
54
|
-
#
|
|
55
|
-
#
|
|
56
|
-
#next = 0;
|
|
57
|
-
#losses = 0;
|
|
47
|
+
#client: NatsClient | undefined;
|
|
48
|
+
#dialing: Promise<NatsClient> | undefined;
|
|
49
|
+
#retries = 0;
|
|
58
50
|
#closed = false;
|
|
59
51
|
|
|
60
52
|
constructor(options: NatsTransportOptions) {
|
|
61
|
-
|
|
53
|
+
// Parsed here rather than at the first publish: a malformed NATS_URL is a boot-time fault, and
|
|
54
|
+
// a container that reports itself healthy on one is a container nothing will ever page about.
|
|
55
|
+
parseNatsUrl(options.url);
|
|
62
56
|
this.#options = options;
|
|
57
|
+
this.#connect = options.connect ?? openNatsClient;
|
|
63
58
|
this.#backoff = options.backoff ?? defaultBackoff;
|
|
64
59
|
this.#attempts = options.maxReconnectAttempts ?? DEFAULT_ATTEMPTS;
|
|
65
60
|
this.#rng = options.rng ?? Math.random;
|
|
66
|
-
this.#sleep = options.sleep ?? ((ms) => Bun.sleep(ms));
|
|
67
61
|
this.shared = new NatsKvSet({
|
|
68
|
-
|
|
62
|
+
client: () => this.#ensure(),
|
|
69
63
|
bucket: options.bucket,
|
|
70
64
|
clock: options.clock ?? systemClock,
|
|
71
65
|
});
|
|
@@ -77,107 +71,88 @@ export class NatsTransport implements Transport {
|
|
|
77
71
|
}
|
|
78
72
|
|
|
79
73
|
get connected(): boolean {
|
|
80
|
-
return this.#
|
|
74
|
+
return this.#client?.connected === true;
|
|
81
75
|
}
|
|
82
76
|
|
|
83
77
|
async publish(subject: string, payload: string): Promise<void> {
|
|
84
|
-
const
|
|
85
|
-
|
|
78
|
+
const client = await this.#ensure();
|
|
79
|
+
// `client.publish` is synchronous and refuses locally: a bad subject, a payload over the
|
|
80
|
+
// server's `max_payload`, a connection torn down between the `#ensure` and this line. Those
|
|
81
|
+
// are the LIBRARY's errors — or an app-supplied `connect`'s — so they arrive uncoded, and
|
|
82
|
+
// `ChannelHub`'s bridge, `SocketRegistry` and the replicator all await this call.
|
|
83
|
+
this.#translating(`publish to ${subject}`, () =>
|
|
84
|
+
client.publish(subject, encoder.encode(payload)),
|
|
85
|
+
);
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
+
/**
|
|
89
|
+
* The subscription is the client's to keep: it survives a drop and comes back with the reconnect,
|
|
90
|
+
* so there is no intent map here to re-bind from — and therefore no way for a re-bind to run
|
|
91
|
+
* twice and double every change on the subject.
|
|
92
|
+
*/
|
|
88
93
|
async subscribe(subject: string, handler: TransportHandler): Promise<TransportSubscription> {
|
|
89
|
-
this.#
|
|
90
|
-
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
}
|
|
102
|
-
return {
|
|
103
|
-
subject,
|
|
104
|
-
unsubscribe: () => {
|
|
105
|
-
this.#wanted.delete(id);
|
|
106
|
-
void wanted.live?.unsubscribe().catch((error: unknown) => this.#report(error, subject));
|
|
107
|
-
wanted.live = undefined;
|
|
108
|
-
},
|
|
109
|
-
};
|
|
94
|
+
const client = await this.#ensure();
|
|
95
|
+
// Same seam as `publish`: a permissions violation on the subject is refused here, not later.
|
|
96
|
+
const live = this.#translating(`subscribe to ${subject}`, () =>
|
|
97
|
+
client.subscribe(subject, (message) => {
|
|
98
|
+
try {
|
|
99
|
+
handler(decoder.decode(message.payload), message.subject);
|
|
100
|
+
} catch (error) {
|
|
101
|
+
this.#report(error, message.subject);
|
|
102
|
+
}
|
|
103
|
+
}),
|
|
104
|
+
);
|
|
105
|
+
return { subject, unsubscribe: () => live.unsubscribe() };
|
|
110
106
|
}
|
|
111
107
|
|
|
112
108
|
async close(): Promise<void> {
|
|
113
109
|
this.#closed = true;
|
|
114
|
-
this.#
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
await connection?.close();
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
#bind(connection: NatsConnection, wanted: Wanted): Promise<NatsSubscription> {
|
|
121
|
-
return connection.subscribe(wanted.subject, (message) => {
|
|
122
|
-
try {
|
|
123
|
-
wanted.handler(decoder.decode(message.payload), message.subject);
|
|
124
|
-
} catch (error) {
|
|
125
|
-
this.#report(error, message.subject);
|
|
126
|
-
}
|
|
127
|
-
});
|
|
110
|
+
const client = this.#client;
|
|
111
|
+
this.#client = undefined;
|
|
112
|
+
await client?.close();
|
|
128
113
|
}
|
|
129
114
|
|
|
130
|
-
|
|
115
|
+
/**
|
|
116
|
+
* One dial, shared by every caller that races it. A client that has already been handed out is
|
|
117
|
+
* reused whatever its state: while it is reconnecting the library is re-establishing that same
|
|
118
|
+
* connection and its subscriptions, and a second dial alongside it would double every delivery.
|
|
119
|
+
* A budget that ran out is a readiness failure, not a reason to start an unbounded retry here.
|
|
120
|
+
*/
|
|
121
|
+
#ensure(): Promise<NatsClient> {
|
|
131
122
|
if (this.#closed) {
|
|
132
123
|
return Promise.reject(
|
|
133
124
|
new TransportUnavailableError({ transport: this.name, reason: 'transport is closed' }),
|
|
134
125
|
);
|
|
135
126
|
}
|
|
136
|
-
const current = this.#
|
|
137
|
-
if (current !== undefined
|
|
127
|
+
const current = this.#client;
|
|
128
|
+
if (current !== undefined) return Promise.resolve(current);
|
|
138
129
|
this.#dialing ??= this.#dial().finally(() => {
|
|
139
130
|
this.#dialing = undefined;
|
|
140
131
|
});
|
|
141
132
|
return this.#dialing;
|
|
142
133
|
}
|
|
143
134
|
|
|
144
|
-
/** Retry is bounded: a bus that is down for longer than the budget is a readiness failure. */
|
|
145
|
-
async #dial(): Promise<NatsConnection> {
|
|
146
|
-
for (let attempt = 0; ; attempt += 1) {
|
|
147
|
-
try {
|
|
148
|
-
return await this.#establish();
|
|
149
|
-
} catch (error) {
|
|
150
|
-
// A protocol mismatch answers the same on every attempt — a server too old for per-message
|
|
151
|
-
// TTL stays too old — so retrying only delays the one report that names the fix.
|
|
152
|
-
const fatal = isUltimateError(error) && error.code === 'X_TRANSPORT_PROTOCOL';
|
|
153
|
-
if (fatal || this.#closed || attempt >= this.#attempts) {
|
|
154
|
-
throw isUltimateError(error)
|
|
155
|
-
? error
|
|
156
|
-
: new TransportUnavailableError({
|
|
157
|
-
transport: this.name,
|
|
158
|
-
reason: `${this.#target.host}:${this.#target.port} — ${String(error)}`,
|
|
159
|
-
});
|
|
160
|
-
}
|
|
161
|
-
await this.#sleep(backoffDelay(attempt, this.#backoff, this.#rng));
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
|
|
166
135
|
/**
|
|
167
|
-
* One attempt, published only once it is whole. A
|
|
168
|
-
*
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
* A failed attempt therefore takes its own socket down with it rather than leaking one per retry.
|
|
136
|
+
* One attempt, published only once it is whole. A client parked in `#client` before its bucket is
|
|
137
|
+
* up answers `connected` for a dial that rejected, and the next caller then writes presence into
|
|
138
|
+
* a bucket that does not exist. A failed attempt therefore closes its own connection rather than
|
|
139
|
+
* leaking one per retry.
|
|
172
140
|
*/
|
|
173
|
-
async #
|
|
174
|
-
const
|
|
141
|
+
async #dial(): Promise<NatsClient> {
|
|
142
|
+
const client = await this.#connect({
|
|
143
|
+
url: this.#options.url,
|
|
144
|
+
name: 'ultimate',
|
|
145
|
+
maxReconnectAttempts: this.#attempts,
|
|
146
|
+
// The library retries; the spread is ours, so a cluster restart does not bring every node
|
|
147
|
+
// back on the same millisecond.
|
|
148
|
+
reconnectDelay: () => backoffDelay(this.#retries++, this.#backoff, this.#rng),
|
|
149
|
+
onError: (error) => this.#report(error, this.name),
|
|
150
|
+
onReconnect: () => this.#recovered(),
|
|
151
|
+
});
|
|
175
152
|
try {
|
|
176
|
-
await
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
// `close()` can land while an attempt is in flight, and it only closes what it can see:
|
|
180
|
-
// publishing now would leave a socket open that nothing will ever close again.
|
|
153
|
+
await this.#ensureBucket(client);
|
|
154
|
+
// `close()` can land while a dial is in flight, and it only closes what it can see:
|
|
155
|
+
// publishing now would leave a connection open that nothing will ever close again.
|
|
181
156
|
if (this.#closed) {
|
|
182
157
|
throw new TransportUnavailableError({
|
|
183
158
|
transport: this.name,
|
|
@@ -185,55 +160,51 @@ export class NatsTransport implements Transport {
|
|
|
185
160
|
});
|
|
186
161
|
}
|
|
187
162
|
} catch (error) {
|
|
188
|
-
|
|
189
|
-
await connection.close();
|
|
163
|
+
await client.close();
|
|
190
164
|
throw error;
|
|
191
165
|
}
|
|
192
|
-
this.#
|
|
193
|
-
this.#
|
|
194
|
-
return
|
|
166
|
+
this.#client = client;
|
|
167
|
+
this.#retries = 0;
|
|
168
|
+
return client;
|
|
195
169
|
}
|
|
196
170
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
held.connection = await NatsConnection.open({
|
|
204
|
-
stream,
|
|
205
|
-
target: this.#target,
|
|
206
|
-
name: 'ultimate',
|
|
207
|
-
rng: this.#options.rng,
|
|
208
|
-
onClose: (error) => this.#lost(error, held.connection),
|
|
209
|
-
onError: (error) => this.#report(error, this.name),
|
|
210
|
-
});
|
|
211
|
-
return held.connection;
|
|
171
|
+
#ensureBucket(client: NatsClient): Promise<void> {
|
|
172
|
+
return ensureKvBucket(
|
|
173
|
+
client,
|
|
174
|
+
this.#options.bucket,
|
|
175
|
+
this.#options.presenceTtlMs ?? DEFAULT_PRESENCE_TTL_MS,
|
|
176
|
+
);
|
|
212
177
|
}
|
|
213
178
|
|
|
214
179
|
/**
|
|
215
|
-
* A
|
|
216
|
-
*
|
|
180
|
+
* A reconnect may have landed on a different cluster — a restarted single node, or a failover to
|
|
181
|
+
* one that never held this bucket. The subscriptions came back with the client; the bucket is the
|
|
182
|
+
* one thing the library knows nothing about, so it is re-asserted here. It is idempotent.
|
|
217
183
|
*/
|
|
218
|
-
#
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
this.#connection = undefined;
|
|
224
|
-
for (const wanted of this.#wanted.values()) wanted.live = undefined;
|
|
225
|
-
this.#report(error, this.name);
|
|
226
|
-
if (this.#closed || this.#wanted.size === 0) return;
|
|
227
|
-
this.#losses += 1;
|
|
228
|
-
if (this.#losses > this.#attempts) return;
|
|
229
|
-
void this.#recover();
|
|
184
|
+
#recovered(): void {
|
|
185
|
+
this.#retries = 0;
|
|
186
|
+
const client = this.#client;
|
|
187
|
+
if (client === undefined) return;
|
|
188
|
+
void this.#ensureBucket(client).catch((error: unknown) => this.#report(error, this.name));
|
|
230
189
|
}
|
|
231
190
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
191
|
+
/**
|
|
192
|
+
* One call into the client, with its refusal translated. An `UltimateError` passes through — the
|
|
193
|
+
* port raises its own for a closed client, and re-wrapping would bury the code a caller branches
|
|
194
|
+
* on — while anything else becomes `X_TRANSPORT_UNAVAILABLE` carrying the library's own words as
|
|
195
|
+
* evidence. Never a bare `Error` out of this file: a raw `NatsError` has no code, no `fix:` and
|
|
196
|
+
* nothing an operator can act on, which is the whole reason the port is here.
|
|
197
|
+
*/
|
|
198
|
+
#translating<T>(what: string, call: () => T): T {
|
|
199
|
+
try {
|
|
200
|
+
return call();
|
|
201
|
+
} catch (error) {
|
|
202
|
+
if (isUltimateError(error)) throw error;
|
|
203
|
+
throw new TransportUnavailableError({
|
|
204
|
+
transport: this.name,
|
|
205
|
+
reason: `${what} was refused: ${renderThrowable(error)}`,
|
|
206
|
+
});
|
|
207
|
+
}
|
|
237
208
|
}
|
|
238
209
|
|
|
239
210
|
/**
|
package/src/offline-queue.ts
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
|
-
// Tier 3: the durable mutation queue.
|
|
1
|
+
// Tier 3: the durable mutation queue. Three invariants, all enforced here rather than documented:
|
|
2
2
|
//
|
|
3
3
|
// 1. **Order.** Mutations drain in client sequence order and stop at the first failure. A mutator
|
|
4
4
|
// that assumed `like` ran before `unlike` must never see them swapped.
|
|
5
5
|
// 2. **Dedupe.** The idempotency key is the identity of the intent. Re-enqueueing a key that is
|
|
6
6
|
// already queued (double click, replay after a crash) collapses onto the existing entry and
|
|
7
7
|
// never gets a new sequence number.
|
|
8
|
+
// 3. **Only the server removes a mutation.** A `send` that returned proves the frame was handed
|
|
9
|
+
// to a socket and nothing more, so a drained mutation is `inflight` — not `acked` — until an
|
|
10
|
+
// `ack`/`fail` frame settles it, or a lost connection returns it to the queue.
|
|
8
11
|
|
|
12
|
+
import { renderThrowable, stringField } from '@ultimat3/core';
|
|
9
13
|
import type { JsonValue } from './json';
|
|
10
14
|
import { type Frame, PROTOCOL_VERSION, type WireError } from './sync-protocol';
|
|
11
15
|
|
|
@@ -50,6 +54,7 @@ export class MemoryQueueStore implements QueueStore {
|
|
|
50
54
|
export interface DrainReport {
|
|
51
55
|
readonly sent: number;
|
|
52
56
|
readonly collapsed: number;
|
|
57
|
+
/** Still to send. Not the queue depth: a sent mutation is unacknowledged, never unsent. */
|
|
53
58
|
readonly remaining: number;
|
|
54
59
|
readonly stoppedAt: string | null;
|
|
55
60
|
}
|
|
@@ -61,6 +66,14 @@ export class OfflineQueue {
|
|
|
61
66
|
#mutations: QueuedMutation[] = [];
|
|
62
67
|
#nextSeq = 1;
|
|
63
68
|
#collapsed = 0;
|
|
69
|
+
/** The drain lane: one pass at a time, in call order. See `drain`. */
|
|
70
|
+
#draining: Promise<DrainReport> | null = null;
|
|
71
|
+
/**
|
|
72
|
+
* Which connection the current pass is draining into. The lane orders passes against each other
|
|
73
|
+
* but `requeueInflight` is not a pass — it is a socket death, and it cannot reach into one that
|
|
74
|
+
* is parked inside `send`. Bumped by every loss so a pass that resumes afterwards claims nothing.
|
|
75
|
+
*/
|
|
76
|
+
#epoch = 0;
|
|
64
77
|
|
|
65
78
|
private constructor(store: QueueStore, state: QueueState) {
|
|
66
79
|
this.#store = store;
|
|
@@ -89,7 +102,10 @@ export class OfflineQueue {
|
|
|
89
102
|
return this.#mutations.find((mutation) => mutation.key === key);
|
|
90
103
|
}
|
|
91
104
|
|
|
92
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* Everything the server has not settled yet, sorted by sequence — this is the only order
|
|
107
|
+
* anything downstream is allowed to use, and the count a UI renders as "unsynced".
|
|
108
|
+
*/
|
|
93
109
|
pending(): readonly QueuedMutation[] {
|
|
94
110
|
return this.#mutations
|
|
95
111
|
.filter((mutation) => mutation.status === 'pending' || mutation.status === 'inflight')
|
|
@@ -107,10 +123,16 @@ export class OfflineQueue {
|
|
|
107
123
|
at?: number;
|
|
108
124
|
}): Promise<QueuedMutation> {
|
|
109
125
|
const existing = this.find(args.key);
|
|
110
|
-
if (existing) {
|
|
126
|
+
if (existing && existing.status !== 'failed') {
|
|
111
127
|
this.#collapsed += 1;
|
|
112
128
|
return existing;
|
|
113
129
|
}
|
|
130
|
+
// A terminally failed entry is a decision the server already made about this key, kept for the
|
|
131
|
+
// UI — collapsing onto it makes an explicit idempotency key unusable for the rest of the
|
|
132
|
+
// session, because nothing ever retries a denial. Re-issuing one is a NEW intent, so the old
|
|
133
|
+
// entry is dropped and this one takes a new sequence at the back of the queue.
|
|
134
|
+
if (existing)
|
|
135
|
+
this.#mutations = this.#mutations.filter((candidate) => candidate.key !== args.key);
|
|
114
136
|
const mutation: QueuedMutation = {
|
|
115
137
|
key: args.key,
|
|
116
138
|
seq: this.#nextSeq,
|
|
@@ -130,31 +152,51 @@ export class OfflineQueue {
|
|
|
130
152
|
/**
|
|
131
153
|
* Drains in sequence order and stops at the first failure. Continuing past a failure is how a
|
|
132
154
|
* sync engine reorders a user's intent — so it does not continue.
|
|
155
|
+
*
|
|
156
|
+
* **One pass at a time, chained rather than joined.** Two passes overlapping read the same entry
|
|
157
|
+
* as sendable and put the same key on the wire twice — with the same seq, and the node dedupes
|
|
158
|
+
* nothing — and a pass that started later could pass a mutation the pass in front of it has not
|
|
159
|
+
* reached yet, which is the ordering guarantee above, gone. Chained rather than joined because a
|
|
160
|
+
* caller that enqueued after the running pass began must still see its own mutation sent: it
|
|
161
|
+
* gets a pass BEHIND that one, not that one's promise. The chain hangs off a settled shadow, so
|
|
162
|
+
* one pass that rejected does not reject every pass behind it.
|
|
133
163
|
*/
|
|
134
164
|
async drain(send: MutationSender): Promise<DrainReport> {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
await this.#persist();
|
|
148
|
-
return {
|
|
149
|
-
sent,
|
|
150
|
-
collapsed: this.#collapsed,
|
|
151
|
-
remaining: this.pending().length,
|
|
152
|
-
stoppedAt: mutation.key,
|
|
153
|
-
};
|
|
154
|
-
}
|
|
165
|
+
const ahead = this.#draining?.then(
|
|
166
|
+
() => undefined,
|
|
167
|
+
() => undefined,
|
|
168
|
+
);
|
|
169
|
+
const pass = (ahead ?? Promise.resolve()).then(() => this.#pass(send));
|
|
170
|
+
this.#draining = pass;
|
|
171
|
+
try {
|
|
172
|
+
return await pass;
|
|
173
|
+
} finally {
|
|
174
|
+
// Cleared only by the last pass in the chain, so the next drain starts fresh instead of
|
|
175
|
+
// queueing behind a promise that settled a lifetime ago.
|
|
176
|
+
if (this.#draining === pass) this.#draining = null;
|
|
155
177
|
}
|
|
156
|
-
|
|
157
|
-
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* A lost connection: everything handed to the dead socket goes back to `pending`, because a
|
|
182
|
+
* `send` that returned is not an acknowledgement and those frames may never have left the tab.
|
|
183
|
+
* At least once by construction — the idempotency key is what makes the resend safe.
|
|
184
|
+
*
|
|
185
|
+
* The epoch is bumped BEFORE the scan, not after: a pass parked at `await send(p1)` on the socket
|
|
186
|
+
* that just died resumes into this same turn and would otherwise mark p2 and p3 `inflight` for a
|
|
187
|
+
* connection that is gone. `#sendable` excludes `inflight`, so the next drain skips them, no ack
|
|
188
|
+
* will ever arrive, and the writes are lost — which is exactly what invariant 3 forbids.
|
|
189
|
+
*/
|
|
190
|
+
async requeueInflight(): Promise<number> {
|
|
191
|
+
this.#epoch += 1;
|
|
192
|
+
let returned = 0;
|
|
193
|
+
for (const mutation of this.#mutations) {
|
|
194
|
+
if (mutation.status !== 'inflight') continue;
|
|
195
|
+
mutation.status = 'pending';
|
|
196
|
+
returned += 1;
|
|
197
|
+
}
|
|
198
|
+
if (returned > 0) await this.#persist();
|
|
199
|
+
return returned;
|
|
158
200
|
}
|
|
159
201
|
|
|
160
202
|
/** Server acknowledged: the mutation leaves the queue and its rebase entry can be committed. */
|
|
@@ -175,13 +217,82 @@ export class OfflineQueue {
|
|
|
175
217
|
await this.#persist();
|
|
176
218
|
}
|
|
177
219
|
|
|
220
|
+
/** Never sent on this connection. `inflight` is excluded: it is already on a socket. */
|
|
221
|
+
#sendable(): readonly QueuedMutation[] {
|
|
222
|
+
return this.#mutations
|
|
223
|
+
.filter((mutation) => mutation.status === 'pending')
|
|
224
|
+
.sort((a, b) => a.seq - b.seq);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** One drain pass. Never called concurrently with itself — `drain` owns that. */
|
|
228
|
+
async #pass(send: MutationSender): Promise<DrainReport> {
|
|
229
|
+
const epoch = this.#epoch;
|
|
230
|
+
const sendable = this.#sendable();
|
|
231
|
+
// Nothing to do: a pass chained behind one that already sent everything must not rewrite the
|
|
232
|
+
// durable state for the privilege of reporting zero.
|
|
233
|
+
if (sendable.length === 0) {
|
|
234
|
+
return { sent: 0, collapsed: this.#collapsed, remaining: 0, stoppedAt: null };
|
|
235
|
+
}
|
|
236
|
+
let sent = 0;
|
|
237
|
+
for (const mutation of sendable) {
|
|
238
|
+
// The connection this pass was draining into is gone, and `requeueInflight` has already
|
|
239
|
+
// handed back what was on it. Everything left stays `pending` for the pass the next
|
|
240
|
+
// connection arms — claiming it here would strand it on a socket that cannot answer.
|
|
241
|
+
if (epoch !== this.#epoch) {
|
|
242
|
+
return {
|
|
243
|
+
sent,
|
|
244
|
+
collapsed: this.#collapsed,
|
|
245
|
+
remaining: this.#sendable().length,
|
|
246
|
+
stoppedAt: mutation.key,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
mutation.status = 'inflight';
|
|
250
|
+
mutation.attempts += 1;
|
|
251
|
+
try {
|
|
252
|
+
await send(mutation);
|
|
253
|
+
// Stays `inflight`. `send` resolving means the frame reached a socket — a browser
|
|
254
|
+
// `WebSocket.send` on a CLOSING socket discards it and returns normally — so calling that
|
|
255
|
+
// an ack drops the mutation on exactly the socket death this queue exists to survive.
|
|
256
|
+
// Only `ack`/`fail` (the server) or `requeueInflight` (a lost connection) moves it on.
|
|
257
|
+
mutation.error = null;
|
|
258
|
+
sent += 1;
|
|
259
|
+
} catch (error) {
|
|
260
|
+
mutation.status = 'pending';
|
|
261
|
+
mutation.error = toQueueError(error);
|
|
262
|
+
await this.#persist();
|
|
263
|
+
return {
|
|
264
|
+
sent,
|
|
265
|
+
collapsed: this.#collapsed,
|
|
266
|
+
remaining: this.#sendable().length,
|
|
267
|
+
stoppedAt: mutation.key,
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
await this.#persist();
|
|
272
|
+
return {
|
|
273
|
+
sent,
|
|
274
|
+
collapsed: this.#collapsed,
|
|
275
|
+
remaining: this.#sendable().length,
|
|
276
|
+
stoppedAt: null,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
|
|
178
280
|
async clear(): Promise<void> {
|
|
179
281
|
this.#mutations = [];
|
|
180
282
|
await this.#persist();
|
|
181
283
|
}
|
|
182
284
|
|
|
285
|
+
/**
|
|
286
|
+
* A snapshot, never the live entries. `save` is a durable write — OPFS, IndexedDB — and it is
|
|
287
|
+
* allowed to await before it reads. Handed the array itself, a store that resolves after the next
|
|
288
|
+
* pass has moved on persists a status that was never true when it was called; `inflight` is the
|
|
289
|
+
* one a reload cannot recover from, because `#sendable` skips it and no ack is coming.
|
|
290
|
+
*/
|
|
183
291
|
async #persist(): Promise<void> {
|
|
184
|
-
await this.#store.save({
|
|
292
|
+
await this.#store.save({
|
|
293
|
+
mutations: this.#mutations.map((mutation) => ({ ...mutation })),
|
|
294
|
+
nextSeq: this.#nextSeq,
|
|
295
|
+
});
|
|
185
296
|
}
|
|
186
297
|
}
|
|
187
298
|
|
|
@@ -197,10 +308,15 @@ export function mutateFrame(mutation: QueuedMutation): Frame {
|
|
|
197
308
|
}
|
|
198
309
|
|
|
199
310
|
function toQueueError(error: unknown): WireError {
|
|
200
|
-
const shape = error as { code?: unknown; cause?: unknown; fix?: unknown } | null;
|
|
201
311
|
return {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
312
|
+
// `stringField`, not `shape?.code`: the sender is a transport the app supplied, so the probe
|
|
313
|
+
// for "did it throw a coded error" is itself a property read on an app value. A getter that
|
|
314
|
+
// throws escaped `drain`'s catch through the probe rather than the render — the same contract
|
|
315
|
+
// break one line earlier than the one the comment below records.
|
|
316
|
+
code: stringField(error, 'code') ?? 'X_TRANSPORT_UNAVAILABLE',
|
|
317
|
+
// Whatever the sender threw. `String()` here escaped `drain`'s own catch, so the queue's
|
|
318
|
+
// stop-at-the-first-failure contract broke on the failure it exists to record.
|
|
319
|
+
cause: stringField(error, 'cause') ?? renderThrowable(error),
|
|
320
|
+
fix: stringField(error, 'fix') ?? 'the queue retries on the next reconnect',
|
|
205
321
|
};
|
|
206
322
|
}
|