@ultimat3/realtime 21.0.0 → 22.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +293 -1009
- package/README.md +78 -12
- package/package.json +4 -4
- package/src/changefeed.ts +7 -1
- package/src/channel-authz.ts +23 -4
- package/src/channel-decl.ts +16 -5
- package/src/channel-describe.ts +7 -5
- package/src/channel-logs.ts +19 -1
- package/src/channel-records.ts +8 -0
- package/src/client-channels.ts +75 -5
- package/src/client.ts +14 -2
- package/src/cursor.ts +5 -0
- package/src/errors.ts +21 -0
- package/src/idb-fake.ts +24 -4
- package/src/idb-types.ts +7 -0
- package/src/index.ts +0 -1
- package/src/live-definition.ts +5 -1
- package/src/live-fanout.ts +51 -2
- package/src/live-query.ts +11 -0
- package/src/live-replicator.ts +160 -0
- package/src/local-store-idb.ts +89 -15
- package/src/matcher-bridge.ts +5 -0
- package/src/nats-fake.ts +10 -1
- package/src/nats-jetstream.ts +36 -14
- package/src/nats-transport.ts +2 -2
- package/src/offline-queue.ts +76 -21
- package/src/page-outbox.ts +80 -10
- package/src/page-socket.ts +39 -8
- package/src/pg-entity-row.ts +37 -184
- package/src/pg-preflight.ts +24 -2
- package/src/pg-replication.ts +19 -6
- package/src/pg-wire.ts +51 -15
- package/src/policy-fake.ts +14 -0
- package/src/query-window.ts +35 -21
- package/src/replicator.ts +13 -3
- package/src/server.ts +8 -3
- package/src/socket-drops.ts +30 -0
- package/src/socket-engine.ts +15 -3
- package/src/socket-host.ts +103 -4
- package/src/socket-idle.ts +21 -0
- package/src/socket.ts +41 -38
- package/src/subscriber-gate.ts +92 -3
- package/src/sync-node.ts +2 -7
- package/src/thundering-herd.ts +12 -11
- package/src/transport-env.ts +55 -14
- package/src/use-mutation.ts +13 -0
- package/src/use-query.ts +10 -5
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// How many dropped frames close a socket: more than `maxDroppedFrames` inside one window, never a
|
|
2
|
+
// lifetime count. Split from `socket.ts`, which counts the drops; this decides what they mean.
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The window `maxDroppedFrames` is counted over. Ten seconds: long enough that a burst of
|
|
6
|
+
* backpressure drops closes a socket that cannot keep up, short enough that a connection open for
|
|
7
|
+
* hours is never closed for a lifetime's worth of isolated drops.
|
|
8
|
+
*/
|
|
9
|
+
export const DROP_WINDOW_MS = 10_000;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The instants of the drops inside the current window. A burst of backpressure drops closes a
|
|
13
|
+
* socket that cannot keep up; the same number spread over a connection open for hours is a flaky
|
|
14
|
+
* link the cursor repairs, and closing for it was a healthy socket lost after 33 lifetime drops.
|
|
15
|
+
*/
|
|
16
|
+
export class DropWindow {
|
|
17
|
+
readonly #max: number;
|
|
18
|
+
readonly #recent: number[] = [];
|
|
19
|
+
|
|
20
|
+
constructor(max: number) {
|
|
21
|
+
this.#max = max;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Records one drop at `now` (monotonic ms) and answers whether the window is over its ceiling. */
|
|
25
|
+
overflowed(now: number): boolean {
|
|
26
|
+
this.#recent.push(now);
|
|
27
|
+
while ((this.#recent[0] ?? now) <= now - DROP_WINDOW_MS) this.#recent.shift();
|
|
28
|
+
return this.#recent.length > this.#max;
|
|
29
|
+
}
|
|
30
|
+
}
|
package/src/socket-engine.ts
CHANGED
|
@@ -13,8 +13,8 @@ import { PortRouter } from './socket-routes';
|
|
|
13
13
|
import { decode, encode, type Frame, PROTOCOL_VERSION } from './sync-protocol';
|
|
14
14
|
import {
|
|
15
15
|
type BackoffPolicy,
|
|
16
|
-
backoffDelay,
|
|
17
16
|
browserBackoff,
|
|
17
|
+
policyDelay,
|
|
18
18
|
type Rng,
|
|
19
19
|
type Scheduler,
|
|
20
20
|
timeoutScheduler,
|
|
@@ -250,7 +250,8 @@ export class SocketEngine {
|
|
|
250
250
|
if (this.#reconnect !== null) return;
|
|
251
251
|
const rng = this.#options.rng ?? Math.random;
|
|
252
252
|
const delay =
|
|
253
|
-
|
|
253
|
+
// `#attempt` counts reconnects already scheduled, from 0; the wait being armed is the next one.
|
|
254
|
+
afterMs ?? policyDelay(this.#options.backoff ?? browserBackoff, this.#attempt + 1, rng);
|
|
254
255
|
this.#attempt += 1;
|
|
255
256
|
this.#reconnect = this.#schedule(() => {
|
|
256
257
|
this.#reconnect = null;
|
|
@@ -284,6 +285,17 @@ export class SocketEngine {
|
|
|
284
285
|
socket?.close(1000, 'no tab left');
|
|
285
286
|
}
|
|
286
287
|
|
|
288
|
+
/**
|
|
289
|
+
* A silent port, released — and TOLD first. Silent is usually a closed tab, but a hidden tab the
|
|
290
|
+
* browser throttled is silent too, and it was released without a word: its virtual socket stayed
|
|
291
|
+
* "open" over a port nobody read, and that tab's realtime was dead until a reload. Told, its
|
|
292
|
+
* client goes offline and redials, and the page re-hosts (`socket-host.ts`'s `rehosting`).
|
|
293
|
+
*/
|
|
294
|
+
#reap(attached: AttachedPort): void {
|
|
295
|
+
if (attached.open) this.#post(attached, { t: 'close', code: 1006 });
|
|
296
|
+
this.#detach(attached);
|
|
297
|
+
}
|
|
298
|
+
|
|
287
299
|
/** A `MessagePort` has no close event: a port silent for three beats is a closed tab. */
|
|
288
300
|
#armReaper(): void {
|
|
289
301
|
if (this.#reaper !== null) return;
|
|
@@ -291,7 +303,7 @@ export class SocketEngine {
|
|
|
291
303
|
this.#reaper = null;
|
|
292
304
|
const cutoff = this.#now() - REAP_AFTER_BEATS * this.#beatMs;
|
|
293
305
|
for (const attached of [...this.#ports.values()]) {
|
|
294
|
-
if (attached.lastSeen < cutoff) this.#
|
|
306
|
+
if (attached.lastSeen < cutoff) this.#reap(attached);
|
|
295
307
|
}
|
|
296
308
|
if (this.#ports.size > 0) this.#armReaper();
|
|
297
309
|
}, this.#beatMs);
|
package/src/socket-host.ts
CHANGED
|
@@ -7,12 +7,15 @@ import { browserSocket, dialUrl } from './browser-socket';
|
|
|
7
7
|
import type { ClientSocket } from './client-contract';
|
|
8
8
|
import type { SyncTarget } from './page-store';
|
|
9
9
|
import { messagePort, type PortLike, SocketEngine } from './socket-engine';
|
|
10
|
+
import { type Scheduler, timeoutScheduler } from './thundering-herd';
|
|
10
11
|
|
|
11
12
|
export interface SocketHostOptions {
|
|
12
13
|
/** The built worker script (`<meta name="ultimate-sync-worker">`); absent = in-page host. */
|
|
13
14
|
readonly workerUrl?: string | undefined;
|
|
14
15
|
/** The principal the page acts for: one worker per principal, never a shared socket across two. */
|
|
15
16
|
readonly scope: string | null;
|
|
17
|
+
/** The build the page was rendered by: one worker per build, too. See `workerName`. */
|
|
18
|
+
readonly buildId?: string | undefined;
|
|
16
19
|
/** Injected for tests; production reads the globals. */
|
|
17
20
|
readonly sharedWorker?: SharedWorkerLike | undefined;
|
|
18
21
|
readonly inPageEngine?: () => SocketEngine;
|
|
@@ -32,9 +35,13 @@ export interface SocketHost {
|
|
|
32
35
|
bye(): void;
|
|
33
36
|
}
|
|
34
37
|
|
|
35
|
-
/**
|
|
36
|
-
|
|
37
|
-
|
|
38
|
+
/**
|
|
39
|
+
* The worker's name carries the scope, so two principals in two tabs get two workers — and the
|
|
40
|
+
* BUILD, so two builds do too. A SharedWorker keeps the first tab's build id for its whole life, so
|
|
41
|
+
* a tab of the new build joined the old engine and was told "update available" about itself.
|
|
42
|
+
*/
|
|
43
|
+
export function workerName(scope: string | null, buildId: string | undefined): string {
|
|
44
|
+
return `ultimate-sync:${encodeURIComponent(scope ?? '')}:${encodeURIComponent(buildId ?? '')}`;
|
|
38
45
|
}
|
|
39
46
|
|
|
40
47
|
export function openHost(options: SocketHostOptions): SocketHost {
|
|
@@ -55,7 +62,9 @@ function workerPort(options: SocketHostOptions): PortLike | undefined {
|
|
|
55
62
|
(typeof SharedWorker === 'function' ? (SharedWorker as SharedWorkerLike) : undefined);
|
|
56
63
|
if (Worker === undefined || options.workerUrl === undefined) return undefined;
|
|
57
64
|
try {
|
|
58
|
-
return messagePort(
|
|
65
|
+
return messagePort(
|
|
66
|
+
new Worker(options.workerUrl, { name: workerName(options.scope, options.buildId) }).port,
|
|
67
|
+
);
|
|
59
68
|
} catch {
|
|
60
69
|
return undefined;
|
|
61
70
|
}
|
|
@@ -124,3 +133,93 @@ class VirtualSocket implements ClientSocket {
|
|
|
124
133
|
}
|
|
125
134
|
}
|
|
126
135
|
}
|
|
136
|
+
|
|
137
|
+
/** A host that can be replaced under the page's one client. */
|
|
138
|
+
export interface RehostingHost extends SocketHost {
|
|
139
|
+
/** Say bye to the current host and build a new one — the next `socket()` dials through it. */
|
|
140
|
+
rehost(): void;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export interface RehostingOptions {
|
|
144
|
+
/** How long a virtual socket may wait for `open` before its host is presumed dead. */
|
|
145
|
+
readonly openTimeoutMs: number;
|
|
146
|
+
readonly schedule?: Scheduler | undefined;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* The page's host, replaceable. A port the engine reaped (a throttled hidden tab), or one the tab
|
|
151
|
+
* itself said `bye` on at `pagehide` before a bfcache restore, is a port nobody reads: the virtual
|
|
152
|
+
* socket over it waited for `open` forever and the tab's realtime was dead until a reload. So an
|
|
153
|
+
* `open` unanswered by `openTimeoutMs` closes that socket (1006, which the client redials on) and
|
|
154
|
+
* re-hosts on a fresh port; `page-socket.ts` also re-hosts on a bfcache `pageshow`.
|
|
155
|
+
*/
|
|
156
|
+
export function rehosting(make: () => SocketHost, options: RehostingOptions): RehostingHost {
|
|
157
|
+
const schedule = options.schedule ?? timeoutScheduler;
|
|
158
|
+
let current = make();
|
|
159
|
+
const self: RehostingHost = {
|
|
160
|
+
get kind(): 'worker' | 'in-page' {
|
|
161
|
+
return current.kind;
|
|
162
|
+
},
|
|
163
|
+
socket: (target) => {
|
|
164
|
+
const inner = current.socket(target);
|
|
165
|
+
return watchOpen(inner, schedule, options.openTimeoutMs, () => self.rehost());
|
|
166
|
+
},
|
|
167
|
+
bye: () => current.bye(),
|
|
168
|
+
rehost: () => {
|
|
169
|
+
current.bye();
|
|
170
|
+
current = make();
|
|
171
|
+
},
|
|
172
|
+
};
|
|
173
|
+
return self;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** `inner`, with a deadline on its `open`: missed, the host is re-made and the socket closes. */
|
|
177
|
+
function watchOpen(
|
|
178
|
+
inner: ClientSocket,
|
|
179
|
+
schedule: Scheduler,
|
|
180
|
+
ms: number,
|
|
181
|
+
rehost: () => void,
|
|
182
|
+
): ClientSocket {
|
|
183
|
+
let opened: (() => void) | null = null;
|
|
184
|
+
let closed: ((code: number) => void) | null = null;
|
|
185
|
+
let settled = false;
|
|
186
|
+
const disarm = schedule(() => {
|
|
187
|
+
if (settled) return;
|
|
188
|
+
settled = true;
|
|
189
|
+
rehost();
|
|
190
|
+
closed?.(1006);
|
|
191
|
+
}, ms);
|
|
192
|
+
inner.onOpen(() => {
|
|
193
|
+
if (settled) return;
|
|
194
|
+
settled = true;
|
|
195
|
+
disarm();
|
|
196
|
+
opened?.();
|
|
197
|
+
});
|
|
198
|
+
inner.onClose((code) => {
|
|
199
|
+
if (!settled) {
|
|
200
|
+
settled = true;
|
|
201
|
+
disarm();
|
|
202
|
+
}
|
|
203
|
+
closed?.(code);
|
|
204
|
+
});
|
|
205
|
+
return {
|
|
206
|
+
get bufferedAmount(): number {
|
|
207
|
+
return inner.bufferedAmount ?? 0;
|
|
208
|
+
},
|
|
209
|
+
send: (data) => inner.send(data),
|
|
210
|
+
close: (code, reason) => {
|
|
211
|
+
if (!settled) {
|
|
212
|
+
settled = true;
|
|
213
|
+
disarm();
|
|
214
|
+
}
|
|
215
|
+
inner.close(code, reason);
|
|
216
|
+
},
|
|
217
|
+
onOpen: (handler) => {
|
|
218
|
+
opened = handler;
|
|
219
|
+
},
|
|
220
|
+
onMessage: (handler) => inner.onMessage(handler),
|
|
221
|
+
onClose: (handler) => {
|
|
222
|
+
closed = handler;
|
|
223
|
+
},
|
|
224
|
+
};
|
|
225
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// The application idle budget a sync node evicts a silent socket on, and how often it asks. Split
|
|
2
|
+
// from `socket.ts` at its line ceiling.
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* How long a socket may route no frame before `sync-node` evicts it. It is an APPLICATION
|
|
6
|
+
* inactivity budget and not Bun's transport one: Bun's `idleTimeout` is renewed by its own
|
|
7
|
+
* ping/pong, so a client whose TCP stack still answers pings while its frame loop is wedged holds
|
|
8
|
+
* its grant, its subscriptions and its topic membership forever. A beating client sends a `hello`
|
|
9
|
+
* every `DEFAULT_HEARTBEAT_MS` (15s), so this is eight missed beats.
|
|
10
|
+
*/
|
|
11
|
+
export const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* How often to ask. A quarter of the budget, floored at a second: a socket is evicted within 25%
|
|
15
|
+
* of its window of going quiet, and a node holding 50,000 of them pays one pass over the table
|
|
16
|
+
* four times per window rather than once a second. Derived rather than configured — a second knob
|
|
17
|
+
* is a second number that can disagree with the one it is a fraction of.
|
|
18
|
+
*/
|
|
19
|
+
export function idleSweepPeriodMs(idleTimeoutMs: number): number {
|
|
20
|
+
return Math.max(1_000, Math.floor(idleTimeoutMs / 4));
|
|
21
|
+
}
|
package/src/socket.ts
CHANGED
|
@@ -19,6 +19,8 @@ import {
|
|
|
19
19
|
import { GapRepairs } from './channel-gaps';
|
|
20
20
|
import type { ChannelRecordsFrame } from './channel-wire';
|
|
21
21
|
import { CLOSE } from './close-codes';
|
|
22
|
+
import { DropWindow } from './socket-drops';
|
|
23
|
+
import { DEFAULT_IDLE_TIMEOUT_MS } from './socket-idle';
|
|
22
24
|
import { encode, type Frame } from './sync-protocol';
|
|
23
25
|
import { AcceptBudget } from './thundering-herd';
|
|
24
26
|
|
|
@@ -156,6 +158,7 @@ export class SyncSocket {
|
|
|
156
158
|
readonly #maxDroppedFrames: number;
|
|
157
159
|
#clientBuildId: string;
|
|
158
160
|
#closed = false;
|
|
161
|
+
readonly #drops: DropWindow;
|
|
159
162
|
|
|
160
163
|
constructor(options: SyncSocketOptions) {
|
|
161
164
|
this.#ws = options.ws;
|
|
@@ -168,6 +171,7 @@ export class SyncSocket {
|
|
|
168
171
|
finiteOption('SyncSocket', 'maxBufferedBytes', this.#maxBufferedBytes);
|
|
169
172
|
this.#maxDroppedFrames = options.maxDroppedFrames ?? 32;
|
|
170
173
|
finiteOption('SyncSocket', 'maxDroppedFrames', this.#maxDroppedFrames);
|
|
174
|
+
this.#drops = new DropWindow(this.#maxDroppedFrames);
|
|
171
175
|
this.frameBudget = new AcceptBudget({
|
|
172
176
|
perSecond: finiteOption(
|
|
173
177
|
'SyncSocket',
|
|
@@ -215,33 +219,34 @@ export class SyncSocket {
|
|
|
215
219
|
|
|
216
220
|
/** `false` means the frame was dropped by backpressure — the caller must mark state stale. */
|
|
217
221
|
send(frame: Frame): boolean {
|
|
222
|
+
return this.sendEncoded(encode(frame));
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** An `encode()`d frame — the fan-outs encode once for every socket. Adds no validation. */
|
|
226
|
+
sendEncoded(text: string): boolean {
|
|
218
227
|
if (this.#closed) return false;
|
|
219
228
|
if (this.#ws.getBufferedAmount() > this.#maxBufferedBytes) {
|
|
220
|
-
this
|
|
221
|
-
if (this.droppedFrames > this.#maxDroppedFrames) {
|
|
222
|
-
this.close(CLOSE.overloaded, 'backpressure');
|
|
223
|
-
}
|
|
229
|
+
this.#dropped();
|
|
224
230
|
return false;
|
|
225
231
|
}
|
|
226
|
-
// `
|
|
227
|
-
//
|
|
228
|
-
//
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
// the exact outcome every other `socket.send` on this node reads its answer to prevent.
|
|
232
|
-
if (this.#ws.send(encode(frame)) <= 0) {
|
|
233
|
-
this.droppedFrames += 1;
|
|
234
|
-
// The same ceiling backpressure takes: a socket the runtime keeps refusing is one to close,
|
|
235
|
-
// and the two are one failure — the write went nowhere either way.
|
|
236
|
-
if (this.droppedFrames > this.#maxDroppedFrames) {
|
|
237
|
-
this.close(CLOSE.overloaded, 'backpressure');
|
|
238
|
-
}
|
|
232
|
+
// Bun answers `0` for a message it DROPPED (the socket closed after the check above) — the one
|
|
233
|
+
// drop. `-1` means QUEUED under backpressure, and it is delivered: counted as a drop it sent a
|
|
234
|
+
// spurious `replay-gap`, desynced a healthy subscriber, and closed the socket after 33.
|
|
235
|
+
if (this.#ws.send(text) === 0) {
|
|
236
|
+
this.#dropped();
|
|
239
237
|
return false;
|
|
240
238
|
}
|
|
241
239
|
this.sentFrames += 1;
|
|
242
240
|
return true;
|
|
243
241
|
}
|
|
244
242
|
|
|
243
|
+
/** One frame that never left: counted for life, judged per `DROP_WINDOW_MS` (`socket-drops.ts`). */
|
|
244
|
+
#dropped(): void {
|
|
245
|
+
this.droppedFrames += 1;
|
|
246
|
+
if (this.#drops.overflowed(this.#clock.monotonic()))
|
|
247
|
+
this.close(CLOSE.overloaded, 'backpressure');
|
|
248
|
+
}
|
|
249
|
+
|
|
245
250
|
/** Record a dropped or invalidated subscription so the next flush re-snapshots it. */
|
|
246
251
|
markDesynced(sid: string): void {
|
|
247
252
|
this.desynced.add(sid);
|
|
@@ -282,25 +287,6 @@ export class SyncSocket {
|
|
|
282
287
|
}
|
|
283
288
|
}
|
|
284
289
|
|
|
285
|
-
/**
|
|
286
|
-
* How long a socket may route no frame before `sync-node` evicts it. It is an APPLICATION
|
|
287
|
-
* inactivity budget and not Bun's transport one: Bun's `idleTimeout` is renewed by its own
|
|
288
|
-
* ping/pong, so a client whose TCP stack still answers pings while its frame loop is wedged holds
|
|
289
|
-
* its grant, its subscriptions and its topic membership forever. A beating client sends a `hello`
|
|
290
|
-
* every `DEFAULT_HEARTBEAT_MS` (15s), so this is eight missed beats.
|
|
291
|
-
*/
|
|
292
|
-
export const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
|
|
293
|
-
|
|
294
|
-
/**
|
|
295
|
-
* How often to ask. A quarter of the budget, floored at a second: a socket is evicted within 25%
|
|
296
|
-
* of its window of going quiet, and a node holding 50,000 of them pays one pass over the table
|
|
297
|
-
* four times per window rather than once a second. Derived rather than configured — a second knob
|
|
298
|
-
* is a second number that can disagree with the one it is a fraction of.
|
|
299
|
-
*/
|
|
300
|
-
export function idleSweepPeriodMs(idleTimeoutMs: number): number {
|
|
301
|
-
return Math.max(1_000, Math.floor(idleTimeoutMs / 4));
|
|
302
|
-
}
|
|
303
|
-
|
|
304
290
|
export interface SocketRegistryOptions {
|
|
305
291
|
readonly clock?: Clock;
|
|
306
292
|
/** Bun's own `idleTimeout` is renewed by its ping/pong; this budget counts routed FRAMES. */
|
|
@@ -425,6 +411,9 @@ export class SocketRegistry {
|
|
|
425
411
|
deliver(topic: string, frame: Frame): number {
|
|
426
412
|
const members = this.#byTopic.get(topic);
|
|
427
413
|
if (!members) return 0;
|
|
414
|
+
// Encoded ONCE for every member: per socket it was 16.9 ms against 0.6 ms for a 2 kB frame to
|
|
415
|
+
// 10,000 sockets, measured.
|
|
416
|
+
const text = encode(frame);
|
|
428
417
|
let sent = 0;
|
|
429
418
|
let dropped = 0;
|
|
430
419
|
for (const socket of members) {
|
|
@@ -432,7 +421,7 @@ export class SocketRegistry {
|
|
|
432
421
|
members.delete(socket);
|
|
433
422
|
continue;
|
|
434
423
|
}
|
|
435
|
-
if (socket.
|
|
424
|
+
if (socket.sendEncoded(text)) sent += 1;
|
|
436
425
|
else dropped += 1;
|
|
437
426
|
}
|
|
438
427
|
if (members.size === 0) this.#byTopic.delete(topic);
|
|
@@ -447,6 +436,7 @@ export class SocketRegistry {
|
|
|
447
436
|
deliverRecords(topic: string, epoch: string, frame: ChannelRecordsFrame): number {
|
|
448
437
|
const members = this.#byTopic.get(topic);
|
|
449
438
|
if (!members) return 0;
|
|
439
|
+
const text = encode(frame);
|
|
450
440
|
let sent = 0;
|
|
451
441
|
let dropped = 0;
|
|
452
442
|
for (const socket of members) {
|
|
@@ -455,7 +445,7 @@ export class SocketRegistry {
|
|
|
455
445
|
continue;
|
|
456
446
|
}
|
|
457
447
|
this.gapRepairs.repair(socket, topic);
|
|
458
|
-
if (socket.
|
|
448
|
+
if (socket.sendEncoded(text)) {
|
|
459
449
|
sent += 1;
|
|
460
450
|
continue;
|
|
461
451
|
}
|
|
@@ -467,6 +457,19 @@ export class SocketRegistry {
|
|
|
467
457
|
return sent;
|
|
468
458
|
}
|
|
469
459
|
|
|
460
|
+
/** Every member owes a re-read (a TRUNCATE): one `replay-gap` each now, or kept as a mark. */
|
|
461
|
+
announceGap(topic: string, epoch: string): number {
|
|
462
|
+
const members = this.#byTopic.get(topic);
|
|
463
|
+
if (!members) return 0;
|
|
464
|
+
let announced = 0;
|
|
465
|
+
for (const socket of members) {
|
|
466
|
+
if (socket.closed) continue;
|
|
467
|
+
socket.gaps.set(topic, epoch);
|
|
468
|
+
if (this.gapRepairs.repair(socket, topic)) announced += 1;
|
|
469
|
+
}
|
|
470
|
+
return announced;
|
|
471
|
+
}
|
|
472
|
+
|
|
470
473
|
#countDropped(topic: string, dropped: number): void {
|
|
471
474
|
this.#droppedChannelFrames += dropped;
|
|
472
475
|
// Two readers, one event, one spelling: the series an operator alerts on and the line that
|
package/src/subscriber-gate.ts
CHANGED
|
@@ -54,6 +54,55 @@ export interface GateTarget {
|
|
|
54
54
|
readonly rows: readonly Row[];
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* The shared window, indexed ONCE per fan-out rather than searched per subscriber per patch: a
|
|
59
|
+
* `rows.find` inside the subscriber loop was O(subscribers × window) — 54.6 ms against 0.7 ms at
|
|
60
|
+
* 1000 subscribers over a 500-row window. `windowIndex` builds it; `filterPatches` builds its own
|
|
61
|
+
* when a caller passes none.
|
|
62
|
+
*/
|
|
63
|
+
export interface WindowIndex {
|
|
64
|
+
readonly byId: ReadonlyMap<string, Row>;
|
|
65
|
+
readonly position: ReadonlyMap<string, number>;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function windowIndex(rows: readonly Row[]): WindowIndex {
|
|
69
|
+
const byId = new Map<string, Row>();
|
|
70
|
+
const position = new Map<string, number>();
|
|
71
|
+
rows.forEach((row, at) => {
|
|
72
|
+
byId.set(row.id, row);
|
|
73
|
+
position.set(row.id, at);
|
|
74
|
+
});
|
|
75
|
+
return { byId, position };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* What a subscriber holds as a patch list is folded: the cursor's set, plus what this list has
|
|
80
|
+
* inserted, minus what it has deleted — never a copy of the set per subscriber.
|
|
81
|
+
*/
|
|
82
|
+
class Holding {
|
|
83
|
+
readonly #base: ReadonlySet<string>;
|
|
84
|
+
readonly #added = new Set<string>();
|
|
85
|
+
readonly #removed = new Set<string>();
|
|
86
|
+
|
|
87
|
+
constructor(base: ReadonlySet<string>) {
|
|
88
|
+
this.#base = base;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
has(id: string): boolean {
|
|
92
|
+
return this.#added.has(id) || (this.#base.has(id) && !this.#removed.has(id));
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
fold(patch: RowPatch): void {
|
|
96
|
+
if (patch.op === 'delete') {
|
|
97
|
+
this.#added.delete(patch.id);
|
|
98
|
+
this.#removed.add(patch.id);
|
|
99
|
+
} else if (patch.op === 'insert') {
|
|
100
|
+
this.#removed.delete(patch.id);
|
|
101
|
+
this.#added.add(patch.id);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
57
106
|
export interface SubscriberGateOptions {
|
|
58
107
|
/**
|
|
59
108
|
* `live.rows_denied`. A row an actor's policy refuses is dropped, never sent and never turned
|
|
@@ -113,11 +162,16 @@ export class SubscriberGate {
|
|
|
113
162
|
who: Subscriber,
|
|
114
163
|
patches: readonly RowPatch[],
|
|
115
164
|
held: ReadonlySet<string>,
|
|
165
|
+
index: WindowIndex = windowIndex(target.rows),
|
|
116
166
|
): Promise<RowPatch[]> {
|
|
117
167
|
const out: RowPatch[] = [];
|
|
168
|
+
const holding = new Holding(held);
|
|
118
169
|
for (const patch of patches) {
|
|
119
|
-
const allowed = await this
|
|
120
|
-
if (allowed
|
|
170
|
+
const allowed = await this.#decide(target, who, patch, holding.has(patch.id), index);
|
|
171
|
+
if (allowed === null) continue;
|
|
172
|
+
const placed = rebase(allowed, holding, target.rows, index);
|
|
173
|
+
holding.fold(placed);
|
|
174
|
+
out.push(placed);
|
|
121
175
|
}
|
|
122
176
|
return out;
|
|
123
177
|
}
|
|
@@ -128,6 +182,16 @@ export class SubscriberGate {
|
|
|
128
182
|
who: Subscriber,
|
|
129
183
|
patch: RowPatch,
|
|
130
184
|
holds: boolean,
|
|
185
|
+
): Promise<RowPatch | null> {
|
|
186
|
+
return await this.#decide(target, who, patch, holds, windowIndex(target.rows));
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
async #decide(
|
|
190
|
+
target: GateTarget,
|
|
191
|
+
who: Subscriber,
|
|
192
|
+
patch: RowPatch,
|
|
193
|
+
holds: boolean,
|
|
194
|
+
index: WindowIndex,
|
|
131
195
|
): Promise<RowPatch | null> {
|
|
132
196
|
// A delete carries no row, so there is nothing to put in front of the rule — `holds` IS the
|
|
133
197
|
// decision, the same one the two branches below take for a row a rule has just refused.
|
|
@@ -147,7 +211,7 @@ export class SubscriberGate {
|
|
|
147
211
|
this.#denied(target.qid, who, patch.id);
|
|
148
212
|
return null;
|
|
149
213
|
}
|
|
150
|
-
const full =
|
|
214
|
+
const full = index.byId.get(patch.id);
|
|
151
215
|
// No whole row means no decision to take. An update patch carries the changed columns only, so
|
|
152
216
|
// a rule reading `row.ownerId` on one reads `undefined` and answers as if the row had said so —
|
|
153
217
|
// fail-closed for `=== actor.id`, and a leak for every `!row.private`. It is not a gate that
|
|
@@ -218,6 +282,31 @@ const actorIdOf = (who: Subscriber): string | null => (who.actor === null ? null
|
|
|
218
282
|
* window stopped holding it. Written once so the two paths cannot answer differently: a client left
|
|
219
283
|
* holding the row instead renders a revoked grant until something else reconnects it.
|
|
220
284
|
*/
|
|
285
|
+
/**
|
|
286
|
+
* The patch as this subscriber must read it. `index` was a position in the SHARED, pre-policy
|
|
287
|
+
* window, and forwarded unchanged it placed the row out of order for anyone who sees fewer rows —
|
|
288
|
+
* and its size told them how many rows they may not see sit ahead of it. Re-based on the rows ahead
|
|
289
|
+
* of it that this subscriber holds; dropped from a delete, which the client applies by id. Bounded
|
|
290
|
+
* by `CURSOR_ID_LIMIT` like `holds` is: a held row past it is not counted.
|
|
291
|
+
*/
|
|
292
|
+
function rebase(
|
|
293
|
+
patch: RowPatch,
|
|
294
|
+
holding: Holding,
|
|
295
|
+
rows: readonly Row[],
|
|
296
|
+
index: WindowIndex,
|
|
297
|
+
): RowPatch {
|
|
298
|
+
if (patch.index === undefined) return patch;
|
|
299
|
+
const { index: _shared, ...rest } = patch;
|
|
300
|
+
if (patch.op === 'delete' || patch.row === null) return rest;
|
|
301
|
+
const at = index.position.get(patch.id) ?? patch.index;
|
|
302
|
+
let local = 0;
|
|
303
|
+
for (let i = 0; i < at && i < rows.length; i += 1) {
|
|
304
|
+
const id = rows[i]?.id;
|
|
305
|
+
if (id !== undefined && id !== patch.id && holding.has(id)) local += 1;
|
|
306
|
+
}
|
|
307
|
+
return { ...rest, index: local };
|
|
308
|
+
}
|
|
309
|
+
|
|
221
310
|
const withdrawn = (patch: RowPatch): RowPatch => ({
|
|
222
311
|
op: 'delete',
|
|
223
312
|
id: patch.id,
|
package/src/sync-node.ts
CHANGED
|
@@ -11,13 +11,8 @@ import { evictInChunks } from './drain-evictions';
|
|
|
11
11
|
import { isClientFault } from './errors';
|
|
12
12
|
import type { TransportSubscription } from './fanout';
|
|
13
13
|
import { CHANGE_SUBJECT_ALL, parseEnvelope, SeqGapDetector } from './replicator';
|
|
14
|
-
import {
|
|
15
|
-
|
|
16
|
-
DEFAULT_MAX_BUFFERED_BYTES,
|
|
17
|
-
idleSweepPeriodMs,
|
|
18
|
-
SocketRegistry,
|
|
19
|
-
SyncSocket,
|
|
20
|
-
} from './socket';
|
|
14
|
+
import { CLOSE, DEFAULT_MAX_BUFFERED_BYTES, SocketRegistry, SyncSocket } from './socket';
|
|
15
|
+
import { idleSweepPeriodMs } from './socket-idle';
|
|
21
16
|
import { GrantBook, sweepGrants } from './sync-auth';
|
|
22
17
|
import { ackRefOf, createFrameRouter } from './sync-frames';
|
|
23
18
|
import { drainGraceMs, socketCeilings, syncNodeBounds } from './sync-node-bounds';
|
package/src/thundering-herd.ts
CHANGED
|
@@ -3,12 +3,12 @@
|
|
|
3
3
|
//
|
|
4
4
|
// Three mechanisms, in the order they fire:
|
|
5
5
|
// 1. drainPlan() — the draining node assigns each client a distinct delay slot before closing
|
|
6
|
-
// 2.
|
|
6
|
+
// 2. policyDelay() — the client's own jittered retry, for failures nobody scheduled
|
|
7
7
|
// 3. AcceptBudget — the receiving node's token bucket, so recovery sheds instead of collapsing
|
|
8
8
|
|
|
9
9
|
import {
|
|
10
|
+
backoffDelay,
|
|
10
11
|
type Clock,
|
|
11
|
-
backoffDelay as coreBackoffDelay,
|
|
12
12
|
finiteOption,
|
|
13
13
|
type JitterMode,
|
|
14
14
|
type Random,
|
|
@@ -59,20 +59,21 @@ export const browserBackoff: BackoffPolicy = {
|
|
|
59
59
|
};
|
|
60
60
|
|
|
61
61
|
/**
|
|
62
|
-
*
|
|
62
|
+
* A {@link BackoffPolicy} mapped onto `@ultimat3/core`'s `backoffDelay` — the one curve. Attempt is
|
|
63
|
+
* 1-BASED, core's count: the wait after the first failure is `attempt: 1` and is `baseMs`.
|
|
63
64
|
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
65
|
+
* Internal to this package and never re-exported from the barrel. Until 22.0.0 `.` exported a
|
|
66
|
+
* 0-based `backoffDelay` of its own that shifted by one before delegating, which made two counting
|
|
67
|
+
* conventions under one name — and a caller that passed a 1-based count to it (the channel
|
|
68
|
+
* catch-up retry did) waited twice as long as it meant to, with no error anywhere.
|
|
68
69
|
*/
|
|
69
|
-
export function
|
|
70
|
+
export function policyDelay(
|
|
71
|
+
policy: BackoffPolicy,
|
|
70
72
|
attempt: number,
|
|
71
|
-
policy: BackoffPolicy = defaultBackoff,
|
|
72
73
|
rng: Rng = Math.random,
|
|
73
74
|
): number {
|
|
74
|
-
return
|
|
75
|
-
attempt
|
|
75
|
+
return backoffDelay({
|
|
76
|
+
attempt,
|
|
76
77
|
base: policy.baseMs,
|
|
77
78
|
max: policy.maxMs,
|
|
78
79
|
factor: policy.factor,
|