@catalyst-cloud/sdk 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -0
- package/dist/browser.d.ts +9 -0
- package/dist/browser.d.ts.map +1 -0
- package/dist/browser.js +24 -0
- package/dist/browser.js.map +1 -0
- package/dist/live-sync-client.d.ts +128 -9
- package/dist/live-sync-client.d.ts.map +1 -1
- package/dist/live-sync-client.js +369 -51
- package/dist/live-sync-client.js.map +1 -1
- package/dist/replica/browser/apply.d.ts +29 -0
- package/dist/replica/browser/apply.d.ts.map +1 -0
- package/dist/replica/browser/apply.js +59 -0
- package/dist/replica/browser/apply.js.map +1 -0
- package/dist/replica/browser/browser-lock.d.ts +25 -0
- package/dist/replica/browser/browser-lock.d.ts.map +1 -0
- package/dist/replica/browser/browser-lock.js +91 -0
- package/dist/replica/browser/browser-lock.js.map +1 -0
- package/dist/replica/browser/browser-replica.d.ts +237 -0
- package/dist/replica/browser/browser-replica.d.ts.map +1 -0
- package/dist/replica/browser/browser-replica.js +952 -0
- package/dist/replica/browser/browser-replica.js.map +1 -0
- package/dist/replica/browser/db.worker.d.ts +2 -0
- package/dist/replica/browser/db.worker.d.ts.map +1 -0
- package/dist/replica/browser/db.worker.js +40 -0
- package/dist/replica/browser/db.worker.js.map +1 -0
- package/dist/replica/browser/delta-queue.d.ts +187 -0
- package/dist/replica/browser/delta-queue.d.ts.map +1 -0
- package/dist/replica/browser/delta-queue.js +328 -0
- package/dist/replica/browser/delta-queue.js.map +1 -0
- package/dist/replica/browser/ports.d.ts +39 -0
- package/dist/replica/browser/ports.d.ts.map +1 -0
- package/dist/replica/browser/ports.js +144 -0
- package/dist/replica/browser/ports.js.map +1 -0
- package/dist/replica/browser/protocol.d.ts +129 -0
- package/dist/replica/browser/protocol.d.ts.map +1 -0
- package/dist/replica/browser/protocol.js +12 -0
- package/dist/replica/browser/protocol.js.map +1 -0
- package/dist/replica/browser/seed-read-gate.d.ts +13 -0
- package/dist/replica/browser/seed-read-gate.d.ts.map +1 -0
- package/dist/replica/browser/seed-read-gate.js +36 -0
- package/dist/replica/browser/seed-read-gate.js.map +1 -0
- package/dist/replica/browser/seed-session.d.ts +23 -0
- package/dist/replica/browser/seed-session.d.ts.map +1 -0
- package/dist/replica/browser/seed-session.js +57 -0
- package/dist/replica/browser/seed-session.js.map +1 -0
- package/dist/replica/browser/snapshot-stream.d.ts +24 -0
- package/dist/replica/browser/snapshot-stream.d.ts.map +1 -0
- package/dist/replica/browser/snapshot-stream.js +113 -0
- package/dist/replica/browser/snapshot-stream.js.map +1 -0
- package/dist/replica/browser/sqlite-db.d.ts +9 -0
- package/dist/replica/browser/sqlite-db.d.ts.map +1 -0
- package/dist/replica/browser/sqlite-db.js +28 -0
- package/dist/replica/browser/sqlite-db.js.map +1 -0
- package/dist/replica/browser/support.d.ts +2 -0
- package/dist/replica/browser/support.d.ts.map +1 -0
- package/dist/replica/browser/support.js +30 -0
- package/dist/replica/browser/support.js.map +1 -0
- package/dist/replica/browser/validate.d.ts +17 -0
- package/dist/replica/browser/validate.d.ts.map +1 -0
- package/dist/replica/browser/validate.js +47 -0
- package/dist/replica/browser/validate.js.map +1 -0
- package/dist/replica/browser/worker-core.d.ts +10 -0
- package/dist/replica/browser/worker-core.d.ts.map +1 -0
- package/dist/replica/browser/worker-core.js +181 -0
- package/dist/replica/browser/worker-core.js.map +1 -0
- package/dist/replica/catalyst-replica.d.ts +6 -9
- package/dist/replica/catalyst-replica.d.ts.map +1 -1
- package/dist/replica/catalyst-replica.js +53 -19
- package/dist/replica/catalyst-replica.js.map +1 -1
- package/dist/replica/migration-shape.d.ts +11 -0
- package/dist/replica/migration-shape.d.ts.map +1 -0
- package/dist/replica/migration-shape.js +25 -0
- package/dist/replica/migration-shape.js.map +1 -0
- package/package.json +16 -3
package/dist/live-sync-client.js
CHANGED
|
@@ -61,9 +61,10 @@
|
|
|
61
61
|
// a client RESTARTED mid-window would have re-latched the disable, so the degrade-not-disable shape is
|
|
62
62
|
// what actually guarantees convergence); (4) a FAILED reseed re-enters the backoff path instead
|
|
63
63
|
// of hot-reopening — and the reseed await itself is bounded by `reseedTimeoutMs` (the injected
|
|
64
|
-
// callback is a trust boundary like the ws impl:
|
|
65
|
-
//
|
|
66
|
-
// can hang, and "resyncing" holds no socket and suppresses
|
|
64
|
+
// callback is a trust boundary like the ws impl: both first-party reseeds self-bound (the node
|
|
65
|
+
// replica's seedFromSnapshot and the browser replica's OPFS seed both abort on an idle body), but an
|
|
66
|
+
// arbitrary consumer-supplied reseed can still hang, and "resyncing" holds no socket and suppresses
|
|
67
|
+
// scheduleReconnect, so without this bound
|
|
67
68
|
// it was the one remaining zero-timer state; a timed-out reseed is ABANDONED, its late settle
|
|
68
69
|
// discarded, and the client re-enters backoff); (5) closeSocket() escalates past `close()` to a duck-typed `terminate()` (Bun /
|
|
69
70
|
// the 'ws' package expose one; undici does not — its close-handshake wait is why teardown must not
|
|
@@ -77,6 +78,16 @@
|
|
|
77
78
|
// beyond-gap change frame (re-request the hole `deliveredSeq+1..head`) but never applies it.
|
|
78
79
|
import { PING_FRAME } from "./types.js";
|
|
79
80
|
import { NOOP_TELEMETRY, createTelemetry, CATALYST_ATTR, REPLICA_LOG, REPLICA_METRIC, REPLICA_SPAN, DEFAULT_SCOPE_NAME, } from "./otel.js";
|
|
81
|
+
/**
|
|
82
|
+
* How long a CANCELLED reseed gets to unwind before the transport settles anyway.
|
|
83
|
+
*
|
|
84
|
+
* Cancelling is asynchronous on the consumer's side — the browser seed aborts a fetch, trips its
|
|
85
|
+
* supersede guard, posts `seedAbort` and resumes its delta queue — and the transport reconnects the
|
|
86
|
+
* moment it settles, so it must wait for that unwind or it reconnects into a still-paused consumer.
|
|
87
|
+
* Bounded because the deadline that triggered this exists precisely for an unresponsive callback: a
|
|
88
|
+
* cleanup that also hangs must not wedge the transport (CTC-114 review round 12).
|
|
89
|
+
*/
|
|
90
|
+
const DEFAULT_CANCEL_CLEANUP_GRACE_MS = 250;
|
|
80
91
|
/** Resolve the runtime global WebSocket, or fail with an actionable message. */
|
|
81
92
|
function defaultWsFactory(url) {
|
|
82
93
|
const Ctor = globalThis.WebSocket;
|
|
@@ -106,8 +117,14 @@ export function buildConnectUrl(opts) {
|
|
|
106
117
|
const params = new URLSearchParams();
|
|
107
118
|
if (opts.auth.kind === "token")
|
|
108
119
|
params.set("token", opts.auth.token);
|
|
109
|
-
|
|
110
|
-
|
|
120
|
+
// Only when a tenant was actually named. `?account=` is NOT the same as no account: the server's
|
|
121
|
+
// consumers are truthiness checks, so empty takes the omitted path anyway — but it would freeze a
|
|
122
|
+
// contract in which "" is a legal mirror name, and it puts `catalyst.tenant=""` on every span.
|
|
123
|
+
if (opts.accountId)
|
|
124
|
+
params.set("account", opts.accountId);
|
|
125
|
+
const query = params.toString();
|
|
126
|
+
// Guard the dangling "?" — with cookie auth and no account there are no params at all.
|
|
127
|
+
return `${origin}${opts.connectPath}${query ? `?${query}` : ""}`;
|
|
111
128
|
}
|
|
112
129
|
/**
|
|
113
130
|
* Consecutive opened-then-never-ponged connections after which the watchdog DEGRADES itself
|
|
@@ -153,6 +170,7 @@ export class LiveSyncClient {
|
|
|
153
170
|
pongTimeoutMs;
|
|
154
171
|
openTimeoutMs;
|
|
155
172
|
reseedTimeoutMs;
|
|
173
|
+
cancelCleanupGraceMs;
|
|
156
174
|
gapTimeoutMs;
|
|
157
175
|
gapRetryLimit;
|
|
158
176
|
wsFactory;
|
|
@@ -160,6 +178,36 @@ export class LiveSyncClient {
|
|
|
160
178
|
telemetryConfig;
|
|
161
179
|
ws = null;
|
|
162
180
|
stopped = false;
|
|
181
|
+
/**
|
|
182
|
+
* Has start() been entered? `stopped` alone cannot answer this — it is false BEFORE the first
|
|
183
|
+
* start() as well as during a run, so every "am I running?" guard read true on a client that had
|
|
184
|
+
* never booted. Only the public `requestResync()` can reach that window (CTC-114 review round 5):
|
|
185
|
+
* it would reseed and open a socket with no lifecycle deferred and no telemetry resolved, and the
|
|
186
|
+
* later real start() would then openSocket() again — overwriting `this.ws`, so the first socket
|
|
187
|
+
* kept delivering duplicate frames and could no longer be closed through the stored reference.
|
|
188
|
+
*/
|
|
189
|
+
started = false;
|
|
190
|
+
/** The in-flight boot task, or null once it settles — `requestResync()` serializes behind it. */
|
|
191
|
+
bootTask = null;
|
|
192
|
+
/** Settles an in-flight `boundedReseed` wrapper on stop(), so an awaited resync cannot hang. */
|
|
193
|
+
abandonReseed = null;
|
|
194
|
+
/**
|
|
195
|
+
* The in-flight resync, so a concurrent caller AWAITS it instead of being handed an
|
|
196
|
+
* already-resolved promise and acting on a store still being rebuilt (round 10).
|
|
197
|
+
*/
|
|
198
|
+
activeResync = null;
|
|
199
|
+
/** Did the boot this request waited on perform a COLD re-seed? Only then may it be absorbed. */
|
|
200
|
+
bootColdSeeded = false;
|
|
201
|
+
/**
|
|
202
|
+
* Did the last boot REJECT? Latched until the next `start()` (CTC-114 review round 9).
|
|
203
|
+
*
|
|
204
|
+
* Round 8 read the failure from the awaited `bootTask`, but that handle is nulled once the boot
|
|
205
|
+
* settles — so a `requestResync()` arriving after that microtask found no record of the failure,
|
|
206
|
+
* with `started` still true and `stopped` still false, and sailed past the guard into a reseed that
|
|
207
|
+
* could open a live socket under an application already told startup had failed. The outcome has to
|
|
208
|
+
* outlive the handle.
|
|
209
|
+
*/
|
|
210
|
+
bootFailed = false;
|
|
163
211
|
resyncing = false;
|
|
164
212
|
backoff;
|
|
165
213
|
reconnectTimer = null;
|
|
@@ -220,10 +268,19 @@ export class LiveSyncClient {
|
|
|
220
268
|
socketOpened = false;
|
|
221
269
|
/** The pending reseed deadline (CTC-281) — the timer that makes "resyncing" (no socket, reconnect
|
|
222
270
|
* suppressed) a bounded state instead of a restart-only wedge. Cleared when the reseed settles in
|
|
223
|
-
* time and by stop() (ask 4: stop() leaves NOTHING pending). At most one reseed is ever in flight
|
|
224
|
-
*
|
|
271
|
+
* time and by stop() (ask 4: stop() leaves NOTHING pending). At most one reseed is ever in flight:
|
|
272
|
+
* `resyncing` guards the resync path, and the boot cold seed SETS that same flag for its duration.
|
|
273
|
+
* (It used to rely on "the boot seed runs before any socket exists" — true only while a resync
|
|
274
|
+
* needed a server frame. The public `requestResync()` added in 0.8.0 needs no socket.) */
|
|
225
275
|
reseedTimer = null;
|
|
226
276
|
constructor(opts) {
|
|
277
|
+
// Fail fast, and fail HERE. A token-authed client has no session to fall back to, so an omitted
|
|
278
|
+
// account is a misconfiguration, not a default. It is checked in the constructor rather than in
|
|
279
|
+
// buildConnectUrl because `connectUrl()` is called from `openSocket()` OUTSIDE its try/catch — a
|
|
280
|
+
// throw down there escapes the reconnect machinery entirely instead of surfacing to the caller.
|
|
281
|
+
if (opts.auth.kind === "token" && !opts.accountId) {
|
|
282
|
+
throw new Error("LiveSyncClient: accountId is required with token auth (only cookie auth can fall back to the session's own tenant)");
|
|
283
|
+
}
|
|
227
284
|
this.baseUrl = stripTrailingSlashes(opts.baseUrl);
|
|
228
285
|
this.accountId = opts.accountId;
|
|
229
286
|
this.connectPath = opts.connectPath ?? "/connect";
|
|
@@ -239,6 +296,8 @@ export class LiveSyncClient {
|
|
|
239
296
|
this.pongTimeoutMs = opts.pongTimeoutMs ?? 15_000;
|
|
240
297
|
this.openTimeoutMs = opts.openTimeoutMs ?? 20_000;
|
|
241
298
|
this.reseedTimeoutMs = opts.reseedTimeoutMs ?? 600_000;
|
|
299
|
+
this.cancelCleanupGraceMs =
|
|
300
|
+
opts.cancelCleanupGraceMs ?? DEFAULT_CANCEL_CLEANUP_GRACE_MS;
|
|
242
301
|
this.gapTimeoutMs = opts.gapTimeoutMs ?? 10_000;
|
|
243
302
|
this.gapRetryLimit = opts.gapRetryLimit ?? 3;
|
|
244
303
|
this.wsFactory = opts.wsFactory ?? defaultWsFactory;
|
|
@@ -256,6 +315,13 @@ export class LiveSyncClient {
|
|
|
256
315
|
*/
|
|
257
316
|
start() {
|
|
258
317
|
this.stopped = false;
|
|
318
|
+
this.started = true;
|
|
319
|
+
// RESET per boot. `start()` is restartable after `stop()`, and a stale `true` from a previous
|
|
320
|
+
// cold boot would make the NEXT boot — warm, and therefore re-seeding nothing — absorb a resync
|
|
321
|
+
// it should have honoured. Found while re-reading this path rather than reported; the same class
|
|
322
|
+
// of staleness as the `bootTask` handle being nulled when it settles.
|
|
323
|
+
this.bootColdSeeded = false;
|
|
324
|
+
this.bootFailed = false;
|
|
259
325
|
// The done deferred is created BEFORE the boot body runs (CTC-281 N2): stop() during the cold-seed
|
|
260
326
|
// await used to find resolveDone still null and leave the returned promise pending forever — a
|
|
261
327
|
// contract violation for a consumer awaiting start(). The boot body below is deliberately its OWN
|
|
@@ -266,31 +332,83 @@ export class LiveSyncClient {
|
|
|
266
332
|
this.resolveDone = resolve;
|
|
267
333
|
});
|
|
268
334
|
const boot = (async () => {
|
|
269
|
-
//
|
|
270
|
-
//
|
|
271
|
-
//
|
|
272
|
-
//
|
|
273
|
-
//
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
335
|
+
// The WHOLE boot is an in-flight resync, not just the cold seed (CTC-114 review rounds 4 + 6).
|
|
336
|
+
//
|
|
337
|
+
// `requestResync()` — public as of 0.8.0 — is callable the moment start() returns its promise,
|
|
338
|
+
// which is before ANY of this settles. Without the latch, `handleResync()`'s re-entrancy guard
|
|
339
|
+
// read false and started a SECOND concurrent reseed: two seeds interleaving writes through a
|
|
340
|
+
// non-reentrant consumer callback, then each completion calling openSocket() — and since
|
|
341
|
+
// openSocket() overwrites `this.ws`, the first socket was orphaned, still delivering duplicate
|
|
342
|
+
// frames and unreachable by stop().
|
|
343
|
+
//
|
|
344
|
+
// Round 4 latched only the cold seed. That was not enough: `createTelemetry()` below is awaited
|
|
345
|
+
// BEFORE the seed, so with telemetry enabled the boot suspends in a window where `started` is
|
|
346
|
+
// already true and the latch is not yet set. The latch therefore has to cover the entire body.
|
|
347
|
+
//
|
|
348
|
+
// Until this release the invariant held for free — a resync could only be driven by a server
|
|
349
|
+
// frame, and a frame needs a socket, which does not exist until openSocket() below.
|
|
350
|
+
//
|
|
351
|
+
// The latch makes the request WAIT; whether it is then absorbed or honoured is decided in
|
|
352
|
+
// requestResync() from `bootColdSeeded`, once this task has settled.
|
|
353
|
+
//
|
|
354
|
+
// Round 6 absorbed it on BOTH arms, arguing that a warm boot's `{type:"sync", after:<cursor>}`
|
|
355
|
+
// is itself the catch-up. That was wrong (round 7), and wrong against this method's whole
|
|
356
|
+
// reason for existing: a consumer calls requestResync() when it has discovered ON ITS OWN SIDE
|
|
357
|
+
// that deltas can no longer catch its store up — the browser replica's dropped overflow buffer
|
|
358
|
+
// is the motivating case. Replaying from the cursor cannot rebuild rows the consumer already
|
|
359
|
+
// lost, so silently swallowing the request left it permanently inconsistent. Only a COLD boot
|
|
360
|
+
// may absorb it, because that boot really is a full re-seed from /snapshot.
|
|
361
|
+
this.resyncing = true;
|
|
362
|
+
try {
|
|
363
|
+
// Resolve the OTel seam ONCE up front (before the first reseed, so the seed span exists on the
|
|
364
|
+
// cold-start path too). Keep the OFF path FULLY SYNCHRONOUS — no `await`, so a caller that opens
|
|
365
|
+
// the socket and inspects it in the same tick still sees it (the boot body runs synchronously up
|
|
366
|
+
// to its first await); only pay the async resolution (guarded dynamic import, or a
|
|
367
|
+
// CatalystReplica passing its already-resolved instance) when telemetry is on.
|
|
368
|
+
this.telemetry =
|
|
369
|
+
this.telemetryConfig === undefined || this.telemetryConfig === false
|
|
370
|
+
? NOOP_TELEMETRY
|
|
371
|
+
: await createTelemetry(this.telemetryConfig, {
|
|
372
|
+
tracerName: DEFAULT_SCOPE_NAME,
|
|
373
|
+
meterName: DEFAULT_SCOPE_NAME,
|
|
374
|
+
});
|
|
375
|
+
this.gapCounter = this.telemetry.counter(REPLICA_METRIC.gaps, {
|
|
376
|
+
description: "Change-feed seq-gap lifecycle events (detected/healed/escalated).",
|
|
377
|
+
unit: "{gap}",
|
|
378
|
+
});
|
|
379
|
+
const saved = this.getCursor();
|
|
380
|
+
if (saved == null) {
|
|
381
|
+
this.setStatus("resyncing");
|
|
382
|
+
// Bounded like the resync-path reseed (CTC-281): a hanging COLD seed surfaces as a start()
|
|
383
|
+
// rejection (the boot arm rejects) instead of a silent forever-"resyncing" start().
|
|
384
|
+
await this.boundedReseed();
|
|
385
|
+
// Only NOW may a request that waited on this boot be absorbed — this really was a full
|
|
386
|
+
// re-seed from /snapshot. A warm boot sets nothing, so the waiter is honoured instead.
|
|
387
|
+
this.bootColdSeeded = true;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
finally {
|
|
391
|
+
// Must clear on the FAILURE arm too, or a failed boot latches the client into a state where
|
|
392
|
+
// every later resync — and scheduleReconnect — is suppressed forever.
|
|
393
|
+
this.resyncing = false;
|
|
291
394
|
}
|
|
292
395
|
this.openSocket();
|
|
293
396
|
})();
|
|
397
|
+
this.bootTask = boot;
|
|
398
|
+
// Clear the handle once boot settles, so a resync arriving LONG after startup is never mistaken
|
|
399
|
+
// for one that raced it — otherwise `bootColdSeeded` would absorb legitimate later requests
|
|
400
|
+
// forever. The catch keeps a boot rejection from surfacing as an unhandled one on this arm; the
|
|
401
|
+
// race below is what actually reports it.
|
|
402
|
+
void boot
|
|
403
|
+
.catch(() => {
|
|
404
|
+
// LATCH the failure before the handle is dropped — `bootTask` is the transient record, this is
|
|
405
|
+
// the durable one, and requestResync() has to be able to see it afterwards (round 9).
|
|
406
|
+
this.bootFailed = true;
|
|
407
|
+
})
|
|
408
|
+
.then(() => {
|
|
409
|
+
if (this.bootTask === boot)
|
|
410
|
+
this.bootTask = null;
|
|
411
|
+
});
|
|
294
412
|
// Settles when stop() resolves the deferred, OR rejects if the boot (cold seed) fails — a boot
|
|
295
413
|
// SUCCESS deliberately keeps waiting on `done` (the "runs forever" contract). Promise.race
|
|
296
414
|
// attaches handlers to both arms, so a boot rejection after stop() is never an unhandled one.
|
|
@@ -312,6 +430,12 @@ export class LiveSyncClient {
|
|
|
312
430
|
const done = this.resolveDone;
|
|
313
431
|
this.resolveDone = null;
|
|
314
432
|
done?.();
|
|
433
|
+
// AFTER resolving start()'s deferred, so the race below settles on `done` and a boot arm that
|
|
434
|
+
// rejects from this abandon lands on an already-settled race rather than surfacing as the
|
|
435
|
+
// outcome of start(). Settles a `requestResync()` a consumer is awaiting — see boundedReseed.
|
|
436
|
+
const abandon = this.abandonReseed;
|
|
437
|
+
this.abandonReseed = null;
|
|
438
|
+
abandon?.();
|
|
315
439
|
}
|
|
316
440
|
/** The ws(s):// URL this client opens, for diagnostics/tests. Re-derived from the options. */
|
|
317
441
|
connectUrl() {
|
|
@@ -322,6 +446,15 @@ export class LiveSyncClient {
|
|
|
322
446
|
auth: this.auth,
|
|
323
447
|
});
|
|
324
448
|
}
|
|
449
|
+
/**
|
|
450
|
+
* The tenant label for telemetry. Attributes are `Record<string, string>`, and an empty string reads
|
|
451
|
+
* as a MISSING attribute in Loki/Tempo — which silently merges every session-scoped browser client
|
|
452
|
+
* into one unlabelled bucket. `"session"` names the case instead: this client is scoped to whatever
|
|
453
|
+
* tenant the cookie resolves to.
|
|
454
|
+
*/
|
|
455
|
+
get tenantAttr() {
|
|
456
|
+
return this.accountId ?? "session";
|
|
457
|
+
}
|
|
325
458
|
setStatus(status) {
|
|
326
459
|
try {
|
|
327
460
|
this.onStatus?.(status);
|
|
@@ -334,10 +467,19 @@ export class LiveSyncClient {
|
|
|
334
467
|
if (this.stopped)
|
|
335
468
|
return;
|
|
336
469
|
this.setStatus("connecting");
|
|
470
|
+
// RE-CHECK after the status callback (CTC-114 review round 13). `setStatus` calls into consumer
|
|
471
|
+
// code, and a consumer may synchronously tear down from it — the browser replica's very first
|
|
472
|
+
// "reconnecting" notification is a documented place to do so. `stop()` then ran while `this.ws`
|
|
473
|
+
// was still null, so it had nothing to close; we resumed here, constructed a socket, stored it in
|
|
474
|
+
// an already-stopped client, and left it open processing frames with the teardown long finished.
|
|
475
|
+
// Every guarded entry point that calls out and then continues needs this; this is the one that
|
|
476
|
+
// creates a resource afterwards.
|
|
477
|
+
if (this.stopped)
|
|
478
|
+
return;
|
|
337
479
|
// One span per connect attempt: started here, ended OK in onopen, ERROR on construct-fail / a close
|
|
338
480
|
// before open. Manual (not active) because the lifecycle spans onopen…onclose callbacks.
|
|
339
481
|
this.connectSpan = this.telemetry.startSpan(REPLICA_SPAN.reconnect, {
|
|
340
|
-
[CATALYST_ATTR.tenant]: this.
|
|
482
|
+
[CATALYST_ATTR.tenant]: this.tenantAttr,
|
|
341
483
|
});
|
|
342
484
|
const wsUrl = this.connectUrl();
|
|
343
485
|
let ws;
|
|
@@ -666,7 +808,7 @@ export class LiveSyncClient {
|
|
|
666
808
|
* KEY ON `escalated` ONLY (logged at ERROR); a gap that heals is routine and boring. */
|
|
667
809
|
recordGap(event, gap) {
|
|
668
810
|
this.gapCounter.add(1, {
|
|
669
|
-
[CATALYST_ATTR.tenant]: this.
|
|
811
|
+
[CATALYST_ATTR.tenant]: this.tenantAttr,
|
|
670
812
|
[CATALYST_ATTR.gapEvent]: event,
|
|
671
813
|
});
|
|
672
814
|
this.log(event === "escalated" ? "error" : "info", REPLICA_LOG.gap, {
|
|
@@ -700,38 +842,175 @@ export class LiveSyncClient {
|
|
|
700
842
|
* The injected callback is a trust boundary like the ws impl: while it runs there is NO socket and
|
|
701
843
|
* scheduleReconnect is suppressed, so an unbounded await here was the last zero-timer wedge — the
|
|
702
844
|
* deadline below is the pending timer that upholds the header invariant for the "resyncing" state.
|
|
703
|
-
* On timeout the attempt is
|
|
704
|
-
*
|
|
705
|
-
*
|
|
845
|
+
* On timeout the attempt is both CANCELLED and abandoned (CTC-114 review round 10). It used to be
|
|
846
|
+
* abandoned only, justified as "the callback owns its own I/O bounds" — but the callback owning
|
|
847
|
+
* bounds is exactly what makes a SECOND, independent deadline here dangerous. The browser seed is
|
|
848
|
+
* bounded by network idleness and by per-RPC worker deadlines, both of which a legitimately slow
|
|
849
|
+
* ~100 MB snapshot satisfies indefinitely; this total deadline could therefore fire on a seed that
|
|
850
|
+
* was making honest progress, whereupon the transport reconnected while the callback kept writing.
|
|
851
|
+
* Frames past the abandoned seed's cursor were then accepted by the socket and discarded by the
|
|
852
|
+
* replica's paused queue, and a late seed completion left the socket advanced over a hole that
|
|
853
|
+
* later deltas sealed for good.
|
|
854
|
+
*
|
|
855
|
+
* So the deadline now fires an AbortSignal FIRST and rejects second: whoever is told the attempt is
|
|
856
|
+
* over is also told to stop. A late settle is still discarded via the `settled` latch, and a late
|
|
857
|
+
* REJECTION is still swallowed so it can never surface as an unhandled rejection.
|
|
706
858
|
*/
|
|
707
859
|
boundedReseed() {
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
860
|
+
// Cancellation is scoped to THIS attempt. Aborting it must not disturb a successor.
|
|
861
|
+
const cancel = new AbortController();
|
|
862
|
+
const seed = this.reseed(cancel.signal);
|
|
711
863
|
void seed.catch(() => { }); // an abandoned attempt's late rejection must never go unhandled
|
|
864
|
+
// ALWAYS wrapped, even with the deadline disabled (CTC-114 review round 8). This used to
|
|
865
|
+
// early-return the raw seed promise when `reseedTimeoutMs <= 0` — the documented way to turn the
|
|
866
|
+
// deadline off — which skipped installing `abandonReseed` and so bypassed round 7's stop() fix
|
|
867
|
+
// entirely on that path. Disabling the DEADLINE must not also disable teardown: the two are
|
|
868
|
+
// independent, and `stop()` has to be able to settle an awaited `requestResync()` either way.
|
|
712
869
|
return new Promise((resolve, reject) => {
|
|
713
870
|
let settled = false;
|
|
714
|
-
|
|
715
|
-
if (settled)
|
|
716
|
-
return;
|
|
717
|
-
settled = true;
|
|
718
|
-
if (this.reseedTimer === timer)
|
|
719
|
-
this.reseedTimer = null;
|
|
720
|
-
reject(new Error(`reseed did not settle within ${this.reseedTimeoutMs}ms; abandoning (CTC-281)`));
|
|
721
|
-
}, this.reseedTimeoutMs);
|
|
722
|
-
this.reseedTimer = timer;
|
|
871
|
+
let timer = null;
|
|
723
872
|
const finish = (fn) => {
|
|
724
873
|
if (settled)
|
|
725
|
-
return; // stale settle: the deadline already
|
|
874
|
+
return; // stale settle: the deadline or stop() already took this attempt
|
|
726
875
|
settled = true;
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
this.reseedTimer
|
|
876
|
+
if (timer !== null) {
|
|
877
|
+
clearTimeout(timer);
|
|
878
|
+
if (this.reseedTimer === timer)
|
|
879
|
+
this.reseedTimer = null;
|
|
880
|
+
}
|
|
881
|
+
this.abandonReseed = null;
|
|
730
882
|
fn();
|
|
731
883
|
};
|
|
884
|
+
/**
|
|
885
|
+
* Give up on this attempt: tell it to stop, WAIT for it to unwind, then settle.
|
|
886
|
+
*
|
|
887
|
+
* Signalling alone was not enough (CTC-114 review round 12, P1). `abort()` only *initiates* the
|
|
888
|
+
* consumer's cleanup — the browser seed still has to abort its fetch, let the supersede guard
|
|
889
|
+
* trip, post its `seedAbort`, and run the `finally` that resumes its delta queue. Settling
|
|
890
|
+
* immediately let `runResync()` reconnect after one backoff while that queue was still PAUSED,
|
|
891
|
+
* so arriving frames were counted as delivered and then discarded: exactly the hole the queue's
|
|
892
|
+
* discard/rollback pairing exists to prevent, re-opened from the other side.
|
|
893
|
+
*
|
|
894
|
+
* Bounded, because the whole point of this deadline is that the callback may be unresponsive: a
|
|
895
|
+
* cleanup that itself hangs must not wedge the transport, so we settle anyway after a grace.
|
|
896
|
+
*/
|
|
897
|
+
const giveUp = (err) => {
|
|
898
|
+
if (settled)
|
|
899
|
+
return;
|
|
900
|
+
cancel.abort();
|
|
901
|
+
// ALWAYS wait for the unwind, bounded by the grace (CTC-114 review round 14).
|
|
902
|
+
//
|
|
903
|
+
// Round 12 skipped the wait when `reseed.length === 0`, reasoning that a zero-arg callback
|
|
904
|
+
// cannot observe the signal. `Function.length` does not support that inference: it counts only
|
|
905
|
+
// parameters before the first default or rest, so `reseed: (signal = undefined) => …` reports
|
|
906
|
+
// 0 while receiving and honouring the signal — and that consumer got no wait at all, which is
|
|
907
|
+
// exactly the hazard the wait exists for. There is no sound way to ask a function whether it
|
|
908
|
+
// will act on a signal, so stop trying: wait for everyone, and let the grace bound it. A
|
|
909
|
+
// consumer that ignores the signal simply hits the grace, which is the pre-round-12 behaviour
|
|
910
|
+
// delayed by `cancelCleanupGraceMs` — the reason that default is small and this is the knob
|
|
911
|
+
// the browser replica raises.
|
|
912
|
+
// 0 DISABLES the wait outright rather than scheduling a zero-delay timer — the same
|
|
913
|
+
// disable-by-zero convention the other bounds use, and the honest meaning of "this consumer
|
|
914
|
+
// has nothing to unwind".
|
|
915
|
+
if (this.cancelCleanupGraceMs <= 0) {
|
|
916
|
+
finish(() => reject(err));
|
|
917
|
+
return;
|
|
918
|
+
}
|
|
919
|
+
const grace = setTimeout(() => finish(() => reject(err)), this.cancelCleanupGraceMs);
|
|
920
|
+
void seed
|
|
921
|
+
.catch(() => undefined)
|
|
922
|
+
.then(() => {
|
|
923
|
+
clearTimeout(grace);
|
|
924
|
+
finish(() => reject(err));
|
|
925
|
+
});
|
|
926
|
+
};
|
|
927
|
+
if (this.reseedTimeoutMs > 0) {
|
|
928
|
+
timer = setTimeout(() => giveUp(new Error(`reseed did not settle within ${this.reseedTimeoutMs}ms; cancelled (CTC-281)`)), this.reseedTimeoutMs);
|
|
929
|
+
this.reseedTimer = timer;
|
|
930
|
+
}
|
|
931
|
+
// stop() settles this wrapper (CTC-114 review round 7). Clearing the deadline is not enough:
|
|
932
|
+
// if the injected reseed() never settles, nothing else ever settles THIS promise, and while
|
|
933
|
+
// that was merely "an irrelevant await nobody holds" when boundedReseed was internal, the
|
|
934
|
+
// public `requestResync()` is now awaited by consumers — so teardown or recovery code holding
|
|
935
|
+
// that await hung forever on stop(). Rejecting rather than resolving keeps the outcome honest;
|
|
936
|
+
// requestResync() catches it and still upholds its never-rejects contract.
|
|
937
|
+
this.abandonReseed = () => giveUp(new Error("client stopped while re-seeding"));
|
|
732
938
|
seed.then((cursor) => finish(() => resolve(cursor)), (err) => finish(() => reject(err instanceof Error ? err : new Error(String(err)))));
|
|
733
939
|
});
|
|
734
940
|
}
|
|
941
|
+
/**
|
|
942
|
+
* Ask the client to drop the socket, re-seed, and reconnect from the fresh cursor — the same path a
|
|
943
|
+
* server `{type:"resync"}` frame drives, exposed for a consumer that discovers ON ITS OWN SIDE that
|
|
944
|
+
* its store can no longer be caught up by deltas.
|
|
945
|
+
*
|
|
946
|
+
* The browser replica is the motivating caller: when its delta queue overflows (or its applies keep
|
|
947
|
+
* rejecting) the buffered frames are dropped, so the transport's notion of what has been delivered is
|
|
948
|
+
* now ahead of what the store actually holds. Re-seeding through here — rather than calling the
|
|
949
|
+
* `reseed` callback directly — is what makes that safe: this CLOSES THE SOCKET FIRST, so no live
|
|
950
|
+
* frame interleaves with the snapshot and lands in the window that is in neither the snapshot nor the
|
|
951
|
+
* store.
|
|
952
|
+
*
|
|
953
|
+
* Delegates to the existing resync path in full — same `resyncing` re-entrancy guard, same
|
|
954
|
+
* `boundedReseed` deadline, same span, same failure→backoff behaviour. NEVER rejects: a failed
|
|
955
|
+
* re-seed is already handled internally by re-entering the reconnect path, and this is called from
|
|
956
|
+
* event handlers and `void` contexts where a rejection would surface as an unhandled promise.
|
|
957
|
+
*/
|
|
958
|
+
async requestResync() {
|
|
959
|
+
if (this.stopped)
|
|
960
|
+
return;
|
|
961
|
+
// BEFORE start() there is nothing to resync (CTC-114 review round 5). `stopped` is false on a
|
|
962
|
+
// never-started client, so it cannot carry this guard by itself. Reseeding here would run with no
|
|
963
|
+
// lifecycle deferred and no telemetry resolved, and — worse — open a socket that the later real
|
|
964
|
+
// start() would orphan: openSocket() overwrites `this.ws`, so the first socket keeps delivering
|
|
965
|
+
// frames and stop() can no longer reach it. Ignore rather than throw: the contract above is that
|
|
966
|
+
// this never rejects, and it is called from `void` contexts.
|
|
967
|
+
if (!this.started) {
|
|
968
|
+
this.log("warn", "requestResync() before start() — ignored");
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
// SERIALIZE behind an in-flight boot (CTC-114 review rounds 6 + 7). Running concurrently with it
|
|
972
|
+
// meant two reseeds and two openSocket() calls, the second overwriting `this.ws` and orphaning the
|
|
973
|
+
// first socket. Waiting — rather than dropping — is what keeps the WARM path correct: a warm boot
|
|
974
|
+
// performs no re-seed, and this method's whole purpose is a consumer that has discovered its store
|
|
975
|
+
// can no longer be caught up by deltas, which replaying from the cursor cannot fix.
|
|
976
|
+
const boot = this.bootTask;
|
|
977
|
+
if (boot) {
|
|
978
|
+
await boot.catch(() => undefined); // the outcome is read from `bootFailed`, latched below
|
|
979
|
+
if (this.stopped)
|
|
980
|
+
return;
|
|
981
|
+
}
|
|
982
|
+
// A FAILED boot must not be recovered from here (CTC-114 review rounds 8 + 9). Round 7 swallowed
|
|
983
|
+
// the rejection and carried straight on into handleResync — so a cold start whose /snapshot failed
|
|
984
|
+
// would reject the caller's start(), sending the application into its boot-error path, and then a
|
|
985
|
+
// later successful reseed here would quietly open a live socket underneath it. Round 8 read the
|
|
986
|
+
// failure from the awaited task, which round 9 showed is not enough: `bootTask` is nulled when the
|
|
987
|
+
// boot settles, so a request arriving after that microtask saw no failure at all. Checked OUTSIDE
|
|
988
|
+
// the `if (boot)` for exactly that reason — the latch outlives the handle, until the next start().
|
|
989
|
+
if (this.bootFailed) {
|
|
990
|
+
this.log("warn", "requestResync() ignored — startup failed");
|
|
991
|
+
return;
|
|
992
|
+
}
|
|
993
|
+
// A COLD boot re-seeded from /snapshot while we waited, which IS what was being asked for.
|
|
994
|
+
if (boot && this.bootColdSeeded) {
|
|
995
|
+
this.log("info", "requestResync() absorbed by the boot's cold seed");
|
|
996
|
+
return;
|
|
997
|
+
}
|
|
998
|
+
// Unlike the frame path, this entry point can be called MID-BACKOFF: the queue overflowed while the
|
|
999
|
+
// client was already waiting to reconnect. handleResync would then reopen the socket itself and the
|
|
1000
|
+
// pending timer would open a second one on top of it.
|
|
1001
|
+
if (this.reconnectTimer != null) {
|
|
1002
|
+
clearTimeout(this.reconnectTimer);
|
|
1003
|
+
this.reconnectTimer = null;
|
|
1004
|
+
}
|
|
1005
|
+
try {
|
|
1006
|
+
await this.handleResync();
|
|
1007
|
+
}
|
|
1008
|
+
catch (err) {
|
|
1009
|
+
// handleResync already catches its own reseed failure; this is the belt-and-braces guard for the
|
|
1010
|
+
// contract above (never reject) against a throw from a consumer callback it invokes.
|
|
1011
|
+
this.log("error", "requestResync failed", err);
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
735
1014
|
/**
|
|
736
1015
|
* Cursor underflow: the deltas we need were evicted from the service's retained change buffer. Close the socket
|
|
737
1016
|
* (so no live frame interleaves with the re-seed), re-seed via the injected callback, then reconnect
|
|
@@ -739,6 +1018,45 @@ export class LiveSyncClient {
|
|
|
739
1018
|
* frame and suppresses scheduleReconnect for the duration so we reopen exactly once.
|
|
740
1019
|
*/
|
|
741
1020
|
async handleResync() {
|
|
1021
|
+
// RETURN THE RUNNING ONE, do not resolve immediately (CTC-114 review round 10). The re-entrancy
|
|
1022
|
+
// guard used to `return` bare, so a caller awaiting the public `requestResync()` while a
|
|
1023
|
+
// server-driven resync was already re-seeding got a promise that resolved AT ONCE — and then read
|
|
1024
|
+
// the store while the replacement snapshot was still being written into it. Awaiting recovery has
|
|
1025
|
+
// to mean recovery finished, whoever started it.
|
|
1026
|
+
//
|
|
1027
|
+
// This is the same seam as the boot serialization, which covered only `bootTask`: one handle for
|
|
1028
|
+
// "a re-seed is in flight", awaited by everything that needs it to be done.
|
|
1029
|
+
// `return await`, not `return` (CodeQL, round 11). Behaviour is identical today, but returning a
|
|
1030
|
+
// bare promise from an async function means a later `try`/`finally` added around this line would
|
|
1031
|
+
// settle BEFORE the awaited work — a foot-gun this file has been bitten by often enough to be
|
|
1032
|
+
// worth foreclosing, and it keeps this frame in the stack trace when the resync rejects.
|
|
1033
|
+
if (this.activeResync)
|
|
1034
|
+
return await this.activeResync;
|
|
1035
|
+
const run = this.runResync();
|
|
1036
|
+
this.activeResync = run;
|
|
1037
|
+
try {
|
|
1038
|
+
await run;
|
|
1039
|
+
}
|
|
1040
|
+
finally {
|
|
1041
|
+
if (this.activeResync === run)
|
|
1042
|
+
this.activeResync = null;
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
/**
|
|
1046
|
+
* The resync body. Never call directly — `handleResync()` owns the in-flight handle.
|
|
1047
|
+
*
|
|
1048
|
+
* TWO FLAGS, ONE FACT — documented rather than consolidated here, deliberately. `resyncing` and
|
|
1049
|
+
* `activeResync` both mean "a re-seed is in flight", and they can disagree: the boot path sets
|
|
1050
|
+
* `resyncing` directly (so a resync cannot race startup) without ever creating an `activeResync`
|
|
1051
|
+
* handle. In that window a call landing here would hit the guard below and resolve silently, which
|
|
1052
|
+
* is the very shape round 10 fixed for the live path.
|
|
1053
|
+
*
|
|
1054
|
+
* It is NOT reachable today, and both reasons are load-bearing: `requestResync()` serialises behind
|
|
1055
|
+
* `bootTask` before it can get here, and the server-frame path needs a socket, which does not exist
|
|
1056
|
+
* until the boot opens one. Consolidating the two into a single handle is the right fix and belongs
|
|
1057
|
+
* with the wider lifecycle rework — not in a release candidate at round eleven.
|
|
1058
|
+
*/
|
|
1059
|
+
async runResync() {
|
|
742
1060
|
if (this.resyncing)
|
|
743
1061
|
return;
|
|
744
1062
|
this.resyncing = true;
|
|
@@ -751,7 +1069,7 @@ export class LiveSyncClient {
|
|
|
751
1069
|
try {
|
|
752
1070
|
// The reseed runs inside an ACTIVE span so the replica's seed span (the injected reseed IS
|
|
753
1071
|
// seedFromSnapshot) auto-parents under this resync span.
|
|
754
|
-
await this.telemetry.withActiveSpan(REPLICA_SPAN.resync, { [CATALYST_ATTR.tenant]: this.
|
|
1072
|
+
await this.telemetry.withActiveSpan(REPLICA_SPAN.resync, { [CATALYST_ATTR.tenant]: this.tenantAttr }, async () => {
|
|
755
1073
|
const cursor = await this.boundedReseed();
|
|
756
1074
|
this.log("info", `resynced, cursor=${cursor}`);
|
|
757
1075
|
});
|