@catalyst-cloud/sdk 0.8.49 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -78,6 +78,35 @@
78
78
  // beyond-gap change frame (re-request the hole `deliveredSeq+1..head`) but never applies it.
79
79
  import { PING_FRAME } from "./types.js";
80
80
  import { NOOP_TELEMETRY, createTelemetry, CATALYST_ATTR, REPLICA_LOG, REPLICA_METRIC, REPLICA_SPAN, DEFAULT_SCOPE_NAME, } from "./otel.js";
81
+ /**
82
+ * A typed authorization failure surfaced through `onAuthError` (CTC-2111). `code` distinguishes the
83
+ * two wire origins: `4401` is the mirror's WebSocket close code for an accept-time token that aged out
84
+ * ("reauthenticate"); `401`/`403` are the HTTP status of a rejected `/snapshot` fetch. It replaces the
85
+ * opaque `Error("/snapshot 401")` a consumer could not act on — an AuthError says re-authenticate,
86
+ * a plain Error says retry.
87
+ */
88
+ export class AuthError extends Error {
89
+ code;
90
+ reason;
91
+ constructor(code, reason) {
92
+ super(reason ? `auth error ${code}: ${reason}` : `auth error ${code}`);
93
+ this.name = "AuthError";
94
+ this.code = code;
95
+ this.reason = reason;
96
+ }
97
+ }
98
+ /** The mirror's WebSocket close code for an accept-time authorization that aged out (CTC-2111): the
99
+ * socket must be re-opened with a fresh token, not blindly reconnected. */
100
+ export const CLOSE_REAUTHENTICATE = 4401;
101
+ /** Internal sentinel (CTC-2111): a bearer `getToken()` that did not settle within the connect deadline.
102
+ * Distinguished from a getToken REJECTION so the two get different treatment — a hang is a bounded
103
+ * backoff reconnect (transient), a rejection is a park in auth-required (re-auth needed). Not exported. */
104
+ class BearerTokenTimeoutError extends Error {
105
+ constructor(ms) {
106
+ super(`bearer getToken() did not settle within ${ms}ms`);
107
+ this.name = "BearerTokenTimeoutError";
108
+ }
109
+ }
81
110
  /**
82
111
  * How long a CANCELLED reseed gets to unwind before the transport settles anyway.
83
112
  *
@@ -90,7 +119,8 @@ import { NOOP_TELEMETRY, createTelemetry, CATALYST_ATTR, REPLICA_LOG, REPLICA_ME
90
119
  const DEFAULT_CANCEL_CLEANUP_GRACE_MS = 250;
91
120
  /** Resolve the runtime global WebSocket, or fail with an actionable message. */
92
121
  function defaultWsFactory(url) {
93
- const Ctor = globalThis.WebSocket;
122
+ const Ctor = globalThis
123
+ .WebSocket;
94
124
  if (!Ctor) {
95
125
  throw new Error("global WebSocket unavailable; pass wsFactory (browser, Bun, or Node >=22 expose one)");
96
126
  }
@@ -105,6 +135,18 @@ export function toWsOrigin(baseUrl) {
105
135
  * is ever appended (the type system + this single construction point make a browser token leak
106
136
  * impossible). Token is ordered FIRST so a truncated log line still reveals the account.
107
137
  */
138
+ /** The WebSocket close event's numeric `code`, if the impl provides one (CTC-2111). The onclose event
139
+ * is typed `unknown` (the structural WebSocketLike carries no CloseEvent shape); a real close event
140
+ * and the FakeWebSocket both expose `.code`. Returns undefined for a code-less `{}` close. */
141
+ function closeCode(ev) {
142
+ const code = ev?.code;
143
+ return typeof code === "number" ? code : undefined;
144
+ }
145
+ /** The WebSocket close event's `reason`, normalized: a non-empty string, else undefined (CTC-2111). */
146
+ function closeReason(ev) {
147
+ const reason = ev?.reason;
148
+ return typeof reason === "string" && reason !== "" ? reason : undefined;
149
+ }
108
150
  /** Strip trailing "/" without a backtracking regex (ReDoS-safe vs `/\/+$/`, CodeQL js/polynomial-redos). */
109
151
  export function stripTrailingSlashes(s) {
110
152
  let end = s.length;
@@ -117,6 +159,10 @@ export function buildConnectUrl(opts) {
117
159
  const params = new URLSearchParams();
118
160
  if (opts.auth.kind === "token")
119
161
  params.set("token", opts.auth.token);
162
+ // CTC-2111 — a bearer's resolved token rides `?token=` identically to the `token` strategy. Only
163
+ // when a value was actually resolved: a diagnostic `connectUrl()` cannot resolve one synchronously.
164
+ else if (opts.auth.kind === "bearer" && opts.bearerToken != null)
165
+ params.set("token", opts.bearerToken);
120
166
  // Only when a tenant was actually named. `?account=` is NOT the same as no account: the server's
121
167
  // consumers are truthiness checks, so empty takes the omitted path anyway — but it would freeze a
122
168
  // contract in which "" is a legal mirror name, and it puts `catalyst.tenant=""` on every span.
@@ -174,6 +220,7 @@ export class LiveSyncClient {
174
220
  onChange;
175
221
  onFrame;
176
222
  onStatus;
223
+ onAuthError;
177
224
  backoffMs;
178
225
  maxBackoffMs;
179
226
  pingIntervalMs;
@@ -219,6 +266,36 @@ export class LiveSyncClient {
219
266
  */
220
267
  bootFailed = false;
221
268
  resyncing = false;
269
+ /** CTC-2111 — parked in `"auth-required"` after a 4401 / a getToken rejection / a 401-403 seed. Only
270
+ * `resume()` (or a fresh `start()`) leaves this state; a normal reconnect never sets it. */
271
+ authRequired = false;
272
+ /** CTC-2111 — an auth failure (401/403) interrupted a RESYNC before its /snapshot could rebuild the
273
+ * store. The cursor is then stale, so recovery must RE-SEED (not just re-open, which would return
274
+ * to `live` with the very inconsistency the resync existed to fix). Distinct from the cursorless
275
+ * initial-seed case, which `getCursor()` already detects. */
276
+ resyncNeededAfterAuth = false;
277
+ /**
278
+ * CTC-2111 — a `resume()` intent awaiting drain. `resume()` from a parked run only RECORDS this (it
279
+ * does not re-drive or clear `authRequired` synchronously), because a synchronous `resume()` from
280
+ * `onAuthError` fires while the failing op (a boot or a resync) is still in flight. The single
281
+ * `drainRecovery()` point fires it once that owning op SETTLES — so a resume during a boot, during a
282
+ * resync, or between them is always captured and re-driven from the CURRENT store state, with no
283
+ * per-path deferral to get wrong and no path that clears the latch without starting recovery.
284
+ */
285
+ pendingRecovery = false;
286
+ /**
287
+ * CTC-2111 — THE single connect-generation guard. A bearer connect resolves getToken() asynchronously,
288
+ * so a late/stale resolution must be dropped rather than open an orphan socket. One counter, ONE check
289
+ * site (`supersededConnect`, in `resolveBearerAndConnect` — the only getToken-gated connect), bumped at
290
+ * the TWO choke points EVERY lifecycle transition funnels through:
291
+ * • `openSocket()` — every connect: initial, reconnect, and resume-reopen (so start → boot →
292
+ * openSocket, and a resume re-open, each stamp a new attempt);
293
+ * • `closeSocket()` — every deliberate teardown/reseed: stop, requestResync/runResync, forceReconnect,
294
+ * and resume-reseed (so entering a resync invalidates any pending connect — RwF).
295
+ * `stop()` also bumps directly, ahead of its closeSocket(), to close the stop()-then-start() window
296
+ * before the new run's openSocket() re-stamps.
297
+ */
298
+ connectAttempt = 0;
222
299
  backoff;
223
300
  reconnectTimer = null;
224
301
  resolveDone = null;
@@ -283,13 +360,18 @@ export class LiveSyncClient {
283
360
  * (It used to rely on "the boot seed runs before any socket exists" — true only while a resync
284
361
  * needed a server frame. The public `requestResync()` added in 0.8.0 needs no socket.) */
285
362
  reseedTimer = null;
363
+ /** CTC-2111 — the in-flight bearer getToken() deadline (awaitTokenBounded). Tracked so stop() clears
364
+ * it (ask 4: teardown leaves NOTHING pending) rather than letting it hold a process alive for up to
365
+ * openTimeoutMs after an otherwise-clean shutdown. */
366
+ bearerTokenTimer = null;
286
367
  constructor(opts) {
287
368
  // Fail fast, and fail HERE. A token-authed client has no session to fall back to, so an omitted
288
369
  // account is a misconfiguration, not a default. It is checked in the constructor rather than in
289
370
  // buildConnectUrl because `connectUrl()` is called from `openSocket()` OUTSIDE its try/catch — a
290
371
  // throw down there escapes the reconnect machinery entirely instead of surfacing to the caller.
291
- if (opts.auth.kind === "token" && !opts.accountId) {
292
- throw new Error("LiveSyncClient: accountId is required with token auth (only cookie auth can fall back to the session's own tenant)");
372
+ if ((opts.auth.kind === "token" || opts.auth.kind === "bearer") &&
373
+ !opts.accountId) {
374
+ throw new Error("LiveSyncClient: accountId is required with token or bearer auth (only cookie auth can fall back to the session's own tenant)");
293
375
  }
294
376
  this.baseUrl = stripTrailingSlashes(opts.baseUrl);
295
377
  this.accountId = opts.accountId;
@@ -301,6 +383,7 @@ export class LiveSyncClient {
301
383
  this.onChange = opts.onChange;
302
384
  this.onFrame = opts.onFrame;
303
385
  this.onStatus = opts.onStatus;
386
+ this.onAuthError = opts.onAuthError;
304
387
  this.backoffMs = opts.backoffMs ?? 1000;
305
388
  this.maxBackoffMs = opts.maxBackoffMs ?? 30_000;
306
389
  this.pingIntervalMs = opts.pingIntervalMs ?? 90_000;
@@ -327,6 +410,9 @@ export class LiveSyncClient {
327
410
  start() {
328
411
  this.stopped = false;
329
412
  this.started = true;
413
+ this.authRequired = false; // a fresh run is never still parked from a previous one (CTC-2111)
414
+ this.resyncNeededAfterAuth = false;
415
+ this.pendingRecovery = false;
330
416
  // RESET per boot. `start()` is restartable after `stop()`, and a stale `true` from a previous
331
417
  // cold boot would make the NEXT boot — warm, and therefore re-seeding nothing — absorb a resync
332
418
  // it should have honoured. Found while re-reading this path rather than reported; the same class
@@ -342,70 +428,100 @@ export class LiveSyncClient {
342
428
  const done = new Promise((resolve) => {
343
429
  this.resolveDone = resolve;
344
430
  });
345
- const boot = (async () => {
346
- // The WHOLE boot is an in-flight resync, not just the cold seed (CTC-114 review rounds 4 + 6).
347
- //
348
- // `requestResync()` public as of 0.8.0 is callable the moment start() returns its promise,
349
- // which is before ANY of this settles. Without the latch, `handleResync()`'s re-entrancy guard
350
- // read false and started a SECOND concurrent reseed: two seeds interleaving writes through a
351
- // non-reentrant consumer callback, then each completion calling openSocket() — and since
352
- // openSocket() overwrites `this.ws`, the first socket was orphaned, still delivering duplicate
353
- // frames and unreachable by stop().
354
- //
355
- // Round 4 latched only the cold seed. That was not enough: `createTelemetry()` below is awaited
356
- // BEFORE the seed, so with telemetry enabled the boot suspends in a window where `started` is
357
- // already true and the latch is not yet set. The latch therefore has to cover the entire body.
358
- //
359
- // Until this release the invariant held for free — a resync could only be driven by a server
360
- // frame, and a frame needs a socket, which does not exist until openSocket() below.
361
- //
362
- // The latch makes the request WAIT; whether it is then absorbed or honoured is decided in
363
- // requestResync() from `bootColdSeeded`, once this task has settled.
364
- //
365
- // Round 6 absorbed it on BOTH arms, arguing that a warm boot's `{type:"sync", after:<cursor>}`
366
- // is itself the catch-up. That was wrong (round 7), and wrong against this method's whole
367
- // reason for existing: a consumer calls requestResync() when it has discovered ON ITS OWN SIDE
368
- // that deltas can no longer catch its store up — the browser replica's dropped overflow buffer
369
- // is the motivating case. Replaying from the cursor cannot rebuild rows the consumer already
370
- // lost, so silently swallowing the request left it permanently inconsistent. Only a COLD boot
371
- // may absorb it, because that boot really is a full re-seed from /snapshot.
372
- this.resyncing = true;
431
+ // Register ownership SYNCHRONOUSLY, before the body runs (CTC-2111 :868): the async body runs
432
+ // synchronously up to its first await, and a bearer cold reseed that throws SYNCHRONOUSLY there
433
+ // fires onAuthError → resume() while `this.bootTask` would otherwise still be unassigned — so
434
+ // drainRecovery would see no in-flight owner and drain into a RECURSIVE boot, which the failing
435
+ // outer boot then clobbers. Assigning the handle first (a promise the body settles) guarantees
436
+ // hasInFlightRun() always sees this boot, so drainRecovery defers to its settle instead.
437
+ let finishBoot;
438
+ let failBoot;
439
+ const boot = new Promise((resolve, reject) => {
440
+ finishBoot = resolve;
441
+ failBoot = reject;
442
+ });
443
+ this.bootTask = boot;
444
+ void (async () => {
373
445
  try {
374
- // Resolve the OTel seam ONCE up front (before the first reseed, so the seed span exists on the
375
- // cold-start path too). Keep the OFF path FULLY SYNCHRONOUS — no `await`, so a caller that opens
376
- // the socket and inspects it in the same tick still sees it (the boot body runs synchronously up
377
- // to its first await); only pay the async resolution (guarded dynamic import, or a
378
- // CatalystReplica passing its already-resolved instance) when telemetry is on.
379
- this.telemetry =
380
- this.telemetryConfig === undefined || this.telemetryConfig === false
381
- ? NOOP_TELEMETRY
382
- : await createTelemetry(this.telemetryConfig, {
383
- tracerName: DEFAULT_SCOPE_NAME,
384
- meterName: DEFAULT_SCOPE_NAME,
385
- });
386
- this.gapCounter = this.telemetry.counter(REPLICA_METRIC.gaps, {
387
- description: "Change-feed seq-gap lifecycle events (detected/healed/escalated).",
388
- unit: "{gap}",
389
- });
390
- const saved = this.getCursor();
391
- if (saved == null) {
392
- this.setStatus("resyncing");
393
- // Bounded like the resync-path reseed (CTC-281): a hanging COLD seed surfaces as a start()
394
- // rejection (the boot arm rejects) instead of a silent forever-"resyncing" start().
395
- await this.boundedReseed();
396
- // Only NOW may a request that waited on this boot be absorbed this really was a full
397
- // re-seed from /snapshot. A warm boot sets nothing, so the waiter is honoured instead.
398
- this.bootColdSeeded = true;
446
+ // The WHOLE boot is an in-flight resync, not just the cold seed (CTC-114 review rounds 4 + 6).
447
+ //
448
+ // `requestResync()` public as of 0.8.0 is callable the moment start() returns its promise,
449
+ // which is before ANY of this settles. Without the latch, `handleResync()`'s re-entrancy guard
450
+ // read false and started a SECOND concurrent reseed: two seeds interleaving writes through a
451
+ // non-reentrant consumer callback, then each completion calling openSocket() — and since
452
+ // openSocket() overwrites `this.ws`, the first socket was orphaned, still delivering duplicate
453
+ // frames and unreachable by stop().
454
+ //
455
+ // Round 4 latched only the cold seed. That was not enough: `createTelemetry()` below is awaited
456
+ // BEFORE the seed, so with telemetry enabled the boot suspends in a window where `started` is
457
+ // already true and the latch is not yet set. The latch therefore has to cover the entire body.
458
+ //
459
+ // Until this release the invariant held for free — a resync could only be driven by a server
460
+ // frame, and a frame needs a socket, which does not exist until openSocket() below.
461
+ //
462
+ // The latch makes the request WAIT; whether it is then absorbed or honoured is decided in
463
+ // requestResync() from `bootColdSeeded`, once this task has settled.
464
+ //
465
+ // Round 6 absorbed it on BOTH arms, arguing that a warm boot's `{type:"sync", after:<cursor>}`
466
+ // is itself the catch-up. That was wrong (round 7), and wrong against this method's whole
467
+ // reason for existing: a consumer calls requestResync() when it has discovered ON ITS OWN SIDE
468
+ // that deltas can no longer catch its store up the browser replica's dropped overflow buffer
469
+ // is the motivating case. Replaying from the cursor cannot rebuild rows the consumer already
470
+ // lost, so silently swallowing the request left it permanently inconsistent. Only a COLD boot
471
+ // may absorb it, because that boot really is a full re-seed from /snapshot.
472
+ this.resyncing = true;
473
+ try {
474
+ // Resolve the OTel seam ONCE up front (before the first reseed, so the seed span exists on the
475
+ // cold-start path too). Keep the OFF path FULLY SYNCHRONOUS — no `await`, so a caller that opens
476
+ // the socket and inspects it in the same tick still sees it (the boot body runs synchronously up
477
+ // to its first await); only pay the async resolution (guarded dynamic import, or a
478
+ // CatalystReplica passing its already-resolved instance) when telemetry is on.
479
+ this.telemetry =
480
+ this.telemetryConfig === undefined || this.telemetryConfig === false
481
+ ? NOOP_TELEMETRY
482
+ : await createTelemetry(this.telemetryConfig, {
483
+ tracerName: DEFAULT_SCOPE_NAME,
484
+ meterName: DEFAULT_SCOPE_NAME,
485
+ });
486
+ this.gapCounter = this.telemetry.counter(REPLICA_METRIC.gaps, {
487
+ description: "Change-feed seq-gap lifecycle events (detected/healed/escalated).",
488
+ unit: "{gap}",
489
+ });
490
+ const saved = this.getCursor();
491
+ if (saved == null) {
492
+ this.setStatus("resyncing");
493
+ // Bounded like the resync-path reseed (CTC-281): a hanging COLD seed surfaces as a start()
494
+ // rejection (the boot arm rejects) instead of a silent forever-"resyncing" start().
495
+ try {
496
+ await this.boundedReseed();
497
+ }
498
+ catch (err) {
499
+ // CTC-2111 — the initial /snapshot was rejected 401/403: for a BEARER client, surface it as
500
+ // "auth-required" + onAuthError so the consumer can refresh and resume() (not an endless
501
+ // retry). BEARER-ONLY (see runResync): a token/cookie initial-seed failure stays today's
502
+ // start() rejection, never auth-required. Either way, still reject the boot — an unseeded
503
+ // store cannot go live, and no socket is opened.
504
+ if (err instanceof AuthError && this.auth.kind === "bearer")
505
+ this.raiseAuthError(err);
506
+ throw err;
507
+ }
508
+ // Only NOW may a request that waited on this boot be absorbed — this really was a full
509
+ // re-seed from /snapshot. A warm boot sets nothing, so the waiter is honoured instead.
510
+ this.bootColdSeeded = true;
511
+ }
399
512
  }
513
+ finally {
514
+ // Must clear on the FAILURE arm too, or a failed boot latches the client into a state where
515
+ // every later resync — and scheduleReconnect — is suppressed forever.
516
+ this.resyncing = false;
517
+ }
518
+ this.openSocket();
519
+ finishBoot();
400
520
  }
401
- finally {
402
- // Must clear on the FAILURE arm too, or a failed boot latches the client into a state where
403
- // every later resync — and scheduleReconnect — is suppressed forever.
404
- this.resyncing = false;
521
+ catch (err) {
522
+ failBoot(err);
405
523
  }
406
- this.openSocket();
407
524
  })();
408
- this.bootTask = boot;
409
525
  // Clear the handle once boot settles, so a resync arriving LONG after startup is never mistaken
410
526
  // for one that raced it — otherwise `bootColdSeeded` would absorb legitimate later requests
411
527
  // forever. The catch keeps a boot rejection from surfacing as an unhandled one on this arm; the
@@ -419,6 +535,9 @@ export class LiveSyncClient {
419
535
  .then(() => {
420
536
  if (this.bootTask === boot)
421
537
  this.bootTask = null;
538
+ // CTC-2111 — the single recovery drain: a resume() that arrived while this boot was in flight
539
+ // is re-driven now that it (and its latch cleanup above) has settled.
540
+ this.drainRecovery();
422
541
  });
423
542
  // Settles when stop() resolves the deferred, OR rejects if the boot (cold seed) fails — a boot
424
543
  // SUCCESS deliberately keeps waiting on `done` (the "runs forever" contract). Promise.race
@@ -428,6 +547,12 @@ export class LiveSyncClient {
428
547
  /** Stop the client: close the socket, cancel any pending reconnect, resolve start(). Idempotent. */
429
548
  stop() {
430
549
  this.stopped = true;
550
+ this.pendingRecovery = false; // CTC-2111 — a stopped client has no run to recover
551
+ // CTC-2111 — invalidate any in-flight bearer connect attempt IMMEDIATELY, so a getToken() resolving
552
+ // after a stop()+start() (its telemetry await can outlast this) is dropped rather than connecting
553
+ // under the new run. The new run bumps this again in openSocket(), but bumping here closes the
554
+ // window between start() resetting `stopped` and openSocket() re-stamping the attempt.
555
+ this.connectAttempt++;
431
556
  if (this.reconnectTimer != null) {
432
557
  clearTimeout(this.reconnectTimer);
433
558
  this.reconnectTimer = null;
@@ -436,6 +561,11 @@ export class LiveSyncClient {
436
561
  // reseedTimeoutMs. With it cleared a still-hanging reseed simply never settles its (now
437
562
  // irrelevant) await — every post-await path in handleResync/boot checks `stopped` first.
438
563
  this.clearReseedTimer();
564
+ // CTC-2111 — a still-pending bearer token deadline must not outlive the client (leak/hold-open).
565
+ if (this.bearerTokenTimer != null) {
566
+ clearTimeout(this.bearerTokenTimer);
567
+ this.bearerTokenTimer = null;
568
+ }
439
569
  this.closeSocket();
440
570
  this.setStatus("stopped");
441
571
  const done = this.resolveDone;
@@ -448,15 +578,94 @@ export class LiveSyncClient {
448
578
  this.abandonReseed = null;
449
579
  abandon?.();
450
580
  }
581
+ /**
582
+ * CTC-2111 — resume a run parked in `"auth-required"` (a 4401 close, a `getToken()` rejection, or a
583
+ * 401/403 initial /snapshot). The supported recovery after `onAuthError`: refresh whatever credential
584
+ * `getToken()` draws on, then call this. A bearer `getToken()` is resolved FRESH on the re-open, so
585
+ * the newly-authorized token is the one used.
586
+ *
587
+ * It REUSES the original run rather than starting a second one — calling `start()` again from a
588
+ * parked-but-still-pending run would overwrite `resolveDone` and strand the promise `start()` first
589
+ * returned (it would never settle). Progress surfaces through `onStatus` (connecting → live), the
590
+ * same way `start()` reports it.
591
+ *
592
+ * • Never started, or already stopped → this is a fresh `start()` (nothing to resume).
593
+ * • Not parked (a live/reconnecting run) → a no-op; there is nothing to resume.
594
+ * • Parked with a durable cursor (the 4401 mid-run case) → re-open the socket, reusing the deferred.
595
+ * • Parked with NO cursor (an initial-connect auth failure, whose `start()` already REJECTED) → a
596
+ * fresh `start()` re-runs the cold seed; its old promise already settled, so nothing is stranded.
597
+ */
598
+ resume() {
599
+ if (this.stopped || !this.started) {
600
+ void this.start().catch(() => { }); // fresh run — its failure surfaces via onStatus / onAuthError
601
+ return;
602
+ }
603
+ if (!this.authRequired)
604
+ return; // a live/reconnecting run has nothing to resume
605
+ // Record the intent ONLY — do not clear `authRequired` and do not re-drive synchronously. A
606
+ // synchronous resume() from onAuthError fires while the failing op (a boot or a resync) is still in
607
+ // flight; `drainRecovery()` re-drives it from the CURRENT store state once that op settles. If
608
+ // nothing is in flight (a later, standalone resume() — e.g. the 4401 mid-run case), drain now.
609
+ this.pendingRecovery = true;
610
+ if (!this.hasInFlightRun())
611
+ this.drainRecovery();
612
+ }
613
+ /** CTC-2111 — is a boot or a resync currently the owner of the client's recovery state? While one is,
614
+ * a `resume()` intent waits for it to settle (that owner's settle calls `drainRecovery`). */
615
+ hasInFlightRun() {
616
+ return this.bootTask != null || this.activeResync != null;
617
+ }
618
+ /**
619
+ * CTC-2111 — THE single recovery drain. Called from `resume()` (when idle) and from the settle of
620
+ * every in-flight owner (the boot chain, `handleResync`'s finally). It re-drives a recorded
621
+ * `resume()` intent exactly once, from the CURRENT store state, and clears `authRequired` only when
622
+ * recovery actually starts — so no path clears the latch without starting recovery, and a resume()
623
+ * during a boot, during a resync, or between them is always captured and correctly re-driven.
624
+ */
625
+ drainRecovery() {
626
+ if (!this.pendingRecovery || this.stopped)
627
+ return;
628
+ // Another owner may still hold the state (a boot settled but a resync is now in flight, or vice
629
+ // versa): let the still-running one drain when IT settles, so recovery never races a live op.
630
+ if (this.hasInFlightRun())
631
+ return;
632
+ this.pendingRecovery = false;
633
+ this.authRequired = false;
634
+ if (this.resyncNeededAfterAuth) {
635
+ // A resync was interrupted by auth before it rebuilt the store: the cursor is stale, so re-open
636
+ // alone would return to `live` with the original inconsistency. Re-seed (handleResync re-seeds
637
+ // regardless of cursor, then re-opens).
638
+ this.resyncNeededAfterAuth = false;
639
+ void this.handleResync().catch(() => { });
640
+ }
641
+ else if (this.getCursor() == null) {
642
+ // The cold seed never completed (the initial /snapshot failed): a fresh run re-seeds. Its old
643
+ // promise already rejected, so a new deferred strands nothing.
644
+ void this.start().catch(() => { });
645
+ }
646
+ else {
647
+ // Seeded already (the 4401 mid-run case): re-open, REUSING the original run's deferred (never a
648
+ // second start(), which would overwrite resolveDone and strand the first promise).
649
+ this.openSocket();
650
+ }
651
+ }
451
652
  /** The ws(s):// URL this client opens, for diagnostics/tests. Re-derived from the options — and,
452
653
  * since CTC-628, re-RESOLVED: `connectParams` is invoked here, so every reconnect reports the
453
- * consumer's CURRENT state rather than the state it had when the client was constructed. */
654
+ * consumer's CURRENT state rather than the state it had when the client was constructed.
655
+ * For a bearer auth this omits `?token=` — the token is resolved asynchronously at connect time
656
+ * (see `connectUrlWith`), which a synchronous diagnostic accessor cannot do. */
454
657
  connectUrl() {
658
+ return this.connectUrlWith(undefined);
659
+ }
660
+ /** The connect URL with a resolved bearer token injected (CTC-2111). Kept private and separate from
661
+ * the public sync `connectUrl()` so the token/cookie path stays byte-identical and same-tick. */
662
+ connectUrlWith(bearerToken) {
455
663
  return buildConnectUrl({
456
664
  baseUrl: this.baseUrl,
457
665
  connectPath: this.connectPath,
458
666
  accountId: this.accountId,
459
667
  auth: this.auth,
668
+ bearerToken,
460
669
  extraParams: this.resolveConnectParams(),
461
670
  });
462
671
  }
@@ -502,6 +711,55 @@ export class LiveSyncClient {
502
711
  this.log("warn", "onStatus handler threw", err);
503
712
  }
504
713
  }
714
+ /** CTC-2111 — enter the parked `"auth-required"` state: no socket, no reconnect scheduled (the CALLER
715
+ * never schedules one), and `authRequired` latched so `resume()` knows there is a run to resume. */
716
+ parkAuthRequired() {
717
+ this.authRequired = true;
718
+ this.setStatus("auth-required");
719
+ }
720
+ /** CTC-2111 — park in `"auth-required"` AND hand the consumer a typed `AuthError` (a wire code:
721
+ * 4401 close, or a 401/403 /snapshot). The onAuthError handler is guarded like onStatus — a throwing
722
+ * consumer callback never wedges the transport. */
723
+ raiseAuthError(err) {
724
+ this.parkAuthRequired();
725
+ try {
726
+ this.onAuthError?.(err);
727
+ }
728
+ catch (e) {
729
+ this.log("warn", "onAuthError handler threw", e);
730
+ }
731
+ }
732
+ /** CTC-2111 — await a bearer `getToken()` but never longer than the connect/open deadline, so a token
733
+ * endpoint that never settles cannot wedge the client in `"connecting"` with no timer pending (the
734
+ * `openTimeoutMs` deadline is armed LATER, inside `connect()`, only once the token is in hand). A
735
+ * disabled deadline (`openTimeoutMs <= 0`) leaves the acquisition unbounded — the caller's own
736
+ * choice to run without connect deadlines. The timer is local and self-clears on settle. */
737
+ awaitTokenBounded(p) {
738
+ if (this.openTimeoutMs <= 0)
739
+ return p;
740
+ return new Promise((resolve, reject) => {
741
+ const timer = setTimeout(() => {
742
+ if (this.bearerTokenTimer === timer)
743
+ this.bearerTokenTimer = null;
744
+ reject(new BearerTokenTimeoutError(this.openTimeoutMs));
745
+ }, this.openTimeoutMs);
746
+ // Never let a pending token deadline hold a supervised process's exit open on its own (CTC-2111).
747
+ timer.unref?.();
748
+ this.bearerTokenTimer = timer;
749
+ const clear = () => {
750
+ clearTimeout(timer);
751
+ if (this.bearerTokenTimer === timer)
752
+ this.bearerTokenTimer = null;
753
+ };
754
+ p.then((v) => {
755
+ clear();
756
+ resolve(v);
757
+ }, (e) => {
758
+ clear();
759
+ reject(e);
760
+ });
761
+ });
762
+ }
505
763
  openSocket() {
506
764
  if (this.stopped)
507
765
  return;
@@ -515,12 +773,68 @@ export class LiveSyncClient {
515
773
  // creates a resource afterwards.
516
774
  if (this.stopped)
517
775
  return;
776
+ // A deliberate (re)connect leaves the parked auth-required state (resume() cleared it too).
777
+ this.authRequired = false;
778
+ // CTC-2111: a bearer strategy must resolve a FRESH token before it can build the connect URL, so
779
+ // its connect leg is async. token/cookie stay fully synchronous — no await between the "connecting"
780
+ // status and the socket construction — so the byte-identical URL and same-tick-socket both hold.
781
+ if (this.auth.kind === "bearer") {
782
+ // Stamp THIS attempt: a late token resolution superseded by a newer attempt / stop() / start()
783
+ // must be dropped rather than connect (which would orphan this.ws).
784
+ const attempt = ++this.connectAttempt;
785
+ void this.resolveBearerAndConnect(this.auth.getToken, attempt);
786
+ return;
787
+ }
788
+ this.connect(undefined);
789
+ }
790
+ /**
791
+ * CTC-2111 — resolve a fresh bearer token (bounded by the connect deadline), then connect IFF this is
792
+ * still the current attempt. Three failure modes, each returning the state machine to an actionable
793
+ * state — never a busy loop, never a permanent wedge:
794
+ * • superseded (a newer attempt, a stop(), or a stop()+start() cycle) → drop the stale resolution;
795
+ * • `getToken()` timed out (never settled) → a bounded backoff reconnect (transient, retryable);
796
+ * • `getToken()` rejected → park in `"auth-required"` (re-auth needed) until `resume()`.
797
+ */
798
+ async resolveBearerAndConnect(getToken, attempt) {
799
+ let token;
800
+ try {
801
+ token = await this.awaitTokenBounded(getToken());
802
+ }
803
+ catch (err) {
804
+ // Superseded: its late value must never open a socket the current run does not expect.
805
+ if (this.supersededConnect(attempt))
806
+ return;
807
+ if (err instanceof BearerTokenTimeoutError) {
808
+ this.log("warn", "bearer getToken() did not settle within the connect deadline; backing off", err);
809
+ this.setStatus("reconnecting");
810
+ this.scheduleReconnect();
811
+ return;
812
+ }
813
+ this.log("warn", "bearer getToken() rejected; auth-required until resume()", err);
814
+ this.parkAuthRequired();
815
+ return;
816
+ }
817
+ // Superseded during the await (a newer attempt, or a stop()/start()) — drop the stale resolution.
818
+ if (this.supersededConnect(attempt))
819
+ return;
820
+ this.connect(token);
821
+ }
822
+ /** CTC-2111 — the ONE connect-generation check: true if `attempt` is no longer the current connect
823
+ * (a stop, a start, a resync, a reconnect, a resume-reopen all bump `connectAttempt`), so a bearer
824
+ * getToken() resolving after any of those must be dropped rather than open an orphan socket. */
825
+ supersededConnect(attempt) {
826
+ return this.stopped || attempt !== this.connectAttempt;
827
+ }
828
+ /** Open a socket with the connect URL, given an already-resolved bearer token (undefined for
829
+ * token/cookie). Everything from the connect span onward — byte-identical to the pre-CTC-2111
830
+ * `openSocket` body, only the URL now carries a resolved bearer token when there is one. */
831
+ connect(bearerToken) {
518
832
  // One span per connect attempt: started here, ended OK in onopen, ERROR on construct-fail / a close
519
833
  // before open. Manual (not active) because the lifecycle spans onopen…onclose callbacks.
520
834
  this.connectSpan = this.telemetry.startSpan(REPLICA_SPAN.reconnect, {
521
835
  [CATALYST_ATTR.tenant]: this.tenantAttr,
522
836
  });
523
- const wsUrl = this.connectUrl();
837
+ const wsUrl = this.connectUrlWith(bearerToken);
524
838
  let ws;
525
839
  try {
526
840
  ws = this.wsFactory(wsUrl);
@@ -574,10 +888,23 @@ export class LiveSyncClient {
574
888
  this.onInboundFrame();
575
889
  void this.handleFrame(ev.data);
576
890
  };
577
- ws.onclose = () => {
891
+ ws.onclose = (ev) => {
578
892
  if (this.ws === ws)
579
893
  this.ws = null;
580
894
  this.clearLivenessTimers(); // this connection's ping/deadline die with its socket
895
+ // CTC-2111 — a BEARER socket closed 4401 is the mirror saying "reauthenticate": the access
896
+ // token was revoked or the session went inactive (a routine 15-minute expiry refreshes silently
897
+ // through getToken on the next connect, and never reaches here). This is NOT a transient drop,
898
+ // so the reconnect loop STOPS — the bug this fixes was reconnecting forever against a dead
899
+ // token. A {kind:"token"} client is UNCHANGED: it ignores the code and reconnects as it always
900
+ // has (regression pin). The next deliberate start() resolves a fresh token via getToken().
901
+ if (this.auth.kind === "bearer" &&
902
+ closeCode(ev) === CLOSE_REAUTHENTICATE &&
903
+ !this.stopped) {
904
+ this.endConnectSpan(new Error("socket closed 4401 (reauthenticate)"));
905
+ this.raiseAuthError(new AuthError(CLOSE_REAUTHENTICATE, closeReason(ev)));
906
+ return; // deliberately NO scheduleReconnect — see above
907
+ }
581
908
  if (!this.stopped && !this.resyncing)
582
909
  this.setStatus("reconnecting");
583
910
  // No-op if onopen already ended it (a normal disconnect of a healthy socket isn't a connect error).
@@ -622,6 +949,10 @@ export class LiveSyncClient {
622
949
  closeSocket() {
623
950
  // A deliberate teardown of an in-flight attempt (stop/resync): end the connect span neutrally.
624
951
  this.endConnectSpan();
952
+ // CTC-2111 — a deliberate teardown SUPERSEDES any pending bearer connect: bump the attempt id so a
953
+ // getToken() resolving after this (e.g. mid-resync) is dropped rather than opening a socket against
954
+ // the store being rebuilt — which the following openSocket() would then orphan by overwriting this.ws.
955
+ this.connectAttempt++;
625
956
  // The single choke point for liveness-timer teardown — covers stop/resync/forceReconnect. (The
626
957
  // server-close path clears them in onclose; both routes null this.ws, so no timer outlives a socket.)
627
958
  this.clearLivenessTimers();
@@ -766,7 +1097,11 @@ export class LiveSyncClient {
766
1097
  onGapFrame(frame) {
767
1098
  if (this.gap)
768
1099
  return; // a re-request is already in flight; the replay redelivers this frame too
769
- this.gap = { seqFrom: this.deliveredSeq + 1, seqTo: frame.seq - 1, retries: 0 };
1100
+ this.gap = {
1101
+ seqFrom: this.deliveredSeq + 1,
1102
+ seqTo: frame.seq - 1,
1103
+ retries: 0,
1104
+ };
770
1105
  this.recordGap("detected", this.gap);
771
1106
  this.sendGapRequest();
772
1107
  }
@@ -898,7 +1233,19 @@ export class LiveSyncClient {
898
1233
  boundedReseed() {
899
1234
  // Cancellation is scoped to THIS attempt. Aborting it must not disturb a successor.
900
1235
  const cancel = new AbortController();
901
- const seed = this.reseed(cancel.signal);
1236
+ // CTC-2111 NORMALIZE a synchronous throw from `reseed` into a rejected promise. A consumer reseed
1237
+ // that throws SYNCHRONOUSLY (rather than rejecting) would otherwise unwind the caller before its
1238
+ // `await` suspends — so the owning boot/resync would still be registering its handle when a sync
1239
+ // resume() from onAuthError ran, and drainRecovery would drain into a recursive op. As a promise,
1240
+ // the failure is always a microtask, after the handle is registered. (Register-first is the primary
1241
+ // guard; this is the belt-and-suspenders that makes the reseed await ALWAYS suspend.)
1242
+ let seed;
1243
+ try {
1244
+ seed = Promise.resolve(this.reseed(cancel.signal));
1245
+ }
1246
+ catch (err) {
1247
+ seed = Promise.reject(err instanceof Error ? err : new Error(String(err)));
1248
+ }
902
1249
  void seed.catch(() => { }); // an abandoned attempt's late rejection must never go unhandled
903
1250
  // ALWAYS wrapped, even with the deadline disabled (CTC-114 review round 8). This used to
904
1251
  // early-return the raw seed promise when `reseedTimeoutMs <= 0` — the documented way to turn the
@@ -1071,14 +1418,27 @@ export class LiveSyncClient {
1071
1418
  // worth foreclosing, and it keeps this frame in the stack trace when the resync rejects.
1072
1419
  if (this.activeResync)
1073
1420
  return await this.activeResync;
1074
- const run = this.runResync();
1421
+ // Register ownership SYNCHRONOUSLY, before runResync's body runs (CTC-2111 :868): a synchronous
1422
+ // resume() from onAuthError (fired by a reseed that rejects) must find `activeResync` already set,
1423
+ // or drainRecovery sees no in-flight owner and drains into a recursive resync. The run promise is
1424
+ // created here and settled by runResync, so the handle exists before any of runResync executes.
1425
+ let finishRun;
1426
+ let failRun;
1427
+ const run = new Promise((resolve, reject) => {
1428
+ finishRun = resolve;
1429
+ failRun = reject;
1430
+ });
1075
1431
  this.activeResync = run;
1432
+ void this.runResync().then(finishRun, failRun);
1076
1433
  try {
1077
1434
  await run;
1078
1435
  }
1079
1436
  finally {
1080
1437
  if (this.activeResync === run)
1081
1438
  this.activeResync = null;
1439
+ // CTC-2111 — the single recovery drain: a resume() that arrived while this resync was in flight
1440
+ // (a synchronous resume() from onAuthError, before runResync returned) is re-driven now.
1441
+ this.drainRecovery();
1082
1442
  }
1083
1443
  }
1084
1444
  /**
@@ -1105,6 +1465,7 @@ export class LiveSyncClient {
1105
1465
  this.setStatus("resyncing");
1106
1466
  this.closeSocket();
1107
1467
  let reseeded = false;
1468
+ let authErr; // CTC-2111 — a 401/403 /snapshot routes to auth-required
1108
1469
  try {
1109
1470
  // The reseed runs inside an ACTIVE span so the replica's seed span (the injected reseed IS
1110
1471
  // seedFromSnapshot) auto-parents under this resync span.
@@ -1115,13 +1476,29 @@ export class LiveSyncClient {
1115
1476
  reseeded = true;
1116
1477
  }
1117
1478
  catch (err) {
1118
- this.log("error", "resync reseed failed; will retry on reconnect", err);
1479
+ // CTC-2111 a 401/403 /snapshot during a RESYNC is the SAME re-auth signal as a 4401 close for a
1480
+ // BEARER client (getToken() is its recovery: park → refresh → resume()), so route it to
1481
+ // auth-required instead of re-entering the reconnect loop. BEARER-ONLY: a token/cookie credential
1482
+ // has no refresh path and nothing calls resume() on a host-sync daemon, so parking it on a WorkOS
1483
+ // blip would strand it — and the mirror's 401 does not separate "invalid" from "unavailable"
1484
+ // (CTC-792-grammar). Legacy clients keep reconnect-with-backoff. Every other failure retries too.
1485
+ if (err instanceof AuthError && this.auth.kind === "bearer")
1486
+ authErr = err;
1487
+ else
1488
+ this.log("error", "resync reseed failed; will retry on reconnect", err);
1119
1489
  }
1120
1490
  finally {
1121
1491
  this.resyncing = false;
1122
1492
  }
1123
1493
  if (this.stopped)
1124
1494
  return;
1495
+ if (authErr) {
1496
+ // CTC-2111 — this resync's /snapshot failed auth BEFORE it could rebuild the store, so the cursor
1497
+ // is now stale: resume() must re-seed, not just re-open (RwI). Latch that, then park.
1498
+ this.resyncNeededAfterAuth = true;
1499
+ this.raiseAuthError(authErr); // NO reconnect — the token must be re-authorized (resume())
1500
+ return;
1501
+ }
1125
1502
  if (reseeded) {
1126
1503
  // A completed re-seed reopens immediately — the store is fresh and the endpoint just served us.
1127
1504
  this.openSocket();