js-bao-wss-client 2.2.0-alpha.7 → 2.2.0-alpha.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,6 +6,7 @@ import type { TypedModelConstructor } from "./types/typed-model-constructor";
6
6
  import { LogLevel } from "./internal/logger";
7
7
  import { type SubscribeOptions } from "./internal/databaseSubscriptions";
8
8
  import { type StorageConfig, type YjsPersistenceFactory } from "./internal/storage/index.js";
9
+ import { type WorkflowStatusSlice } from "./internal/workflowSlice";
9
10
  import { BlobManager } from "./internal/blobManager";
10
11
  import { type AnalyticsEventInput } from "./internal/analyticsQueue";
11
12
  import { type GoogleClientsConfig } from "./internal/authController";
@@ -497,11 +498,23 @@ export interface SyncEvent {
497
498
  * loader that needs to reload on every remote write can subscribe here
498
499
  * instead of observing the underlying document directly.
499
500
  *
500
- * It also fires with `state: "error"` when a remote update for an open
501
- * document arrives but cannot be applied — currently when downloading a
502
- * large update from R2 is rejected (401/403), which leaves the document
503
- * behind the server. It does not fire at all for updates addressed to a
504
- * document this client does not have open.
501
+ * It also fires with `state: "error"` when an open document cannot be kept
502
+ * in sync:
503
+ *
504
+ * - a remote update for it arrives but cannot be applied — when downloading
505
+ * a large update from R2 is rejected (401/403), which leaves the document
506
+ * behind the server;
507
+ * - its sync handshake goes unanswered for the whole handshake budget
508
+ * (`sync.handshakeTimeoutMs`, default 10s), so what the app is rendering
509
+ * has stopped converging. The client keeps retrying underneath, at a
510
+ * backoff that caps at 15s, and repeats `"error"` on each timeout it
511
+ * hits. Once such a document does sync, the event fires `"synced"` once
512
+ * more so the app can clear whatever it surfaced — but not for a
513
+ * completion that only scheduled a stale-local-state reset, since that
514
+ * document still has no server state.
515
+ *
516
+ * It does not fire at all for updates addressed to a document this client
517
+ * does not have open.
505
518
  *
506
519
  * `state` also carries `"syncing"` and `"stale"` for cross-client
507
520
  * compatibility; this client emits only `"synced"` and `"error"`. For a
@@ -1344,7 +1357,16 @@ export interface WorkflowStatusResult<O = any> {
1344
1357
  /** Why the run did not run, when `status === "skipped"`. */
1345
1358
  skipReason?: WorkflowRunSkipReason;
1346
1359
  run?: WorkflowRun;
1360
+ /**
1361
+ * The task slice's record, for a durable function run. Additive and present
1362
+ * only when the run has one (a DSL run and a request invocation have none)
1363
+ * and the client could read the block the server sent.
1364
+ * `refreshCount` is how many times the slice's token was refreshed through
1365
+ * the gateway instead of yielding to renew its budget.
1366
+ */
1367
+ slice?: WorkflowStatusSlice;
1347
1368
  }
1369
+ export type { WorkflowStatusSlice };
1348
1370
  /** Options for claiming a workflow apply */
1349
1371
  export interface ClaimApplyOptions {
1350
1372
  workflowKey: string;
@@ -1959,8 +1981,47 @@ export declare class JsBaoClient extends Observable<any> {
1959
1981
  private syncRetryBackoffMs;
1960
1982
  private syncRetryTimeoutHandles;
1961
1983
  private syncWatchdogTimeoutMs;
1962
- private readonly syncRetryInitialMs;
1963
- private readonly syncRetryMaxMs;
1984
+ private syncRetryInitialMs;
1985
+ private syncRetryMaxMs;
1986
+ /**
1987
+ * Documents whose failing sync has been reported to the app as a
1988
+ * `documentSyncStateChanged` "error" (#3390). Kept so the matching
1989
+ * "synced" recovery is emitted only for a document the app was actually
1990
+ * warned about, and once.
1991
+ */
1992
+ private syncErrorReported;
1993
+ /**
1994
+ * The outcome of the `syncStep2` payload of a document's current cycle,
1995
+ * still resolving or resolved (#3390). `syncStep2ReceivedFor` records the
1996
+ * frame before its payload is fetched, so a `syncComplete` can pass the
1997
+ * stale-state check while the download is in flight — or after it failed.
1998
+ * The recovery report waits on this instead, so a cycle whose server state
1999
+ * never reached the Y.Doc is not reported to the app as the recovery.
2000
+ */
2001
+ private syncStep2PayloadFor;
2002
+ /**
2003
+ * Consecutive handshake-budget timeouts per document, cleared when a cycle
2004
+ * completes or the document closes (#3390).
2005
+ */
2006
+ private syncTimeoutStreak;
2007
+ /**
2008
+ * Documents whose stall has already asked for a fresh socket. It outlives
2009
+ * the reconnect it triggers — only a completed sync or a close clears it —
2010
+ * so one stalled document rebuilds the connection once rather than every
2011
+ * few budgets (#3390).
2012
+ */
2013
+ private syncStallReconnectRequested;
2014
+ /**
2015
+ * Whether the current connection has already been rebuilt for a stalled
2016
+ * document. Reset on connect, so a burst of stalled documents costs one
2017
+ * reconnect between them (#3390).
2018
+ */
2019
+ private syncStallReconnectOnThisConnection;
2020
+ /**
2021
+ * Consecutive handshake-budget timeouts on a live socket before the client
2022
+ * stops trusting the connection and rebuilds it (#3390).
2023
+ */
2024
+ private static readonly syncStallReconnectAfterTimeouts;
1964
2025
  private jsBaoGetModelDocumentId;
1965
2026
  private jsBaoSetDefaultDocumentIdRaw;
1966
2027
  private jsBaoClearDefaultDocumentIdRaw;
@@ -2382,7 +2443,7 @@ export declare class JsBaoClient extends Observable<any> {
2382
2443
  private runSyncWorkflow;
2383
2444
  private getWorkflowStatus;
2384
2445
  /**
2385
- * Unpack a `{ status, run }` status response into a
2446
+ * Unpack a `{ status, run, slice? }` status response into a
2386
2447
  * {@link WorkflowStatusResult}. Shared by `getStatus` (workflowKey/runKey
2387
2448
  * route), `terminate`, and `waitFor`'s reconcile fetch (runId route).
2388
2449
  *
@@ -2583,6 +2644,71 @@ export declare class JsBaoClient extends Observable<any> {
2583
2644
  private clearSyncWatchdog;
2584
2645
  private clearAllSyncWatchdogs;
2585
2646
  private handleSyncWatchdogTimeout;
2647
+ /**
2648
+ * Rebuild the socket for a document that has missed the handshake budget
2649
+ * several times in a row while the transport still reads as open (#3390).
2650
+ *
2651
+ * Retrying is only recovery while the connection still carries answers. A
2652
+ * half-open socket reads as open from here — `isSocketOpen` is only
2653
+ * `readyState`, and this transport sends no heartbeat of its own — so every
2654
+ * `syncStep1` goes out, nothing comes back, and no amount of re-sending
2655
+ * converges on it. So the client builds a new socket itself; the
2656
+ * reconnect's document sweep re-syncs everything open.
2657
+ *
2658
+ * This is NOT what recovered the field report, and could not have been:
2659
+ * the client keeps one `connectionId` for the life of the process, and the
2660
+ * stall there was a server-side sync-in-progress marker keyed on that id,
2661
+ * which a new socket carries along — only a process restart (a new id)
2662
+ * ended it. That marker, and the broadcast mapping a reconnect lost, are
2663
+ * fixed on the server (#3390). The rebuild stays for the transport failure
2664
+ * it does address. Same shape and bounds as the Swift client's
2665
+ * `rebuildConnectionForStalledSync`.
2666
+ *
2667
+ * Bounded on purpose. One document rebuilds the connection once — the
2668
+ * claim outlives the reconnect and is only dropped when the document
2669
+ * finally syncs or closes — and one connection is rebuilt once however
2670
+ * many documents are stalled on it. A document that still cannot sync on
2671
+ * the fresh socket is left to the retry chain rather than reconnected in a
2672
+ * loop.
2673
+ */
2674
+ private rebuildConnectionForStalledSync;
2675
+ /**
2676
+ * Tell the app that a document's sync is failing (#3390).
2677
+ *
2678
+ * Before this the timeout only logged, so an app had no way to know that
2679
+ * what it was rendering had stopped converging — the session just looked
2680
+ * like a hung peer. The client keeps retrying underneath; this is what lets
2681
+ * the app say so meanwhile. Emitted on every timeout: the retry cadence
2682
+ * backs off to `syncRetryMaxMs`, so it is a slow repeat of "still not
2683
+ * synced" rather than a flood. Mirrors the Swift client's `reportSyncError`.
2684
+ */
2685
+ private reportSyncError;
2686
+ /**
2687
+ * A document the app was warned about has synced. Report the recovery once
2688
+ * so the app can clear the error it surfaced (#3390). Mirrors the Swift
2689
+ * client's `reportSyncRecovered`.
2690
+ */
2691
+ private reportSyncRecovered;
2692
+ /**
2693
+ * Arm the next `syncStep1` for a document whose cycle did not complete,
2694
+ * after the current backoff — then double it, capped.
2695
+ *
2696
+ * The retry keeps the chain alive (#3390). `sendSyncStep1` returns without
2697
+ * arming anything whenever it cannot start a cycle: the transport is down,
2698
+ * a claim from an earlier cycle is still held, the document is no longer
2699
+ * open, or the send itself throws. Before this, `handleSyncWatchdogTimeout`
2700
+ * scheduled exactly one retry, so an attempt that hit any of those left the
2701
+ * document unsynced with no watchdog and no pending retry — nothing to
2702
+ * re-drive it short of a reconnect, a remote update, an explicit
2703
+ * `startNetworkSync` or a process restart. Here the retry checks whether
2704
+ * its attempt actually put a cycle in flight and re-arms itself when it did
2705
+ * not, so an open document keeps trying at the capped backoff until it
2706
+ * syncs. Mirrors the Swift client's `scheduleSyncRetry`.
2707
+ *
2708
+ * Idempotent: a retry already armed for the document wins, so the re-arm
2709
+ * and a concurrent watchdog timeout cannot stack two chains.
2710
+ */
2711
+ private scheduleSyncRetry;
2586
2712
  /** Compute a hash of the document's current state.
2587
2713
  * @param documentId - The document to hash
2588
2714
  * @group Documents */
@@ -8,6 +8,7 @@ import { DatabaseSubscriptionRegistry, } from "./internal/databaseSubscriptions"
8
8
  import { OfflineStore } from "./internal/offlineStore";
9
9
  import { createStorageProvider, } from "./internal/storage/index.js";
10
10
  import { deleteIdbByName } from "./internal/storage/idbUtils.js";
11
+ import { readWorkflowSlice, } from "./internal/workflowSlice";
11
12
  import { BlobManager } from "./internal/blobManager";
12
13
  import { BrowserConnectivityMonitor, CONNECTIVITY_LOST, CONNECTIVITY_RESTORED, USER_SET, } from "./internal/connectivityMonitor";
13
14
  import { createAnalyticsQueue, } from "./internal/analyticsQueue";
@@ -531,8 +532,44 @@ export class JsBaoClient extends Observable {
531
532
  this.syncRetryBackoffMs = new Map();
532
533
  this.syncRetryTimeoutHandles = new Map();
533
534
  this.syncWatchdogTimeoutMs = 10000;
535
+ // Not `readonly`: the #3390 tests run the same retry mechanism on a shorter
536
+ // cadence, as the Swift client's `internal var`s let its tests do.
534
537
  this.syncRetryInitialMs = 2000;
535
538
  this.syncRetryMaxMs = 15000;
539
+ /**
540
+ * Documents whose failing sync has been reported to the app as a
541
+ * `documentSyncStateChanged` "error" (#3390). Kept so the matching
542
+ * "synced" recovery is emitted only for a document the app was actually
543
+ * warned about, and once.
544
+ */
545
+ this.syncErrorReported = new Set();
546
+ /**
547
+ * The outcome of the `syncStep2` payload of a document's current cycle,
548
+ * still resolving or resolved (#3390). `syncStep2ReceivedFor` records the
549
+ * frame before its payload is fetched, so a `syncComplete` can pass the
550
+ * stale-state check while the download is in flight — or after it failed.
551
+ * The recovery report waits on this instead, so a cycle whose server state
552
+ * never reached the Y.Doc is not reported to the app as the recovery.
553
+ */
554
+ this.syncStep2PayloadFor = new Map();
555
+ /**
556
+ * Consecutive handshake-budget timeouts per document, cleared when a cycle
557
+ * completes or the document closes (#3390).
558
+ */
559
+ this.syncTimeoutStreak = new Map();
560
+ /**
561
+ * Documents whose stall has already asked for a fresh socket. It outlives
562
+ * the reconnect it triggers — only a completed sync or a close clears it —
563
+ * so one stalled document rebuilds the connection once rather than every
564
+ * few budgets (#3390).
565
+ */
566
+ this.syncStallReconnectRequested = new Set();
567
+ /**
568
+ * Whether the current connection has already been rebuilt for a stalled
569
+ * document. Reset on connect, so a burst of stalled documents costs one
570
+ * reconnect between them (#3390).
571
+ */
572
+ this.syncStallReconnectOnThisConnection = false;
536
573
  this.jsBaoGetModelDocumentId = null;
537
574
  this.jsBaoSetDefaultDocumentIdRaw = null;
538
575
  this.jsBaoClearDefaultDocumentIdRaw = null;
@@ -3016,7 +3053,7 @@ export class JsBaoClient extends Observable {
3016
3053
  return this.normalizeWorkflowStatusResponse(response);
3017
3054
  }
3018
3055
  /**
3019
- * Unpack a `{ status, run }` status response into a
3056
+ * Unpack a `{ status, run, slice? }` status response into a
3020
3057
  * {@link WorkflowStatusResult}. Shared by `getStatus` (workflowKey/runKey
3021
3058
  * route), `terminate`, and `waitFor`'s reconcile fetch (runId route).
3022
3059
  *
@@ -3029,6 +3066,7 @@ export class JsBaoClient extends Observable {
3029
3066
  normalizeWorkflowStatusResponse(response) {
3030
3067
  const serverStatus = response?.status;
3031
3068
  const run = response?.run;
3069
+ const slice = readWorkflowSlice(response?.slice);
3032
3070
  return {
3033
3071
  status: serverStatus?.status,
3034
3072
  output: serverStatus?.output,
@@ -3040,6 +3078,11 @@ export class JsBaoClient extends Observable {
3040
3078
  run?.skipReason ??
3041
3079
  undefined),
3042
3080
  run,
3081
+ // #3388 — a function run IS a run row, so the run read through these
3082
+ // surfaces is the same run `functions.getStatus` reads, and it answers
3083
+ // the same slice block. Read by the shared reader, which drops a block
3084
+ // it cannot vouch for rather than failing the status read.
3085
+ ...(slice ? { slice } : {}),
3043
3086
  };
3044
3087
  }
3045
3088
  /**
@@ -4672,10 +4715,15 @@ export class JsBaoClient extends Observable {
4672
4715
  if (this.syncRetryBackoffMs.delete(documentId)) {
4673
4716
  debugLog("[dbg][sync] watchdog-reset-backoff", { documentId, reason });
4674
4717
  }
4718
+ // The stall is over (the cycle completed, or the document closed), so
4719
+ // both stall claims start fresh for the next one (#3390).
4720
+ this.syncTimeoutStreak.delete(documentId);
4721
+ this.syncStallReconnectRequested.delete(documentId);
4675
4722
  }
4676
4723
  clearAllSyncWatchdogs(reason) {
4677
4724
  for (const documentId of this.syncWatchdogTimers.keys()) {
4678
- this.clearSyncWatchdog(documentId, reason);
4725
+ this.clearSyncWatchdogTimer(documentId);
4726
+ this.clearSyncRetryTimer(documentId);
4679
4727
  }
4680
4728
  this.syncWatchdogTimers.clear();
4681
4729
  this.syncWatchdogStartedAt.clear();
@@ -4684,6 +4732,11 @@ export class JsBaoClient extends Observable {
4684
4732
  }
4685
4733
  this.syncRetryTimeoutHandles.clear();
4686
4734
  this.syncRetryBackoffMs.clear();
4735
+ // No cycle is in flight to time out, so no document is carrying a
4736
+ // streak. `syncStallReconnectRequested` is deliberately kept: a document
4737
+ // that asked for this reconnect must not ask for another one as soon as
4738
+ // it has counted up again on the new socket (#3390).
4739
+ this.syncTimeoutStreak.clear();
4687
4740
  debugLog("[dbg][sync] watchdog-clear-all", { reason });
4688
4741
  }
4689
4742
  handleSyncWatchdogTimeout(documentId) {
@@ -4697,11 +4750,117 @@ export class JsBaoClient extends Observable {
4697
4750
  });
4698
4751
  this.docManager.completePendingSyncOperation(documentId);
4699
4752
  this._updateSynced(documentId, false);
4700
- const currentBackoff = this.syncRetryBackoffMs.get(documentId) ?? this.syncRetryInitialMs;
4701
- const nextBackoff = Math.min(currentBackoff * 2, this.syncRetryMaxMs);
4702
- this.syncRetryBackoffMs.set(documentId, nextBackoff);
4753
+ this.reportSyncError(documentId);
4754
+ // The timeout owns the next re-send, so a retry an earlier round armed
4755
+ // gives way to the one scheduled below.
4703
4756
  this.clearSyncRetryTimer(documentId);
4704
- const retryHandle = setTimeout(() => {
4757
+ this.scheduleSyncRetry(documentId);
4758
+ const streak = (this.syncTimeoutStreak.get(documentId) ?? 0) + 1;
4759
+ this.syncTimeoutStreak.set(documentId, streak);
4760
+ if (streak >= JsBaoClient.syncStallReconnectAfterTimeouts) {
4761
+ this.rebuildConnectionForStalledSync(documentId);
4762
+ }
4763
+ }
4764
+ /**
4765
+ * Rebuild the socket for a document that has missed the handshake budget
4766
+ * several times in a row while the transport still reads as open (#3390).
4767
+ *
4768
+ * Retrying is only recovery while the connection still carries answers. A
4769
+ * half-open socket reads as open from here — `isSocketOpen` is only
4770
+ * `readyState`, and this transport sends no heartbeat of its own — so every
4771
+ * `syncStep1` goes out, nothing comes back, and no amount of re-sending
4772
+ * converges on it. So the client builds a new socket itself; the
4773
+ * reconnect's document sweep re-syncs everything open.
4774
+ *
4775
+ * This is NOT what recovered the field report, and could not have been:
4776
+ * the client keeps one `connectionId` for the life of the process, and the
4777
+ * stall there was a server-side sync-in-progress marker keyed on that id,
4778
+ * which a new socket carries along — only a process restart (a new id)
4779
+ * ended it. That marker, and the broadcast mapping a reconnect lost, are
4780
+ * fixed on the server (#3390). The rebuild stays for the transport failure
4781
+ * it does address. Same shape and bounds as the Swift client's
4782
+ * `rebuildConnectionForStalledSync`.
4783
+ *
4784
+ * Bounded on purpose. One document rebuilds the connection once — the
4785
+ * claim outlives the reconnect and is only dropped when the document
4786
+ * finally syncs or closes — and one connection is rebuilt once however
4787
+ * many documents are stalled on it. A document that still cannot sync on
4788
+ * the fresh socket is left to the retry chain rather than reconnected in a
4789
+ * loop.
4790
+ */
4791
+ rebuildConnectionForStalledSync(documentId) {
4792
+ // Nothing to rebuild on a transport that is already down: the reconnect
4793
+ // path owns that case and its sweep re-syncs on the way back up.
4794
+ if (this._destroyed || !this.isWebSocketOpen())
4795
+ return;
4796
+ if (this.syncStallReconnectOnThisConnection)
4797
+ return;
4798
+ if (this.syncStallReconnectRequested.has(documentId))
4799
+ return;
4800
+ this.syncStallReconnectRequested.add(documentId);
4801
+ this.syncStallReconnectOnThisConnection = true;
4802
+ logger.warn(`Repeated handshake-budget timeouts on a live socket; rebuilding the connection for doc ${documentId}`);
4803
+ this.forceReconnect();
4804
+ }
4805
+ /**
4806
+ * Tell the app that a document's sync is failing (#3390).
4807
+ *
4808
+ * Before this the timeout only logged, so an app had no way to know that
4809
+ * what it was rendering had stopped converging — the session just looked
4810
+ * like a hung peer. The client keeps retrying underneath; this is what lets
4811
+ * the app say so meanwhile. Emitted on every timeout: the retry cadence
4812
+ * backs off to `syncRetryMaxMs`, so it is a slow repeat of "still not
4813
+ * synced" rather than a flood. Mirrors the Swift client's `reportSyncError`.
4814
+ */
4815
+ reportSyncError(documentId) {
4816
+ this.syncErrorReported.add(documentId);
4817
+ try {
4818
+ this.emit("documentSyncStateChanged", [{ documentId, state: "error" }]);
4819
+ }
4820
+ catch (err) {
4821
+ logger.debug("[dbg][sync] sync error emit failed", { documentId, error: err });
4822
+ }
4823
+ }
4824
+ /**
4825
+ * A document the app was warned about has synced. Report the recovery once
4826
+ * so the app can clear the error it surfaced (#3390). Mirrors the Swift
4827
+ * client's `reportSyncRecovered`.
4828
+ */
4829
+ reportSyncRecovered(documentId) {
4830
+ if (!this.syncErrorReported.delete(documentId))
4831
+ return;
4832
+ try {
4833
+ this.emit("documentSyncStateChanged", [{ documentId, state: "synced" }]);
4834
+ }
4835
+ catch (err) {
4836
+ logger.debug("[dbg][sync] sync recovery emit failed", { documentId, error: err });
4837
+ }
4838
+ }
4839
+ /**
4840
+ * Arm the next `syncStep1` for a document whose cycle did not complete,
4841
+ * after the current backoff — then double it, capped.
4842
+ *
4843
+ * The retry keeps the chain alive (#3390). `sendSyncStep1` returns without
4844
+ * arming anything whenever it cannot start a cycle: the transport is down,
4845
+ * a claim from an earlier cycle is still held, the document is no longer
4846
+ * open, or the send itself throws. Before this, `handleSyncWatchdogTimeout`
4847
+ * scheduled exactly one retry, so an attempt that hit any of those left the
4848
+ * document unsynced with no watchdog and no pending retry — nothing to
4849
+ * re-drive it short of a reconnect, a remote update, an explicit
4850
+ * `startNetworkSync` or a process restart. Here the retry checks whether
4851
+ * its attempt actually put a cycle in flight and re-arms itself when it did
4852
+ * not, so an open document keeps trying at the capped backoff until it
4853
+ * syncs. Mirrors the Swift client's `scheduleSyncRetry`.
4854
+ *
4855
+ * Idempotent: a retry already armed for the document wins, so the re-arm
4856
+ * and a concurrent watchdog timeout cannot stack two chains.
4857
+ */
4858
+ scheduleSyncRetry(documentId) {
4859
+ if (this._destroyed || this.syncRetryTimeoutHandles.has(documentId))
4860
+ return;
4861
+ const delayMs = this.syncRetryBackoffMs.get(documentId) ?? this.syncRetryInitialMs;
4862
+ this.syncRetryBackoffMs.set(documentId, Math.min(delayMs * 2, this.syncRetryMaxMs));
4863
+ const retryHandle = setTimeout(async () => {
4705
4864
  this.syncRetryTimeoutHandles.delete(documentId);
4706
4865
  if (!this.docManager.hasOpenDoc(documentId)) {
4707
4866
  this.syncRetryBackoffMs.delete(documentId);
@@ -4711,12 +4870,25 @@ export class JsBaoClient extends Observable {
4711
4870
  });
4712
4871
  return;
4713
4872
  }
4714
- debugLog("[dbg][sync] watchdog-retry", {
4715
- documentId,
4716
- delayMs: currentBackoff,
4717
- });
4718
- void this.sendSyncStep1(documentId);
4719
- }, currentBackoff);
4873
+ debugLog("[dbg][sync] watchdog-retry", { documentId, delayMs });
4874
+ try {
4875
+ await this.sendSyncStep1(documentId);
4876
+ }
4877
+ catch (err) {
4878
+ logger.warn(`sendSyncStep1 retry failed for doc ${documentId}`, err);
4879
+ }
4880
+ // A watchdog armed for this document is the evidence that a cycle did
4881
+ // go out; its own timeout then owns the next round. Without one, and
4882
+ // with the document still open and unsynced, this attempt produced
4883
+ // nothing and the chain has to re-arm.
4884
+ if (this.syncWatchdogTimers.has(documentId) ||
4885
+ !this.docManager.hasOpenDoc(documentId) ||
4886
+ this.docManager.isSynced(documentId)) {
4887
+ return;
4888
+ }
4889
+ debugLog("[dbg][sync] watchdog-retry-rearm", { documentId });
4890
+ this.scheduleSyncRetry(documentId);
4891
+ }, delayMs);
4720
4892
  this.syncRetryTimeoutHandles.set(documentId, retryHandle);
4721
4893
  }
4722
4894
  /** Compute a hash of the document's current state.
@@ -4863,15 +5035,31 @@ export class JsBaoClient extends Observable {
4863
5035
  }
4864
5036
  this.scheduleSyncWatchdog(documentId);
4865
5037
  this.syncStep2ReceivedFor.delete(documentId);
5038
+ this.syncStep2PayloadFor.delete(documentId);
4866
5039
  this.docManager.clearSyncTimings(documentId);
4867
5040
  this.docManager.setSyncTiming(documentId, "syncStep1SentAt", Date.now());
4868
- this.ws.send(JSON.stringify(syncMessage));
5041
+ try {
5042
+ this.ws.send(JSON.stringify(syncMessage));
5043
+ }
5044
+ catch (err) {
5045
+ logger.warn(`Failed to send syncStep1 for doc ${documentId}`, err);
5046
+ // No frame went out, so no syncComplete will release the claim.
5047
+ this.docManager.completePendingSyncOperation(documentId);
5048
+ // Drop the timer this cycle armed — there is nothing for it to time
5049
+ // out against — but keep the backoff and re-drive the document
5050
+ // (#3390). A send that threw used to reject straight through to a
5051
+ // `void` caller, leaving the document open, unsynced and with
5052
+ // nothing scheduled to try again.
5053
+ this.clearSyncWatchdogTimer(documentId);
5054
+ this.scheduleSyncRetry(documentId);
5055
+ }
4869
5056
  }
4870
5057
  else {
4871
5058
  logger.warn(`sendSyncStep1: Skipped send; socket not OPEN for doc ${documentId}`);
4872
5059
  this._logWsState(`sendSyncStep1: skipped for ${documentId}`);
4873
- // Ensure we do not leave a stale pending flag
4874
- this.clearSyncWatchdog(documentId, "sendStep1-not-open");
5060
+ // Ensure we do not leave a stale pending flag. The backoff and any
5061
+ // pending retry stay: this attempt started no cycle, and the chain
5062
+ // re-drives the document at the capped cadence (#3390).
4875
5063
  this.docManager.completePendingSyncOperation(documentId);
4876
5064
  debugLog("[dbg][sync] skip-sendStep1 not-open", {
4877
5065
  documentId,
@@ -5390,6 +5578,9 @@ export class JsBaoClient extends Observable {
5390
5578
  // socket's token was accepted in the handshake, so whatever the previous
5391
5579
  // one spent says nothing about this one.
5392
5580
  this.wsAuthChallengeRefreshes = 0;
5581
+ // A fresh socket gets a fresh claim: whatever a stalled document spent
5582
+ // on the previous connection says nothing about this one (#3390).
5583
+ this.syncStallReconnectOnThisConnection = false;
5393
5584
  void this.flushAllLocalUpdates("ws-open");
5394
5585
  logger.debug("[WS Auth][debug] handleWebSocketOpen", {
5395
5586
  wsconnected: this.wsconnected,
@@ -6862,7 +7053,16 @@ export class JsBaoClient extends Observable {
6862
7053
  }
6863
7054
  else if (data.type === "syncStep2" || data.type === "update") {
6864
7055
  logger.log(`📥 Handling ${data.type} message`);
6865
- const updateOutcome = await this.handleUpdate(data);
7056
+ const payload = this.handleUpdate(data);
7057
+ if (data.type === "syncStep2" && data.documentId) {
7058
+ // Frames are not processed one at a time, so the cycle's
7059
+ // `syncComplete` can be handled while this payload is still being
7060
+ // fetched. Recorded so the recovery report can wait for what the
7061
+ // payload actually did (#3390); a fetch that throws is a payload
7062
+ // that did not apply, and the throw still reaches the catch below.
7063
+ this.syncStep2PayloadFor.set(data.documentId, payload.then((outcome) => outcome, () => "error"));
7064
+ }
7065
+ const updateOutcome = await payload;
6866
7066
  if (data.type === "update" &&
6867
7067
  data.documentId &&
6868
7068
  updateOutcome !== "skipped") {
@@ -6917,10 +7117,16 @@ export class JsBaoClient extends Observable {
6917
7117
  catch { }
6918
7118
  const receivedSyncStep2 = this.syncStep2ReceivedFor.has(data.documentId);
6919
7119
  this.syncStep2ReceivedFor.delete(data.documentId);
7120
+ // Not consumed here: the server closes one cycle with more than one
7121
+ // syncComplete (another follows the client's answer to the server's
7122
+ // own syncStep1), and every one of them belongs to the cycle whose
7123
+ // payload this is. `sendSyncStep1` drops it when the next goes out.
7124
+ const syncStep2Payload = this.syncStep2PayloadFor.get(data.documentId);
6920
7125
  logger.log("[load-debug] syncComplete received", {
6921
7126
  documentId: data.documentId,
6922
7127
  ydocSharedTypes: syncCompleteDebugInfo,
6923
7128
  });
7129
+ let staleResetScheduled = false;
6924
7130
  if (!receivedSyncStep2) {
6925
7131
  logger.warn(`[sync] syncComplete WITHOUT syncStep2 doc=${data.documentId} ydocTypes=[${syncCompleteDebugInfo.join(",")}] ydocEmpty=${syncCompleteDebugInfo.length === 0}`);
6926
7132
  // If this doc was flagged as suspected stale AND the server sent no data,
@@ -6928,6 +7134,7 @@ export class JsBaoClient extends Observable {
6928
7134
  // the client is in sync. Reset the persistence and re-sync fresh.
6929
7135
  if (this.suspectedStaleIdbDocs.has(data.documentId)) {
6930
7136
  this.suspectedStaleIdbDocs.delete(data.documentId);
7137
+ staleResetScheduled = true;
6931
7138
  void this.resetYjsPersistenceAndResync(data.documentId);
6932
7139
  }
6933
7140
  }
@@ -6941,6 +7148,27 @@ export class JsBaoClient extends Observable {
6941
7148
  resolvedSynced,
6942
7149
  });
6943
7150
  this._updateSynced(data.documentId, resolvedSynced);
7151
+ // The document converged after the app was told its sync was
7152
+ // failing, so the app hears that it can drop the warning (#3390).
7153
+ // Not when this completion is the stale-state case: the document
7154
+ // is about to be reset and re-synced and still has no server
7155
+ // state, so the error it was warned about stands until the re-sync
7156
+ // completes for real. Nor when the cycle's syncStep2 carried an
7157
+ // `updateUrl` whose download failed: the frame counted as received
7158
+ // above, but no server state reached the document, so the app
7159
+ // would be told to clear its warning over a document still stale.
7160
+ // The error stands and a later cycle that applies reports it.
7161
+ if (resolvedSynced && !staleResetScheduled) {
7162
+ const payloadOutcome = syncStep2Payload
7163
+ ? await syncStep2Payload
7164
+ : "skipped";
7165
+ if (payloadOutcome === "error") {
7166
+ logger.warn(`[sync] syncComplete after a syncStep2 payload that did not apply; not reporting recovery doc=${data.documentId}`);
7167
+ }
7168
+ else {
7169
+ this.reportSyncRecovered(data.documentId);
7170
+ }
7171
+ }
6944
7172
  }
6945
7173
  }
6946
7174
  else if (data.type === "availability") {
@@ -8056,6 +8284,9 @@ export class JsBaoClient extends Observable {
8056
8284
  });
8057
8285
  this.clearLocalUpdateState(documentId);
8058
8286
  this.clearSyncWatchdog(documentId, "closeDocument");
8287
+ // A closed document has no sync error left to clear (#3390).
8288
+ this.syncErrorReported.delete(documentId);
8289
+ this.syncStep2PayloadFor.delete(documentId);
8059
8290
  // Free discovered-schema cache so the map doesn't grow unbounded across
8060
8291
  // document sessions.
8061
8292
  this.discoveredSchemas.delete(documentId);
@@ -9644,8 +9875,12 @@ export class JsBaoClient extends Observable {
9644
9875
  // Clear remaining user-scoped in-memory state
9645
9876
  this.lastEmittedPermission.clear();
9646
9877
  this.syncStep2ReceivedFor.clear();
9878
+ this.syncStep2PayloadFor.clear();
9647
9879
  this.suspectedStaleIdbDocs.clear();
9648
9880
  this.yjsIdbResetInProgress.clear();
9881
+ this.syncErrorReported.clear();
9882
+ this.syncTimeoutStreak.clear();
9883
+ this.syncStallReconnectRequested.clear();
9649
9884
  this.workflowApplyHandlers.clear();
9650
9885
  // Clear service worker bridge state
9651
9886
  this.serviceWorkerBridgePendingMessages.clear();
@@ -10381,6 +10616,11 @@ export class JsBaoClient extends Observable {
10381
10616
  return this.discoveredSchemas.get(documentId) ?? null;
10382
10617
  }
10383
10618
  }
10619
+ /**
10620
+ * Consecutive handshake-budget timeouts on a live socket before the client
10621
+ * stops trusting the connection and rebuilds it (#3390).
10622
+ */
10623
+ JsBaoClient.syncStallReconnectAfterTimeouts = 3;
10384
10624
  /**
10385
10625
  * Documents per page for the whole-scope walk. The server caps `limit` at
10386
10626
  * 100 (`parseListLimitParam`), so asking for more buys nothing.
@@ -1,4 +1,5 @@
1
1
  import { JsBaoError } from "../errors";
2
+ import { readWorkflowSlice } from "../internal/workflowSlice";
2
3
  /**
3
4
  * Server functions, available as `client.functions`.
4
5
  *
@@ -78,9 +79,19 @@ export class FunctionsAPI {
78
79
  // The route nests the reconciled status under `status` and the row under
79
80
  // `run`; `workflows.getStatus` flattens the same shape, so this one does
80
81
  // too rather than exposing a second envelope for the same fact.
82
+ //
83
+ // The flatten stays local — spreading `status` passes a server-added key
84
+ // through, where the workflow normalizer names the fields it reads — but
85
+ // the SLICE is read by the one shared reader both surfaces use (#3388),
86
+ // so which method a caller read the run through cannot decide what the
87
+ // block means.
88
+ const slice = readWorkflowSlice(response?.slice);
81
89
  return {
82
90
  ...(response?.status ?? {}),
83
91
  run: response?.run,
92
+ // #3381 — the slice block is a sibling of `status` and `run`, additive,
93
+ // present only for a durable function run with a record.
94
+ ...(slice ? { slice } : {}),
84
95
  };
85
96
  }
86
97
  /**
@@ -2426,6 +2426,101 @@
2426
2426
  });
2427
2427
  }
2428
2428
 
2429
+ /**
2430
+ * The task run's `slice` block, as the client is willing to publish it.
2431
+ *
2432
+ * `GET /workflows/runs/:runId/status` carries `slice` beside `status` and
2433
+ * `run` for a task run that has a record (#3381). The block is
2434
+ * OBSERVABILITY, never the answer, and that decides both halves of how it is
2435
+ * read here (#3388):
2436
+ *
2437
+ * - A block whose shape surprises the client is DROPPED, and the status read
2438
+ * still succeeds. Publishing a partial record would be worse than
2439
+ * publishing none: a `refreshCount` the client filled in with `0` because
2440
+ * the payload said `"three"` reads to the caller as "this run never
2441
+ * refreshed", which is telemetry the client does not have. It is the same
2442
+ * reading the server takes — `readSliceBlock` in `workflows-controller.ts`
2443
+ * returns null for a record it cannot read rather than 500ing the route —
2444
+ * and the same one the Swift client's `WorkflowSliceInfo` takes.
2445
+ * - The tolerance that is actually wanted stays: an absent or `null`
2446
+ * `refreshCount` reads `0`, and `null` timestamps read `null` without
2447
+ * costing the caller the record.
2448
+ *
2449
+ * Shared by `functions.getStatus` and by `normalizeWorkflowStatusResponse`,
2450
+ * so which method a caller read the run through cannot decide whether they
2451
+ * can see its slice.
2452
+ */
2453
+ /** A value the client cannot read, as distinct from one that is absent. */
2454
+ const UNREADABLE = Symbol("unreadable slice value");
2455
+ /**
2456
+ * One epoch-millisecond field. Absent and `null` alike read `null`; a
2457
+ * non-integral number is rounded rather than refused; anything that is not a
2458
+ * number is UNREADABLE, and drops the block it belongs to.
2459
+ */
2460
+ function epochMillis(raw) {
2461
+ if (raw === undefined || raw === null)
2462
+ return null;
2463
+ if (typeof raw !== "number")
2464
+ return UNREADABLE;
2465
+ // No timestamp is worth failing a status read for: a value JSON should not
2466
+ // have been able to carry reads as "not recorded".
2467
+ if (!Number.isFinite(raw))
2468
+ return null;
2469
+ return Math.round(raw);
2470
+ }
2471
+ /**
2472
+ * Read the response's `slice` key into a record, or `undefined` when there is
2473
+ * none to publish — an absent key (a request invocation, a DSL run) and a
2474
+ * block the client cannot read are both "no record".
2475
+ */
2476
+ function readWorkflowSlice(raw) {
2477
+ if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
2478
+ return undefined;
2479
+ }
2480
+ const block = raw;
2481
+ // `sliceId` is what makes this a record: a block without one is not a slice.
2482
+ if (typeof block.sliceId !== "string")
2483
+ return undefined;
2484
+ const startedAt = epochMillis(block.startedAt);
2485
+ const ceilingAt = epochMillis(block.ceilingAt);
2486
+ const settledAt = epochMillis(block.settledAt);
2487
+ const lastRefreshAt = epochMillis(block.lastRefreshAt);
2488
+ if (startedAt === UNREADABLE ||
2489
+ ceilingAt === UNREADABLE ||
2490
+ settledAt === UNREADABLE ||
2491
+ lastRefreshAt === UNREADABLE) {
2492
+ return undefined;
2493
+ }
2494
+ // A plain string for the same reason every other status on this surface is
2495
+ // one: a server-added spelling must never cost the caller the record.
2496
+ const rawStatus = block.settledStatus;
2497
+ if (rawStatus !== undefined &&
2498
+ rawStatus !== null &&
2499
+ typeof rawStatus !== "string") {
2500
+ return undefined;
2501
+ }
2502
+ // Absent and `null` alike read 0 — the server always sends the count, so
2503
+ // this is tolerance for a payload that predates it. A value that is not a
2504
+ // whole number drops the block rather than being invented as `0`.
2505
+ const rawCount = block.refreshCount;
2506
+ let refreshCount = 0;
2507
+ if (rawCount !== undefined && rawCount !== null) {
2508
+ if (typeof rawCount !== "number" || !Number.isInteger(rawCount)) {
2509
+ return undefined;
2510
+ }
2511
+ refreshCount = rawCount;
2512
+ }
2513
+ return {
2514
+ sliceId: block.sliceId,
2515
+ startedAt,
2516
+ ceilingAt,
2517
+ settledAt,
2518
+ settledStatus: rawStatus ?? null,
2519
+ lastRefreshAt,
2520
+ refreshCount,
2521
+ };
2522
+ }
2523
+
2429
2524
  /**
2430
2525
  * Normalizes a list response envelope to the unified pagination contract.
2431
2526
  *
@@ -15032,9 +15127,19 @@
15032
15127
  // The route nests the reconciled status under `status` and the row under
15033
15128
  // `run`; `workflows.getStatus` flattens the same shape, so this one does
15034
15129
  // too rather than exposing a second envelope for the same fact.
15130
+ //
15131
+ // The flatten stays local — spreading `status` passes a server-added key
15132
+ // through, where the workflow normalizer names the fields it reads — but
15133
+ // the SLICE is read by the one shared reader both surfaces use (#3388),
15134
+ // so which method a caller read the run through cannot decide what the
15135
+ // block means.
15136
+ const slice = readWorkflowSlice(response?.slice);
15035
15137
  return {
15036
15138
  ...(response?.status ?? {}),
15037
15139
  run: response?.run,
15140
+ // #3381 — the slice block is a sibling of `status` and `run`, additive,
15141
+ // present only for a durable function run with a record.
15142
+ ...(slice ? { slice } : {}),
15038
15143
  };
15039
15144
  }
15040
15145
  /**
@@ -16379,8 +16484,44 @@
16379
16484
  this.syncRetryBackoffMs = new Map();
16380
16485
  this.syncRetryTimeoutHandles = new Map();
16381
16486
  this.syncWatchdogTimeoutMs = 10000;
16487
+ // Not `readonly`: the #3390 tests run the same retry mechanism on a shorter
16488
+ // cadence, as the Swift client's `internal var`s let its tests do.
16382
16489
  this.syncRetryInitialMs = 2000;
16383
16490
  this.syncRetryMaxMs = 15000;
16491
+ /**
16492
+ * Documents whose failing sync has been reported to the app as a
16493
+ * `documentSyncStateChanged` "error" (#3390). Kept so the matching
16494
+ * "synced" recovery is emitted only for a document the app was actually
16495
+ * warned about, and once.
16496
+ */
16497
+ this.syncErrorReported = new Set();
16498
+ /**
16499
+ * The outcome of the `syncStep2` payload of a document's current cycle,
16500
+ * still resolving or resolved (#3390). `syncStep2ReceivedFor` records the
16501
+ * frame before its payload is fetched, so a `syncComplete` can pass the
16502
+ * stale-state check while the download is in flight — or after it failed.
16503
+ * The recovery report waits on this instead, so a cycle whose server state
16504
+ * never reached the Y.Doc is not reported to the app as the recovery.
16505
+ */
16506
+ this.syncStep2PayloadFor = new Map();
16507
+ /**
16508
+ * Consecutive handshake-budget timeouts per document, cleared when a cycle
16509
+ * completes or the document closes (#3390).
16510
+ */
16511
+ this.syncTimeoutStreak = new Map();
16512
+ /**
16513
+ * Documents whose stall has already asked for a fresh socket. It outlives
16514
+ * the reconnect it triggers — only a completed sync or a close clears it —
16515
+ * so one stalled document rebuilds the connection once rather than every
16516
+ * few budgets (#3390).
16517
+ */
16518
+ this.syncStallReconnectRequested = new Set();
16519
+ /**
16520
+ * Whether the current connection has already been rebuilt for a stalled
16521
+ * document. Reset on connect, so a burst of stalled documents costs one
16522
+ * reconnect between them (#3390).
16523
+ */
16524
+ this.syncStallReconnectOnThisConnection = false;
16384
16525
  this.jsBaoGetModelDocumentId = null;
16385
16526
  this.jsBaoSetDefaultDocumentIdRaw = null;
16386
16527
  this.jsBaoClearDefaultDocumentIdRaw = null;
@@ -18864,7 +19005,7 @@
18864
19005
  return this.normalizeWorkflowStatusResponse(response);
18865
19006
  }
18866
19007
  /**
18867
- * Unpack a `{ status, run }` status response into a
19008
+ * Unpack a `{ status, run, slice? }` status response into a
18868
19009
  * {@link WorkflowStatusResult}. Shared by `getStatus` (workflowKey/runKey
18869
19010
  * route), `terminate`, and `waitFor`'s reconcile fetch (runId route).
18870
19011
  *
@@ -18877,6 +19018,7 @@
18877
19018
  normalizeWorkflowStatusResponse(response) {
18878
19019
  const serverStatus = response?.status;
18879
19020
  const run = response?.run;
19021
+ const slice = readWorkflowSlice(response?.slice);
18880
19022
  return {
18881
19023
  status: serverStatus?.status,
18882
19024
  output: serverStatus?.output,
@@ -18888,6 +19030,11 @@
18888
19030
  run?.skipReason ??
18889
19031
  undefined),
18890
19032
  run,
19033
+ // #3388 — a function run IS a run row, so the run read through these
19034
+ // surfaces is the same run `functions.getStatus` reads, and it answers
19035
+ // the same slice block. Read by the shared reader, which drops a block
19036
+ // it cannot vouch for rather than failing the status read.
19037
+ ...(slice ? { slice } : {}),
18891
19038
  };
18892
19039
  }
18893
19040
  /**
@@ -20520,10 +20667,15 @@
20520
20667
  if (this.syncRetryBackoffMs.delete(documentId)) {
20521
20668
  debugLog("[dbg][sync] watchdog-reset-backoff", { documentId, reason });
20522
20669
  }
20670
+ // The stall is over (the cycle completed, or the document closed), so
20671
+ // both stall claims start fresh for the next one (#3390).
20672
+ this.syncTimeoutStreak.delete(documentId);
20673
+ this.syncStallReconnectRequested.delete(documentId);
20523
20674
  }
20524
20675
  clearAllSyncWatchdogs(reason) {
20525
20676
  for (const documentId of this.syncWatchdogTimers.keys()) {
20526
- this.clearSyncWatchdog(documentId, reason);
20677
+ this.clearSyncWatchdogTimer(documentId);
20678
+ this.clearSyncRetryTimer(documentId);
20527
20679
  }
20528
20680
  this.syncWatchdogTimers.clear();
20529
20681
  this.syncWatchdogStartedAt.clear();
@@ -20532,6 +20684,11 @@
20532
20684
  }
20533
20685
  this.syncRetryTimeoutHandles.clear();
20534
20686
  this.syncRetryBackoffMs.clear();
20687
+ // No cycle is in flight to time out, so no document is carrying a
20688
+ // streak. `syncStallReconnectRequested` is deliberately kept: a document
20689
+ // that asked for this reconnect must not ask for another one as soon as
20690
+ // it has counted up again on the new socket (#3390).
20691
+ this.syncTimeoutStreak.clear();
20535
20692
  debugLog("[dbg][sync] watchdog-clear-all", { reason });
20536
20693
  }
20537
20694
  handleSyncWatchdogTimeout(documentId) {
@@ -20545,11 +20702,117 @@
20545
20702
  });
20546
20703
  this.docManager.completePendingSyncOperation(documentId);
20547
20704
  this._updateSynced(documentId, false);
20548
- const currentBackoff = this.syncRetryBackoffMs.get(documentId) ?? this.syncRetryInitialMs;
20549
- const nextBackoff = Math.min(currentBackoff * 2, this.syncRetryMaxMs);
20550
- this.syncRetryBackoffMs.set(documentId, nextBackoff);
20705
+ this.reportSyncError(documentId);
20706
+ // The timeout owns the next re-send, so a retry an earlier round armed
20707
+ // gives way to the one scheduled below.
20551
20708
  this.clearSyncRetryTimer(documentId);
20552
- const retryHandle = setTimeout(() => {
20709
+ this.scheduleSyncRetry(documentId);
20710
+ const streak = (this.syncTimeoutStreak.get(documentId) ?? 0) + 1;
20711
+ this.syncTimeoutStreak.set(documentId, streak);
20712
+ if (streak >= JsBaoClient.syncStallReconnectAfterTimeouts) {
20713
+ this.rebuildConnectionForStalledSync(documentId);
20714
+ }
20715
+ }
20716
+ /**
20717
+ * Rebuild the socket for a document that has missed the handshake budget
20718
+ * several times in a row while the transport still reads as open (#3390).
20719
+ *
20720
+ * Retrying is only recovery while the connection still carries answers. A
20721
+ * half-open socket reads as open from here — `isSocketOpen` is only
20722
+ * `readyState`, and this transport sends no heartbeat of its own — so every
20723
+ * `syncStep1` goes out, nothing comes back, and no amount of re-sending
20724
+ * converges on it. So the client builds a new socket itself; the
20725
+ * reconnect's document sweep re-syncs everything open.
20726
+ *
20727
+ * This is NOT what recovered the field report, and could not have been:
20728
+ * the client keeps one `connectionId` for the life of the process, and the
20729
+ * stall there was a server-side sync-in-progress marker keyed on that id,
20730
+ * which a new socket carries along — only a process restart (a new id)
20731
+ * ended it. That marker, and the broadcast mapping a reconnect lost, are
20732
+ * fixed on the server (#3390). The rebuild stays for the transport failure
20733
+ * it does address. Same shape and bounds as the Swift client's
20734
+ * `rebuildConnectionForStalledSync`.
20735
+ *
20736
+ * Bounded on purpose. One document rebuilds the connection once — the
20737
+ * claim outlives the reconnect and is only dropped when the document
20738
+ * finally syncs or closes — and one connection is rebuilt once however
20739
+ * many documents are stalled on it. A document that still cannot sync on
20740
+ * the fresh socket is left to the retry chain rather than reconnected in a
20741
+ * loop.
20742
+ */
20743
+ rebuildConnectionForStalledSync(documentId) {
20744
+ // Nothing to rebuild on a transport that is already down: the reconnect
20745
+ // path owns that case and its sweep re-syncs on the way back up.
20746
+ if (this._destroyed || !this.isWebSocketOpen())
20747
+ return;
20748
+ if (this.syncStallReconnectOnThisConnection)
20749
+ return;
20750
+ if (this.syncStallReconnectRequested.has(documentId))
20751
+ return;
20752
+ this.syncStallReconnectRequested.add(documentId);
20753
+ this.syncStallReconnectOnThisConnection = true;
20754
+ logger.warn(`Repeated handshake-budget timeouts on a live socket; rebuilding the connection for doc ${documentId}`);
20755
+ this.forceReconnect();
20756
+ }
20757
+ /**
20758
+ * Tell the app that a document's sync is failing (#3390).
20759
+ *
20760
+ * Before this the timeout only logged, so an app had no way to know that
20761
+ * what it was rendering had stopped converging — the session just looked
20762
+ * like a hung peer. The client keeps retrying underneath; this is what lets
20763
+ * the app say so meanwhile. Emitted on every timeout: the retry cadence
20764
+ * backs off to `syncRetryMaxMs`, so it is a slow repeat of "still not
20765
+ * synced" rather than a flood. Mirrors the Swift client's `reportSyncError`.
20766
+ */
20767
+ reportSyncError(documentId) {
20768
+ this.syncErrorReported.add(documentId);
20769
+ try {
20770
+ this.emit("documentSyncStateChanged", [{ documentId, state: "error" }]);
20771
+ }
20772
+ catch (err) {
20773
+ logger.debug("[dbg][sync] sync error emit failed", { documentId, error: err });
20774
+ }
20775
+ }
20776
+ /**
20777
+ * A document the app was warned about has synced. Report the recovery once
20778
+ * so the app can clear the error it surfaced (#3390). Mirrors the Swift
20779
+ * client's `reportSyncRecovered`.
20780
+ */
20781
+ reportSyncRecovered(documentId) {
20782
+ if (!this.syncErrorReported.delete(documentId))
20783
+ return;
20784
+ try {
20785
+ this.emit("documentSyncStateChanged", [{ documentId, state: "synced" }]);
20786
+ }
20787
+ catch (err) {
20788
+ logger.debug("[dbg][sync] sync recovery emit failed", { documentId, error: err });
20789
+ }
20790
+ }
20791
+ /**
20792
+ * Arm the next `syncStep1` for a document whose cycle did not complete,
20793
+ * after the current backoff — then double it, capped.
20794
+ *
20795
+ * The retry keeps the chain alive (#3390). `sendSyncStep1` returns without
20796
+ * arming anything whenever it cannot start a cycle: the transport is down,
20797
+ * a claim from an earlier cycle is still held, the document is no longer
20798
+ * open, or the send itself throws. Before this, `handleSyncWatchdogTimeout`
20799
+ * scheduled exactly one retry, so an attempt that hit any of those left the
20800
+ * document unsynced with no watchdog and no pending retry — nothing to
20801
+ * re-drive it short of a reconnect, a remote update, an explicit
20802
+ * `startNetworkSync` or a process restart. Here the retry checks whether
20803
+ * its attempt actually put a cycle in flight and re-arms itself when it did
20804
+ * not, so an open document keeps trying at the capped backoff until it
20805
+ * syncs. Mirrors the Swift client's `scheduleSyncRetry`.
20806
+ *
20807
+ * Idempotent: a retry already armed for the document wins, so the re-arm
20808
+ * and a concurrent watchdog timeout cannot stack two chains.
20809
+ */
20810
+ scheduleSyncRetry(documentId) {
20811
+ if (this._destroyed || this.syncRetryTimeoutHandles.has(documentId))
20812
+ return;
20813
+ const delayMs = this.syncRetryBackoffMs.get(documentId) ?? this.syncRetryInitialMs;
20814
+ this.syncRetryBackoffMs.set(documentId, Math.min(delayMs * 2, this.syncRetryMaxMs));
20815
+ const retryHandle = setTimeout(async () => {
20553
20816
  this.syncRetryTimeoutHandles.delete(documentId);
20554
20817
  if (!this.docManager.hasOpenDoc(documentId)) {
20555
20818
  this.syncRetryBackoffMs.delete(documentId);
@@ -20559,12 +20822,25 @@
20559
20822
  });
20560
20823
  return;
20561
20824
  }
20562
- debugLog("[dbg][sync] watchdog-retry", {
20563
- documentId,
20564
- delayMs: currentBackoff,
20565
- });
20566
- void this.sendSyncStep1(documentId);
20567
- }, currentBackoff);
20825
+ debugLog("[dbg][sync] watchdog-retry", { documentId, delayMs });
20826
+ try {
20827
+ await this.sendSyncStep1(documentId);
20828
+ }
20829
+ catch (err) {
20830
+ logger.warn(`sendSyncStep1 retry failed for doc ${documentId}`, err);
20831
+ }
20832
+ // A watchdog armed for this document is the evidence that a cycle did
20833
+ // go out; its own timeout then owns the next round. Without one, and
20834
+ // with the document still open and unsynced, this attempt produced
20835
+ // nothing and the chain has to re-arm.
20836
+ if (this.syncWatchdogTimers.has(documentId) ||
20837
+ !this.docManager.hasOpenDoc(documentId) ||
20838
+ this.docManager.isSynced(documentId)) {
20839
+ return;
20840
+ }
20841
+ debugLog("[dbg][sync] watchdog-retry-rearm", { documentId });
20842
+ this.scheduleSyncRetry(documentId);
20843
+ }, delayMs);
20568
20844
  this.syncRetryTimeoutHandles.set(documentId, retryHandle);
20569
20845
  }
20570
20846
  /** Compute a hash of the document's current state.
@@ -20711,15 +20987,31 @@
20711
20987
  }
20712
20988
  this.scheduleSyncWatchdog(documentId);
20713
20989
  this.syncStep2ReceivedFor.delete(documentId);
20990
+ this.syncStep2PayloadFor.delete(documentId);
20714
20991
  this.docManager.clearSyncTimings(documentId);
20715
20992
  this.docManager.setSyncTiming(documentId, "syncStep1SentAt", Date.now());
20716
- this.ws.send(JSON.stringify(syncMessage));
20993
+ try {
20994
+ this.ws.send(JSON.stringify(syncMessage));
20995
+ }
20996
+ catch (err) {
20997
+ logger.warn(`Failed to send syncStep1 for doc ${documentId}`, err);
20998
+ // No frame went out, so no syncComplete will release the claim.
20999
+ this.docManager.completePendingSyncOperation(documentId);
21000
+ // Drop the timer this cycle armed — there is nothing for it to time
21001
+ // out against — but keep the backoff and re-drive the document
21002
+ // (#3390). A send that threw used to reject straight through to a
21003
+ // `void` caller, leaving the document open, unsynced and with
21004
+ // nothing scheduled to try again.
21005
+ this.clearSyncWatchdogTimer(documentId);
21006
+ this.scheduleSyncRetry(documentId);
21007
+ }
20717
21008
  }
20718
21009
  else {
20719
21010
  logger.warn(`sendSyncStep1: Skipped send; socket not OPEN for doc ${documentId}`);
20720
21011
  this._logWsState(`sendSyncStep1: skipped for ${documentId}`);
20721
- // Ensure we do not leave a stale pending flag
20722
- this.clearSyncWatchdog(documentId, "sendStep1-not-open");
21012
+ // Ensure we do not leave a stale pending flag. The backoff and any
21013
+ // pending retry stay: this attempt started no cycle, and the chain
21014
+ // re-drives the document at the capped cadence (#3390).
20723
21015
  this.docManager.completePendingSyncOperation(documentId);
20724
21016
  debugLog("[dbg][sync] skip-sendStep1 not-open", {
20725
21017
  documentId,
@@ -21238,6 +21530,9 @@
21238
21530
  // socket's token was accepted in the handshake, so whatever the previous
21239
21531
  // one spent says nothing about this one.
21240
21532
  this.wsAuthChallengeRefreshes = 0;
21533
+ // A fresh socket gets a fresh claim: whatever a stalled document spent
21534
+ // on the previous connection says nothing about this one (#3390).
21535
+ this.syncStallReconnectOnThisConnection = false;
21241
21536
  void this.flushAllLocalUpdates("ws-open");
21242
21537
  logger.debug("[WS Auth][debug] handleWebSocketOpen", {
21243
21538
  wsconnected: this.wsconnected,
@@ -22710,7 +23005,16 @@
22710
23005
  }
22711
23006
  else if (data.type === "syncStep2" || data.type === "update") {
22712
23007
  logger.log(`📥 Handling ${data.type} message`);
22713
- const updateOutcome = await this.handleUpdate(data);
23008
+ const payload = this.handleUpdate(data);
23009
+ if (data.type === "syncStep2" && data.documentId) {
23010
+ // Frames are not processed one at a time, so the cycle's
23011
+ // `syncComplete` can be handled while this payload is still being
23012
+ // fetched. Recorded so the recovery report can wait for what the
23013
+ // payload actually did (#3390); a fetch that throws is a payload
23014
+ // that did not apply, and the throw still reaches the catch below.
23015
+ this.syncStep2PayloadFor.set(data.documentId, payload.then((outcome) => outcome, () => "error"));
23016
+ }
23017
+ const updateOutcome = await payload;
22714
23018
  if (data.type === "update" &&
22715
23019
  data.documentId &&
22716
23020
  updateOutcome !== "skipped") {
@@ -22765,10 +23069,16 @@
22765
23069
  catch { }
22766
23070
  const receivedSyncStep2 = this.syncStep2ReceivedFor.has(data.documentId);
22767
23071
  this.syncStep2ReceivedFor.delete(data.documentId);
23072
+ // Not consumed here: the server closes one cycle with more than one
23073
+ // syncComplete (another follows the client's answer to the server's
23074
+ // own syncStep1), and every one of them belongs to the cycle whose
23075
+ // payload this is. `sendSyncStep1` drops it when the next goes out.
23076
+ const syncStep2Payload = this.syncStep2PayloadFor.get(data.documentId);
22768
23077
  logger.log("[load-debug] syncComplete received", {
22769
23078
  documentId: data.documentId,
22770
23079
  ydocSharedTypes: syncCompleteDebugInfo,
22771
23080
  });
23081
+ let staleResetScheduled = false;
22772
23082
  if (!receivedSyncStep2) {
22773
23083
  logger.warn(`[sync] syncComplete WITHOUT syncStep2 doc=${data.documentId} ydocTypes=[${syncCompleteDebugInfo.join(",")}] ydocEmpty=${syncCompleteDebugInfo.length === 0}`);
22774
23084
  // If this doc was flagged as suspected stale AND the server sent no data,
@@ -22776,6 +23086,7 @@
22776
23086
  // the client is in sync. Reset the persistence and re-sync fresh.
22777
23087
  if (this.suspectedStaleIdbDocs.has(data.documentId)) {
22778
23088
  this.suspectedStaleIdbDocs.delete(data.documentId);
23089
+ staleResetScheduled = true;
22779
23090
  void this.resetYjsPersistenceAndResync(data.documentId);
22780
23091
  }
22781
23092
  }
@@ -22789,6 +23100,27 @@
22789
23100
  resolvedSynced,
22790
23101
  });
22791
23102
  this._updateSynced(data.documentId, resolvedSynced);
23103
+ // The document converged after the app was told its sync was
23104
+ // failing, so the app hears that it can drop the warning (#3390).
23105
+ // Not when this completion is the stale-state case: the document
23106
+ // is about to be reset and re-synced and still has no server
23107
+ // state, so the error it was warned about stands until the re-sync
23108
+ // completes for real. Nor when the cycle's syncStep2 carried an
23109
+ // `updateUrl` whose download failed: the frame counted as received
23110
+ // above, but no server state reached the document, so the app
23111
+ // would be told to clear its warning over a document still stale.
23112
+ // The error stands and a later cycle that applies reports it.
23113
+ if (resolvedSynced && !staleResetScheduled) {
23114
+ const payloadOutcome = syncStep2Payload
23115
+ ? await syncStep2Payload
23116
+ : "skipped";
23117
+ if (payloadOutcome === "error") {
23118
+ logger.warn(`[sync] syncComplete after a syncStep2 payload that did not apply; not reporting recovery doc=${data.documentId}`);
23119
+ }
23120
+ else {
23121
+ this.reportSyncRecovered(data.documentId);
23122
+ }
23123
+ }
22792
23124
  }
22793
23125
  }
22794
23126
  else if (data.type === "availability") {
@@ -23901,6 +24233,9 @@
23901
24233
  });
23902
24234
  this.clearLocalUpdateState(documentId);
23903
24235
  this.clearSyncWatchdog(documentId, "closeDocument");
24236
+ // A closed document has no sync error left to clear (#3390).
24237
+ this.syncErrorReported.delete(documentId);
24238
+ this.syncStep2PayloadFor.delete(documentId);
23904
24239
  // Free discovered-schema cache so the map doesn't grow unbounded across
23905
24240
  // document sessions.
23906
24241
  this.discoveredSchemas.delete(documentId);
@@ -25489,8 +25824,12 @@
25489
25824
  // Clear remaining user-scoped in-memory state
25490
25825
  this.lastEmittedPermission.clear();
25491
25826
  this.syncStep2ReceivedFor.clear();
25827
+ this.syncStep2PayloadFor.clear();
25492
25828
  this.suspectedStaleIdbDocs.clear();
25493
25829
  this.yjsIdbResetInProgress.clear();
25830
+ this.syncErrorReported.clear();
25831
+ this.syncTimeoutStreak.clear();
25832
+ this.syncStallReconnectRequested.clear();
25494
25833
  this.workflowApplyHandlers.clear();
25495
25834
  // Clear service worker bridge state
25496
25835
  this.serviceWorkerBridgePendingMessages.clear();
@@ -26226,6 +26565,11 @@
26226
26565
  return this.discoveredSchemas.get(documentId) ?? null;
26227
26566
  }
26228
26567
  }
26568
+ /**
26569
+ * Consecutive handshake-budget timeouts on a live socket before the client
26570
+ * stops trusting the connection and rebuilds it (#3390).
26571
+ */
26572
+ JsBaoClient.syncStallReconnectAfterTimeouts = 3;
26229
26573
  /**
26230
26574
  * Documents per page for the whole-scope walk. The server caps `limit` at
26231
26575
  * 100 (`parseListLimitParam`), so asking for more buys nothing.
@@ -0,0 +1,70 @@
1
+ /**
2
+ * The task run's `slice` block, as the client is willing to publish it.
3
+ *
4
+ * `GET /workflows/runs/:runId/status` carries `slice` beside `status` and
5
+ * `run` for a task run that has a record (#3381). The block is
6
+ * OBSERVABILITY, never the answer, and that decides both halves of how it is
7
+ * read here (#3388):
8
+ *
9
+ * - A block whose shape surprises the client is DROPPED, and the status read
10
+ * still succeeds. Publishing a partial record would be worse than
11
+ * publishing none: a `refreshCount` the client filled in with `0` because
12
+ * the payload said `"three"` reads to the caller as "this run never
13
+ * refreshed", which is telemetry the client does not have. It is the same
14
+ * reading the server takes — `readSliceBlock` in `workflows-controller.ts`
15
+ * returns null for a record it cannot read rather than 500ing the route —
16
+ * and the same one the Swift client's `WorkflowSliceInfo` takes.
17
+ * - The tolerance that is actually wanted stays: an absent or `null`
18
+ * `refreshCount` reads `0`, and `null` timestamps read `null` without
19
+ * costing the caller the record.
20
+ *
21
+ * Shared by `functions.getStatus` and by `normalizeWorkflowStatusResponse`,
22
+ * so which method a caller read the run through cannot decide whether they
23
+ * can see its slice.
24
+ */
25
+ /**
26
+ * A slice record the client can vouch for. Field for field with the Swift
27
+ * client's `WorkflowSliceInfo`.
28
+ *
29
+ * Timestamps are epoch milliseconds (what the route sends), not the ISO-8601
30
+ * strings a run record carries — the slice record is the platform's own
31
+ * bookkeeping, and the client passes its numbers through verbatim.
32
+ */
33
+ export interface WorkflowStatusSlice {
34
+ /** Id of the slice record. The one field the server always fills. */
35
+ sliceId: string;
36
+ /** When this slice started, epoch ms. */
37
+ startedAt: number | null;
38
+ /**
39
+ * The 12-hour bound THIS slice ends at, epoch ms — `startedAt` plus the
40
+ * slice maximum, not a deadline for the run, which can continue in a later
41
+ * slice with a ceiling of its own.
42
+ */
43
+ ceilingAt: number | null;
44
+ /**
45
+ * When THIS slice settled, epoch ms; `null` while the slice is open. A
46
+ * settled slice is not a finished run: a slice that yielded settles here
47
+ * and the run continues in the next slice, so a caller watching a run
48
+ * still reads `status`, never this field.
49
+ */
50
+ settledAt: number | null;
51
+ /**
52
+ * How THIS slice settled (`"completed"`, `"failed"`, `"cpu-yield"`, …);
53
+ * `null` while the slice is open. `"cpu-yield"` is the slice that gave up
54
+ * its CPU budget for the run to carry on in another one.
55
+ */
56
+ settledStatus: string | null;
57
+ /** When the credential was last refreshed, epoch ms. */
58
+ lastRefreshAt: number | null;
59
+ /**
60
+ * How many times the credential was refreshed through the gateway instead
61
+ * of yielding. `0` for a run short enough never to need one.
62
+ */
63
+ refreshCount: number;
64
+ }
65
+ /**
66
+ * Read the response's `slice` key into a record, or `undefined` when there is
67
+ * none to publish — an absent key (a request invocation, a DSL run) and a
68
+ * block the client cannot read are both "no record".
69
+ */
70
+ export declare function readWorkflowSlice(raw: unknown): WorkflowStatusSlice | undefined;
@@ -0,0 +1,94 @@
1
+ /**
2
+ * The task run's `slice` block, as the client is willing to publish it.
3
+ *
4
+ * `GET /workflows/runs/:runId/status` carries `slice` beside `status` and
5
+ * `run` for a task run that has a record (#3381). The block is
6
+ * OBSERVABILITY, never the answer, and that decides both halves of how it is
7
+ * read here (#3388):
8
+ *
9
+ * - A block whose shape surprises the client is DROPPED, and the status read
10
+ * still succeeds. Publishing a partial record would be worse than
11
+ * publishing none: a `refreshCount` the client filled in with `0` because
12
+ * the payload said `"three"` reads to the caller as "this run never
13
+ * refreshed", which is telemetry the client does not have. It is the same
14
+ * reading the server takes — `readSliceBlock` in `workflows-controller.ts`
15
+ * returns null for a record it cannot read rather than 500ing the route —
16
+ * and the same one the Swift client's `WorkflowSliceInfo` takes.
17
+ * - The tolerance that is actually wanted stays: an absent or `null`
18
+ * `refreshCount` reads `0`, and `null` timestamps read `null` without
19
+ * costing the caller the record.
20
+ *
21
+ * Shared by `functions.getStatus` and by `normalizeWorkflowStatusResponse`,
22
+ * so which method a caller read the run through cannot decide whether they
23
+ * can see its slice.
24
+ */
25
+ /** A value the client cannot read, as distinct from one that is absent. */
26
+ const UNREADABLE = Symbol("unreadable slice value");
27
+ /**
28
+ * One epoch-millisecond field. Absent and `null` alike read `null`; a
29
+ * non-integral number is rounded rather than refused; anything that is not a
30
+ * number is UNREADABLE, and drops the block it belongs to.
31
+ */
32
+ function epochMillis(raw) {
33
+ if (raw === undefined || raw === null)
34
+ return null;
35
+ if (typeof raw !== "number")
36
+ return UNREADABLE;
37
+ // No timestamp is worth failing a status read for: a value JSON should not
38
+ // have been able to carry reads as "not recorded".
39
+ if (!Number.isFinite(raw))
40
+ return null;
41
+ return Math.round(raw);
42
+ }
43
+ /**
44
+ * Read the response's `slice` key into a record, or `undefined` when there is
45
+ * none to publish — an absent key (a request invocation, a DSL run) and a
46
+ * block the client cannot read are both "no record".
47
+ */
48
+ export function readWorkflowSlice(raw) {
49
+ if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
50
+ return undefined;
51
+ }
52
+ const block = raw;
53
+ // `sliceId` is what makes this a record: a block without one is not a slice.
54
+ if (typeof block.sliceId !== "string")
55
+ return undefined;
56
+ const startedAt = epochMillis(block.startedAt);
57
+ const ceilingAt = epochMillis(block.ceilingAt);
58
+ const settledAt = epochMillis(block.settledAt);
59
+ const lastRefreshAt = epochMillis(block.lastRefreshAt);
60
+ if (startedAt === UNREADABLE ||
61
+ ceilingAt === UNREADABLE ||
62
+ settledAt === UNREADABLE ||
63
+ lastRefreshAt === UNREADABLE) {
64
+ return undefined;
65
+ }
66
+ // A plain string for the same reason every other status on this surface is
67
+ // one: a server-added spelling must never cost the caller the record.
68
+ const rawStatus = block.settledStatus;
69
+ if (rawStatus !== undefined &&
70
+ rawStatus !== null &&
71
+ typeof rawStatus !== "string") {
72
+ return undefined;
73
+ }
74
+ // Absent and `null` alike read 0 — the server always sends the count, so
75
+ // this is tolerance for a payload that predates it. A value that is not a
76
+ // whole number drops the block rather than being invented as `0`.
77
+ const rawCount = block.refreshCount;
78
+ let refreshCount = 0;
79
+ if (rawCount !== undefined && rawCount !== null) {
80
+ if (typeof rawCount !== "number" || !Number.isInteger(rawCount)) {
81
+ return undefined;
82
+ }
83
+ refreshCount = rawCount;
84
+ }
85
+ return {
86
+ sliceId: block.sliceId,
87
+ startedAt,
88
+ ceilingAt,
89
+ settledAt,
90
+ settledStatus: rawStatus ?? null,
91
+ lastRefreshAt,
92
+ refreshCount,
93
+ };
94
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "js-bao-wss-client",
3
- "version": "2.2.0-alpha.7",
3
+ "version": "2.2.0-alpha.9",
4
4
  "description": "Client library for js-bao-wss Yjs WebSocket service",
5
5
  "author": "Primitive LLC",
6
6
  "license": "UNLICENSED",
@@ -36,7 +36,7 @@
36
36
  "peerDependencies": {
37
37
  "lib0": "^0.2.0",
38
38
  "yjs": "^13.6.0",
39
- "js-bao": "0.7.0-alpha.2",
39
+ "js-bao": "0.7.0-alpha.4",
40
40
  "better-sqlite3": "^11.0.0",
41
41
  "y-sqlite3": "^0.1.0",
42
42
  "react": ">=17.0.0",