@brass-build/client 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/AGENTS.md +72 -63
  2. package/CHANGELOG.md +97 -0
  3. package/README.md +325 -235
  4. package/dist/auth-flow.d.ts +10 -1
  5. package/dist/auth-flow.d.ts.map +1 -1
  6. package/dist/auth-flow.js +64 -20
  7. package/dist/auth-flow.js.map +1 -1
  8. package/dist/client.d.ts +4 -0
  9. package/dist/client.d.ts.map +1 -1
  10. package/dist/client.js +245 -76
  11. package/dist/client.js.map +1 -1
  12. package/dist/document-checkpoints.d.ts +1 -0
  13. package/dist/document-checkpoints.d.ts.map +1 -1
  14. package/dist/document-checkpoints.js +27 -0
  15. package/dist/document-checkpoints.js.map +1 -1
  16. package/dist/document-engine.d.ts +4 -0
  17. package/dist/document-engine.d.ts.map +1 -0
  18. package/dist/document-engine.js +9 -0
  19. package/dist/document-engine.js.map +1 -0
  20. package/dist/document-signals.d.ts +31 -1
  21. package/dist/document-signals.d.ts.map +1 -1
  22. package/dist/document-signals.js +59 -0
  23. package/dist/document-signals.js.map +1 -1
  24. package/dist/document-streams.d.ts +8 -1
  25. package/dist/document-streams.d.ts.map +1 -1
  26. package/dist/document-streams.js +79 -8
  27. package/dist/document-streams.js.map +1 -1
  28. package/dist/document-thumbnail-capture.d.ts +20 -0
  29. package/dist/document-thumbnail-capture.d.ts.map +1 -1
  30. package/dist/document-thumbnail-capture.js +247 -9
  31. package/dist/document-thumbnail-capture.js.map +1 -1
  32. package/dist/documents-namespace.d.ts.map +1 -1
  33. package/dist/documents-namespace.js +22 -3
  34. package/dist/documents-namespace.js.map +1 -1
  35. package/dist/documents.d.ts +49 -16
  36. package/dist/documents.d.ts.map +1 -1
  37. package/dist/documents.js +981 -237
  38. package/dist/documents.js.map +1 -1
  39. package/dist/embed-protocol.d.ts +3 -0
  40. package/dist/embed-protocol.d.ts.map +1 -0
  41. package/dist/embed-protocol.js +26 -0
  42. package/dist/embed-protocol.js.map +1 -0
  43. package/dist/emitter.d.ts.map +1 -1
  44. package/dist/emitter.js +8 -1
  45. package/dist/emitter.js.map +1 -1
  46. package/dist/export-guest.d.ts.map +1 -1
  47. package/dist/export-guest.js +14 -6
  48. package/dist/export-guest.js.map +1 -1
  49. package/dist/fetch-deadline.d.ts +22 -0
  50. package/dist/fetch-deadline.d.ts.map +1 -0
  51. package/dist/fetch-deadline.js +96 -0
  52. package/dist/fetch-deadline.js.map +1 -0
  53. package/dist/fetch-retry.d.ts +1 -0
  54. package/dist/fetch-retry.d.ts.map +1 -1
  55. package/dist/fetch-retry.js +12 -5
  56. package/dist/fetch-retry.js.map +1 -1
  57. package/dist/handle-types.d.ts +150 -0
  58. package/dist/handle-types.d.ts.map +1 -0
  59. package/dist/handle-types.js +8 -0
  60. package/dist/handle-types.js.map +1 -0
  61. package/dist/handles.d.ts +48 -48
  62. package/dist/handles.d.ts.map +1 -1
  63. package/dist/handles.js +3122 -501
  64. package/dist/handles.js.map +1 -1
  65. package/dist/hosted-runtime.d.ts.map +1 -1
  66. package/dist/hosted-runtime.js +8 -12
  67. package/dist/hosted-runtime.js.map +1 -1
  68. package/dist/hosted-ui.d.ts +0 -1
  69. package/dist/hosted-ui.d.ts.map +1 -1
  70. package/dist/hosted-ui.js +29 -24
  71. package/dist/hosted-ui.js.map +1 -1
  72. package/dist/importer.d.ts.map +1 -1
  73. package/dist/importer.js +204 -8
  74. package/dist/importer.js.map +1 -1
  75. package/dist/index.d.ts +10 -10
  76. package/dist/index.d.ts.map +1 -1
  77. package/dist/index.js +11 -2
  78. package/dist/index.js.map +1 -1
  79. package/dist/internal.d.ts +2 -1
  80. package/dist/internal.d.ts.map +1 -1
  81. package/dist/internal.js +1 -0
  82. package/dist/internal.js.map +1 -1
  83. package/dist/join-coverage.d.ts +12 -0
  84. package/dist/join-coverage.d.ts.map +1 -0
  85. package/dist/join-coverage.js +37 -0
  86. package/dist/join-coverage.js.map +1 -0
  87. package/dist/jwt.d.ts.map +1 -1
  88. package/dist/jwt.js +5 -1
  89. package/dist/jwt.js.map +1 -1
  90. package/dist/launch-context.js +2 -2
  91. package/dist/launch-context.js.map +1 -1
  92. package/dist/mutate-batching.d.ts +25 -2
  93. package/dist/mutate-batching.d.ts.map +1 -1
  94. package/dist/mutate-batching.js +57 -10
  95. package/dist/mutate-batching.js.map +1 -1
  96. package/dist/open-document.d.ts +30 -1
  97. package/dist/open-document.d.ts.map +1 -1
  98. package/dist/open-document.js +1905 -322
  99. package/dist/open-document.js.map +1 -1
  100. package/dist/presence.d.ts +28 -0
  101. package/dist/presence.d.ts.map +1 -1
  102. package/dist/presence.js +149 -7
  103. package/dist/presence.js.map +1 -1
  104. package/dist/record-paging.d.ts +335 -71
  105. package/dist/record-paging.d.ts.map +1 -1
  106. package/dist/record-paging.js +590 -127
  107. package/dist/record-paging.js.map +1 -1
  108. package/dist/selection-persistence.d.ts.map +1 -1
  109. package/dist/selection-persistence.js +11 -1
  110. package/dist/selection-persistence.js.map +1 -1
  111. package/dist/serializer.d.ts +30 -1
  112. package/dist/serializer.d.ts.map +1 -1
  113. package/dist/session-persistence.d.ts.map +1 -1
  114. package/dist/session-persistence.js +9 -8
  115. package/dist/session-persistence.js.map +1 -1
  116. package/dist/session-toolbar.d.ts +0 -1
  117. package/dist/session-toolbar.d.ts.map +1 -1
  118. package/dist/session-toolbar.js +356 -83
  119. package/dist/session-toolbar.js.map +1 -1
  120. package/dist/session.d.ts.map +1 -1
  121. package/dist/session.js +87 -18
  122. package/dist/session.js.map +1 -1
  123. package/dist/tail-ops.d.ts +14 -4
  124. package/dist/tail-ops.d.ts.map +1 -1
  125. package/dist/tail-ops.js +75 -15
  126. package/dist/tail-ops.js.map +1 -1
  127. package/dist/topic-lifecycle.d.ts +4 -0
  128. package/dist/topic-lifecycle.d.ts.map +1 -1
  129. package/dist/topic-lifecycle.js +25 -1
  130. package/dist/topic-lifecycle.js.map +1 -1
  131. package/dist/types.d.ts +47 -1
  132. package/dist/types.d.ts.map +1 -1
  133. package/dist/undo.d.ts +2 -3
  134. package/dist/undo.d.ts.map +1 -1
  135. package/dist/undo.js +11 -0
  136. package/dist/undo.js.map +1 -1
  137. package/dist/unload-guard.d.ts +5 -0
  138. package/dist/unload-guard.d.ts.map +1 -0
  139. package/dist/unload-guard.js +88 -0
  140. package/dist/unload-guard.js.map +1 -0
  141. package/dist/version-bar.d.ts +4 -3
  142. package/dist/version-bar.d.ts.map +1 -1
  143. package/dist/version-bar.js +162 -20
  144. package/dist/version-bar.js.map +1 -1
  145. package/dist/version.d.ts +2 -2
  146. package/dist/version.js +1 -1
  147. package/dist/vite.d.ts +18 -2
  148. package/dist/vite.d.ts.map +1 -1
  149. package/dist/vite.js +133 -8
  150. package/dist/vite.js.map +1 -1
  151. package/dist/when-saved.d.ts +2 -0
  152. package/dist/when-saved.d.ts.map +1 -0
  153. package/dist/when-saved.js +70 -0
  154. package/dist/when-saved.js.map +1 -0
  155. package/dist/ws-client.d.ts +27 -3
  156. package/dist/ws-client.d.ts.map +1 -1
  157. package/dist/ws-client.js +408 -89
  158. package/dist/ws-client.js.map +1 -1
  159. package/dist/ws-frame-limit.d.ts.map +1 -1
  160. package/dist/ws-frame-limit.js +23 -17
  161. package/dist/ws-frame-limit.js.map +1 -1
  162. package/docs/app-manifest.md +7 -8
  163. package/docs/custom-chrome.md +21 -22
  164. package/docs/hosted-ui-advanced.md +14 -17
  165. package/docs/imported-documents.md +53 -197
  166. package/docs/presence.md +45 -21
  167. package/docs/schemas.md +12 -17
  168. package/docs/streams.md +55 -57
  169. package/package.json +4 -3
  170. package/src/auth-flow.ts +90 -23
  171. package/src/client.ts +262 -80
  172. package/src/document-checkpoints.ts +30 -0
  173. package/src/document-engine.ts +15 -0
  174. package/src/document-signals.ts +120 -8
  175. package/src/document-streams.ts +95 -9
  176. package/src/document-thumbnail-capture.ts +253 -9
  177. package/src/documents-namespace.ts +38 -13
  178. package/src/documents.ts +1103 -290
  179. package/src/embed-protocol.ts +25 -0
  180. package/src/emitter.ts +8 -1
  181. package/src/export-guest.ts +15 -9
  182. package/src/fetch-deadline.ts +119 -0
  183. package/src/fetch-retry.ts +14 -5
  184. package/src/handle-types.ts +303 -0
  185. package/src/handles.ts +3407 -683
  186. package/src/hosted-runtime.ts +8 -12
  187. package/src/hosted-ui.ts +28 -32
  188. package/src/importer.ts +216 -8
  189. package/src/index.ts +29 -18
  190. package/src/internal.ts +7 -0
  191. package/src/join-coverage.ts +42 -0
  192. package/src/jwt.ts +8 -1
  193. package/src/launch-context.ts +2 -2
  194. package/src/mutate-batching.ts +60 -11
  195. package/src/open-document.ts +2441 -502
  196. package/src/presence.ts +210 -11
  197. package/src/record-paging.ts +686 -137
  198. package/src/selection-persistence.ts +10 -1
  199. package/src/serializer.ts +33 -1
  200. package/src/session-persistence.ts +9 -8
  201. package/src/session-toolbar.ts +387 -89
  202. package/src/session.ts +91 -17
  203. package/src/tail-ops.ts +76 -10
  204. package/src/topic-lifecycle.ts +31 -1
  205. package/src/types.ts +56 -4
  206. package/src/undo.ts +13 -5
  207. package/src/unload-guard.ts +91 -0
  208. package/src/version-bar.ts +183 -26
  209. package/src/version.ts +1 -1
  210. package/src/vite.ts +175 -10
  211. package/src/when-saved.ts +66 -0
  212. package/src/ws-client.ts +461 -92
  213. package/src/ws-frame-limit.ts +23 -17
@@ -8,21 +8,90 @@ import { createPresenceController } from './presence.js';
8
8
  import { applyAppAssertionHeader, applyClientVersionHeader } from './api-headers.js';
9
9
  import { brassErrorFromBody } from './api-fetch.js';
10
10
  import { fetchWithRetry, RETRY_IDEMPOTENT_WRITE } from './fetch-retry.js';
11
+ import { deadlineForUpload, fetchWithDeadline, OP_WRITE_TIMEOUT_MS, } from './fetch-deadline.js';
11
12
  import { BrassError } from './errors.js';
12
13
  import { perfStart } from './perf.js';
13
- import { resolveTailOps } from './tail-ops.js';
14
- import { opBlobExceedsDurableLimit, opFrameExceedsWsLimit, opBlobNeedsStaging, } from './ws-frame-limit.js';
14
+ import { joinCoverageFloor, snapshotSeqOf } from './join-coverage.js';
15
+ import { declaredTailBytes, resolveTailOps } from './tail-ops.js';
16
+ import { opBlobExceedsDurableLimit, opBlobNeedsStaging, opFrameExceedsWsLimit, } from './ws-frame-limit.js';
17
+ // The terminal frame the platform sends when it clears a document's work
18
+ // status: the work finished and the content is in. The SDK emits the same
19
+ // shape wherever it settles that transition off its own import (a
20
+ // subscribe-time reconcile, a retried pull), so the app cannot tell the two
21
+ // apart. `blocking: false` is what releases the load lock.
22
+ const WORK_READY_STATUS = {
23
+ message: 'Ready',
24
+ pending: false,
25
+ blocking: false,
26
+ };
27
+ const CONTENT_PULL_ATTEMPTS = 5;
28
+ const CONTENT_PULL_BACKOFF_MS = 1000;
29
+ // Pull a document's content in after the platform reports its inbound work
30
+ // finished, retrying a transient fetch failure (a network blip, a dropped
31
+ // presigned download): the platform asserts the content exists, so a silent
32
+ // give-up would leave the doc empty behind a terminal status, reading as "the
33
+ // model never loaded" and invisible in every log. Emits `finalStatus` only
34
+ // once the import lands, so an app re-reading on `onStatus` sees the content.
35
+ // On exhaustion it reports the failure and emits a terminal failure status,
36
+ // which keeps the doc read-only. Returns whether the content is in.
37
+ async function pullContentWithRetry(args) {
38
+ let lastError = null;
39
+ for (let attempt = 1; attempt <= CONTENT_PULL_ATTEMPTS; attempt += 1) {
40
+ try {
41
+ const outcome = await args.fetchAndImport();
42
+ if (outcome.ok) {
43
+ args.doc._emitStatus(args.finalStatus, { contentLoaded: true });
44
+ return true;
45
+ }
46
+ lastError = new Error(outcome.reason);
47
+ }
48
+ catch (err) {
49
+ lastError = err instanceof Error ? err : new Error(String(err));
50
+ }
51
+ // A retrying attempt is exceptional, so this warn is quiet in normal
52
+ // operation and decisive in a bug report; without it a failure here is
53
+ // undiagnosable from any log.
54
+ console.warn(`[Brass] document content reload attempt ${attempt}/${CONTENT_PULL_ATTEMPTS} failed (doc ${args.docId}): ${lastError.message}`);
55
+ if (attempt < CONTENT_PULL_ATTEMPTS) {
56
+ await new Promise((r) => setTimeout(r, CONTENT_PULL_BACKOFF_MS * attempt));
57
+ }
58
+ }
59
+ const error = new BrassError(`Document content reload failed after ${CONTENT_PULL_ATTEMPTS} attempts: ${lastError?.message ?? 'unknown error'}`, { status: 0 });
60
+ if (args.reportError)
61
+ args.reportError(error);
62
+ else
63
+ console.error('[Brass]', error);
64
+ args.doc._emitStatus({
65
+ message: 'The model could not be loaded. Reload to try again.',
66
+ pending: false,
67
+ // The content never arrived, so this terminal status still locks the
68
+ // doc: an edit here would write into a document holding nothing.
69
+ blocking: true,
70
+ });
71
+ return false;
72
+ }
15
73
  // Both join fetches and the `doc_init` frame funnel through this, so a
16
74
  // tail op whose bytes spilled to storage is fetched once, here. Every
17
75
  // reader downstream then sees the inline shape it already handles, and
18
76
  // none of the seq bookkeeping has to learn a second representation.
19
- async function resolveParsedTailOps(parsed) {
77
+ async function resolveParsedTailOps(parsed, onProgress) {
20
78
  if (typeof parsed !== 'object' || parsed === null)
21
79
  return;
22
80
  const body = parsed;
23
81
  if (!Array.isArray(body.tail_ops))
24
82
  return;
25
- body.tail_ops = await resolveTailOps(body.tail_ops);
83
+ const tailOps = body.tail_ops;
84
+ // The whole download, stated before any of it starts. A presigned snapshot
85
+ // names its decompressed size and each spilled op names its blob's, and both
86
+ // are fetched over the network with a reader waiting, so the load reports one
87
+ // figure against their sum. An inline snapshot contributes nothing: its bytes
88
+ // came with this response.
89
+ const snapshot = body.snapshot;
90
+ const snapshotBytes = typeof snapshot?.raw_bytes === 'number' ? snapshot.raw_bytes : 0;
91
+ const totalBytes = declaredTailBytes(tailOps) + snapshotBytes;
92
+ body.tail_ops = await resolveTailOps(tailOps, fetch, onProgress === undefined
93
+ ? undefined
94
+ : (loadedBytes) => onProgress(loadedBytes, totalBytes > 0 ? totalBytes : null, 'tail'));
26
95
  }
27
96
  // Map the snake_case wire `drive` block to the public camelCase
28
97
  // `DocumentDriveStatus`. `canPush` is derived once here (unbroken, two-way,
@@ -42,7 +111,9 @@ function toDriveStatus(wire) {
42
111
  // update) must not be treated as unbroken and offered a push the server
43
112
  // would reject. Require an explicit `false` to enable, and an explicit
44
113
  // `true` to flag broken. Behaviour is unchanged for today's frames, which
45
- // always carry the boolean.
114
+ // always carry the boolean. Whether the file is written server-side or
115
+ // from the app's own serializer is a REQUEST SHAPE the push path resolves
116
+ // for itself, so it says nothing about whether this document can publish.
46
117
  canPush: wire.broken === false &&
47
118
  mirrorDirection === 'two_way' &&
48
119
  (state === 'ahead' || state === 'diverged'),
@@ -56,6 +127,29 @@ function toDriveStatus(wire) {
56
127
  itemName: wire.item_name,
57
128
  itemUrl: wire.item_url,
58
129
  publishing: wire.publishing === true,
130
+ // Spread rather than assigned, because `exactOptionalPropertyTypes`
131
+ // separates an absent optional field from one holding `undefined`.
132
+ ...(wire.source_unreadable !== undefined
133
+ ? { sourceUnreadable: wire.source_unreadable }
134
+ : {}),
135
+ };
136
+ }
137
+ // Map the wire target to the public shape, dropping one that does not carry
138
+ // what a publish needs. A malformed target is skipped rather than failing
139
+ // the open: the document is readable whatever its publish offer says.
140
+ function toPublishTarget(wire) {
141
+ if (typeof wire !== 'object' || wire === null)
142
+ return null;
143
+ const driveName = wire.drive_name;
144
+ if (typeof driveName !== 'string' || driveName === '')
145
+ return null;
146
+ return {
147
+ driveName,
148
+ driveType: typeof wire.drive_type === 'string' ? wire.drive_type : '',
149
+ containerName: typeof wire.container_name === 'string' ? wire.container_name : '',
150
+ detail: typeof wire.detail === 'string' ? wire.detail : '',
151
+ publishing: wire.publishing === true,
152
+ publishFailed: wire.publish_failed === true,
59
153
  };
60
154
  }
61
155
  function isDocOpFrame(frame) {
@@ -67,37 +161,63 @@ function isDocOpFrame(frame) {
67
161
  typeof f.op_blob === 'string' &&
68
162
  typeof f.seq === 'number');
69
163
  }
70
- function isDocInitFrame(frame) {
164
+ // The frame in the shape its readers act on, or null when it carries nothing
165
+ // they can act on.
166
+ //
167
+ // Gated on the discriminant plus `current_seq`, with the tail degraded entry
168
+ // by entry. Most of what a `doc_init` does is carried by the frame itself
169
+ // rather than by any one entry: `current_seq` is the cursor, the tail's lower
170
+ // edge decides whether the reader re-imports the join, and the frame is the
171
+ // (re)subscribe signal that re-asserts every outstanding op and runs the
172
+ // durability handshake. So an entry whose shape this SDK does not recognise
173
+ // costs the one op it carries, which stays uncovered and is pulled over the
174
+ // HTTP join; refusing the frame over it would cost all of the above.
175
+ //
176
+ // `snapshot_version_vector` is likewise tolerated as absent when it is not a
177
+ // string: the handshake needs a vector it can decode, and skipping the
178
+ // handshake is a smaller loss than dropping the frame that drives it.
179
+ function parseDocInitFrame(frame) {
71
180
  if (typeof frame !== 'object' || frame === null)
72
- return false;
181
+ return null;
73
182
  const f = frame;
74
183
  if (f.type !== 'doc_init' || typeof f.topic !== 'string')
75
- return false;
184
+ return null;
76
185
  if (typeof f.current_seq !== 'number')
77
- return false;
78
- // `snapshot_version_vector` is an optional durability-handshake input, read
79
- // defensively at its use site. A present-but-non-string value (a future
80
- // server retyping it) is tolerated as absent rather than rejecting the whole
81
- // frame, which would silently break the reconnect / durability path.
82
- if (!Array.isArray(f.tail_ops))
83
- return false;
84
- for (const entry of f.tail_ops) {
85
- if (typeof entry !== 'object' || entry === null)
86
- return false;
87
- const e = entry;
88
- if (typeof e.seq !== 'number')
89
- return false;
90
- // A tail op is normally inline here: the server withholds a tail
91
- // carrying a signed URL from this frame, because fetching one needs
92
- // an await this reader does not have, and sends `current_seq` alone
93
- // so the client re-imports the join over HTTP instead. Tolerating
94
- // the shape anyway keeps such a frame's `current_seq` usable rather
95
- // than dropping the whole frame and with it the resync trigger.
96
- if (typeof e.op_blob_b64 !== 'string' && typeof e.op_blob_url !== 'string') {
97
- return false;
186
+ return null;
187
+ const tailOps = [];
188
+ if (Array.isArray(f.tail_ops)) {
189
+ for (const entry of f.tail_ops) {
190
+ if (typeof entry !== 'object' || entry === null)
191
+ continue;
192
+ const e = entry;
193
+ // The seq is what places an entry in the sequence the cursors and the
194
+ // resync trigger are computed from, so an entry without one is not an
195
+ // entry any reader here can use.
196
+ if (typeof e.seq !== 'number')
197
+ continue;
198
+ tailOps.push(typeof e.op_blob_b64 === 'string'
199
+ ? { seq: e.seq, op_blob_b64: e.op_blob_b64 }
200
+ : { seq: e.seq });
98
201
  }
99
202
  }
100
- return true;
203
+ return {
204
+ type: 'doc_init',
205
+ topic: f.topic,
206
+ current_seq: f.current_seq,
207
+ tail_ops: tailOps,
208
+ ...(typeof f.snapshot_seq === 'number' ? { snapshot_seq: f.snapshot_seq } : {}),
209
+ ...(typeof f.snapshot_version_vector === 'string'
210
+ ? { snapshot_version_vector: f.snapshot_version_vector }
211
+ : {}),
212
+ };
213
+ }
214
+ function isDocCatchUpFrame(frame) {
215
+ if (typeof frame !== 'object' || frame === null)
216
+ return false;
217
+ const f = frame;
218
+ return (f.type === 'doc_catch_up' &&
219
+ typeof f.topic === 'string' &&
220
+ typeof f.current_seq === 'number');
101
221
  }
102
222
  function isOpAckFrame(frame) {
103
223
  if (typeof frame !== 'object' || frame === null)
@@ -116,6 +236,59 @@ function isOpRejectedFrame(frame) {
116
236
  typeof f.topic === 'string' &&
117
237
  typeof f.client_op_id === 'string');
118
238
  }
239
+ // Whether re-sending the same op is the right move. Two statuses refuse to
240
+ // write rather than answer about the op, and both leave the row untouched,
241
+ // so the op is still the client's to send: 503 (the store refused on
242
+ // capacity) and 429 (the caller is over the op-write cap, whose window
243
+ // resets). The WS transport reads its own rate refusal the same way, as an
244
+ // `unavailable` the shed recovery re-asserts, and the two transports carry
245
+ // the same ops, so a refusal cannot mean "later" on one and "never" on the
246
+ // other. Every other status is an answer about the op itself and repeats
247
+ // however many times it is sent.
248
+ function opWriteIsRetryable(status) {
249
+ return status === 503 || status === 429;
250
+ }
251
+ // The HTTP transport's rendering of the same refusal `op_rejected` carries on
252
+ // the socket: the document is full. 507 is the status, and the body's `code`
253
+ // is checked beside it so the classification does not rest on a status a
254
+ // proxy in front of the api could rewrite.
255
+ function isCapacityRefusal(status, parsed) {
256
+ if (status === 507)
257
+ return true;
258
+ return parsed?.code === 'at_capacity';
259
+ }
260
+ // A failure uploading an oversized op's blob to storage. It carries its own
261
+ // type because staging runs strictly BEFORE the op write, so what a replay
262
+ // means differs from a throw out of the write itself: nothing durable exists
263
+ // yet and the op is still the client's to send, where a throw mid-write
264
+ // leaves it unknown whether the row landed.
265
+ class OpStagingError extends Error {
266
+ name = 'OpStagingError';
267
+ // The upstream status, or null for a request that drew no answer at all.
268
+ status;
269
+ constructor(status, message) {
270
+ super(message);
271
+ this.status = status;
272
+ }
273
+ }
274
+ // Whether staging the blob again is the right move: a status the op write
275
+ // itself replays replays here for the same reason, and a request that drew
276
+ // no answer wrote nothing either.
277
+ function opStagingIsRetryable(err) {
278
+ return err.status === null || opWriteIsRetryable(err.status);
279
+ }
280
+ function asOpStagingError(err) {
281
+ if (err instanceof OpStagingError)
282
+ return err;
283
+ const status = err instanceof BrassError ? err.status ?? null : null;
284
+ return new OpStagingError(status, err instanceof Error ? err.message : String(err));
285
+ }
286
+ function isServerUnavailableFrame(frame) {
287
+ if (typeof frame !== 'object' || frame === null)
288
+ return false;
289
+ const f = frame;
290
+ return f.type === 'error' && f.reason === 'unavailable';
291
+ }
119
292
  // The message surfaced through the error reporter for a terminal op rejection.
120
293
  // Prefers the server's copy; falls back to a generic, actionable line. The
121
294
  // local edit already applied but can never persist, so the app's cue is to
@@ -126,6 +299,40 @@ function opRejectedMessage(frame) {
126
299
  }
127
300
  return 'A change could not be saved and will not be retried. Reload the document to continue.';
128
301
  }
302
+ // The one rejection reason that describes the DOCUMENT rather than the op:
303
+ // every later write meets it too, so the app is told once about the document
304
+ // instead of once per refused edit. Matched on the reason string rather than
305
+ // on the message, which is copy and may be reworded.
306
+ const OP_REJECTED_AT_CAPACITY = 'at_capacity';
307
+ // Whether the document refused this write because of its size. The server
308
+ // distinguishes a branch that refuses everything from one that refuses only
309
+ // growth, and picks the wording accordingly; the client does not, because both
310
+ // leave refused work on this copy and nowhere else and the answer to that is
311
+ // the same one.
312
+ function isCapacityRejection(frame) {
313
+ return frame.reason === OP_REJECTED_AT_CAPACITY;
314
+ }
315
+ // Which refusals describe the WRITER rather than the op. The server reads
316
+ // write authority once, when the connection subscribes, so a refusal on
317
+ // either of these grounds answers every later op the same way: the reader
318
+ // cannot write to this document at all, and the Document says so through
319
+ // `readOnly` instead of letting the app offer edits that keep failing. A
320
+ // refusal about the op itself (too large, a bad topic) leaves the document
321
+ // writable, because the next op may well be accepted.
322
+ //
323
+ // A Map rather than an object literal because the key is a wire string: an
324
+ // object answers for the names it inherits, so a reason reading `constructor`
325
+ // or `toString` would resolve a function and lock the document, which is the
326
+ // opposite of what an unclassified refusal means.
327
+ const READ_ONLY_REJECTION_REASONS = new Map([
328
+ ['role_denied', 'grant'],
329
+ ['scope_denied', 'app'],
330
+ ]);
331
+ function readOnlyReasonForRejection(frame) {
332
+ if (typeof frame.reason !== 'string')
333
+ return null;
334
+ return READ_ONLY_REJECTION_REASONS.get(frame.reason) ?? null;
335
+ }
129
336
  function isDocGoneFrame(frame) {
130
337
  if (typeof frame !== 'object' || frame === null)
131
338
  return false;
@@ -164,6 +371,25 @@ function isDocStatusFrame(frame) {
164
371
  // unvalidated, so this keeps the WS path from being the stricter of the two.
165
372
  return typeof st.pending === 'boolean';
166
373
  }
374
+ // `message` is typed non-null and rendered verbatim, while the wire gates
375
+ // only on `pending`, so default it here.
376
+ function normalizeWorkStatus(status) {
377
+ return {
378
+ ...status,
379
+ message: typeof status.message === 'string' ? status.message : '',
380
+ };
381
+ }
382
+ // The opaque work status a join response carries, or `null` when it carries
383
+ // none. Gates on `pending` like `isDocStatusFrame`; the rest is display data
384
+ // read defensively at the use site.
385
+ function parseWorkStatus(value) {
386
+ if (typeof value !== 'object' || value === null)
387
+ return null;
388
+ const status = value;
389
+ if (typeof status.pending !== 'boolean')
390
+ return null;
391
+ return normalizeWorkStatus(status);
392
+ }
167
393
  function isDocMetaFrame(frame) {
168
394
  if (typeof frame !== 'object' || frame === null)
169
395
  return false;
@@ -172,6 +398,15 @@ function isDocMetaFrame(frame) {
172
398
  typeof f.topic === 'string' &&
173
399
  typeof f.name === 'string');
174
400
  }
401
+ function isDocDrivePublishFrame(frame) {
402
+ if (typeof frame !== 'object' || frame === null)
403
+ return false;
404
+ const f = frame;
405
+ if (f.type !== 'doc_drive_publish' || typeof f.topic !== 'string') {
406
+ return false;
407
+ }
408
+ return typeof f.publish === 'object' && f.publish !== null;
409
+ }
175
410
  function isDocDriveStatusFrame(frame) {
176
411
  if (typeof frame !== 'object' || frame === null)
177
412
  return false;
@@ -194,10 +429,10 @@ function isDocOpsDroppedFrame(frame) {
194
429
  if (typeof frame !== 'object' || frame === null)
195
430
  return false;
196
431
  const f = frame;
197
- return (f.type === 'doc_ops_dropped' &&
198
- typeof f.topic === 'string' &&
199
- Array.isArray(f.dropped_seqs) &&
200
- typeof f.actor === 'string');
432
+ // Gate on the discriminant alone. The handler reads no other field, so a
433
+ // frame whose informational half changed shape (or went away) must still
434
+ // prompt the durability re-check it is sent for.
435
+ return f.type === 'doc_ops_dropped' && typeof f.topic === 'string';
201
436
  }
202
437
  function defaultClientOpIdMinter() {
203
438
  const g = globalThis;
@@ -241,6 +476,15 @@ function isSnapshotResponse(body) {
241
476
  // snapshot validated structurally in materialise(); a null is fine.
242
477
  return true;
243
478
  }
479
+ // Fold the join's write verdict into the Document options, so a reader the
480
+ // server will not accept writes from opens read-only instead of learning it
481
+ // from the first refused op. Only ever ADDS the restriction: a caller that
482
+ // asked for a read-only Document keeps it whatever the grant says.
483
+ function readOnlyForGrant(body, options) {
484
+ if (body.can_edit !== false)
485
+ return options;
486
+ return { ...options, readOnly: true, readOnlyReason: 'grant' };
487
+ }
244
488
  function isAtVersionResponse(body) {
245
489
  if (typeof body !== 'object' || body === null)
246
490
  return false;
@@ -285,13 +529,27 @@ export function createOpenDocument(opts) {
285
529
  applyClientVersionHeader(headers);
286
530
  await applyAppAssertionHeader(headers, opts.getAppAssertion);
287
531
  const endFetch = perfStart('open: GET /documents (http + read body)');
288
- // Idempotent read: ride out a cold-API transient 5xx / 429 (a 503
289
- // while the API scales into a concurrent open burst is the common
290
- // one) instead of failing the open on a blip that clears in a second.
532
+ // Idempotent read: ride out a transient 5xx / 429 instead of failing the
533
+ // open on something that clears on its own.
534
+ //
535
+ // Longer than the default handful of attempts because of what a 503 means
536
+ // on this route: the document is intact and its storage is busy. Every op
537
+ // row a document has shares one partition, so the document most likely to
538
+ // answer 503 is one that was just written hard, which is exactly the
539
+ // moment a person reloads to see their work. That pressure clears over
540
+ // tens of seconds, not the one second the default budget spans, and the
541
+ // cost of giving up early is telling someone their document will not open
542
+ // when waiting would have opened it.
543
+ //
544
+ // A thrown request gets a short budget of its own: it never reached the
545
+ // API, so the busy partition is not what is wrong, and a blocked origin
546
+ // or a host that does not resolve answers the seventh attempt as it
547
+ // answered the first, with the reader watching a loading surface
548
+ // throughout.
291
549
  const response = await fetchWithRetry(() => fetch(url, {
292
550
  method: 'GET',
293
551
  headers,
294
- }));
552
+ }), { maxAttempts: 7, baseDelayMs: 700, maxNetworkAttempts: 3 });
295
553
  const text = await response.text();
296
554
  endFetch({ status: response.status, responseChars: text.length });
297
555
  let parsed = undefined;
@@ -304,7 +562,7 @@ export function createOpenDocument(opts) {
304
562
  parseFailed = true;
305
563
  }
306
564
  }
307
- await resolveParsedTailOps(parsed);
565
+ await resolveParsedTailOps(parsed, options?.onSnapshotProgress);
308
566
  return { status: response.status, parsed, parseFailed };
309
567
  }
310
568
  // `GET /documents/{docId}/at/{version}`: read-only materialization at a
@@ -430,35 +688,45 @@ export function createOpenDocument(opts) {
430
688
  // key the op write names. The PUT carries no Brass headers: the URL is
431
689
  // signed, and an extra header falls outside that signature.
432
690
  async function stageOpBlob(docId, opBlobB64, token) {
433
- const headers = {
434
- authorization: `Bearer ${token}`,
435
- 'content-type': 'application/json',
436
- };
437
- applyClientVersionHeader(headers);
438
- await applyAppAssertionHeader(headers, opts.getAppAssertion);
439
- const mint = await fetch(`${baseUrl}/documents/${encodeURIComponent(docId)}/ops/upload`, { method: 'POST', headers, body: '{}' });
440
- if (!mint.ok) {
441
- const text = await mint.text();
442
- let parsed = undefined;
691
+ // Everything in here happens before the op write, so every failure it
692
+ // can take is tagged as one: what the tag buys the writer is the right
693
+ // to send the op again, which a failure of the write itself does not
694
+ // carry.
695
+ try {
696
+ const headers = {
697
+ authorization: `Bearer ${token}`,
698
+ 'content-type': 'application/json',
699
+ };
700
+ applyClientVersionHeader(headers);
701
+ await applyAppAssertionHeader(headers, opts.getAppAssertion);
702
+ const mint = await fetchWithDeadline(`${baseUrl}/documents/${encodeURIComponent(docId)}/ops/upload`, { method: 'POST', headers, body: '{}' }, OP_WRITE_TIMEOUT_MS);
703
+ let minted = undefined;
443
704
  try {
444
- parsed = text ? JSON.parse(text) : undefined;
705
+ minted = mint.text ? JSON.parse(mint.text) : undefined;
445
706
  }
446
707
  catch {
447
- parsed = undefined;
708
+ minted = undefined;
448
709
  }
449
- throw brassErrorFromBody(mint.status, parsed);
450
- }
451
- const body = (await mint.json());
452
- if (typeof body.upload_key !== 'string' || typeof body.url !== 'string') {
453
- throw new BrassError('Malformed op upload response', { status: 500 });
710
+ if (!mint.ok) {
711
+ throw brassErrorFromBody(mint.status, minted);
712
+ }
713
+ const body = (minted ?? {});
714
+ if (typeof body.upload_key !== 'string' || typeof body.url !== 'string') {
715
+ throw new BrassError('Malformed op upload response', { status: 500 });
716
+ }
717
+ const endUpload = perfStart('op: PUT staged op blob');
718
+ const put = await fetchWithDeadline(body.url, { method: 'PUT', body: opBlobB64 }, deadlineForUpload(opBlobB64.length));
719
+ endUpload({ status: put.status, opChars: opBlobB64.length });
720
+ if (!put.ok) {
721
+ throw new BrassError('Could not upload the update', {
722
+ status: put.status,
723
+ });
724
+ }
725
+ return body.upload_key;
454
726
  }
455
- const endUpload = perfStart('op: PUT staged op blob');
456
- const put = await fetch(body.url, { method: 'PUT', body: opBlobB64 });
457
- endUpload({ status: put.status, opChars: opBlobB64.length });
458
- if (!put.ok) {
459
- throw new BrassError('Could not upload the update', { status: put.status });
727
+ catch (err) {
728
+ throw asOpStagingError(err);
460
729
  }
461
- return body.upload_key;
462
730
  }
463
731
  async function postDocumentOp(docId, branchId, opBlobB64, clientOpId, token) {
464
732
  const headers = {
@@ -475,7 +743,7 @@ export function createOpenDocument(opts) {
475
743
  stagedKey = await stageOpBlob(docId, opBlobB64, token);
476
744
  }
477
745
  const endFetch = perfStart('op: POST /documents/{id}/ops');
478
- const response = await fetch(`${baseUrl}/documents/${encodeURIComponent(docId)}/ops`, {
746
+ const response = await fetchWithDeadline(`${baseUrl}/documents/${encodeURIComponent(docId)}/ops`, {
479
747
  method: 'POST',
480
748
  headers,
481
749
  body: JSON.stringify({
@@ -485,12 +753,14 @@ export function createOpenDocument(opts) {
485
753
  branch_id: branchId,
486
754
  client_op_id: clientOpId,
487
755
  }),
488
- });
489
- const text = await response.text();
756
+ },
757
+ // An op under the staging threshold travels inline, so this request
758
+ // carries the blob and its deadline has to allow for sending it.
759
+ deadlineForUpload(stagedKey === undefined ? opBlobB64.length : 0));
490
760
  endFetch({ status: response.status, opChars: opBlobB64.length });
491
761
  let parsed = undefined;
492
762
  try {
493
- parsed = text ? JSON.parse(text) : undefined;
763
+ parsed = response.text ? JSON.parse(response.text) : undefined;
494
764
  }
495
765
  catch {
496
766
  parsed = undefined;
@@ -638,7 +908,9 @@ export function createOpenDocument(opts) {
638
908
  status: 500,
639
909
  });
640
910
  }
641
- const doc = await materialise(parsed);
911
+ // A stream's write authority is the document's, so the line opens
912
+ // read-only for the same reader the document opens read-only for.
913
+ const doc = await materialise(parsed, readOnlyForGrant(parsed, undefined));
642
914
  // A stream line materializes at a historical seq the same way the
643
915
  // default line does, so the parent's version preview can bring this
644
916
  // stream to a point in time (`doc.stream(name)._previewAt(seq)`). The
@@ -649,7 +921,8 @@ export function createOpenDocument(opts) {
649
921
  // line (see the comments there); scoped per line because each stream
650
922
  // has its own seq space.
651
923
  let currentSeq = Math.max(parsed.tail_ops.reduce((acc, entry) => (entry.seq > acc ? entry.seq : acc), 0), typeof parsed.current_seq === 'number' ? parsed.current_seq : 0);
652
- let coveredSeq = typeof parsed.current_seq === 'number' ? parsed.current_seq : 0;
924
+ const lineTailSeqs = parsed.tail_ops.map((entry) => entry.seq);
925
+ let coveredSeq = joinCoverageFloor(snapshotSeqOf(parsed.snapshot_seq, lineTailSeqs, typeof parsed.current_seq === 'number' ? parsed.current_seq : 0), lineTailSeqs);
653
926
  const coverageParked = new Set();
654
927
  const advanceCoverage = (seq) => {
655
928
  if (seq <= coveredSeq)
@@ -660,50 +933,79 @@ export function createOpenDocument(opts) {
660
933
  coveredSeq += 1;
661
934
  }
662
935
  };
936
+ const raiseCoverage = (floor) => {
937
+ if (floor > coveredSeq)
938
+ coveredSeq = floor;
939
+ for (const seq of coverageParked) {
940
+ if (seq <= coveredSeq)
941
+ coverageParked.delete(seq);
942
+ }
943
+ while (coverageParked.has(coveredSeq + 1)) {
944
+ coverageParked.delete(coveredSeq + 1);
945
+ coveredSeq += 1;
946
+ }
947
+ };
663
948
  for (const entry of parsed.tail_ops)
664
949
  advanceCoverage(entry.seq);
665
- // Reconnect-across-compaction recovery: re-import the line's join.
666
- // Loro imports are idempotent, so re-applying over live state is a
667
- // clean merge.
668
- const resyncFromSnapshot = () => {
669
- void (async () => {
670
- try {
671
- const t = await requireToken();
672
- const res = await fetchSnapshot(docId, { branch: lineId }, t);
673
- if (res.status < 200 || res.status >= 300)
674
- return;
675
- if (!isSnapshotResponse(res.parsed))
676
- return;
677
- const snap = res.parsed.snapshot;
678
- if (snap !== null) {
679
- const bytes = snap.kind === 'inline'
680
- ? base64ToBytes(snap.data_b64)
681
- : await fetchPresignedSnapshot(snap.url);
682
- doc._applySnapshot(bytes);
683
- }
684
- for (const entry of res.parsed.tail_ops) {
685
- doc.applyRemoteOp(base64ToBytes(entry.op_blob_b64));
686
- if (entry.seq > currentSeq)
687
- currentSeq = entry.seq;
688
- advanceCoverage(entry.seq);
689
- }
690
- const joined = res.parsed.current_seq;
691
- if (typeof joined === 'number') {
692
- if (joined > currentSeq)
950
+ // Re-import the line's join: the reconnect-across-compaction recovery,
951
+ // and the answer to a catch-up prompt. Loro imports are idempotent, so
952
+ // re-applying over live state is a clean merge.
953
+ //
954
+ // Coalesced and delivery-honest for the same reasons the default line
955
+ // is: a prompt during a fetch is the only notice of that op the line
956
+ // gets, and a join's `current_seq` names seqs whose rows the tail query
957
+ // could not see yet.
958
+ let lineResyncRun = null;
959
+ let lineResyncAgain = false;
960
+ const runLineResync = async () => {
961
+ try {
962
+ do {
963
+ lineResyncAgain = false;
964
+ try {
965
+ const t = await requireToken();
966
+ const res = await fetchSnapshot(docId, { branch: lineId }, t);
967
+ if (res.status < 200 || res.status >= 300)
968
+ continue;
969
+ if (!isSnapshotResponse(res.parsed))
970
+ continue;
971
+ const snap = res.parsed.snapshot;
972
+ if (snap !== null) {
973
+ const bytes = snap.kind === 'inline'
974
+ ? base64ToBytes(snap.data_b64)
975
+ : await fetchPresignedSnapshot(snap.url);
976
+ doc._applySnapshot(bytes);
977
+ }
978
+ const seqs = [];
979
+ for (const entry of res.parsed.tail_ops) {
980
+ doc.applyRemoteOp(base64ToBytes(entry.op_blob_b64));
981
+ if (entry.seq > currentSeq)
982
+ currentSeq = entry.seq;
983
+ seqs.push(entry.seq);
984
+ }
985
+ const joined = res.parsed.current_seq;
986
+ if (typeof joined === 'number' && joined > currentSeq) {
693
987
  currentSeq = joined;
694
- if (joined > coveredSeq) {
695
- coveredSeq = joined;
696
- for (const seq of coverageParked) {
697
- if (seq <= coveredSeq)
698
- coverageParked.delete(seq);
699
- }
700
988
  }
989
+ raiseCoverage(joinCoverageFloor(snapshotSeqOf(res.parsed.snapshot_seq, seqs, currentSeq), seqs));
990
+ for (const seq of seqs)
991
+ advanceCoverage(seq);
701
992
  }
702
- }
703
- catch {
704
- // Best-effort; the next doc_init retries.
705
- }
706
- })();
993
+ catch {
994
+ // Best-effort; the next prompt or doc_init retries.
995
+ }
996
+ } while (lineResyncAgain);
997
+ }
998
+ finally {
999
+ lineResyncRun = null;
1000
+ }
1001
+ };
1002
+ const resyncFromSnapshot = () => {
1003
+ if (lineResyncRun !== null) {
1004
+ lineResyncAgain = true;
1005
+ return lineResyncRun;
1006
+ }
1007
+ lineResyncRun = runLineResync();
1008
+ return lineResyncRun;
707
1009
  };
708
1010
  // Write an op the socket cannot carry. The op stays one commit and
709
1011
  // one seq; only the transport differs, and the response carries the
@@ -712,11 +1014,23 @@ export function createOpenDocument(opts) {
712
1014
  const token = await requireToken();
713
1015
  const { status, parsed: opResult } = await postDocumentOp(docId, lineId, opBlobB64, clientOpId, token);
714
1016
  if (status < 200 || status >= 300) {
715
- return { ok: false, message: opWriteFailureMessage(status, opResult) };
1017
+ return {
1018
+ ok: false,
1019
+ message: opWriteFailureMessage(status, opResult),
1020
+ retryable: opWriteIsRetryable(status),
1021
+ atCapacity: isCapacityRefusal(status, opResult),
1022
+ // The route answers 403 when the caller's grant carries no write
1023
+ // access, which is the HTTP face of the socket's `role_denied`.
1024
+ ...(status === 403 ? { readOnly: 'grant' } : {}),
1025
+ };
716
1026
  }
717
1027
  const seq = opResult?.seq;
718
1028
  if (typeof seq !== 'number') {
719
- return { ok: false, message: 'The server did not report a version for this change.' };
1029
+ return {
1030
+ ok: false,
1031
+ message: 'The server did not report a version for this change.',
1032
+ retryable: false,
1033
+ };
720
1034
  }
721
1035
  return { ok: true, seq };
722
1036
  };
@@ -750,17 +1064,27 @@ export function createOpenDocument(opts) {
750
1064
  if (!isSnapshotResponse(parsed)) {
751
1065
  throw new BrassError('Malformed document snapshot response', { status: 500 });
752
1066
  }
753
- const doc = await materialise(parsed, options?.documentOptions, options?.onSnapshotProgress);
1067
+ const doc = await materialise(parsed, readOnlyForGrant(parsed, options?.documentOptions), options?.onSnapshotProgress);
754
1068
  // Populate `doc.status` from the join before the first `doc_status` frame
755
1069
  // (a stub opened mid-hydration returns `snapshot: null` + a `pending`
756
1070
  // status).
757
1071
  if (parsed.status !== undefined)
758
1072
  doc._emitStatus(parsed.status);
1073
+ // Whether this app is the one to serialize the document, which is what
1074
+ // decides whether a push carries bytes. Answered by the server for the
1075
+ // caller's own app and only on the join, since the fanout frame is one
1076
+ // payload for every subscriber and has no caller to answer for.
1077
+ doc._setExportRunsHere(parsed.calling_app_exports === true);
759
1078
  // Seed `doc.driveStatus` from the join before the first
760
1079
  // `doc_drive_status` frame.
761
1080
  if (parsed.drive !== undefined) {
762
1081
  doc._emitDriveStatus(toDriveStatus(parsed.drive));
763
1082
  }
1083
+ else {
1084
+ // Only for a document that mirrors nothing: a mirror publishes through
1085
+ // its link, and `_emitDriveStatus` clears any targets anyway.
1086
+ doc._setPublishTarget(toPublishTarget(parsed.drive_publish_target));
1087
+ }
764
1088
  // Highest seq seen; later `doc_op` / `op_ack` frames advance it. Kept in
765
1089
  // a closure, not on the Document: it is a transport concern, not CRDT
766
1090
  // state.
@@ -771,9 +1095,14 @@ export function createOpenDocument(opts) {
771
1095
  // then can't tell "caught up" from "missed ops since compacted into the
772
1096
  // snapshot". The floor never overestimates, so the `doc_init` resync
773
1097
  // check always detects such a hole (at worst a transient out-of-order
774
- // arrival costs one idempotent re-fetch). Seeded from the consistent-read
775
- // join's `current_seq`.
776
- let coveredSeq = typeof parsed.current_seq === 'number' ? parsed.current_seq : 0;
1098
+ // arrival costs one idempotent re-fetch).
1099
+ //
1100
+ // Seeded from what this join DELIVERED, which is its snapshot plus the
1101
+ // unbroken run of tail seqs above it, never from `current_seq`: a join
1102
+ // read while someone writes in bulk reports a counter above the rows it
1103
+ // could see, and seeding from the counter opens the document already
1104
+ // claiming ops it does not hold.
1105
+ let coveredSeq = joinCoverageFloor(snapshotSeqOf(parsed.snapshot_seq, parsed.tail_ops.map((entry) => entry.seq), typeof parsed.current_seq === 'number' ? parsed.current_seq : 0), parsed.tail_ops.map((entry) => entry.seq));
777
1106
  // Seqs applied above the floor, parked until the gap below fills.
778
1107
  const coverageParked = new Set();
779
1108
  const drainCoverage = () => {
@@ -788,9 +1117,11 @@ export function createOpenDocument(opts) {
788
1117
  coverageParked.add(seq);
789
1118
  drainCoverage();
790
1119
  };
791
- // Mark everything at or below `floor` covered (a full consistent join was
792
- // imported). Also steps the floor past any abandoned server-side seq
793
- // (allocated, never written) that `advanceCoverage` would stall under.
1120
+ // Mark everything at or below `floor` covered. The caller owes the
1121
+ // guarantee that goes with it: every seq under `floor` is applied, or
1122
+ // folded into a snapshot this doc imported. Coverage is what retires a
1123
+ // catch-up prompt, so a floor taken from anything looser than that is a
1124
+ // reader quietly deciding it holds ops nobody sent it.
794
1125
  const raiseCoverage = (floor) => {
795
1126
  if (floor > coveredSeq)
796
1127
  coveredSeq = floor;
@@ -859,14 +1190,34 @@ export function createOpenDocument(opts) {
859
1190
  if (entry.seq > maxTailSeq)
860
1191
  maxTailSeq = entry.seq;
861
1192
  }
862
- // Raise both cursors so the floor reflects the import (this clears a
863
- // detected missed-compaction hole).
864
1193
  const fetchedSeq = typeof result.parsed.current_seq === 'number'
865
1194
  ? result.parsed.current_seq
866
1195
  : maxTailSeq;
867
1196
  if (fetchedSeq > currentSeq)
868
1197
  currentSeq = fetchedSeq;
869
- raiseCoverage(fetchedSeq);
1198
+ // Coverage moves to what this response DELIVERED, not to the head it
1199
+ // reported. The two differ exactly when it matters: the head is the
1200
+ // branch's seq counter, bumped when a seq is handed out rather than
1201
+ // when its row is written, so a join read during a bulk write names
1202
+ // ops the tail query could not see. Crediting the head marks those
1203
+ // seqs covered, and the prompts that would fetch them then short
1204
+ // circuit, which is a reader left permanently short of a write it
1205
+ // watched land.
1206
+ //
1207
+ // The cost is that a seq allocated and never written (a failed op
1208
+ // write leaves a hole rather than a duplicate) now stalls the floor
1209
+ // under it. That resolves at the next compaction, which folds the tail
1210
+ // past the hole and raises `snapshot_seq` over it; until then the gap
1211
+ // pull's own budget is what stops the reader asking forever.
1212
+ const tailSeqs = sortedTail.map((entry) => entry.seq);
1213
+ // A snapshot only counts as a floor once its bytes are in: a shape this
1214
+ // pin does not decode leaves `snapshot_seq` describing content the doc
1215
+ // never received, and the tail alone is then what it holds.
1216
+ if (snapshotBytes !== null || result.parsed.snapshot === null) {
1217
+ raiseCoverage(joinCoverageFloor(snapshotSeqOf(result.parsed.snapshot_seq, tailSeqs, fetchedSeq), tailSeqs));
1218
+ }
1219
+ for (const seq of tailSeqs)
1220
+ advanceCoverage(seq);
870
1221
  endPerf({
871
1222
  ok: true,
872
1223
  snapshotBytes: snapshotBytes?.byteLength ?? 0,
@@ -897,90 +1248,106 @@ export function createOpenDocument(opts) {
897
1248
  }
898
1249
  return out;
899
1250
  };
900
- let resyncInFlight = false;
901
- const resyncFromSnapshot = () => {
902
- if (resyncInFlight)
903
- return;
904
- resyncInFlight = true;
905
- void (async () => {
906
- try {
907
- const outcome = await refetchAndImport();
908
- if (!outcome.ok) {
909
- console.error('Brass: document resync after reconnect failed; state may lag until the next reconnect or reload');
1251
+ // Coalesced, not skipped. A prompt that lands while a join is in flight
1252
+ // is a statement that the document has moved past what that join can
1253
+ // possibly carry, so dropping it loses the only notice this reader gets:
1254
+ // the server posts each prompt once, and a passive reader has no ack or
1255
+ // op of its own to notice the shortfall from. A bulk write is a run of
1256
+ // prompts (every op past the frame bound takes the HTTP route and
1257
+ // prompts each subscriber), and under a plain skip-if-in-flight the
1258
+ // whole run collapses into the one or two joins that happened not to be
1259
+ // busy, which is a reader stranded partway through a write it watched.
1260
+ //
1261
+ // `resyncAgain` is a flag rather than a queue because a join reads the
1262
+ // whole document: one more pass answers every prompt that arrived during
1263
+ // this one. The loop therefore runs at most one extra fetch after the
1264
+ // last prompt, and settles when the writer stops.
1265
+ //
1266
+ // The returned promise settles when the last pass this call is
1267
+ // responsible for has landed, so a caller deciding whether the reader is
1268
+ // still short reads the floor after the work rather than during it.
1269
+ let resyncRun = null;
1270
+ let resyncAgain = false;
1271
+ const runResync = async () => {
1272
+ try {
1273
+ do {
1274
+ resyncAgain = false;
1275
+ try {
1276
+ const outcome = await refetchAndImport();
1277
+ if (!outcome.ok) {
1278
+ console.error('Brass: document resync after reconnect failed; state may lag until the next reconnect or reload');
1279
+ }
910
1280
  }
911
- }
912
- catch (err) {
913
- console.error('Brass: document resync after reconnect failed; state may lag until the next reconnect or reload', err);
914
- }
915
- finally {
916
- resyncInFlight = false;
917
- }
918
- })();
1281
+ catch (err) {
1282
+ console.error('Brass: document resync after reconnect failed; state may lag until the next reconnect or reload', err);
1283
+ }
1284
+ } while (resyncAgain);
1285
+ }
1286
+ finally {
1287
+ resyncRun = null;
1288
+ }
1289
+ };
1290
+ const resyncFromSnapshot = () => {
1291
+ if (resyncRun !== null) {
1292
+ resyncAgain = true;
1293
+ return resyncRun;
1294
+ }
1295
+ resyncRun = runResync();
1296
+ return resyncRun;
919
1297
  };
920
1298
  // When a doc opened mid-load finishes, pull the now-available content
921
1299
  // into the live doc so it appears without the app reopening. Only ever
922
1300
  // called on the server's authoritative `reload` hint, so it never has to
923
1301
  // tell success from failure: a failure carries no `reload`, so no racy
924
1302
  // read and no loop against a failed hydration.
925
- //
926
- // A failed fetch here is transient (network, dropped presigned download)
927
- // and is retried with backoff: a silent give-up would strand the doc
928
- // empty-but-editable, reading as "the model never loaded" and invisible
929
- // in every log. On exhaustion the failure is reported and the terminal
930
- // status is an honest failure, not the server's success copy.
931
1303
  let reloadInFlight = false;
932
- const RELOAD_ATTEMPTS = 5;
933
- const RELOAD_BACKOFF_MS = 1000;
1304
+ let reconcileInFlight = false;
1305
+ // A terminal frame that lands while the subscribe-time reconcile is still
1306
+ // fetching waits for it: two concurrent imports each clear the undo stack
1307
+ // and race their terminal emits, and dropping the frame would leave a
1308
+ // reconcile that read the row too early sitting on the pending overlay.
1309
+ let deferredReload = null;
1310
+ // Armed when the pull below gives up. The work is terminal server-side, so
1311
+ // no further status frame arrives, and later ops are causally dependent on
1312
+ // a snapshot this client never imported (Loro buffers them and the doc
1313
+ // stays empty), so the next frame on the topic drives one more attempt.
1314
+ let retryPullOnNextFrame = false;
934
1315
  const reloadAfterHydration = (finalStatus) => {
935
1316
  if (reloadInFlight)
936
1317
  return;
1318
+ if (reconcileInFlight) {
1319
+ deferredReload = finalStatus;
1320
+ return;
1321
+ }
937
1322
  reloadInFlight = true;
938
1323
  void (async () => {
939
- let lastError = null;
940
- for (let attempt = 1; attempt <= RELOAD_ATTEMPTS; attempt += 1) {
941
- try {
942
- const outcome = await refetchAndImport();
943
- if (outcome.ok) {
944
- reloadInFlight = false;
945
- // Emit the terminal status AFTER the import settles, so an app
946
- // re-reading on `onStatus` sees the content present and (with
947
- // `readOnly` now false) clears its loading overlay.
948
- doc._emitStatus(finalStatus);
949
- return;
950
- }
951
- lastError = new Error(outcome.reason);
952
- }
953
- catch (err) {
954
- lastError = err instanceof Error ? err : new Error(String(err));
955
- }
956
- // A retrying attempt is exceptional, so this warn is quiet in
957
- // normal operation and decisive in a bug report; without it a
958
- // failure here is undiagnosable from any log.
959
- console.warn(`[Brass] document content reload attempt ${attempt}/${RELOAD_ATTEMPTS} failed (doc ${docId}): ${lastError.message}`);
960
- if (attempt < RELOAD_ATTEMPTS) {
961
- await new Promise((r) => setTimeout(r, RELOAD_BACKOFF_MS * attempt));
962
- }
963
- }
964
- reloadInFlight = false;
965
- const error = new BrassError(`Document content reload failed after ${RELOAD_ATTEMPTS} attempts: ${lastError?.message ?? 'unknown error'}`, { status: 0 });
966
- if (opts.reportError)
967
- opts.reportError(error);
968
- else
969
- console.error('[Brass]', error);
970
- doc._emitStatus({
971
- message: 'The model could not be loaded. Reload to try again.',
972
- pending: false,
1324
+ const imported = await pullContentWithRetry({
1325
+ doc,
1326
+ docId,
1327
+ finalStatus,
1328
+ fetchAndImport: refetchAndImport,
1329
+ ...(opts.reportError ? { reportError: opts.reportError } : {}),
973
1330
  });
1331
+ reloadInFlight = false;
1332
+ retryPullOnNextFrame = !imported;
974
1333
  })();
975
1334
  };
1335
+ // Evidence the document moved on the server after a give-up: one more
1336
+ // pull, which on success imports the content and releases the lock the
1337
+ // failed terminal status left in place.
1338
+ const retryContentPull = () => {
1339
+ if (!retryPullOnNextFrame)
1340
+ return;
1341
+ retryPullOnNextFrame = false;
1342
+ reloadAfterHydration(WORK_READY_STATUS);
1343
+ };
976
1344
  // Reconcile a pending open once the live subscription exists. The
977
1345
  // terminal `doc_status` frame is best-effort: if hydration finishes
978
- // before this client's SUB# row exists, the fan-out finds no subscriber
1346
+ // before this client's subscription record exists, the fan-out finds no subscriber
979
1347
  // and drops the frame, stranding the app on the pending overlay. `doc_init`
980
- // is posted only after the SUB# row is written, so one re-fetch here
1348
+ // is posted only after the subscription record is written, so one re-fetch here
981
1349
  // closes the gap for a completion that landed before the row existed.
982
1350
  // Only a pending open pays the extra fetch.
983
- let reconcileInFlight = false;
984
1351
  const reconcileStatusOnInit = () => {
985
1352
  const current = doc.status;
986
1353
  if (!current || !current.pending)
@@ -989,6 +1356,7 @@ export function createOpenDocument(opts) {
989
1356
  return;
990
1357
  reconcileInFlight = true;
991
1358
  void (async () => {
1359
+ let settled = false;
992
1360
  try {
993
1361
  const outcome = await refetchAndImport();
994
1362
  if (!outcome.ok)
@@ -997,10 +1365,15 @@ export function createOpenDocument(opts) {
997
1365
  // fan-out delivers the terminal frame when the work finishes.
998
1366
  if (outcome.status?.pending)
999
1367
  return;
1000
- // Terminal. A present status is a failure (server message,
1001
- // pending:false); an absent status is success. On success retain
1002
- // the last server message and flip pending so the overlay clears.
1003
- doc._emitStatus(outcome.status ?? { message: current.message, pending: false });
1368
+ settled = true;
1369
+ // Terminal. A status the server still holds is a failure (success
1370
+ // clears the row), and its absence is the success this import just
1371
+ // pulled the content for, reported as the same terminal frame the
1372
+ // platform sends on that transition.
1373
+ if (outcome.status)
1374
+ doc._emitStatus(outcome.status);
1375
+ else
1376
+ doc._emitStatus(WORK_READY_STATUS, { contentLoaded: true });
1004
1377
  }
1005
1378
  catch {
1006
1379
  // Best-effort; the live fan-out remains the primary path when
@@ -1008,6 +1381,12 @@ export function createOpenDocument(opts) {
1008
1381
  }
1009
1382
  finally {
1010
1383
  reconcileInFlight = false;
1384
+ const deferred = deferredReload;
1385
+ deferredReload = null;
1386
+ // A settled reconcile imported the content itself, so the held
1387
+ // frame has nothing left to pull.
1388
+ if (deferred && !settled)
1389
+ reloadAfterHydration(deferred);
1011
1390
  }
1012
1391
  })();
1013
1392
  };
@@ -1019,13 +1398,22 @@ export function createOpenDocument(opts) {
1019
1398
  const token = await requireToken();
1020
1399
  const { status, parsed: opResult } = await postDocumentOp(docId, parsed.branch_id, opBlobB64, clientOpId, token);
1021
1400
  if (status < 200 || status >= 300) {
1022
- return { ok: false, message: opWriteFailureMessage(status, opResult) };
1401
+ return {
1402
+ ok: false,
1403
+ message: opWriteFailureMessage(status, opResult),
1404
+ retryable: opWriteIsRetryable(status),
1405
+ atCapacity: isCapacityRefusal(status, opResult),
1406
+ // The route answers 403 when the caller's grant carries no write
1407
+ // access, which is the HTTP face of the socket's `role_denied`.
1408
+ ...(status === 403 ? { readOnly: 'grant' } : {}),
1409
+ };
1023
1410
  }
1024
1411
  const seq = opResult?.seq;
1025
1412
  if (typeof seq !== 'number') {
1026
1413
  return {
1027
1414
  ok: false,
1028
1415
  message: 'The server did not report a version for this change.',
1416
+ retryable: false,
1029
1417
  };
1030
1418
  }
1031
1419
  return { ok: true, seq };
@@ -1043,6 +1431,10 @@ export function createOpenDocument(opts) {
1043
1431
  ? { requestedBranchId: options.branch }
1044
1432
  : {}),
1045
1433
  ...(options?.stream !== undefined ? { requestedStream: options.stream } : {}),
1434
+ ...(options?.documentOptions !== undefined
1435
+ ? { documentOptions: options.documentOptions }
1436
+ : {}),
1437
+ ...(opts.identity !== undefined ? { identity: opts.identity } : {}),
1046
1438
  wsClient: opts.wsClient,
1047
1439
  postOversizedOp,
1048
1440
  newClientOpId: opts.newClientOpId ?? defaultClientOpIdMinter,
@@ -1057,6 +1449,7 @@ export function createOpenDocument(opts) {
1057
1449
  refetchDurableFrontier,
1058
1450
  reloadAfterHydration,
1059
1451
  reconcileStatusOnInit,
1452
+ onDocumentMoved: retryContentPull,
1060
1453
  });
1061
1454
  }
1062
1455
  // `doc._checkoutVersion(v)` calls back here to materialise historical
@@ -1081,6 +1474,7 @@ export function createOpenDocument(opts) {
1081
1474
  await uploader(doc, image, width, height);
1082
1475
  });
1083
1476
  }
1477
+ doc._setThumbnailVersion(parsed.thumbnail?.version ?? null);
1084
1478
  if (opts.checkpointCreator) {
1085
1479
  const creator = opts.checkpointCreator;
1086
1480
  doc.setCheckpointCreator(() => creator(doc));
@@ -1189,6 +1583,11 @@ export function createOpenDocument(opts) {
1189
1583
  parseFailed = true;
1190
1584
  }
1191
1585
  }
1586
+ // A tail op whose bytes spilled to storage arrives as a signed URL, and
1587
+ // a guest join carries them exactly as a signed-in one does. The URL
1588
+ // carries its own signature, so an anonymous reader fetches it with the
1589
+ // same call. Resolving here is what lets `materialise` see one shape.
1590
+ await resolveParsedTailOps(parsed);
1192
1591
  return { status: response.status, parsed, parseFailed };
1193
1592
  }
1194
1593
  // `GET /links/{token}/streams`: the anonymous stream manifest, the guest
@@ -1242,7 +1641,7 @@ export function createOpenDocument(opts) {
1242
1641
  // The link's grant decides read vs write. An `editor` link is anonymous
1243
1642
  // EDIT: a live Document (not read-only) whose local mutations ship over
1244
1643
  // the same token-authenticated socket, with the server admitting the
1245
- // op-write because the guest's SUB# row carries `role: editor`. A
1644
+ // op-write because the guest's subscription record carries `role: editor`. A
1246
1645
  // `viewer` link (the default) is read-only: mutate throws and only
1247
1646
  // inbound frames apply.
1248
1647
  const canEdit = b.link_role === 'editor';
@@ -1256,7 +1655,20 @@ export function createOpenDocument(opts) {
1256
1655
  // sees the title (the signed-in open gets this from `finalise`).
1257
1656
  if (typeof b.name === 'string')
1258
1657
  doc._setName(b.name);
1259
- wireGuestTransport(doc, b.doc_id, b.branch_id, canEdit);
1658
+ // A link opened while the platform is still preparing the document
1659
+ // carries the same opaque work status the signed-in join does, and the
1660
+ // guest's own topic delivers the phases after it. Seeded before the
1661
+ // transport so the app renders the wait rather than an empty document.
1662
+ const linkStatus = parseWorkStatus(b.status);
1663
+ if (linkStatus)
1664
+ doc._emitStatus(linkStatus);
1665
+ // An edit link's writes fail the same ways a member's do (a shed the
1666
+ // budget outlived, a link revoked mid-edit), and the app hears those
1667
+ // through the same channel it hears a member's, so the reporter is
1668
+ // wired here as `finalise` wires it for a signed-in open.
1669
+ if (opts.reportError)
1670
+ doc.setErrorReporter(opts.reportError);
1671
+ wireGuestTransport(doc, token, b.doc_id, b.branch_id, canEdit);
1260
1672
  // Streams: a guest resolves `doc.stream(name)` through the token
1261
1673
  // routes. `list` hits the anonymous manifest; `openLine` joins the
1262
1674
  // line read/write exactly like the default stream above, over the same
@@ -1284,7 +1696,9 @@ export function createOpenDocument(opts) {
1284
1696
  snapshot: lb.snapshot,
1285
1697
  tail_ops: lb.tail_ops,
1286
1698
  }, { readOnly: !canEdit });
1287
- wireGuestTransport(inner, lb.doc_id, lb.branch_id, canEdit);
1699
+ if (opts.reportError)
1700
+ inner.setErrorReporter(opts.reportError);
1701
+ wireGuestTransport(inner, token, lb.doc_id, lb.branch_id, canEdit);
1288
1702
  return inner;
1289
1703
  },
1290
1704
  create: () => {
@@ -1295,68 +1709,263 @@ export function createOpenDocument(opts) {
1295
1709
  });
1296
1710
  return doc;
1297
1711
  };
1712
+ // Write a guest's op over HTTP: `POST /links/{token}/ops`, the anonymous
1713
+ // counterpart of `POST /documents/{docId}/ops`. Sends NO Authorization
1714
+ // header (the token is the credential) and names the line the guest
1715
+ // opened, which is the branch its socket topic is scoped to.
1716
+ async function postLinkOp(token, lineId, opBlobB64, clientOpId) {
1717
+ const response = await fetch(`${baseUrl}/links/${encodeURIComponent(token)}/ops`, {
1718
+ method: 'POST',
1719
+ headers: { 'content-type': 'application/json' },
1720
+ body: JSON.stringify({
1721
+ op_blob: opBlobB64,
1722
+ branch_id: lineId,
1723
+ client_op_id: clientOpId,
1724
+ }),
1725
+ });
1726
+ const text = await response.text();
1727
+ let parsed = undefined;
1728
+ try {
1729
+ parsed = text ? JSON.parse(text) : undefined;
1730
+ }
1731
+ catch {
1732
+ parsed = undefined;
1733
+ }
1734
+ return { status: response.status, parsed };
1735
+ }
1298
1736
  // Wire a guest Document (the default stream or a stream line) to the
1299
1737
  // token-authenticated socket. Inbound frames (`doc_init` / `doc_op`)
1300
1738
  // apply for both link kinds; an edit link ALSO wires the outbound path
1301
1739
  // (`op_ack` resolves the outstanding-op tracker, `setLocalOpHook` ships
1302
- // each local commit as an `op` frame). Best-effort (no resync / durability
1740
+ // each local commit). Best-effort on the read side (no resync / durability
1303
1741
  // handshake, which need a signed-in token); a guest that misses ops
1304
- // re-opens. Patches `close` to detach, mirroring `wireDocTransport`, so
1305
- // `Document` stays free of WS knowledge. A no-op with no WS client (a
1306
- // snapshot-only guest).
1307
- function wireGuestTransport(doc, docId, lineId, canEdit) {
1742
+ // re-opens. The WRITE side answers a refusal the way the signed-in
1743
+ // transport does, because a guest editor's edit is the same person's work
1744
+ // as a member's and the same two things can happen to it: a frame the
1745
+ // socket cannot carry takes the HTTP route, and a capacity shed is
1746
+ // re-asserted on the shared backoff and reported once the budget is out.
1747
+ // Patches `close` to detach, mirroring `wireDocTransport`, so `Document`
1748
+ // stays free of WS knowledge. A no-op with no WS client (a snapshot-only
1749
+ // guest).
1750
+ function wireGuestTransport(doc, token, docId, lineId, canEdit) {
1308
1751
  const wsClient = opts.wsClient;
1309
1752
  if (!wsClient)
1310
1753
  return;
1311
1754
  const topic = `doc:${docId}:${lineId}`;
1755
+ // Ops on the socket awaiting an answer, keyed by `client_op_id`. An
1756
+ // `unavailable` shed names no op, so re-asserting one needs the frames
1757
+ // in hand. Scoped to what the socket carried: an op that took the HTTP
1758
+ // write must never reach the socket, and that write re-asserts itself.
1759
+ const outstanding = new Map();
1760
+ let shedRounds = 0;
1761
+ let shedTimer = null;
1762
+ function clearShedRecovery() {
1763
+ if (shedTimer !== null) {
1764
+ clearTimeout(shedTimer);
1765
+ shedTimer = null;
1766
+ }
1767
+ shedRounds = 0;
1768
+ }
1769
+ // Every outstanding op is dead: drop the tracker entries and tell the
1770
+ // Document, which stops counting them in `savingCount` and reports each
1771
+ // through the error channel the app reads. The ids are taken first
1772
+ // because `_failOutstandingOp` re-enters app code.
1773
+ function failAllOutstanding(message) {
1774
+ const clientOpIds = [...outstanding.keys()];
1775
+ outstanding.clear();
1776
+ for (const clientOpId of clientOpIds) {
1777
+ doc._failOutstandingOp(clientOpId, message);
1778
+ }
1779
+ }
1780
+ // Put every op still awaiting an answer back on the wire. The client
1781
+ // DROPS an op frame it cannot send right now rather than buffering one
1782
+ // (`ownsFrameRetry` below), so this is the only thing that carries a
1783
+ // shed or reconnect-orphaned op across the window, and it is also what
1784
+ // keeps a frame from being written twice: a copy the client held would
1785
+ // flush on the `subscribed` ack while this re-asserts on the `doc_init`
1786
+ // behind it, and the server allocates a fresh seq per send.
1787
+ const resendAllOutstanding = () => {
1788
+ for (const frame of outstanding.values())
1789
+ wsClient.send(frame);
1790
+ };
1791
+ const onServerUnavailable = () => {
1792
+ if (outstanding.size === 0)
1793
+ return;
1794
+ if (shedTimer !== null)
1795
+ return;
1796
+ shedRounds += 1;
1797
+ if (shedRounds > UNAVAILABLE_MAX_ROUNDS) {
1798
+ clearShedRecovery();
1799
+ failAllOutstanding(UNAVAILABLE_GIVE_UP_MESSAGE);
1800
+ return;
1801
+ }
1802
+ shedTimer = setTimeout(() => {
1803
+ shedTimer = null;
1804
+ resendAllOutstanding();
1805
+ }, unavailableDelayMs(shedRounds));
1806
+ shedTimer.unref?.();
1807
+ };
1312
1808
  const unsub = wsClient.subscribe(topic, (frame) => {
1313
- if (isDocInitFrame(frame)) {
1314
- for (const entry of frame.tail_ops) {
1809
+ const init = parseDocInitFrame(frame);
1810
+ if (init) {
1811
+ for (const entry of init.tail_ops) {
1315
1812
  if (entry.op_blob_b64 === undefined)
1316
1813
  continue;
1317
1814
  doc.applyRemoteOp(base64ToBytes(entry.op_blob_b64));
1318
1815
  }
1816
+ // `doc_init` follows every `subscribed` ack, so this is the point
1817
+ // the topic can carry frames again. A fresh connection also returns
1818
+ // the shed budget to full: this re-assert is the round a pending
1819
+ // one would have run.
1820
+ clearShedRecovery();
1821
+ resendAllOutstanding();
1822
+ return;
1319
1823
  }
1320
- else if (isDocOpFrame(frame)) {
1824
+ if (isDocOpFrame(frame)) {
1321
1825
  doc.applyRemoteOp(base64ToBytes(frame.op_blob));
1322
1826
  }
1323
1827
  else if (isDocGoneFrame(frame)) {
1324
1828
  // The server fans `doc_gone` to EVERY subscriber on the branched
1325
- // content topic, guest-link connections included (`fanoutDocGone` /
1326
- // `fanoutBranchGone`), on a soft-delete / move of the opened doc.
1829
+ // content topic, guest-link connections included, on a soft-delete /
1830
+ // move of the opened doc.
1327
1831
  // Without this branch the guest keeps showing removed content and
1328
1832
  // `doc.onGone(...)` never fires; for an edit link the next local op
1329
- // would ship onto the evicted SUB# row and bounce as a misleading
1330
- // `forbidden`. Mark the doc gone (mapping the reason like the
1331
- // signed-in transport); the guest path has no resend / reverify
1332
- // timers to drain.
1833
+ // would ship onto the evicted subscription record and bounce as a misleading
1834
+ // `forbidden`. Drop the shed recovery first so nothing re-asserts an
1835
+ // op onto an evicted subscription record, then mark the doc gone (mapping the
1836
+ // reason like the signed-in transport).
1837
+ clearShedRecovery();
1838
+ outstanding.clear();
1333
1839
  doc._markGone(goneReasonOf(frame));
1334
1840
  }
1841
+ else if (isDocStatusFrame(frame)) {
1842
+ // A guest can open a document whose inbound work is still running,
1843
+ // and hydration writes a SNAPSHOT rather than ops, so nothing on the
1844
+ // op stream would ever fill the view in. Same split as the signed-in
1845
+ // transport: the `reload` hint pulls the content and emits the
1846
+ // terminal status after the import, and anything else renders.
1847
+ const status = normalizeWorkStatus(frame.status);
1848
+ if (frame.reload)
1849
+ pullGuestContent(doc, token, lineId, status);
1850
+ else
1851
+ doc._emitStatus(status);
1852
+ }
1853
+ else if (isDocMetaFrame(frame)) {
1854
+ // `_setName` de-dupes, so a re-emit of the current name is a no-op.
1855
+ doc._setName(frame.name);
1856
+ }
1335
1857
  else if (canEdit && isOpAckFrame(frame)) {
1858
+ outstanding.delete(frame.client_op_id);
1336
1859
  doc.resolveOutstandingOp(frame.client_op_id);
1860
+ // The server is durably accepting writes again, so the shed budget
1861
+ // starts over. A round already scheduled still runs: the ops it
1862
+ // re-asserts are the ones this ack did not cover.
1863
+ shedRounds = 0;
1864
+ }
1865
+ else if (canEdit && isServerUnavailableFrame(frame)) {
1866
+ onServerUnavailable();
1337
1867
  }
1338
1868
  else if (canEdit && isOpRejectedFrame(frame)) {
1339
- // A guest edit the server terminally rejected. No resend tracker here
1340
- // (guest transport is best-effort), so just fail the op on the doc so
1341
- // "could not be saved" surfaces instead of a stuck saving count.
1869
+ // Terminal, not transient: stop re-asserting this op (retrying
1870
+ // would only draw the same rejection) and fail it on the Document
1871
+ // so "could not be saved" surfaces instead of a stuck saving count.
1872
+ // A refusal naming the writer (an edit link downgraded or revoked
1873
+ // while the guest had it open) turns the Document read-only, the
1874
+ // same way it does for a signed-in reader.
1875
+ outstanding.delete(frame.client_op_id);
1876
+ if (isCapacityRejection(frame))
1877
+ doc._markAtCapacity();
1878
+ const readOnly = readOnlyReasonForRejection(frame);
1879
+ if (readOnly !== null)
1880
+ doc._markReadOnly(readOnly);
1342
1881
  doc._failOutstandingOp(frame.client_op_id, opRejectedMessage(frame));
1343
1882
  }
1344
- });
1883
+ },
1884
+ // The guest owns its outstanding ops: `resendAllOutstanding` on the
1885
+ // `doc_init` behind every `subscribed` ack IS this topic's
1886
+ // re-assertion, so the client drops an op frame it cannot send now
1887
+ // rather than buffering a second copy of it.
1888
+ { ownsFrameRetry: true });
1889
+ // An op the socket cannot carry, written over HTTP on the same policy
1890
+ // the signed-in path runs: refuse locally what no write can hold, and
1891
+ // re-assert a capacity refusal on the shared backoff before reporting it.
1892
+ async function writeOversizedGuestOp(opBlobB64, clientOpId) {
1893
+ // Yield before any outcome resolves the op. The hook's caller records
1894
+ // the outstanding op AFTER the hook returns, and an async function body
1895
+ // runs synchronously up to its first await, so an exit taken inline
1896
+ // would find nothing tracked and leave the entry behind with
1897
+ // `savingCount` stuck above zero.
1898
+ await Promise.resolve();
1899
+ // Both sizes are known here, so refuse without spending a round trip
1900
+ // on a blob that can only come back 400.
1901
+ if (opBlobExceedsDurableLimit(opBlobB64)) {
1902
+ doc._failOutstandingOp(clientOpId, TOO_LARGE_MESSAGE);
1903
+ return;
1904
+ }
1905
+ if (opBlobNeedsStaging(opBlobB64)) {
1906
+ doc._failOutstandingOp(clientOpId, LINK_OP_TOO_LARGE_MESSAGE);
1907
+ return;
1908
+ }
1909
+ let status = 0;
1910
+ let parsed = undefined;
1911
+ for (let round = 0; round <= UNAVAILABLE_MAX_ROUNDS; round += 1) {
1912
+ if (round > 0) {
1913
+ await new Promise((resolve) => {
1914
+ const timer = setTimeout(resolve, unavailableDelayMs(round));
1915
+ timer.unref?.();
1916
+ });
1917
+ }
1918
+ try {
1919
+ ({ status, parsed } = await postLinkOp(token, lineId, opBlobB64, clientOpId));
1920
+ }
1921
+ catch (err) {
1922
+ // A throw leaves it unknown whether the handler ran, and this op
1923
+ // carries no idempotency the server dedupes on, so replaying it
1924
+ // could double-apply.
1925
+ doc._failOutstandingOp(clientOpId, err instanceof Error ? err.message : String(err));
1926
+ return;
1927
+ }
1928
+ if (status >= 200 && status < 300)
1929
+ break;
1930
+ if (!opWriteIsRetryable(status))
1931
+ break;
1932
+ }
1933
+ if (status < 200 || status >= 300) {
1934
+ doc._failOutstandingOp(clientOpId, opWriteIsRetryable(status)
1935
+ ? UNAVAILABLE_GIVE_UP_MESSAGE
1936
+ : opWriteFailureMessage(status, parsed));
1937
+ return;
1938
+ }
1939
+ doc.resolveOutstandingOp(clientOpId);
1940
+ }
1345
1941
  if (canEdit) {
1346
1942
  const minter = opts.newClientOpId ?? defaultClientOpIdMinter;
1347
1943
  doc.setLocalOpHook((opBlob) => {
1348
1944
  const clientOpId = minter();
1349
- wsClient.send({
1945
+ const opBlobB64 = bytesToBase64(opBlob);
1946
+ const sendFrame = {
1350
1947
  type: 'op',
1351
1948
  topic,
1352
- op_blob: bytesToBase64(opBlob),
1949
+ op_blob: opBlobB64,
1353
1950
  client_op_id: clientOpId,
1354
- });
1951
+ };
1952
+ // A frame past the transport's cap is not delivered and takes the
1953
+ // connection down with it, so it must never reach the socket: a
1954
+ // guest's document, presence and every other op ride that one
1955
+ // connection. `mutate` batches to a budget well above what a frame
1956
+ // carries, so an ordinary bulk edit arrives here.
1957
+ if (opFrameExceedsWsLimit(sendFrame)) {
1958
+ void writeOversizedGuestOp(opBlobB64, clientOpId);
1959
+ return clientOpId;
1960
+ }
1961
+ outstanding.set(clientOpId, sendFrame);
1962
+ wsClient.send(sendFrame);
1355
1963
  return clientOpId;
1356
1964
  });
1357
1965
  }
1358
1966
  const originalClose = doc.close.bind(doc);
1359
1967
  doc.close = () => {
1968
+ clearShedRecovery();
1360
1969
  try {
1361
1970
  unsub();
1362
1971
  }
@@ -1366,13 +1975,71 @@ export function createOpenDocument(opts) {
1366
1975
  originalClose();
1367
1976
  };
1368
1977
  }
1978
+ // Guest documents with a content pull running, so a repeated terminal frame
1979
+ // does not stack a second pull on one document.
1980
+ const guestPullInFlight = new WeakSet();
1981
+ function pullGuestContent(doc, token, lineId, finalStatus) {
1982
+ if (guestPullInFlight.has(doc))
1983
+ return;
1984
+ guestPullInFlight.add(doc);
1985
+ void (async () => {
1986
+ await pullContentWithRetry({
1987
+ doc,
1988
+ docId: doc.id ?? '',
1989
+ finalStatus,
1990
+ fetchAndImport: () => importLinkView(doc, token, lineId),
1991
+ ...(opts.reportError ? { reportError: opts.reportError } : {}),
1992
+ });
1993
+ guestPullInFlight.delete(doc);
1994
+ })();
1995
+ }
1996
+ // Re-read the link view and import it into the live guest Document: the
1997
+ // anonymous counterpart of the signed-in `refetchAndImport`, over the same
1998
+ // envelope. Named on the line the guest opened, which is the line its topic
1999
+ // is scoped to.
2000
+ async function importLinkView(doc, token, lineId) {
2001
+ const result = await fetchLinkView(token, lineId);
2002
+ if (result.status < 200 || result.status >= 300) {
2003
+ return { ok: false, reason: `link view returned ${result.status}` };
2004
+ }
2005
+ const b = result.parsed;
2006
+ if (!Array.isArray(b.tail_ops)) {
2007
+ return { ok: false, reason: 'malformed link view response' };
2008
+ }
2009
+ const snap = b.snapshot;
2010
+ let snapshotBytes = null;
2011
+ if (snap !== null && snap !== undefined && snap.kind === 'inline') {
2012
+ snapshotBytes = base64ToBytes(snap.data_b64);
2013
+ }
2014
+ else if (snap !== null && snap !== undefined && snap.kind === 'presigned') {
2015
+ try {
2016
+ snapshotBytes = await fetchPresignedSnapshot(snap.url);
2017
+ }
2018
+ catch (err) {
2019
+ const status = err instanceof BrassError ? err.status ?? 0 : 0;
2020
+ return {
2021
+ ok: false,
2022
+ reason: status > 0
2023
+ ? `presigned snapshot fetch returned ${status}`
2024
+ : `presigned snapshot fetch failed: ${err instanceof Error ? err.message : String(err)}`,
2025
+ };
2026
+ }
2027
+ }
2028
+ if (snapshotBytes)
2029
+ doc._applySnapshot(snapshotBytes);
2030
+ const tail = b.tail_ops;
2031
+ for (const entry of [...tail].sort((x, y) => x.seq - y.seq)) {
2032
+ doc.applyRemoteOp(base64ToBytes(entry.op_blob_b64));
2033
+ }
2034
+ return { ok: true };
2035
+ }
1369
2036
  // Strict open: a 404 propagates as `BrassError`. A `version` option
1370
2037
  // diverts to the read-only at-version path.
1371
2038
  const open = async (docId, options) => {
1372
2039
  // Cache hit: return the parked Document. Its subscription stayed alive
1373
2040
  // while parked, and the UndoController + outstanding-op map survived, so
1374
2041
  // undo works as if the user never switched away.
1375
- const parked = takeParkedDocument(docId, options);
2042
+ const parked = takeParkedDocument(docId, options, opts.identity?.() ?? null);
1376
2043
  if (parked) {
1377
2044
  parked._resume();
1378
2045
  return parked;
@@ -1438,6 +2105,13 @@ export function createOpenDocument(opts) {
1438
2105
  // state to keep. Module-scoped so it persists across factory invocations in
1439
2106
  // one process; keys are app-supplied so no cross-app leak within a tab.
1440
2107
  const PARK_CACHE_CAPACITY = 4;
2108
+ function documentOptionsSig(options) {
2109
+ return JSON.stringify([
2110
+ options?.rootKey ?? null,
2111
+ options?.readOnly ?? null,
2112
+ options?.undo ?? null,
2113
+ ]);
2114
+ }
1441
2115
  const parkedDocs = new Map();
1442
2116
  function parkCacheKey(docId, branchId, streamName) {
1443
2117
  // Skip caching when the branch is unknown: the next open() may resolve to
@@ -1451,7 +2125,23 @@ function parkCacheKey(docId, branchId, streamName) {
1451
2125
  // caller's document out from under it.
1452
2126
  return `${docId}/${branchId}/${streamName ?? DEFAULT_STREAM_NAME}`;
1453
2127
  }
2128
+ function evictParked(entry) {
2129
+ if (!entry)
2130
+ return;
2131
+ parkedDocs.delete(entry.key);
2132
+ try {
2133
+ entry.teardown();
2134
+ }
2135
+ catch {
2136
+ // Teardown errors must not strand the cache; the evicted Doc is gone
2137
+ // from the user's perspective either way.
2138
+ }
2139
+ }
1454
2140
  function parkDocument(entry) {
2141
+ // One Document per key: whatever sat here is unreachable the moment this
2142
+ // entry replaces it, so it is torn down rather than left holding a live
2143
+ // subscription nothing can close.
2144
+ evictParked(parkedDocs.get(entry.key));
1455
2145
  parkedDocs.set(entry.key, entry);
1456
2146
  // Map iteration is insertion order, so evicting the front entry is LRU
1457
2147
  // (a newly parked entry is the most-recent; a hit removes the entry
@@ -1460,18 +2150,10 @@ function parkDocument(entry) {
1460
2150
  const oldestKey = parkedDocs.keys().next().value;
1461
2151
  if (oldestKey === undefined)
1462
2152
  break;
1463
- const oldest = parkedDocs.get(oldestKey);
1464
- parkedDocs.delete(oldestKey);
1465
- try {
1466
- oldest?.teardown();
1467
- }
1468
- catch {
1469
- // Teardown errors must not strand the cache; the evicted Doc is gone
1470
- // from the user's perspective either way.
1471
- }
2153
+ evictParked(parkedDocs.get(oldestKey));
1472
2154
  }
1473
2155
  }
1474
- function takeParkedDocument(docId, options) {
2156
+ function takeParkedDocument(docId, options, identity) {
1475
2157
  if (options?.version !== undefined)
1476
2158
  return null;
1477
2159
  const key = parkCacheKey(docId, options?.branch, options?.stream);
@@ -1480,22 +2162,36 @@ function takeParkedDocument(docId, options) {
1480
2162
  const entry = parkedDocs.get(key);
1481
2163
  if (!entry)
1482
2164
  return null;
2165
+ // A Document the server evicted while it was parked describes a document
2166
+ // that is no longer there, one built from different `documentOptions` is
2167
+ // not the Document this caller asked for, and one opened for a different
2168
+ // person is not this caller's to read. All three take the fresh open, which
2169
+ // for the last is the request that establishes the new caller may have it;
2170
+ // the parked copy goes with its subscription.
2171
+ if (entry.doc._goneReason !== null ||
2172
+ entry.optionsSig !== documentOptionsSig(options?.documentOptions) ||
2173
+ entry.identity !== identity) {
2174
+ evictParked(entry);
2175
+ return null;
2176
+ }
1483
2177
  parkedDocs.delete(key);
1484
2178
  return entry.doc;
1485
2179
  }
1486
- // Test-only: drop the parking cache. Used by unit tests that need
1487
- // a clean slate between cases.
1488
- export function _resetParkCacheForTesting() {
2180
+ /** Retire every parked Document. The client calls this as an identity leaves
2181
+ * the page, so what the previous person opened is torn down rather than left
2182
+ * resident for the rest of the tab's life.
2183
+ * @internal */
2184
+ export function clearParkedDocuments() {
1489
2185
  for (const entry of parkedDocs.values()) {
1490
- try {
1491
- entry.teardown();
1492
- }
1493
- catch {
1494
- // ignore
1495
- }
2186
+ evictParked(entry);
1496
2187
  }
1497
2188
  parkedDocs.clear();
1498
2189
  }
2190
+ // Test-only: drop the parking cache. Used by unit tests that need
2191
+ // a clean slate between cases.
2192
+ export function _resetParkCacheForTesting() {
2193
+ clearParkedDocuments();
2194
+ }
1499
2195
  // Wire the WS transport onto a materialised Document. Three pieces:
1500
2196
  // 1. WS subscribe on the doc topic; the server sends `doc_op` (other
1501
2197
  // clients' edits) and `op_ack` (our ops), topic-scoped so the
@@ -1519,17 +2215,76 @@ export const PERIODIC_REVERIFY_INTERVAL_MS = 5 * 60_000;
1519
2215
  // delay is an op the server will not send again. Exported for the
1520
2216
  // fake-timer test.
1521
2217
  export const MISSED_OP_RESYNC_DELAY_MS = 2_000;
2218
+ // Per-op resend backoff: 1.5s, ~1.6x per attempt, to a 30s ceiling. The
2219
+ // ceiling keeps a doc on a dead connection from hammering; the
2220
+ // reconnect-driven resend (`resendAllOutstanding`) fires the instant
2221
+ // `doc_init` arrives, so the ceiling never delays recovery past the next
2222
+ // reconnect. Exported for the fake-timer tests.
2223
+ export const RESEND_BASE_DELAY_MS = 1_500;
2224
+ export const RESEND_MAX_DELAY_MS = 30_000;
2225
+ // How long the oldest op on the socket may go unacked before its resend
2226
+ // timer spends an attempt even though the connection is answering other
2227
+ // ops. See `scheduleResend` for what this bounds and why only the oldest op
2228
+ // reaches it.
2229
+ //
2230
+ // The value sits above every latency a working connection produces. Acks
2231
+ // measured on a deployed environment run 175 ms at p99 and 1.6 s at their
2232
+ // worst, and an op whose own backoff has grown to `RESEND_MAX_DELAY_MS` is
2233
+ // still well inside this, so a queue that is merely slow never reaches it and
2234
+ // the timer keeps re-arming free. Sitting above the backoff ceiling is also
2235
+ // what makes this value, rather than the backoff, the interval a forced
2236
+ // resend is paced at: the timer fires several times inside one window, and
2237
+ // the first expiry past the window is the one that spends.
2238
+ //
2239
+ // What a false positive costs bounds the risk of setting it at all: one
2240
+ // duplicate op per document per window, which is a Loro no-op on import. The
2241
+ // failure this replaces is unbounded in the other direction, an op the
2242
+ // server never received and nothing ever sends again.
2243
+ // Exported for the fake-timer tests.
2244
+ export const RESEND_FORCE_CEILING_MS = 60_000;
2245
+ // Server-shed recovery policy, driven by `unavailable` error frames. The
2246
+ // frame names no op, so a round re-asserts every outstanding op at once and
2247
+ // any further frame arriving before that round runs is part of the same shed
2248
+ // batch: coalescing them is what keeps N shed ops from fanning out into N
2249
+ // rounds of N resends. Backoff is 1s doubling to a 30s ceiling, so the budget
2250
+ // spans roughly a minute and a half of sustained shedding. An `op_ack` (the
2251
+ // server taking writes again) or a reconnect returns the budget to full.
2252
+ //
2253
+ // Module-scoped because both transports answer a shed on it: the signed-in
2254
+ // one, the HTTP write for an op past the frame cap, and the guest-link one.
2255
+ // A shed costs an edit the same nothing whoever wrote it and whichever
2256
+ // transport carried it.
2257
+ const UNAVAILABLE_MAX_ROUNDS = 6;
2258
+ const UNAVAILABLE_BASE_DELAY_MS = 1000;
2259
+ const UNAVAILABLE_MAX_DELAY_MS = 30_000;
2260
+ const UNAVAILABLE_GIVE_UP_MESSAGE = 'The server could not store this change, and re-sending it did not ' +
2261
+ 'help. Reload the document to continue from the last saved state.';
2262
+ // Reported for every op left behind when the server ends this document's
2263
+ // subscription (it refused the subscribe, or evicted it again after the one
2264
+ // silent re-subscribe a connection grants). The ops can never be acked on a
2265
+ // subscription that is over, and a fresh one arrives only with a fresh open,
2266
+ // so the app's cue is to reload.
2267
+ const SUBSCRIPTION_ENDED_MESSAGE = 'The live connection to this document ended, so this change was not ' +
2268
+ 'saved. Reload the document to continue from the last saved state.';
2269
+ /** How long the Nth (1-based) recovery round waits before re-asserting. */
2270
+ function unavailableDelayMs(round) {
2271
+ return Math.min(UNAVAILABLE_BASE_DELAY_MS * Math.pow(2, round - 1), UNAVAILABLE_MAX_DELAY_MS);
2272
+ }
1522
2273
  // A mutation carrying many records commits as it goes, so the only write
1523
2274
  // that reaches this is one record nothing can split.
1524
2275
  const TOO_LARGE_MESSAGE = 'This record is too large to save. A single record must fit in one ' +
1525
2276
  'update; store the bulk of it as a file instead, or split it across ' +
1526
2277
  'several records.';
2278
+ // What a guest hears for an op no request body can carry. A signed-in
2279
+ // editor stages such an op to storage first; that route is behind the
2280
+ // sign-in gate, so the anonymous tier writes ops that fit one request.
2281
+ const LINK_OP_TOO_LARGE_MESSAGE = 'This change is too large to save over a shared link. Sign in to the ' +
2282
+ 'document to make a change this size.';
1527
2283
  function wireDocTransport(args) {
1528
2284
  const topic = `doc:${args.docId}:${args.branchId}`;
1529
- // Per-op retry tracker. The op-write endpoint sometimes returns
1530
- // `{"message": "Internal server error"}` without reaching the handler
1531
- // (burst input against a cold-started server giving up before scale-up).
1532
- // That frame has no `type`, so the multiplexer drops it,
2285
+ // Per-op retry tracker. The op-write endpoint can answer with an untyped
2286
+ // error frame instead of reaching the handler. That frame has no `type`, so
2287
+ // the multiplexer drops it,
1533
2288
  // leaving the op stuck in `Document._outstandingByClientId` and the
1534
2289
  // actor's lamport stream with a gap that blocks other subscribers' Loro
1535
2290
  // imports from converging (Loro requires per-actor causal contiguity).
@@ -1541,7 +2296,130 @@ function wireDocTransport(args) {
1541
2296
  // retry lands the dependent chain converges everywhere. `doc.savingCount`
1542
2297
  // stays above zero until the `op_ack` lands (the "Saving…" surface); there
1543
2298
  // is no give-up to surface because the SDK keeps trying.
2299
+ // When each op this transport shipped was first committed, keyed the same
2300
+ // way the tracker is. The tracker itself carries no commit time (its
2301
+ // `dispatchedAt` restarts on every resend, which is the right basis for
2302
+ // deciding a resend and the wrong one for reporting how long a reader has
2303
+ // been waiting), and an op past the frame cap is never in the tracker at
2304
+ // all. Entries are dropped as their op settles, so the map's size is the
2305
+ // work outstanding on either transport.
2306
+ const opStartedAt = new Map();
2307
+ // Reconnects the socket had made when this transport last saw a
2308
+ // `doc_init`. The frame itself is not a reconnect signal: it rides every
2309
+ // `subscribed` ack, including the opening one and the silent re-subscribe
2310
+ // a connection grants after a mid-stream eviction, neither of which
2311
+ // involved the socket dropping. A report keyed off the frame therefore
2312
+ // fires when nothing is wrong, which is the fault this latch exists to
2313
+ // avoid, so it is keyed off the socket instead.
2314
+ let docInitReconnectCount = null;
1544
2315
  const outstandingResends = new Map();
2316
+ // How many bytes of op frames may be on the socket unacked at once.
2317
+ //
2318
+ // The transport drops the connection when a client pushes at it hard
2319
+ // enough, and it takes everything multiplexed onto that socket with it;
2320
+ // every op then goes unacked, the reconnect re-asserts them, and that
2321
+ // burst drops the connection again, so nothing is ever persisted.
2322
+ // Measured against one write, a client pushing 26 frames in 320 ms lost
2323
+ // the socket where one spreading 90 frames over 5.3 s did not, so what
2324
+ // this bounds is the RATE rather than any one frame's size.
2325
+ //
2326
+ // Which shape reaches it is worth knowing before tuning it. A bulk write
2327
+ // does not: `mutate` batches to a budget far above what a frame carries,
2328
+ // so its ops take the HTTP write and never enter this queue. What does is
2329
+ // an app issuing many separate mutations that each land under the frame
2330
+ // cap, a drag loop or a per-row import, where the op count is set by how
2331
+ // often the app calls rather than by how much it writes.
2332
+ //
2333
+ // A byte budget rather than a rate is what makes this self-tuning: sends
2334
+ // resume as acks land, so the client goes exactly as fast as the server
2335
+ // is answering and no faster, whatever that speed turns out to be. The
2336
+ // budget is generous enough to keep several ops in flight, so an ack's
2337
+ // round trip is not serialised into the write.
2338
+ const IN_FLIGHT_BUDGET_BYTES = 256 * 1024;
2339
+ // Ops committed but not yet on the socket, oldest first.
2340
+ const sendQueue = [];
2341
+ let inFlightBytes = 0;
2342
+ /** Put an op on the socket if the budget allows, else queue it. */
2343
+ function dispatchOp(clientOpId) {
2344
+ const entry = outstandingResends.get(clientOpId);
2345
+ if (!entry || entry.inFlight > 0)
2346
+ return;
2347
+ const size = entry.frame.op_blob.length;
2348
+ // An op larger than the whole budget goes on its own rather than
2349
+ // waiting for room that can never appear.
2350
+ if (inFlightBytes > 0 && inFlightBytes + size > IN_FLIGHT_BUDGET_BYTES) {
2351
+ if (!entry.queued) {
2352
+ entry.queued = true;
2353
+ entry.dispatchedAt = 0;
2354
+ sendQueue.push(clientOpId);
2355
+ }
2356
+ return;
2357
+ }
2358
+ entry.queued = false;
2359
+ entry.inFlight = size;
2360
+ entry.dispatchedAt = Date.now();
2361
+ inFlightBytes += size;
2362
+ args.wsClient.send(entry.frame);
2363
+ scheduleResend(clientOpId);
2364
+ }
2365
+ /** The oldest op currently on the socket, which is the one a gap blocks
2366
+ * the rest of this actor's stream behind. Map order is commit order, and
2367
+ * an op waiting on the in-flight budget carries no timer, so the first
2368
+ * dispatched entry is that op. */
2369
+ function isOldestDispatchedOp(clientOpId) {
2370
+ for (const [id, entry] of outstandingResends) {
2371
+ if (entry.dispatchedAt === 0)
2372
+ continue;
2373
+ return id === clientOpId;
2374
+ }
2375
+ return false;
2376
+ }
2377
+ /** Send whatever the freed budget now has room for. */
2378
+ function drainSendQueue() {
2379
+ while (sendQueue.length > 0) {
2380
+ const next = sendQueue[0];
2381
+ const entry = outstandingResends.get(next);
2382
+ // An op acked, rejected or abandoned while queued leaves its id behind;
2383
+ // dropping it here is what keeps `forgetOp` off a queue scan.
2384
+ if (!entry || !entry.queued) {
2385
+ sendQueue.shift();
2386
+ continue;
2387
+ }
2388
+ const size = entry.frame.op_blob.length;
2389
+ if (inFlightBytes > 0 && inFlightBytes + size > IN_FLIGHT_BUDGET_BYTES) {
2390
+ return;
2391
+ }
2392
+ sendQueue.shift();
2393
+ entry.queued = false;
2394
+ dispatchOp(next);
2395
+ }
2396
+ }
2397
+ /** Drop one op from the tracker, returning its budget. */
2398
+ function forgetOp(clientOpId) {
2399
+ const entry = outstandingResends.get(clientOpId);
2400
+ if (!entry)
2401
+ return;
2402
+ clearTimeout(entry.timer);
2403
+ inFlightBytes -= entry.inFlight;
2404
+ entry.inFlight = 0;
2405
+ entry.queued = false;
2406
+ outstandingResends.delete(clientOpId);
2407
+ opStartedAt.delete(clientOpId);
2408
+ drainSendQueue();
2409
+ }
2410
+ /** Drop every tracked op and reset the budget with them. */
2411
+ function forgetAllOps() {
2412
+ for (const [clientOpId, entry] of outstandingResends) {
2413
+ clearTimeout(entry.timer);
2414
+ // Only the ops this tracker holds. An op past the frame cap is
2415
+ // travelling over HTTP and is not in here, so clearing the whole
2416
+ // timing map would report it settled while its write is still going.
2417
+ opStartedAt.delete(clientOpId);
2418
+ }
2419
+ outstandingResends.clear();
2420
+ sendQueue.length = 0;
2421
+ inFlightBytes = 0;
2422
+ }
1545
2423
  // Ops this client is writing over HTTP, and the highest seq a catch-up
1546
2424
  // prompt named while one was in flight.
1547
2425
  //
@@ -1564,44 +2442,388 @@ function wireDocTransport(args) {
1564
2442
  const seq = heldCatchUpSeq;
1565
2443
  heldCatchUpSeq = 0;
1566
2444
  if (seq > 0 && args.getCoveredSeq() < seq)
1567
- args.resyncFromSnapshot?.();
1568
- }
1569
- // Backoff: 1.5s, ~1.6x per attempt, ceiling 30s. The ceiling keeps a doc
1570
- // on a dead connection from hammering; the reconnect-driven resend below
1571
- // fires the instant `doc_init` arrives, so the ceiling never delays
1572
- // recovery past the next reconnect.
1573
- const RESEND_BASE_DELAY_MS = 1500;
1574
- const RESEND_MAX_DELAY_MS = 30_000;
2445
+ pullJoin();
2446
+ }
2447
+ // Pull the join, and ask again if it left the reader short.
2448
+ //
2449
+ // A join read while a peer writes in bulk is a torn cut: the branch's seq
2450
+ // counter has run ahead of the op rows, so the response reports a head it
2451
+ // could not deliver the tail for. The reader applies what arrived and its
2452
+ // floor lands below that head, and nothing re-sends the difference. So
2453
+ // whatever the reader still lacks after an import is a hole it has to ask
2454
+ // about itself, which is what a passive reader has instead of the `doc_op`
2455
+ // and `op_ack` frames a writing client arms its checks from.
2456
+ function pullJoin() {
2457
+ void Promise.resolve(args.resyncFromSnapshot?.()).then(() => {
2458
+ scheduleGapCheck();
2459
+ }, () => {
2460
+ // `resyncFromSnapshot` reports its own failures and resolves; a
2461
+ // rejection is a caller that does not, and asking again is still the
2462
+ // right move.
2463
+ scheduleGapCheck();
2464
+ });
2465
+ }
2466
+ // Re-import the join for ops this client cannot bridge to from what it
2467
+ // holds, deferring the judgement while one of its own HTTP writes is still
2468
+ // in flight. Shared by the catch-up prompt and by the reconnect handshake,
2469
+ // which reaches the same conclusion off its tail.
2470
+ // Re-import when the durable frontier names an author whose changes never
2471
+ // came through the op stream. Separate from the coverage check because it
2472
+ // answers a different question: not "did I miss an op" but "did what I hold
2473
+ // stop being how this document is stored".
2474
+ function maybeResyncPastRewrite(vectorB64) {
2475
+ if (!vectorB64)
2476
+ return;
2477
+ let unseen = false;
2478
+ try {
2479
+ unseen = args.doc._snapshotHasUnseenAuthor(base64ToBytes(vectorB64));
2480
+ }
2481
+ catch {
2482
+ return;
2483
+ }
2484
+ if (unseen)
2485
+ pullJoin();
2486
+ }
2487
+ function resyncPastSeq(seq) {
2488
+ if (httpOpWritesInFlight > 0) {
2489
+ if (seq > heldCatchUpSeq)
2490
+ heldCatchUpSeq = seq;
2491
+ return;
2492
+ }
2493
+ pullJoin();
2494
+ }
1575
2495
  function resendDelayMs(attempts) {
1576
2496
  const grown = RESEND_BASE_DELAY_MS * Math.pow(1.6, attempts - 1);
1577
2497
  return Math.min(grown, RESEND_MAX_DELAY_MS);
1578
2498
  }
2499
+ // When this connection last proved it was working: any `op_ack`, from any
2500
+ // op on this topic.
2501
+ //
2502
+ // What this timer is for is a server that answered NOTHING, which is a
2503
+ // real case (an op frame can draw the opaque `{"message":"Internal server
2504
+ // error"}` reply the multiplexer drops). What it must not read as silence
2505
+ // is an ack that is merely behind, because with many ops in flight one
2506
+ // op's latency is the queue's depth times its service time, not evidence
2507
+ // about that op. Resending on that reading writes the op a second time,
2508
+ // which deepens the queue, which slows the next ack, which fires more
2509
+ // resends.
2510
+ //
2511
+ // Measured on a 500,000-element write: the same content went through in
2512
+ // 1,320 op writes on a run whose acks stayed under 375 ms, and in 3,600 on
2513
+ // a run whose acks reached 1,615 ms, just past this backoff's first step.
2514
+ // The extra writes are durable duplicates, so they inflate the document's
2515
+ // history, and every later compaction pays for them: the run that tripled
2516
+ // its ops left a tail compaction could not absorb inside its memory, and
2517
+ // the document stopped opening.
2518
+ //
2519
+ // So the timer fires on silence rather than on slowness: an ack that
2520
+ // arrived after this op's timer was armed re-arms it without spending an
2521
+ // attempt, within the bound `forceResendDue` puts on how long one op may
2522
+ // ride that evidence.
2523
+ let lastAckAt = 0;
1579
2524
  function scheduleResend(clientOpId) {
1580
2525
  const entry = outstandingResends.get(clientOpId);
1581
2526
  if (!entry)
1582
2527
  return;
2528
+ entry.armedAt = Date.now();
1583
2529
  entry.timer = setTimeout(() => {
1584
2530
  const current = outstandingResends.get(clientOpId);
1585
2531
  if (!current)
1586
2532
  return;
2533
+ const forced = forceResendDue(clientOpId, current);
2534
+ if (lastAckAt >= current.armedAt && !forced) {
2535
+ scheduleResend(clientOpId);
2536
+ return;
2537
+ }
2538
+ // A forced resend is the transport concluding this op was lost, which
2539
+ // is the one condition here the server cannot infer, so it is what the
2540
+ // report is pinned to.
2541
+ if (forced)
2542
+ reportSyncHealth('op_stuck');
1587
2543
  current.attempts += 1;
2544
+ current.dispatchedAt = Date.now();
1588
2545
  args.wsClient.send(current.frame);
1589
2546
  scheduleResend(clientOpId);
1590
2547
  }, resendDelayMs(entry.attempts));
1591
2548
  }
2549
+ // The one case where an answering connection is still evidence this op was
2550
+ // lost: it is the oldest op on the socket and it has been there longer than
2551
+ // any working connection takes to answer.
2552
+ //
2553
+ // The evidence `lastAckAt` carries is that the connection is working, not
2554
+ // that it received this op, so an app committing faster than the backoff
2555
+ // supplies that evidence continuously and a dropped op would re-arm free
2556
+ // forever. The server's copy of the document then stops at the gap (Loro
2557
+ // requires per-actor causal contiguity, so every later op is stored and
2558
+ // never applied), the tracker entry never reaches a terminal state, and the
2559
+ // app is told nothing.
2560
+ //
2561
+ // Ack ORDER cannot stand in for that evidence. Acks overtake heavily: 24
2562
+ // ops sent back to back on a deployed environment came back with 10 of 23
2563
+ // adjacent pairs out of send order, and the op sent first was acked 23rd. A
2564
+ // rule reading a later op's ack as proof this one was skipped re-sends ops
2565
+ // that are merely in flight, which is the write-it-twice failure the gate
2566
+ // above exists to prevent.
2567
+ //
2568
+ // Restricting this to the oldest dispatched op is what keeps the traffic
2569
+ // bounded and is sufficient. Bounded: one op per ceiling window however
2570
+ // many ops are outstanding, which matters because this resend hands the
2571
+ // frame straight to `wsClient.send` and so spends none of the in-flight
2572
+ // budget, and a simultaneous round of them is the burst that drops the
2573
+ // socket. Sufficient: the gap at the oldest op is what holds the rest of
2574
+ // this actor's stream unapplied, and an op behind it becomes the oldest in
2575
+ // its turn as the ops in front retire.
2576
+ function forceResendDue(clientOpId, entry) {
2577
+ if (entry.dispatchedAt === 0)
2578
+ return false;
2579
+ if (Date.now() - entry.dispatchedAt <= RESEND_FORCE_CEILING_MS)
2580
+ return false;
2581
+ return isOldestDispatchedOp(clientOpId);
2582
+ }
2583
+ // What this transport knows about its own sync, for a reader whose save
2584
+ // is taking longer than it should. Everything here is closure state the
2585
+ // transport already keeps; collecting it costs a walk of the outstanding
2586
+ // ops, so it is safe to call from a render.
2587
+ function collectDiagnostics() {
2588
+ const now = Date.now();
2589
+ let oldestClientOpId = null;
2590
+ let oldestStartedAt = 0;
2591
+ // Insertion order is commit order, so the first entry is the oldest.
2592
+ for (const [clientOpId, startedAt] of opStartedAt) {
2593
+ oldestClientOpId = clientOpId;
2594
+ oldestStartedAt = startedAt;
2595
+ break;
2596
+ }
2597
+ return {
2598
+ unackedOps: opStartedAt.size,
2599
+ oldestUnackedMs: oldestClientOpId === null ? 0 : Math.max(0, now - oldestStartedAt),
2600
+ oldestClientOpId,
2601
+ reconnects: args.wsClient.reconnectCount ?? 0,
2602
+ lastAckAgeMs: lastAckAt === 0 ? null : Math.max(0, now - lastAckAt),
2603
+ inFlightBytes,
2604
+ httpOpsInFlight: httpOpWritesInFlight,
2605
+ healRounds: divergenceHealRound,
2606
+ hidden: args.wsClient.pageHidden ?? false,
2607
+ };
2608
+ }
2609
+ // Push the same picture to the server, so an incident on a device with no
2610
+ // console attached leaves a record an operator can find. Sent on the
2611
+ // conditions the client can recognise and the server cannot, each of
2612
+ // which is already bounded by the machinery that detects it: a forced
2613
+ // resend fires at most once per ceiling window, a reconnect is one
2614
+ // socket, and a heal round is one push. The server drops the excess
2615
+ // anyway.
2616
+ //
2617
+ // Fire-and-forget by design. The frame carries no `client_op_id` of its
2618
+ // own and draws no reply, so it never joins the outstanding-op tracker
2619
+ // and a report cannot become one more thing a stuck document is waiting
2620
+ // on.
2621
+ function reportSyncHealth(reason) {
2622
+ // Only worth sending while the socket can carry it. Held through an
2623
+ // outage, a report flushes on the re-subscribe describing state from
2624
+ // before the outage, and it arrives on the new connection ahead of the
2625
+ // accurate one the reconnect itself produces, so the stale figures are
2626
+ // what the server's one-per-connection budget accepts.
2627
+ if (args.wsClient.connected === false)
2628
+ return;
2629
+ const d = collectDiagnostics();
2630
+ args.wsClient.send({
2631
+ type: 'client_report',
2632
+ topic,
2633
+ client_report: {
2634
+ reason,
2635
+ unacked_ops: d.unackedOps,
2636
+ oldest_unacked_ms: d.oldestUnackedMs,
2637
+ ...(d.oldestClientOpId === null
2638
+ ? {}
2639
+ : { oldest_client_op_id: d.oldestClientOpId }),
2640
+ reconnects: d.reconnects,
2641
+ last_ack_age_ms: d.lastAckAgeMs,
2642
+ in_flight_bytes: d.inFlightBytes,
2643
+ http_ops_in_flight: d.httpOpsInFlight,
2644
+ heal_rounds: d.healRounds,
2645
+ hidden: d.hidden,
2646
+ },
2647
+ });
2648
+ }
2649
+ args.doc._setDiagnosticsProvider(collectDiagnostics);
1592
2650
  // Resend every outstanding op now and restart its backoff. Called on
1593
2651
  // reconnect (`doc_init`): a dropped-and-restored connection is the most
1594
2652
  // common reason an op went unacked, and the backoff may have grown to its
1595
2653
  // ceiling while the socket was down, so re-asserting immediately is the
1596
2654
  // fast recovery path.
1597
2655
  function resendAllOutstanding() {
2656
+ // Nothing is on the wire any more, whether this follows a reconnect or a
2657
+ // shed, so the budget starts empty and every op re-enters through it.
2658
+ // Sending them all at once is what turns one dropped connection into a
2659
+ // loop that never persists anything: the burst that lost the socket is
2660
+ // exactly the burst the reconnect would repeat.
2661
+ inFlightBytes = 0;
2662
+ sendQueue.length = 0;
2663
+ // Insertion order is the order the ops were committed in, so a round
2664
+ // re-asserts them oldest-first.
1598
2665
  for (const [clientOpId, entry] of outstandingResends) {
1599
2666
  clearTimeout(entry.timer);
1600
2667
  entry.attempts += 1;
1601
- args.wsClient.send(entry.frame);
1602
- scheduleResend(clientOpId);
2668
+ entry.inFlight = 0;
2669
+ entry.queued = false;
2670
+ dispatchOp(clientOpId);
1603
2671
  }
1604
2672
  }
2673
+ let unavailableRounds = 0;
2674
+ let unavailableTimer = null;
2675
+ function clearUnavailableRecovery() {
2676
+ if (unavailableTimer !== null) {
2677
+ clearTimeout(unavailableTimer);
2678
+ unavailableTimer = null;
2679
+ }
2680
+ unavailableRounds = 0;
2681
+ }
2682
+ // Every outstanding op is dead: drop the tracker entries and tell the
2683
+ // Document, which stops counting them in `savingCount` and reports each
2684
+ // through the error channel the app reads (`Brass.onError`). The ids are
2685
+ // taken first because `_failOutstandingOp` re-enters app code.
2686
+ //
2687
+ // Dropping the ops is what opens the divergence, so the heal is armed here
2688
+ // rather than at the call site: the local doc keeps the abandoned ops and
2689
+ // keeps accepting edits that depend on them, and only the handshake can
2690
+ // hand the server the slice it needs to apply any of it.
2691
+ function failAllOutstanding(message) {
2692
+ const clientOpIds = [...outstandingResends.keys()];
2693
+ forgetAllOps();
2694
+ for (const clientOpId of clientOpIds) {
2695
+ args.doc._failOutstandingOp(clientOpId, message);
2696
+ }
2697
+ if (clientOpIds.length > 0) {
2698
+ divergenceHealRound = 0;
2699
+ scheduleDivergenceHeal();
2700
+ }
2701
+ }
2702
+ // One op is dead: report it and arm the heal, for the same reason
2703
+ // `failAllOutstanding` does. The op stays applied locally and every edit
2704
+ // made after it depends on it, so the server holds a stream it can store
2705
+ // and never apply, and its own escape for an op nothing can apply is to
2706
+ // discard it. Most refusals do not change that: the handshake pushes
2707
+ // whatever slice the durable frontier is missing and splits it under the
2708
+ // frame cap, so it heals the dependents even where the op itself is
2709
+ // irreducibly too large to re-send.
2710
+ //
2711
+ // `healable: false` is for the refusal where pushing again is asking the
2712
+ // same question. A full document refuses every op whatever its size, so
2713
+ // the handshake's own chunks come back refused, and the counter reset
2714
+ // below turns that into a loop rather than a bounded retry. What recovers
2715
+ // that document is the periodic re-verify, once the server can take a
2716
+ // write at all.
2717
+ // A refusal the heal cannot work around leaves the writer holding a change
2718
+ // that reached no database and no other reader, so the writer is the only
2719
+ // copy of the document that shows it. Left editable it reads as applied: the
2720
+ // element is on screen, the count includes it, every later edit builds on
2721
+ // it, and a reader who opens the document sees none of it. Nothing local
2722
+ // removes it either, because a CRDT merges rather than replaces and an undo
2723
+ // inverts an operation instead of discarding it, so the only state holding
2724
+ // exactly what is durable is one built from the server.
2725
+ //
2726
+ // Stopping the divergence and resolving it are separated, because they cost
2727
+ // different things and doing them in one silent step is what made a refusal
2728
+ // read as the platform losing work. Stopping is free and immediate:
2729
+ // `readOnly` turns true, which closes an app's existing edit gate, so the
2730
+ // copy cannot drift further. Resolving discards the refused work and swaps
2731
+ // the document under whoever was working in it, so the user is told first
2732
+ // and the reload follows their acknowledgement.
2733
+ //
2734
+ // The telling is the SDK's job rather than each app's. A copy left showing a
2735
+ // change nothing else has is the failure being closed, so an app that
2736
+ // renders no surface for this cannot be the reason a user stays on one.
2737
+ //
2738
+ // Once per document. The first refusal is what diverged this copy, so the
2739
+ // later ones describe the same state and have nothing to add.
2740
+ let lockedForRefusal = false;
2741
+ function lockForUnhealableRefusal(message) {
2742
+ if (lockedForRefusal)
2743
+ return;
2744
+ lockedForRefusal = true;
2745
+ // Torn down in the order the `doc_gone` handler uses, and for the same
2746
+ // reason: every one of these keeps working against a copy that is now
2747
+ // read-only, and the resync push in particular would re-offer the refused
2748
+ // op this exists to stop.
2749
+ forgetAllOps();
2750
+ clearUnavailableRecovery();
2751
+ if (reverifyTimer !== null) {
2752
+ clearInterval(reverifyTimer);
2753
+ reverifyTimer = null;
2754
+ }
2755
+ clearDivergenceHeal();
2756
+ if (gapTimer !== null) {
2757
+ clearTimeout(gapTimer);
2758
+ gapTimer = null;
2759
+ }
2760
+ args.doc._markReadOnly('unsaved');
2761
+ // The user is told by the SDK, not by each app, because an app that
2762
+ // renders nothing is exactly the case this has to cover: a document that
2763
+ // reloaded and a document that reloaded for a reason look the same
2764
+ // otherwise. `alert` holds the main thread until it is acknowledged, so
2765
+ // the reason cannot be missed and the reload lands on a page the user
2766
+ // is looking at. A browser that suppresses the dialog returns from it
2767
+ // immediately, which reloads rather than skipping, and a caller outside
2768
+ // a browser (a worker, a test) has nobody to tell and does the same.
2769
+ //
2770
+ // The reload survives a dialog that throws, which is the one way telling
2771
+ // the user could cost them the fix: `alert` is a page-global an app is
2772
+ // free to replace, and the copy this leaves behind is the diverged one
2773
+ // the reload exists to discard.
2774
+ try {
2775
+ if (typeof alert === 'function')
2776
+ alert(message);
2777
+ }
2778
+ catch {
2779
+ // Told or not, the copy is resolved.
2780
+ }
2781
+ args.doc.reload();
2782
+ }
2783
+ function failOpTerminally(clientOpId, message, opts = {}) {
2784
+ // Read before `_failOutstandingOp`, which clears the id from the
2785
+ // resync-push set on its way out.
2786
+ const wasHealPush = args.doc._isResyncPushOp(clientOpId);
2787
+ opStartedAt.delete(clientOpId);
2788
+ // A refusal the heal cannot answer is about the document, so it answers
2789
+ // every op in flight the same way and is reported once, by the lock. Per
2790
+ // op, a bulk write's outstanding ops each produce an identical error, so
2791
+ // the reader watches the same message arrive dozens of times before the
2792
+ // one dialog that explains it.
2793
+ args.doc._failOutstandingOp(clientOpId, message, {
2794
+ report: opts.report ?? true,
2795
+ });
2796
+ if (opts.healable === false) {
2797
+ lockForUnhealableRefusal(message);
2798
+ return;
2799
+ }
2800
+ // Restart the backoff only for a failure the heal did not itself
2801
+ // produce. A refused heal chunk that zeroed the round count would keep
2802
+ // the bound from ever binding: each round pushes, each push is refused,
2803
+ // and each refusal buys another round. Leaving the count alone lets the
2804
+ // rounds run out the way an unanswered handshake already does.
2805
+ if (!wasHealPush)
2806
+ divergenceHealRound = 0;
2807
+ scheduleDivergenceHeal();
2808
+ }
2809
+ function onServerUnavailable() {
2810
+ if (outstandingResends.size === 0)
2811
+ return;
2812
+ if (unavailableTimer !== null)
2813
+ return;
2814
+ unavailableRounds += 1;
2815
+ if (unavailableRounds > UNAVAILABLE_MAX_ROUNDS) {
2816
+ clearUnavailableRecovery();
2817
+ failAllOutstanding(UNAVAILABLE_GIVE_UP_MESSAGE);
2818
+ return;
2819
+ }
2820
+ const delay = unavailableDelayMs(unavailableRounds);
2821
+ unavailableTimer = setTimeout(() => {
2822
+ unavailableTimer = null;
2823
+ resendAllOutstanding();
2824
+ }, delay);
2825
+ unavailableTimer.unref?.();
2826
+ }
1605
2827
  // Durability handshake, run on every `doc_init`. The persisted version
1606
2828
  // vector + durable tail ops define the durable frontier; if the local doc
1607
2829
  // holds CRDT knowledge beyond it (an op acked then discarded by the
@@ -1613,7 +2835,12 @@ function wireDocTransport(args) {
1613
2835
  function maybeResyncPush(snapshotVersionVectorB64, durableTailBlobs) {
1614
2836
  if (!snapshotVersionVectorB64)
1615
2837
  return;
1616
- args.doc.maybeResyncPush(base64ToBytes(snapshotVersionVectorB64), durableTailBlobs);
2838
+ const outcome = args.doc.maybeResyncPush(base64ToBytes(snapshotVersionVectorB64), durableTailBlobs);
2839
+ // Reported here as well as from the re-verify, or the record holds only
2840
+ // the heals that took the slower route and an operator reading it cannot
2841
+ // tell that the ordinary reconnect handshake is missing from the set.
2842
+ if (outcome === 'pushed')
2843
+ reportSyncHealth('divergence_healed');
1617
2844
  }
1618
2845
  // Re-fetch the durable frontier from the HTTP join and run the handshake.
1619
2846
  // Shared by the `doc_ops_dropped` prompt path and the periodic re-verify:
@@ -1630,66 +2857,217 @@ function wireDocTransport(args) {
1630
2857
  let reverifyAgain = false;
1631
2858
  // Armed after the subscription is wired, cleared on `doc_gone` and teardown.
1632
2859
  let reverifyTimer = null;
2860
+ // The last run's verdict, so a caller healing a known divergence can tell a
2861
+ // finished handshake from one that fetched nothing.
1633
2862
  async function reverifyDurability() {
1634
2863
  if (reverifyInFlight) {
1635
2864
  reverifyAgain = true;
1636
- return;
2865
+ return 'in-flight';
1637
2866
  }
1638
2867
  reverifyInFlight = true;
2868
+ let outcome = 'no-fetch';
1639
2869
  try {
1640
2870
  do {
1641
2871
  reverifyAgain = false;
1642
2872
  try {
1643
2873
  const frontier = await args.refetchDurableFrontier?.();
1644
2874
  if (frontier && frontier.snapshotVersionVector) {
1645
- args.doc.maybeResyncPush(base64ToBytes(frontier.snapshotVersionVector), frontier.tailBlobs);
2875
+ outcome = args.doc.maybeResyncPush(base64ToBytes(frontier.snapshotVersionVector), frontier.tailBlobs);
2876
+ // The handshake having work to do means the server's copy was
2877
+ // behind this client's, which is the state a lost op leaves and
2878
+ // which nothing on the server side reports.
2879
+ if (outcome === 'pushed')
2880
+ reportSyncHealth('divergence_healed');
1646
2881
  }
1647
2882
  }
1648
2883
  catch {
1649
2884
  // Best-effort: a failed re-fetch defers the check to the next tick
1650
2885
  // or reconnect's `doc_init` handshake. Swallowed so the timer's
1651
2886
  // fire-and-forget never surfaces an unhandled rejection and one
1652
- // failed tick does not wedge the loop guard.
2887
+ // failed tick does not wedge the loop guard. The `no-fetch` verdict
2888
+ // is what keeps a divergence heal asking.
2889
+ outcome = 'no-fetch';
1653
2890
  }
1654
2891
  } while (reverifyAgain);
1655
2892
  }
1656
2893
  finally {
1657
2894
  reverifyInFlight = false;
1658
2895
  }
2896
+ return outcome;
2897
+ }
2898
+ // Divergence heal. Giving up on an op leaves the local doc holding CRDT
2899
+ // knowledge the server does not, and Loro requires per-actor causal
2900
+ // contiguity, so every op the app writes afterwards reaches the server as
2901
+ // causally unapplyable and stays that way until the missing slice arrives.
2902
+ // The server's own escape for an op nothing can apply is to discard it, so
2903
+ // the window between the give-up and the next handshake is a window where
2904
+ // the app writes into a document that keeps none of it.
2905
+ //
2906
+ // The periodic re-verify would eventually close that window, but on its own
2907
+ // schedule: a bulk write finishes and the reader reloads well inside one
2908
+ // tick. A give-up is the moment the client knows it has diverged, so it
2909
+ // drives the handshake itself, and keeps driving it on a backoff until one
2910
+ // run reports the durable frontier covering the local doc. The first delay
2911
+ // is not zero because the give-up follows sustained shedding, so the fetch
2912
+ // this schedules would join the burst it is recovering from.
2913
+ const DIVERGENCE_HEAL_BASE_DELAY_MS = 2_000;
2914
+ const DIVERGENCE_HEAL_MAX_DELAY_MS = 30_000;
2915
+ // Roughly four minutes of attempts, after which the periodic re-verify is
2916
+ // the remaining backstop and the tab has long since reported the failure.
2917
+ const DIVERGENCE_HEAL_MAX_ROUNDS = 12;
2918
+ let divergenceHealRound = 0;
2919
+ let divergenceHealTimer = null;
2920
+ function scheduleDivergenceHeal() {
2921
+ if (divergenceHealTimer !== null)
2922
+ return;
2923
+ if (divergenceHealRound >= DIVERGENCE_HEAL_MAX_ROUNDS)
2924
+ return;
2925
+ divergenceHealRound += 1;
2926
+ const delay = Math.min(DIVERGENCE_HEAL_BASE_DELAY_MS * Math.pow(2, divergenceHealRound - 1), DIVERGENCE_HEAL_MAX_DELAY_MS);
2927
+ divergenceHealTimer = setTimeout(() => {
2928
+ divergenceHealTimer = null;
2929
+ void reverifyDurability().then((outcome) => {
2930
+ // `covered` is the only verdict that proves the hole is closed. A
2931
+ // push that just went out is not yet durable, so the next round
2932
+ // re-checks it rather than assuming its acks land, and the backoff
2933
+ // it comes back on is what keeps a push settling from being polled.
2934
+ if (outcome !== 'covered')
2935
+ scheduleDivergenceHeal();
2936
+ else
2937
+ divergenceHealRound = 0;
2938
+ });
2939
+ }, delay);
2940
+ divergenceHealTimer.unref?.();
2941
+ }
2942
+ function clearDivergenceHeal() {
2943
+ if (divergenceHealTimer !== null) {
2944
+ clearTimeout(divergenceHealTimer);
2945
+ divergenceHealTimer = null;
2946
+ }
2947
+ divergenceHealRound = 0;
1659
2948
  }
1660
2949
  // A coverage floor below the highest seq seen records ops this client
1661
2950
  // applied nothing for, and the server posts each op once, so nothing
1662
2951
  // re-sends them. Pull the join instead, after a delay long enough for an
1663
2952
  // out-of-order arrival to close the hole itself.
2953
+ //
2954
+ // The delay is only meaningful against the hole this check was ARMED on,
2955
+ // which is why the seq it waits to reach is captured here rather than read
2956
+ // again when it fires. Holes are the ordinary state of a client that is
2957
+ // writing: acks overtake heavily, so the newest acked seq sits above the
2958
+ // contiguous floor almost continuously, and each such hole closes as the ack
2959
+ // below it lands. Asking on expiry whether ANY hole exists reads a
2960
+ // brand-new one, opened by ops sent since, as the original persisting, and a
2961
+ // client that keeps writing always has one. That answer costs a join
2962
+ // re-import, which discards the undo and redo stacks, so the reader loses
2963
+ // their history every couple of seconds while nothing is wrong.
2964
+ //
2965
+ // Rounds that move nothing back off and then stop, and any advance in the
2966
+ // floor starts the budget over. What separates the two cases is exactly
2967
+ // that: a hole left by a write still in flight fills as the rows land, so
2968
+ // each pull delivers more and earns the next one, while a seq the server
2969
+ // allocated and never wrote (a failed op write leaves a hole rather than a
2970
+ // duplicate) delivers nothing however often it is asked for. Without the
2971
+ // budget that second hole is a join re-import every couple of seconds for
2972
+ // the life of the tab, each one discarding the reader's undo history. It
2973
+ // clears at the next compaction, which folds the tail past the hole.
2974
+ const GAP_PULL_MAX_ROUNDS = 6;
2975
+ const GAP_PULL_MAX_DELAY_MS = 30_000;
1664
2976
  let gapTimer = null;
2977
+ let gapPullRounds = 0;
2978
+ let gapPullFloor = -1;
1665
2979
  function scheduleGapCheck() {
1666
2980
  if (gapTimer !== null)
1667
2981
  return;
1668
- if (args.getCoveredSeq() >= args.getCurrentSeq())
2982
+ const awaited = args.getCurrentSeq();
2983
+ const floor = args.getCoveredSeq();
2984
+ if (floor >= awaited) {
2985
+ gapPullRounds = 0;
2986
+ return;
2987
+ }
2988
+ if (floor > gapPullFloor)
2989
+ gapPullRounds = 0;
2990
+ if (gapPullRounds >= GAP_PULL_MAX_ROUNDS)
1669
2991
  return;
2992
+ gapPullRounds += 1;
2993
+ gapPullFloor = floor;
2994
+ const delay = Math.min(MISSED_OP_RESYNC_DELAY_MS * Math.pow(2, gapPullRounds - 1), GAP_PULL_MAX_DELAY_MS);
1670
2995
  gapTimer = setTimeout(() => {
1671
2996
  gapTimer = null;
1672
- if (args.getCoveredSeq() >= args.getCurrentSeq())
2997
+ // Covered up to where the hole was: it closed on its own, whatever has
2998
+ // been sent since. Anything newer arms its own check.
2999
+ if (args.getCoveredSeq() >= awaited) {
3000
+ gapPullRounds = 0;
1673
3001
  return;
1674
- args.resyncFromSnapshot?.();
1675
- }, MISSED_OP_RESYNC_DELAY_MS);
3002
+ }
3003
+ pullJoin();
3004
+ }, delay);
1676
3005
  gapTimer.unref?.();
1677
3006
  }
3007
+ // The server ended this subscription: it refused the subscribe (a revoked
3008
+ // grant), or evicted it again after the one silent re-subscribe a
3009
+ // connection grants. The client detaches this transport from the topic as
3010
+ // it reports that, so no ack, no fanout and no reconnect reaches it again.
3011
+ //
3012
+ // Everything the transport is waiting on is therefore dead. The ops it
3013
+ // holds can never be acked, and re-sending them puts frames on a
3014
+ // subscription the server has refused while `savingCount` climbs and the
3015
+ // app hears nothing, so they are failed here and every later edit fails
3016
+ // the same way: the app reads one actionable error per unsaved change and
3017
+ // its cue to reload, rather than a document that looks like it is saving
3018
+ // forever. The ids are taken before the reports because
3019
+ // `_failOutstandingOp` re-enters app code.
3020
+ let subscriptionEnded = false;
3021
+ function endSubscription() {
3022
+ if (subscriptionEnded)
3023
+ return;
3024
+ subscriptionEnded = true;
3025
+ const clientOpIds = [...outstandingResends.keys()];
3026
+ forgetAllOps();
3027
+ clearUnavailableRecovery();
3028
+ // The heal and the gap pull both work through this transport, so they
3029
+ // are stopped rather than armed: what they would push or fetch has no
3030
+ // subscription left to travel on.
3031
+ clearDivergenceHeal();
3032
+ if (reverifyTimer !== null) {
3033
+ clearInterval(reverifyTimer);
3034
+ reverifyTimer = null;
3035
+ }
3036
+ if (gapTimer !== null) {
3037
+ clearTimeout(gapTimer);
3038
+ gapTimer = null;
3039
+ }
3040
+ for (const clientOpId of clientOpIds) {
3041
+ args.doc._failOutstandingOp(clientOpId, SUBSCRIPTION_ENDED_MESSAGE);
3042
+ }
3043
+ }
1678
3044
  // 1. Subscribe to the doc topic. Frame routing:
1679
3045
  // - `doc_init`: bridge the join-vs-fanout race. Ops committed between
1680
3046
  // the HTTP join (T0) and the WS subscribe (T1) miss the HTTP tail and
1681
- // can race the SUB# row so the first `doc_op` fanout may arrive BEFORE
3047
+ // can race the subscription record so the first `doc_op` fanout may arrive BEFORE
1682
3048
  // doc_init (separate server invocations, no wire ordering). tail_ops
1683
3049
  // apply unconditionally: filtering on `seq > currentSeq` would mask
1684
3050
  // tail_ops a racing `doc_op` inflated the cursor past, leaving lamport
1685
3051
  // gaps Loro cannot converge across (re-applying is an idempotent
1686
3052
  // no-op).
3053
+ // - `doc_catch_up`: the document moved past this client's coverage on
3054
+ // an op the socket could not carry; re-import the join and nothing
3055
+ // more (see the branch).
1687
3056
  // - `doc_op`: apply, advance cursor.
1688
3057
  // - `op_ack`: resolve the outstanding-op tracker, advance cursor, cancel
1689
3058
  // that op's resend timer.
1690
3059
  // - everything else: handled by `BrassWsClient`.
1691
3060
  const unsub = args.wsClient.subscribe(topic, (frame) => {
1692
- if (isDocInitFrame(frame)) {
3061
+ const init = parseDocInitFrame(frame);
3062
+ if (init) {
3063
+ // Recorded as the frame arrives rather than after the tail applies.
3064
+ // Importing a tail op can throw, and the client swallows it, so a
3065
+ // latch written further down would stay behind and read the NEXT
3066
+ // reconnect as the opening subscribe, suppressing the report this
3067
+ // exists to make.
3068
+ const reconnects = args.wsClient.reconnectCount ?? 0;
3069
+ const socketDropped = docInitReconnectCount !== null && reconnects > docInitReconnectCount;
3070
+ docInitReconnectCount = reconnects;
1693
3071
  // Resync check BEFORE applying. The tail covers (snapshot_seq,
1694
3072
  // current_seq]; if the coverage floor sits below the tail's lower edge
1695
3073
  // (or below current_seq when the tail is empty), ops in between were
@@ -1697,16 +3075,23 @@ function wireDocTransport(args) {
1697
3075
  // this tail or future fanout, so re-import the join's snapshot. The
1698
3076
  // initial subscribe never trips this (the floor was just seeded), so
1699
3077
  // the cost is paid only on a reconnect that crossed a compaction.
1700
- const minTailSeq = frame.tail_ops.reduce((acc, entry) => (acc === null || entry.seq < acc ? entry.seq : acc), null);
1701
- const snapshotFloor = minTailSeq !== null ? minTailSeq - 1 : frame.current_seq;
3078
+ const snapshotFloor = snapshotSeqOf(init.snapshot_seq, init.tail_ops.map((entry) => entry.seq), init.current_seq);
1702
3079
  const needsResync = args.getCoveredSeq() < snapshotFloor;
1703
3080
  const tailBlobs = [];
1704
- for (const entry of frame.tail_ops) {
3081
+ // Whether the frame's own entries can describe the durable frontier.
3082
+ // An entry the server withheld the bytes of still NAMES an op the
3083
+ // server holds, so a frontier folded from the inline entries alone
3084
+ // reports every withheld op as missing from the server. See the
3085
+ // handshake call below.
3086
+ let tailComplete = true;
3087
+ for (const entry of init.tail_ops) {
1705
3088
  // An op with no inline bytes is deliberately left uncovered: it
1706
3089
  // has not been applied, so marking it covered would retire the
1707
3090
  // very resync that fetches it.
1708
- if (entry.op_blob_b64 === undefined)
3091
+ if (entry.op_blob_b64 === undefined) {
3092
+ tailComplete = false;
1709
3093
  continue;
3094
+ }
1710
3095
  const blob = base64ToBytes(entry.op_blob_b64);
1711
3096
  tailBlobs.push(blob);
1712
3097
  args.doc.applyRemoteOp(blob);
@@ -1714,31 +3099,111 @@ function wireDocTransport(args) {
1714
3099
  args.setCurrentSeq(entry.seq);
1715
3100
  args.advanceCoveredSeq(entry.seq);
1716
3101
  }
1717
- if (frame.current_seq > args.getCurrentSeq()) {
1718
- args.setCurrentSeq(frame.current_seq);
3102
+ // The frame also has to ACCOUNT for every seq the head names, not
3103
+ // merely carry bytes for the entries it lists. The server drops an
3104
+ // oversized or spilled tail from this frame ENTIRELY rather than
3105
+ // listing it byte-less (`document.tail.dropped`), so an empty tail on
3106
+ // a branch whose head sits above its snapshot is not a branch with
3107
+ // nothing in flight: it is the whole tail withheld. Measured on a
3108
+ // real document: snapshot_seq 565, current_seq 1089, 524 ops dropped
3109
+ // from the frame and every one of them already imported by the HTTP
3110
+ // join, so a frontier folded from this frame alone sat 524 ops behind
3111
+ // the local doc and pushed them back on every open.
3112
+ if (typeof init.snapshot_seq === 'number' &&
3113
+ tailBlobs.length < init.current_seq - init.snapshot_seq) {
3114
+ tailComplete = false;
3115
+ }
3116
+ if (init.current_seq > args.getCurrentSeq()) {
3117
+ args.setCurrentSeq(init.current_seq);
1719
3118
  }
1720
3119
  if (needsResync) {
1721
- if (httpOpWritesInFlight > 0) {
1722
- if (frame.current_seq > heldCatchUpSeq)
1723
- heldCatchUpSeq = frame.current_seq;
1724
- }
1725
- else {
1726
- args.resyncFromSnapshot?.();
1727
- }
3120
+ resyncPastSeq(init.current_seq);
3121
+ }
3122
+ else {
3123
+ // A frame that named a head it did not carry the tail for leaves
3124
+ // the reader short with nothing further coming: the server drops an
3125
+ // oversized or spilled tail from this frame and posts each op once.
3126
+ // For a reader that never writes, this and the catch-up prompt are
3127
+ // the only two places a gap gets noticed at all.
3128
+ scheduleGapCheck();
1728
3129
  }
1729
3130
  // `doc_init` is the reconnect signal the retry-forever recovery hangs
1730
3131
  // off: re-assert every outstanding op and run the durability handshake
1731
3132
  // now, so a client holding work the server lost re-asserts it on
1732
- // reconnect rather than waiting out the per-op backoff.
3133
+ // reconnect rather than waiting out the per-op backoff. A fresh
3134
+ // connection also returns the shed budget to full: this re-assert is
3135
+ // the round the pending one would have run.
3136
+ clearUnavailableRecovery();
3137
+ // A reconnect, rather than the initial subscribe: the socket dropped
3138
+ // while this document held work, which is the condition the server
3139
+ // has no way to see. The first `doc_init` arrives on the opening
3140
+ // subscribe, and an app that writes as soon as it opens already has
3141
+ // ops outstanding by then, so reporting on every `doc_init` would
3142
+ // fire routinely on a healthy first open and train a reader of the
3143
+ // log to discount the event.
3144
+ //
3145
+ // Before the round rather than after, so the counters describe the
3146
+ // work the reconnect found rather than the work it has just put back
3147
+ // on the wire.
3148
+ if (socketDropped && outstandingResends.size > 0) {
3149
+ reportSyncHealth('reconnected_unsaved');
3150
+ }
1733
3151
  resendAllOutstanding();
1734
- // Tolerate a non-string vector (a future retyping) as absent: the
1735
- // handshake is skipped, but the frame still delivers.
1736
- maybeResyncPush(typeof frame.snapshot_version_vector === 'string'
1737
- ? frame.snapshot_version_vector
1738
- : undefined, tailBlobs);
1739
- // Posted after the SUB# row exists, so this is the safe point to
3152
+ // The handshake needs the WHOLE durable tail, and this frame carries
3153
+ // it only when every entry came inline. A spilled op is named here
3154
+ // with its bytes withheld, and the HTTP join the reader already ran
3155
+ // resolves and imports it, so a frontier built from the inline
3156
+ // entries alone sits behind the local doc by exactly the ops that
3157
+ // spilled, and the handshake pushes the reader's copy of the server's
3158
+ // own tail back at it. On a branch at capacity every chunk is refused
3159
+ // and reported, which is an error on screen for someone who only
3160
+ // reopened the document; on any other branch it lands, writing a
3161
+ // second durable copy of every spilled op and growing the document
3162
+ // toward the ceiling on each open. `reverifyDurability` fetches
3163
+ // through `fetchSnapshot`, which resolves spilled entries, so the
3164
+ // frontier it diffs against is complete.
3165
+ if (tailComplete) {
3166
+ maybeResyncPush(init.snapshot_version_vector, tailBlobs);
3167
+ }
3168
+ else {
3169
+ void reverifyDurability();
3170
+ }
3171
+ // The tail has applied, so every author this client can hear from is
3172
+ // in its doc. An author left in the durable frontier is one whose
3173
+ // changes exist only inside the snapshot, which is the platform having
3174
+ // rewritten how the document stores what it stores. A client cannot
3175
+ // undo across that: its stack inverts the operations it made, and the
3176
+ // values those operations wrote have moved, so an undo would remove
3177
+ // nothing and report success. Re-import instead, which drops both
3178
+ // stacks and leaves the reader on fresh state with undo honestly
3179
+ // unavailable.
3180
+ //
3181
+ // Coverage does not catch this on its own, and the case it misses is
3182
+ // the ordinary one: the fold runs on a branch nobody has open, so the
3183
+ // client that returns to it missed no ops and has no gap to notice.
3184
+ if (!needsResync)
3185
+ maybeResyncPastRewrite(init.snapshot_version_vector);
3186
+ // Posted after the subscription record exists, so this is the safe point to
1740
3187
  // reconcile a pending open whose terminal status frame was dropped.
1741
3188
  args.reconcileStatusOnInit?.();
3189
+ args.onDocumentMoved?.();
3190
+ return;
3191
+ }
3192
+ if (isDocCatchUpFrame(frame)) {
3193
+ // The document moved past what this client can bridge to, so pull the
3194
+ // join. Nothing else: the prompt describes the document rather than
3195
+ // this subscription, and every subscriber gets one on each op the
3196
+ // socket could not carry. Re-asserting outstanding ops here writes a
3197
+ // second durable copy of each of them (the server allocates a fresh
3198
+ // seq per send) for every such op any writer makes, and cancelling
3199
+ // the shed recovery here parks a shed op on its per-op backoff
3200
+ // instead of the schedule the shed armed.
3201
+ if (frame.current_seq > args.getCurrentSeq()) {
3202
+ args.setCurrentSeq(frame.current_seq);
3203
+ }
3204
+ if (args.getCoveredSeq() < frame.current_seq) {
3205
+ resyncPastSeq(frame.current_seq);
3206
+ }
1742
3207
  return;
1743
3208
  }
1744
3209
  if (isDocOpFrame(frame)) {
@@ -1753,20 +3218,28 @@ function wireDocTransport(args) {
1753
3218
  args.setCurrentSeq(frame.seq);
1754
3219
  args.advanceCoveredSeq(frame.seq);
1755
3220
  scheduleGapCheck();
3221
+ args.onDocumentMoved?.();
1756
3222
  return;
1757
3223
  }
1758
3224
  if (isOpAckFrame(frame)) {
3225
+ // Evidence the connection is working through its queue, which is what
3226
+ // the resend timers read to tell a slow ack from a silent server.
3227
+ lastAckAt = Date.now();
1759
3228
  args.doc.resolveOutstandingOp(frame.client_op_id);
1760
3229
  args.setCurrentSeq(frame.seq);
1761
3230
  args.advanceCoveredSeq(frame.seq);
1762
3231
  // An ack for a local op sits above any remote op this client never
1763
3232
  // received, so the user's own next edit is a second chance to notice.
1764
3233
  scheduleGapCheck();
1765
- const pending = outstandingResends.get(frame.client_op_id);
1766
- if (pending) {
1767
- clearTimeout(pending.timer);
1768
- outstandingResends.delete(frame.client_op_id);
1769
- }
3234
+ forgetOp(frame.client_op_id);
3235
+ // The server is durably accepting writes again, so the shed budget
3236
+ // starts over. A round already scheduled still runs: the ops it
3237
+ // re-asserts are the ones this ack did not cover.
3238
+ unavailableRounds = 0;
3239
+ return;
3240
+ }
3241
+ if (isServerUnavailableFrame(frame)) {
3242
+ onServerUnavailable();
1770
3243
  return;
1771
3244
  }
1772
3245
  if (isOpRejectedFrame(frame)) {
@@ -1774,29 +3247,52 @@ function wireDocTransport(args) {
1774
3247
  // draw the same rejection forever and pin `savingCount`) and fail it on
1775
3248
  // the Document so the app hears "could not be saved" instead of a silent
1776
3249
  // wedge. Scoped to this one op; the topic subscription stays live.
1777
- const pending = outstandingResends.get(frame.client_op_id);
1778
- if (pending) {
1779
- clearTimeout(pending.timer);
1780
- outstandingResends.delete(frame.client_op_id);
1781
- }
1782
- args.doc._failOutstandingOp(frame.client_op_id, opRejectedMessage(frame));
3250
+ forgetOp(frame.client_op_id);
3251
+ // The flip is the one moment the refusal is worth reporting: every op
3252
+ // in flight draws the same sentence, and a bulk write has as many of
3253
+ // those as it has unacked writes.
3254
+ const atCapacity = isCapacityRejection(frame);
3255
+ const firstCapacityRefusal = atCapacity && args.doc._markAtCapacity();
3256
+ // A refusal that names the writer rather than the op applies to
3257
+ // every op this reader will send, so it turns the whole Document
3258
+ // read-only before the failure is reported: the app's own
3259
+ // `doc.readOnly` gate then disables the editor in the same tick the
3260
+ // reader hears why, rather than letting them keep typing into a
3261
+ // surface nothing will save.
3262
+ const readOnly = readOnlyReasonForRejection(frame);
3263
+ if (readOnly !== null)
3264
+ args.doc._markReadOnly(readOnly);
3265
+ // NO capacity refusal is healable, whichever kind it is. The refused
3266
+ // work sits on this copy and nowhere else, and the heal is the only
3267
+ // thing that could place it: when it runs out of rounds it stops
3268
+ // silently, so a copy the handshake could not settle is one showing
3269
+ // work that a reload will not, with nothing said. Both kinds therefore
3270
+ // lock, tell and reload, which is what the server's own wording for
3271
+ // each of them already promises.
3272
+ //
3273
+ // The other refusals here stay healable: the handshake splits the
3274
+ // slice under the frame cap, which is what reaches the dependents of
3275
+ // an op that is itself too large to re-send.
3276
+ failOpTerminally(frame.client_op_id, opRejectedMessage(frame), {
3277
+ healable: !atCapacity,
3278
+ report: !atCapacity || firstCapacityRefusal,
3279
+ });
1783
3280
  return;
1784
3281
  }
1785
3282
  if (isDocGoneFrame(frame)) {
1786
3283
  // Cancel every resend timer FIRST so a doc that was saving when delete
1787
3284
  // landed stops re-trying ops the server will only reject as `forbidden`
1788
- // (the SUB# row was evicted at fanout). Draining before the `_markGone`
3285
+ // (the subscription record was evicted at fanout). Draining before the `_markGone`
1789
3286
  // listeners keeps the lifecycle observably clean.
1790
- for (const pending of outstandingResends.values()) {
1791
- clearTimeout(pending.timer);
1792
- }
1793
- outstandingResends.clear();
3287
+ forgetAllOps();
3288
+ clearUnavailableRecovery();
1794
3289
  // Stop re-verifying durability against an evicted doc: a resync push
1795
3290
  // would only bounce as `forbidden`, and a gap pull would only 404.
1796
3291
  if (reverifyTimer !== null) {
1797
3292
  clearInterval(reverifyTimer);
1798
3293
  reverifyTimer = null;
1799
3294
  }
3295
+ clearDivergenceHeal();
1800
3296
  if (gapTimer !== null) {
1801
3297
  clearTimeout(gapTimer);
1802
3298
  gapTimer = null;
@@ -1805,13 +3301,7 @@ function wireDocTransport(args) {
1805
3301
  return;
1806
3302
  }
1807
3303
  if (isDocStatusFrame(frame)) {
1808
- // The validator gates only on `pending`; default `message` here so a
1809
- // message-less status still carries a valid `DocumentWorkStatus`
1810
- // (`message` is typed non-null and rendered verbatim).
1811
- const status = {
1812
- ...frame.status,
1813
- message: typeof frame.status.message === 'string' ? frame.status.message : '',
1814
- };
3304
+ const status = normalizeWorkStatus(frame.status);
1815
3305
  // `reload` is the server's authoritative "succeeded, content ready"
1816
3306
  // hint: pull the content in and emit the terminal status only AFTER the
1817
3307
  // import lands, so an app re-reading on `onStatus` sees content present.
@@ -1835,6 +3325,14 @@ function wireDocTransport(args) {
1835
3325
  args.doc._emitDriveStatus(toDriveStatus(frame.drive));
1836
3326
  return;
1837
3327
  }
3328
+ if (isDocDrivePublishFrame(frame)) {
3329
+ args.doc._emitPublishState({
3330
+ publishing: frame.publish.publishing === true,
3331
+ failed: frame.publish.failed === true,
3332
+ detail: typeof frame.publish.detail === 'string' ? frame.publish.detail : null,
3333
+ });
3334
+ return;
3335
+ }
1838
3336
  if (isDocOpsDroppedFrame(frame)) {
1839
3337
  // Prompt durability re-check: the discard sealed a new snapshot, so
1840
3338
  // re-fetch the moved frontier and run the handshake now instead of at
@@ -1844,8 +3342,14 @@ function wireDocTransport(args) {
1844
3342
  void reverifyDurability();
1845
3343
  return;
1846
3344
  }
1847
- // Unknown frame: ignore, so server-side schema growth doesn't break older
1848
- // SDK builds.
3345
+ // Unknown frame: ignore, so server-side schema growth doesn't break
3346
+ // older SDK builds.
3347
+ }, {
3348
+ // `resendAllOutstanding` on the `doc_init` that follows every
3349
+ // `subscribed` ack IS this topic's re-assertion, so the client drops an
3350
+ // op frame it cannot send now rather than buffering a second copy.
3351
+ ownsFrameRetry: true,
3352
+ onEnded: endSubscription,
1849
3353
  });
1850
3354
  // Arm the periodic durability re-verify (the long-lived-tab backstop).
1851
3355
  // `unref` so the interval never keeps a Node host alive (no-op in the
@@ -1859,6 +3363,22 @@ function wireDocTransport(args) {
1859
3363
  // tracker so a silently-dropped server response triggers a retry.
1860
3364
  args.doc.setLocalOpHook((opBlob) => {
1861
3365
  const clientOpId = args.newClientOpId();
3366
+ // Nothing carries an op once the subscription is over, so it is failed
3367
+ // instead of tracked: an entry nothing can ack pins `savingCount` and
3368
+ // its resends reach a topic the server refused. The caller records the
3369
+ // entry AFTER this hook returns, so the failure waits a microtask for
3370
+ // there to be one to fail.
3371
+ //
3372
+ // Taken before the op is timed, because this path fails the op through
3373
+ // the Document directly rather than through `failOpTerminally`, so a
3374
+ // timing entry written here is one nothing ever drops.
3375
+ if (subscriptionEnded) {
3376
+ void Promise.resolve().then(() => {
3377
+ args.doc._failOutstandingOp(clientOpId, SUBSCRIPTION_ENDED_MESSAGE);
3378
+ });
3379
+ return clientOpId;
3380
+ }
3381
+ opStartedAt.set(clientOpId, Date.now());
1862
3382
  const opBlobB64 = bytesToBase64(opBlob);
1863
3383
  const sendFrame = {
1864
3384
  type: 'op',
@@ -1876,14 +3396,18 @@ function wireDocTransport(args) {
1876
3396
  void writeOversizedOp(opBlobB64, clientOpId);
1877
3397
  return clientOpId;
1878
3398
  }
1879
- args.wsClient.send(sendFrame);
1880
3399
  outstandingResends.set(clientOpId, {
1881
3400
  frame: sendFrame,
1882
3401
  attempts: 1,
1883
- // Placeholder, overwritten immediately below.
3402
+ // Placeholders, overwritten by `dispatchOp` and `scheduleResend` once
3403
+ // the op is actually on the socket.
1884
3404
  timer: setTimeout(() => undefined, 0),
3405
+ armedAt: 0,
3406
+ dispatchedAt: 0,
3407
+ inFlight: 0,
3408
+ queued: false,
1885
3409
  });
1886
- scheduleResend(clientOpId);
3410
+ dispatchOp(clientOpId);
1887
3411
  return clientOpId;
1888
3412
  });
1889
3413
  async function writeOversizedOp(opBlobB64, clientOpId) {
@@ -1907,32 +3431,68 @@ function wireDocTransport(args) {
1907
3431
  // The size is known here, so refuse without spending a round trip on a
1908
3432
  // blob that can only come back 400.
1909
3433
  if (opBlobExceedsDurableLimit(opBlobB64)) {
1910
- args.doc._failOutstandingOp(clientOpId, TOO_LARGE_MESSAGE);
3434
+ failOpTerminally(clientOpId, TOO_LARGE_MESSAGE);
1911
3435
  return;
1912
3436
  }
1913
3437
  if (!args.postOversizedOp) {
1914
- args.doc._failOutstandingOp(clientOpId, TOO_LARGE_MESSAGE);
3438
+ failOpTerminally(clientOpId, TOO_LARGE_MESSAGE);
1915
3439
  return;
1916
3440
  }
3441
+ // An oversized op is deliberately kept out of `outstandingResends` (a
3442
+ // frame past the cap must never reach the socket), so the shed recovery
3443
+ // that re-asserts the tracker cannot reach this one. It runs its own
3444
+ // round of the same policy instead, on the same budget and backoff, so a
3445
+ // capacity refusal costs an op the same nothing whichever transport
3446
+ // carried it.
1917
3447
  let result;
1918
- try {
1919
- result = await args.postOversizedOp(opBlobB64, clientOpId);
3448
+ for (let round = 0; round <= UNAVAILABLE_MAX_ROUNDS; round += 1) {
3449
+ if (round > 0) {
3450
+ await new Promise((resolve) => {
3451
+ const timer = setTimeout(resolve, unavailableDelayMs(round));
3452
+ timer.unref?.();
3453
+ });
3454
+ }
3455
+ try {
3456
+ result = await args.postOversizedOp(opBlobB64, clientOpId);
3457
+ }
3458
+ catch (err) {
3459
+ result = {
3460
+ ok: false,
3461
+ message: err instanceof Error ? err.message : String(err),
3462
+ // Staging the blob runs strictly before the op write, so a failure
3463
+ // there wrote nothing and the op is still the client's to send: it
3464
+ // is priced the way the write prices its own refusals. Any other
3465
+ // throw leaves it unknown whether the handler ran, and this op
3466
+ // carries no idempotency the server dedupes on, so replaying it
3467
+ // could double-apply.
3468
+ retryable: err instanceof OpStagingError && opStagingIsRetryable(err),
3469
+ };
3470
+ }
3471
+ if (result.ok || !result.retryable)
3472
+ break;
1920
3473
  }
1921
- catch (err) {
1922
- result = {
1923
- ok: false,
1924
- message: err instanceof Error ? err.message : String(err),
1925
- };
3474
+ if (result === undefined) {
3475
+ opStartedAt.delete(clientOpId);
3476
+ return;
1926
3477
  }
1927
3478
  if (!result.ok) {
1928
3479
  // Terminal for this op, the same way `op_rejected` is: the local
1929
3480
  // commit stays applied but never persists, so the app hears about it
1930
- // instead of watching `savingCount` sit above zero forever.
1931
- args.doc._failOutstandingOp(clientOpId, result.message);
3481
+ // instead of watching `savingCount` sit above zero forever. A refusal
3482
+ // about the writer turns the Document read-only first, so the app's
3483
+ // own gate closes the editor as the reader is told why.
3484
+ const firstCapacityRefusal = result.atCapacity === true && args.doc._markAtCapacity();
3485
+ if (result.readOnly)
3486
+ args.doc._markReadOnly(result.readOnly);
3487
+ failOpTerminally(clientOpId, result.retryable ? UNAVAILABLE_GIVE_UP_MESSAGE : result.message, {
3488
+ healable: result.atCapacity !== true,
3489
+ report: result.atCapacity !== true || firstCapacityRefusal,
3490
+ });
1932
3491
  return;
1933
3492
  }
1934
3493
  // Same bookkeeping an `op_ack` does: resolve the outstanding entry and
1935
3494
  // advance both cursors past the seq the server allocated.
3495
+ opStartedAt.delete(clientOpId);
1936
3496
  args.doc.resolveOutstandingOp(clientOpId);
1937
3497
  args.setCurrentSeq(result.seq);
1938
3498
  args.advanceCoveredSeq(result.seq);
@@ -1985,14 +3545,13 @@ function wireDocTransport(args) {
1985
3545
  clearInterval(reverifyTimer);
1986
3546
  reverifyTimer = null;
1987
3547
  }
3548
+ clearDivergenceHeal();
1988
3549
  if (gapTimer !== null) {
1989
3550
  clearTimeout(gapTimer);
1990
3551
  gapTimer = null;
1991
3552
  }
1992
- for (const entry of outstandingResends.values()) {
1993
- clearTimeout(entry.timer);
1994
- }
1995
- outstandingResends.clear();
3553
+ forgetAllOps();
3554
+ clearUnavailableRecovery();
1996
3555
  originalClose();
1997
3556
  };
1998
3557
  args.doc.close = () => {
@@ -2006,15 +3565,39 @@ function wireDocTransport(args) {
2006
3565
  // (its lifecycle is the parent Document's). Keying on the REQUESTED branch
2007
3566
  // (not the resolved one) keeps park and take symmetric; a default-branch
2008
3567
  // open has no requested branch and so skips the cache.
2009
- const cacheKey = args.doc.readOnly || args.parkable === false
2010
- ? null
2011
- : parkCacheKey(args.docId, args.requestedBranchId, args.requestedStream);
3568
+ //
3569
+ // A doc with work still running skips it too: parking clears the status
3570
+ // listeners, and the SDK's own waiters on that work (the import runner
3571
+ // waits for the phase that produces the file it imports) are keyed to
3572
+ // this instance, so a reopen would hand back a Document whose wait can
3573
+ // never fire. A fresh open re-attaches everything.
3574
+ //
3575
+ // So does one the server has evicted. `takeParkedDocument` refuses such an
3576
+ // entry, so parking it buys nothing and costs everything parking keeps
3577
+ // alive: the topic subscription (which goes on delivering every op of a
3578
+ // document nobody is reading) and the LoroDoc's heap, until some later
3579
+ // open or eviction reaches that key. A move is the case that makes this
3580
+ // routine rather than terminal, because the session follows it by
3581
+ // reopening the same id, so the evicted copy is retired on every move.
3582
+ const parkable = !args.doc.readOnly &&
3583
+ args.parkable !== false &&
3584
+ args.doc.status?.pending !== true &&
3585
+ args.doc._goneReason === null;
3586
+ const cacheKey = parkable
3587
+ ? parkCacheKey(args.docId, args.requestedBranchId, args.requestedStream)
3588
+ : null;
2012
3589
  if (cacheKey === null) {
2013
3590
  fullTeardown();
2014
3591
  return;
2015
3592
  }
2016
3593
  args.doc._suspend();
2017
- parkDocument({ doc: args.doc, teardown: fullTeardown, key: cacheKey });
3594
+ parkDocument({
3595
+ doc: args.doc,
3596
+ teardown: fullTeardown,
3597
+ key: cacheKey,
3598
+ optionsSig: documentOptionsSig(args.documentOptions),
3599
+ identity: args.identity?.() ?? null,
3600
+ });
2018
3601
  };
2019
3602
  }
2020
3603
  /**