@brass-build/client 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +72 -63
- package/CHANGELOG.md +86 -0
- package/README.md +317 -231
- package/dist/auth-flow.d.ts +10 -1
- package/dist/auth-flow.d.ts.map +1 -1
- package/dist/auth-flow.js +64 -20
- package/dist/auth-flow.js.map +1 -1
- package/dist/client.d.ts +3 -0
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +217 -75
- package/dist/client.js.map +1 -1
- package/dist/document-engine.d.ts +4 -0
- package/dist/document-engine.d.ts.map +1 -0
- package/dist/document-engine.js +9 -0
- package/dist/document-engine.js.map +1 -0
- package/dist/document-streams.d.ts +8 -1
- package/dist/document-streams.d.ts.map +1 -1
- package/dist/document-streams.js +79 -8
- package/dist/document-streams.js.map +1 -1
- package/dist/documents-namespace.d.ts.map +1 -1
- package/dist/documents-namespace.js +19 -3
- package/dist/documents-namespace.js.map +1 -1
- package/dist/documents.d.ts +43 -14
- package/dist/documents.d.ts.map +1 -1
- package/dist/documents.js +871 -223
- package/dist/documents.js.map +1 -1
- package/dist/embed-protocol.d.ts +3 -0
- package/dist/embed-protocol.d.ts.map +1 -0
- package/dist/embed-protocol.js +26 -0
- package/dist/embed-protocol.js.map +1 -0
- package/dist/emitter.d.ts.map +1 -1
- package/dist/emitter.js +8 -1
- package/dist/emitter.js.map +1 -1
- package/dist/export-guest.d.ts.map +1 -1
- package/dist/export-guest.js +14 -6
- package/dist/export-guest.js.map +1 -1
- package/dist/fetch-deadline.d.ts +22 -0
- package/dist/fetch-deadline.d.ts.map +1 -0
- package/dist/fetch-deadline.js +96 -0
- package/dist/fetch-deadline.js.map +1 -0
- package/dist/fetch-retry.d.ts +1 -0
- package/dist/fetch-retry.d.ts.map +1 -1
- package/dist/fetch-retry.js +12 -5
- package/dist/fetch-retry.js.map +1 -1
- package/dist/handle-types.d.ts +150 -0
- package/dist/handle-types.d.ts.map +1 -0
- package/dist/handle-types.js +8 -0
- package/dist/handle-types.js.map +1 -0
- package/dist/handles.d.ts +48 -48
- package/dist/handles.d.ts.map +1 -1
- package/dist/handles.js +3122 -501
- package/dist/handles.js.map +1 -1
- package/dist/hosted-runtime.d.ts.map +1 -1
- package/dist/hosted-runtime.js +8 -12
- package/dist/hosted-runtime.js.map +1 -1
- package/dist/hosted-ui.d.ts +0 -1
- package/dist/hosted-ui.d.ts.map +1 -1
- package/dist/hosted-ui.js +29 -24
- package/dist/hosted-ui.js.map +1 -1
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +253 -8
- package/dist/importer.js.map +1 -1
- package/dist/index.d.ts +10 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -2
- package/dist/index.js.map +1 -1
- package/dist/internal.d.ts +1 -1
- package/dist/internal.d.ts.map +1 -1
- package/dist/internal.js.map +1 -1
- package/dist/join-coverage.d.ts +12 -0
- package/dist/join-coverage.d.ts.map +1 -0
- package/dist/join-coverage.js +37 -0
- package/dist/join-coverage.js.map +1 -0
- package/dist/jwt.d.ts.map +1 -1
- package/dist/jwt.js +5 -1
- package/dist/jwt.js.map +1 -1
- package/dist/launch-context.js +2 -2
- package/dist/launch-context.js.map +1 -1
- package/dist/mutate-batching.d.ts +25 -2
- package/dist/mutate-batching.d.ts.map +1 -1
- package/dist/mutate-batching.js +57 -10
- package/dist/mutate-batching.js.map +1 -1
- package/dist/open-document.d.ts +15 -1
- package/dist/open-document.d.ts.map +1 -1
- package/dist/open-document.js +1851 -321
- package/dist/open-document.js.map +1 -1
- package/dist/presence.d.ts +28 -0
- package/dist/presence.d.ts.map +1 -1
- package/dist/presence.js +149 -7
- package/dist/presence.js.map +1 -1
- package/dist/record-paging.d.ts +336 -71
- package/dist/record-paging.d.ts.map +1 -1
- package/dist/record-paging.js +591 -127
- package/dist/record-paging.js.map +1 -1
- package/dist/selection-persistence.d.ts.map +1 -1
- package/dist/selection-persistence.js +11 -1
- package/dist/selection-persistence.js.map +1 -1
- package/dist/serializer.d.ts +30 -1
- package/dist/serializer.d.ts.map +1 -1
- package/dist/session-persistence.d.ts.map +1 -1
- package/dist/session-persistence.js +9 -8
- package/dist/session-persistence.js.map +1 -1
- package/dist/session-toolbar.d.ts +0 -1
- package/dist/session-toolbar.d.ts.map +1 -1
- package/dist/session-toolbar.js +226 -75
- package/dist/session-toolbar.js.map +1 -1
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +87 -18
- package/dist/session.js.map +1 -1
- package/dist/tail-ops.d.ts +14 -4
- package/dist/tail-ops.d.ts.map +1 -1
- package/dist/tail-ops.js +75 -15
- package/dist/tail-ops.js.map +1 -1
- package/dist/topic-lifecycle.d.ts +4 -0
- package/dist/topic-lifecycle.d.ts.map +1 -1
- package/dist/topic-lifecycle.js +25 -1
- package/dist/topic-lifecycle.js.map +1 -1
- package/dist/types.d.ts +47 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/undo.d.ts +2 -3
- package/dist/undo.d.ts.map +1 -1
- package/dist/undo.js +11 -0
- package/dist/undo.js.map +1 -1
- package/dist/unload-guard.d.ts +5 -0
- package/dist/unload-guard.d.ts.map +1 -0
- package/dist/unload-guard.js +88 -0
- package/dist/unload-guard.js.map +1 -0
- package/dist/version-bar.d.ts +4 -3
- package/dist/version-bar.d.ts.map +1 -1
- package/dist/version-bar.js +140 -19
- package/dist/version-bar.js.map +1 -1
- package/dist/version.d.ts +2 -2
- package/dist/version.js +1 -1
- package/dist/vite.d.ts +18 -2
- package/dist/vite.d.ts.map +1 -1
- package/dist/vite.js +133 -8
- package/dist/vite.js.map +1 -1
- package/dist/ws-client.d.ts +23 -3
- package/dist/ws-client.d.ts.map +1 -1
- package/dist/ws-client.js +339 -89
- package/dist/ws-client.js.map +1 -1
- package/dist/ws-frame-limit.d.ts.map +1 -1
- package/dist/ws-frame-limit.js +19 -12
- package/dist/ws-frame-limit.js.map +1 -1
- package/docs/app-manifest.md +7 -8
- package/docs/custom-chrome.md +17 -20
- package/docs/hosted-ui-advanced.md +14 -17
- package/docs/imported-documents.md +53 -197
- package/docs/presence.md +45 -21
- package/docs/schemas.md +12 -17
- package/docs/streams.md +55 -57
- package/package.json +4 -3
- package/src/auth-flow.ts +90 -23
- package/src/client.ts +225 -78
- package/src/document-engine.ts +15 -0
- package/src/document-signals.ts +7 -7
- package/src/document-streams.ts +95 -9
- package/src/documents-namespace.ts +18 -3
- package/src/documents.ts +983 -277
- package/src/embed-protocol.ts +25 -0
- package/src/emitter.ts +8 -1
- package/src/export-guest.ts +15 -9
- package/src/fetch-deadline.ts +119 -0
- package/src/fetch-retry.ts +14 -5
- package/src/handle-types.ts +303 -0
- package/src/handles.ts +3406 -683
- package/src/hosted-runtime.ts +8 -12
- package/src/hosted-ui.ts +28 -32
- package/src/importer.ts +261 -8
- package/src/index.ts +29 -18
- package/src/internal.ts +1 -0
- package/src/join-coverage.ts +42 -0
- package/src/jwt.ts +8 -1
- package/src/launch-context.ts +2 -2
- package/src/mutate-batching.ts +60 -11
- package/src/open-document.ts +2355 -515
- package/src/presence.ts +210 -11
- package/src/record-paging.ts +687 -137
- package/src/selection-persistence.ts +10 -1
- package/src/serializer.ts +33 -1
- package/src/session-persistence.ts +9 -8
- package/src/session-toolbar.ts +249 -80
- package/src/session.ts +91 -17
- package/src/tail-ops.ts +76 -10
- package/src/topic-lifecycle.ts +31 -1
- package/src/types.ts +56 -4
- package/src/undo.ts +13 -5
- package/src/unload-guard.ts +91 -0
- package/src/version-bar.ts +163 -25
- package/src/version.ts +1 -1
- package/src/vite.ts +175 -10
- package/src/ws-client.ts +392 -92
- package/src/ws-frame-limit.ts +19 -12
package/src/open-document.ts
CHANGED
|
@@ -14,19 +14,38 @@ import {
|
|
|
14
14
|
type DocumentDriveTrackingState,
|
|
15
15
|
type DriveMirrorDirection,
|
|
16
16
|
type HostedUiProvider,
|
|
17
|
+
type ReadOnlyReason,
|
|
18
|
+
type ResyncPushOutcome,
|
|
17
19
|
} from './documents.js';
|
|
18
20
|
import { createPresenceController } from './presence.js';
|
|
19
21
|
import type { WsTopicCallback } from './ws-client.js';
|
|
20
22
|
import { applyAppAssertionHeader, applyClientVersionHeader } from './api-headers.js';
|
|
21
23
|
import { brassErrorFromBody } from './api-fetch.js';
|
|
22
24
|
import { fetchWithRetry, RETRY_IDEMPOTENT_WRITE } from './fetch-retry.js';
|
|
25
|
+
import {
|
|
26
|
+
deadlineForUpload,
|
|
27
|
+
fetchWithDeadline,
|
|
28
|
+
OP_WRITE_TIMEOUT_MS,
|
|
29
|
+
} from './fetch-deadline.js';
|
|
30
|
+
import type { DocSyncDiagnostics } from './types.js';
|
|
31
|
+
|
|
32
|
+
// Why a client is reporting its sync health. Mirrors the server's
|
|
33
|
+
// `CLIENT_REPORT_REASONS`; the published SDK cannot import the workspace
|
|
34
|
+
// contracts package, so the values are restated here and the server refuses
|
|
35
|
+
// any it does not recognise.
|
|
36
|
+
type ClientReportReason =
|
|
37
|
+
| 'op_stuck'
|
|
38
|
+
| 'reconnected_unsaved'
|
|
39
|
+
| 'divergence_healed';
|
|
23
40
|
import { BrassError } from './errors.js';
|
|
24
41
|
import { perfStart } from './perf.js';
|
|
25
|
-
import {
|
|
42
|
+
import { joinCoverageFloor, snapshotSeqOf } from './join-coverage.js';
|
|
43
|
+
import { declaredTailBytes, resolveTailOps, type WireTailOp } from './tail-ops.js';
|
|
26
44
|
import {
|
|
27
45
|
opBlobExceedsDurableLimit,
|
|
28
|
-
opFrameExceedsWsLimit,
|
|
29
46
|
opBlobNeedsStaging,
|
|
47
|
+
opFrameExceedsWsLimit,
|
|
48
|
+
type OpFrame,
|
|
30
49
|
} from './ws-frame-limit.js';
|
|
31
50
|
|
|
32
51
|
export interface OpenDocumentByIdOptions {
|
|
@@ -73,22 +92,48 @@ export interface OpenDocumentByIdOptions {
|
|
|
73
92
|
*/
|
|
74
93
|
version?: number;
|
|
75
94
|
/**
|
|
76
|
-
* SDK-internal
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
95
|
+
* SDK-internal load-progress sink. The session threads it through to
|
|
96
|
+
* surface a progress state before the Document exists; apps read that via
|
|
97
|
+
* `Brass.session`, never this.
|
|
98
|
+
*
|
|
99
|
+
* Both phases count BYTES, so a reader watching a load sees one figure
|
|
100
|
+
* rather than two bars filling in turn. `phase` says which stretch is on
|
|
101
|
+
* the wire, because the two are paced by different things and a document
|
|
102
|
+
* can spend the whole of the first in what looks like silence: `tail` is
|
|
103
|
+
* the spilled ops the join fetches, `snapshot` the snapshot itself, and
|
|
104
|
+
* absent means `snapshot`.
|
|
105
|
+
*
|
|
106
|
+
* `total` is the whole download in both phases. The tail declares it from
|
|
107
|
+
* each spilled op's own size plus the snapshot still to come, so the
|
|
108
|
+
* percentage runs across the pair; the snapshot phase reports only its own
|
|
109
|
+
* (the decompressed size from `x-amz-meta-raw-bytes`, or null), and the
|
|
110
|
+
* session adds the tail back.
|
|
80
111
|
* @internal
|
|
81
112
|
*/
|
|
82
|
-
onSnapshotProgress?:
|
|
113
|
+
onSnapshotProgress?: LoadProgress;
|
|
83
114
|
}
|
|
84
115
|
|
|
116
|
+
/**
|
|
117
|
+
* What a load reports as it runs. See `onSnapshotProgress` for the phases.
|
|
118
|
+
* @internal
|
|
119
|
+
*/
|
|
120
|
+
export type LoadProgress = (
|
|
121
|
+
loaded: number,
|
|
122
|
+
total: number | null,
|
|
123
|
+
phase?: 'snapshot' | 'tail',
|
|
124
|
+
) => void;
|
|
125
|
+
|
|
85
126
|
// `Brass.createDocument(options?)` creates a new document and returns it
|
|
86
|
-
// fully editable. `
|
|
87
|
-
//
|
|
88
|
-
//
|
|
89
|
-
//
|
|
127
|
+
// fully editable. `name` defaults to "Untitled". Placement is resolved from
|
|
128
|
+
// context rather than named here (see `CreateDocumentArgs`): a caller who
|
|
129
|
+
// belongs to one organization lands there, and a caller who belongs to
|
|
130
|
+
// several is asked, so this shape carries no organization or project.
|
|
90
131
|
export interface CreateDocumentOptions {
|
|
91
132
|
name?: string;
|
|
133
|
+
// Snake_case because the SDK relays this object to `POST /documents`
|
|
134
|
+
// verbatim and `Brass.documents.get` hands it back under the same key, so
|
|
135
|
+
// the key an app writes is the key it reads. The camelCase sibling below
|
|
136
|
+
// never leaves the process, which is why the two conventions sit together.
|
|
92
137
|
app_metadata?: Record<string, unknown>;
|
|
93
138
|
// Forwarded to `Document.openFromSnapshot` / `Document.create`; overrides
|
|
94
139
|
// the root-map key for a doc created against a non-default rootKey.
|
|
@@ -140,14 +185,28 @@ export interface DocumentSnapshotResponse {
|
|
|
140
185
|
// Tail ops since the last snapshot, seq ascending, each a base64 Loro op
|
|
141
186
|
// blob. Applied in order; `applyRemoteOp` is idempotent.
|
|
142
187
|
tail_ops: Array<{ seq: number; op_blob_b64: string }>;
|
|
143
|
-
// The branch's seq counter at join time
|
|
144
|
-
//
|
|
145
|
-
//
|
|
188
|
+
// The branch's seq counter at join time: the highest seq the document
|
|
189
|
+
// has handed out. Not the same as the highest seq this response carries,
|
|
190
|
+
// because a seq is allocated before its op row is written, so a join read
|
|
191
|
+
// during a bulk write names seqs whose rows it could not see. Pair it with
|
|
192
|
+
// `snapshot_seq` to work out what was delivered. Optional (older fixtures
|
|
193
|
+
// omit it).
|
|
146
194
|
current_seq?: number;
|
|
195
|
+
// The seq the snapshot was sealed at: everything at or below it is inside
|
|
196
|
+
// `snapshot`, and `tail_ops` carries the rows above it. Optional (older
|
|
197
|
+
// servers and fixtures omit it).
|
|
198
|
+
snapshot_seq?: number;
|
|
147
199
|
// Base64 version vector of the persisted snapshot (durability-handshake
|
|
148
200
|
// ground truth). The handshake runs off the `doc_init` frame's copy; this
|
|
149
201
|
// is carried for wire parallelism. Absent until the branch is compacted.
|
|
150
202
|
snapshot_version_vector?: string;
|
|
203
|
+
// Whether the caller may write to this document, as the server's own
|
|
204
|
+
// write gate answers it. `false` opens the Document read-only, so
|
|
205
|
+
// `doc.readOnly` covers a view-only grant at bind time rather than
|
|
206
|
+
// leaving the app to discover it from a refused op. Optional: a server
|
|
207
|
+
// that does not send it is one whose refusals the transport still
|
|
208
|
+
// catches, so its absence opens the document editable as before.
|
|
209
|
+
can_edit?: boolean;
|
|
151
210
|
// Opaque status of slow server-side work (e.g. a drive stub hydrating on
|
|
152
211
|
// first open, with `snapshot` then `null`). Surfaced on `doc.status`;
|
|
153
212
|
// phases stream in via `doc_status` frames.
|
|
@@ -199,8 +258,24 @@ export interface DocumentAtVersionResponse {
|
|
|
199
258
|
// Structural subset of `BrassWsClient` the factory uses, so it can be
|
|
200
259
|
// unit-tested against a mock.
|
|
201
260
|
export interface OpenDocumentWsClient {
|
|
202
|
-
subscribe(
|
|
261
|
+
subscribe(
|
|
262
|
+
topic: string,
|
|
263
|
+
callback: WsTopicCallback,
|
|
264
|
+
options?: {
|
|
265
|
+
ownsFrameRetry?: boolean;
|
|
266
|
+
onEnded?: (info: { reason: string }) => void;
|
|
267
|
+
},
|
|
268
|
+
): () => void;
|
|
203
269
|
send(frame: unknown): void;
|
|
270
|
+
// Connection-level facts the transport reports but does not own. Optional
|
|
271
|
+
// so a transport-only test can supply a bare stub; absent reads as a
|
|
272
|
+
// client that has never reconnected on a visible page, which is what a
|
|
273
|
+
// test harness is.
|
|
274
|
+
readonly reconnectCount?: number;
|
|
275
|
+
readonly pageHidden?: boolean;
|
|
276
|
+
// Whether a socket is open right now. Absent reads as connected, which is
|
|
277
|
+
// what a transport-only test stub is.
|
|
278
|
+
readonly connected?: boolean;
|
|
204
279
|
}
|
|
205
280
|
|
|
206
281
|
export interface OpenDocumentFactoryOptions {
|
|
@@ -272,6 +347,12 @@ export interface OpenDocumentFactoryOptions {
|
|
|
272
347
|
// through a getter because `init` resolves it after the factory's
|
|
273
348
|
// options are written. See `createDocumentId`.
|
|
274
349
|
launchOrganizationId?: () => string | null;
|
|
350
|
+
// Who is signed in, as the park cache's ownership stamp (see
|
|
351
|
+
// `ParkedDocEntry.identity`). Read at park and at take, so a Document only
|
|
352
|
+
// ever comes back to the person it was opened for. Absent (tests, the
|
|
353
|
+
// anonymous link-guest path) parks under a null identity, which is
|
|
354
|
+
// consistent with itself and so behaves as it always has.
|
|
355
|
+
identity?: () => string | null;
|
|
275
356
|
// Async-error sink for built-in command failures; `null` falls back to
|
|
276
357
|
// console.
|
|
277
358
|
reportError?: (err: Error) => void;
|
|
@@ -287,6 +368,73 @@ export type OpenedDocument = Document & {
|
|
|
287
368
|
readonly id: string;
|
|
288
369
|
};
|
|
289
370
|
|
|
371
|
+
// The terminal frame the platform sends when it clears a document's work
|
|
372
|
+
// status: the work finished and the content is in. The SDK emits the same
|
|
373
|
+
// shape wherever it settles that transition off its own import (a
|
|
374
|
+
// subscribe-time reconcile, a retried pull), so the app cannot tell the two
|
|
375
|
+
// apart. `blocking: false` is what releases the load lock.
|
|
376
|
+
const WORK_READY_STATUS: DocumentWorkStatus = {
|
|
377
|
+
message: 'Ready',
|
|
378
|
+
pending: false,
|
|
379
|
+
blocking: false,
|
|
380
|
+
};
|
|
381
|
+
|
|
382
|
+
const CONTENT_PULL_ATTEMPTS = 5;
|
|
383
|
+
const CONTENT_PULL_BACKOFF_MS = 1000;
|
|
384
|
+
|
|
385
|
+
// Pull a document's content in after the platform reports its inbound work
|
|
386
|
+
// finished, retrying a transient fetch failure (a network blip, a dropped
|
|
387
|
+
// presigned download): the platform asserts the content exists, so a silent
|
|
388
|
+
// give-up would leave the doc empty behind a terminal status, reading as "the
|
|
389
|
+
// model never loaded" and invisible in every log. Emits `finalStatus` only
|
|
390
|
+
// once the import lands, so an app re-reading on `onStatus` sees the content.
|
|
391
|
+
// On exhaustion it reports the failure and emits a terminal failure status,
|
|
392
|
+
// which keeps the doc read-only. Returns whether the content is in.
|
|
393
|
+
async function pullContentWithRetry(args: {
|
|
394
|
+
doc: Document;
|
|
395
|
+
docId: string;
|
|
396
|
+
finalStatus: DocumentWorkStatus;
|
|
397
|
+
fetchAndImport: () => Promise<{ ok: true } | { ok: false; reason: string }>;
|
|
398
|
+
reportError?: (err: Error) => void;
|
|
399
|
+
}): Promise<boolean> {
|
|
400
|
+
let lastError: Error | null = null;
|
|
401
|
+
for (let attempt = 1; attempt <= CONTENT_PULL_ATTEMPTS; attempt += 1) {
|
|
402
|
+
try {
|
|
403
|
+
const outcome = await args.fetchAndImport();
|
|
404
|
+
if (outcome.ok) {
|
|
405
|
+
args.doc._emitStatus(args.finalStatus, { contentLoaded: true });
|
|
406
|
+
return true;
|
|
407
|
+
}
|
|
408
|
+
lastError = new Error(outcome.reason);
|
|
409
|
+
} catch (err) {
|
|
410
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
411
|
+
}
|
|
412
|
+
// A retrying attempt is exceptional, so this warn is quiet in normal
|
|
413
|
+
// operation and decisive in a bug report; without it a failure here is
|
|
414
|
+
// undiagnosable from any log.
|
|
415
|
+
console.warn(
|
|
416
|
+
`[Brass] document content reload attempt ${attempt}/${CONTENT_PULL_ATTEMPTS} failed (doc ${args.docId}): ${lastError.message}`,
|
|
417
|
+
);
|
|
418
|
+
if (attempt < CONTENT_PULL_ATTEMPTS) {
|
|
419
|
+
await new Promise((r) => setTimeout(r, CONTENT_PULL_BACKOFF_MS * attempt));
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
const error = new BrassError(
|
|
423
|
+
`Document content reload failed after ${CONTENT_PULL_ATTEMPTS} attempts: ${lastError?.message ?? 'unknown error'}`,
|
|
424
|
+
{ status: 0 },
|
|
425
|
+
);
|
|
426
|
+
if (args.reportError) args.reportError(error);
|
|
427
|
+
else console.error('[Brass]', error);
|
|
428
|
+
args.doc._emitStatus({
|
|
429
|
+
message: 'The model could not be loaded. Reload to try again.',
|
|
430
|
+
pending: false,
|
|
431
|
+
// The content never arrived, so this terminal status still locks the
|
|
432
|
+
// doc: an edit here would write into a document holding nothing.
|
|
433
|
+
blocking: true,
|
|
434
|
+
});
|
|
435
|
+
return false;
|
|
436
|
+
}
|
|
437
|
+
|
|
290
438
|
export type OpenDocumentByIdFn = (
|
|
291
439
|
docId: string,
|
|
292
440
|
options?: OpenDocumentByIdOptions,
|
|
@@ -323,9 +471,8 @@ export interface DocumentFactory {
|
|
|
323
471
|
openByLink: OpenDocumentByLinkFn;
|
|
324
472
|
}
|
|
325
473
|
|
|
326
|
-
// Wire shapes on the `doc:` topic.
|
|
327
|
-
//
|
|
328
|
-
// type tests stay synced.
|
|
474
|
+
// Wire shapes on the `doc:` topic. Frame names match the server's exactly, so
|
|
475
|
+
// the CONTRACT.md type tests stay synced.
|
|
329
476
|
interface DocOpFrame {
|
|
330
477
|
type: 'doc_op';
|
|
331
478
|
topic: string;
|
|
@@ -360,29 +507,56 @@ interface OpRejectedFrame {
|
|
|
360
507
|
// tail op whose bytes spilled to storage is fetched once, here. Every
|
|
361
508
|
// reader downstream then sees the inline shape it already handles, and
|
|
362
509
|
// none of the seq bookkeeping has to learn a second representation.
|
|
363
|
-
async function resolveParsedTailOps(
|
|
510
|
+
async function resolveParsedTailOps(
|
|
511
|
+
parsed: unknown,
|
|
512
|
+
onProgress?: LoadProgress,
|
|
513
|
+
): Promise<void> {
|
|
364
514
|
if (typeof parsed !== 'object' || parsed === null) return;
|
|
365
|
-
const body = parsed as { tail_ops?: unknown };
|
|
515
|
+
const body = parsed as { tail_ops?: unknown; snapshot?: unknown };
|
|
366
516
|
if (!Array.isArray(body.tail_ops)) return;
|
|
367
|
-
|
|
517
|
+
const tailOps = body.tail_ops as WireTailOp[];
|
|
518
|
+
// The whole download, stated before any of it starts. A presigned snapshot
|
|
519
|
+
// names its decompressed size and each spilled op names its blob's, and both
|
|
520
|
+
// are fetched over the network with a reader waiting, so the load reports one
|
|
521
|
+
// figure against their sum. An inline snapshot contributes nothing: its bytes
|
|
522
|
+
// came with this response.
|
|
523
|
+
const snapshot = body.snapshot as { raw_bytes?: unknown } | null | undefined;
|
|
524
|
+
const snapshotBytes =
|
|
525
|
+
typeof snapshot?.raw_bytes === 'number' ? snapshot.raw_bytes : 0;
|
|
526
|
+
const totalBytes = declaredTailBytes(tailOps) + snapshotBytes;
|
|
527
|
+
body.tail_ops = await resolveTailOps(
|
|
528
|
+
tailOps,
|
|
529
|
+
fetch,
|
|
530
|
+
onProgress === undefined
|
|
531
|
+
? undefined
|
|
532
|
+
: (loadedBytes) =>
|
|
533
|
+
onProgress(loadedBytes, totalBytes > 0 ? totalBytes : null, 'tail'),
|
|
534
|
+
);
|
|
368
535
|
}
|
|
369
536
|
|
|
370
|
-
// Post-subscribe server frame mirroring the HTTP join payload.
|
|
371
|
-
//
|
|
372
|
-
//
|
|
537
|
+
// Post-subscribe server frame mirroring the HTTP join payload. The SDK needs
|
|
538
|
+
// the tail_ops + current_seq to cover the join-vs-fanout race (see
|
|
539
|
+
// `wireDocTransport`).
|
|
540
|
+
// A tail entry as `parseDocInitFrame` normalises it: the seq every reader
|
|
541
|
+
// needs for the cursors, and the inline bytes when the entry carries them.
|
|
373
542
|
interface DocInitTailOp {
|
|
374
543
|
seq: number;
|
|
375
|
-
// Absent when the op's blob spilled to storage
|
|
376
|
-
//
|
|
377
|
-
//
|
|
544
|
+
// Absent when the op's blob spilled to storage (the server withholds such
|
|
545
|
+
// a tail from this frame, since fetching one needs an await the frame
|
|
546
|
+
// reader does not have) or when the entry names its bytes in a shape this
|
|
547
|
+
// SDK does not recognise. A reader that finds one skips it and picks the
|
|
548
|
+
// op up on the HTTP re-import.
|
|
378
549
|
op_blob_b64?: string;
|
|
379
|
-
op_blob_url?: string;
|
|
380
550
|
}
|
|
381
551
|
interface DocInitFrame {
|
|
382
552
|
type: 'doc_init';
|
|
383
553
|
topic: string;
|
|
384
554
|
tail_ops: DocInitTailOp[];
|
|
385
555
|
current_seq: number;
|
|
556
|
+
// The seq the snapshot was sealed at, which is where this frame's tail
|
|
557
|
+
// begins. Absent from an older server's frame, and then inferred from the
|
|
558
|
+
// tail's own lower edge.
|
|
559
|
+
snapshot_seq?: number;
|
|
386
560
|
// Base64 version vector of the persisted snapshot, stamped at seal time.
|
|
387
561
|
// Drives the durability handshake: fold the tail ops in for the durable
|
|
388
562
|
// frontier and resync-push anything local holds beyond it. Absent on
|
|
@@ -390,6 +564,19 @@ interface DocInitFrame {
|
|
|
390
564
|
snapshot_version_vector?: string;
|
|
391
565
|
}
|
|
392
566
|
|
|
567
|
+
// Server-pushed "the document moved past you" prompt, naming the seq to
|
|
568
|
+
// reach and carrying none of the content. The server sends it to every
|
|
569
|
+
// subscriber of a document on each op the socket could not carry, so it is
|
|
570
|
+
// the frequent frame of the two: it says nothing about the subscription it
|
|
571
|
+
// arrived on, and a reader that treats it as a (re)subscribe re-asserts
|
|
572
|
+
// every unacked op on somebody else's write. Wire shape pinned in
|
|
573
|
+
// CONTRACT.md "Server-pushed `doc_catch_up`".
|
|
574
|
+
interface DocCatchUpFrame {
|
|
575
|
+
type: 'doc_catch_up';
|
|
576
|
+
topic: string;
|
|
577
|
+
current_seq: number;
|
|
578
|
+
}
|
|
579
|
+
|
|
393
580
|
// Server-pushed eviction frame. Carried on both the content and presence
|
|
394
581
|
// topics; the SDK collapses the duplicate via `Document._markGone`'s
|
|
395
582
|
// idempotency. Wire shape pinned in CONTRACT.md "Server-pushed `doc_gone`".
|
|
@@ -433,8 +620,8 @@ interface DocDriveStatusFrame {
|
|
|
433
620
|
drive: DocumentDriveSummaryWire;
|
|
434
621
|
}
|
|
435
622
|
|
|
436
|
-
// Server-pushed frame fired when the
|
|
437
|
-
//
|
|
623
|
+
// Server-pushed frame fired when the platform discards ops it cannot apply.
|
|
624
|
+
// The prompt half of the durability
|
|
438
625
|
// handshake: the SDK re-checks durability against ground truth now rather
|
|
439
626
|
// than waiting for the author's next reconnect, and resync-pushes anything
|
|
440
627
|
// it holds. Best-effort and connected-only; the `doc_init` handshake is the
|
|
@@ -443,8 +630,6 @@ interface DocDriveStatusFrame {
|
|
|
443
630
|
interface DocOpsDroppedFrame {
|
|
444
631
|
type: 'doc_ops_dropped';
|
|
445
632
|
topic: string;
|
|
446
|
-
dropped_seqs: number[];
|
|
447
|
-
actor: string;
|
|
448
633
|
}
|
|
449
634
|
|
|
450
635
|
// Map the snake_case wire `drive` block to the public camelCase
|
|
@@ -495,31 +680,62 @@ function isDocOpFrame(frame: unknown): frame is DocOpFrame {
|
|
|
495
680
|
);
|
|
496
681
|
}
|
|
497
682
|
|
|
498
|
-
|
|
499
|
-
|
|
683
|
+
// The frame in the shape its readers act on, or null when it carries nothing
|
|
684
|
+
// they can act on.
|
|
685
|
+
//
|
|
686
|
+
// Gated on the discriminant plus `current_seq`, with the tail degraded entry
|
|
687
|
+
// by entry. Most of what a `doc_init` does is carried by the frame itself
|
|
688
|
+
// rather than by any one entry: `current_seq` is the cursor, the tail's lower
|
|
689
|
+
// edge decides whether the reader re-imports the join, and the frame is the
|
|
690
|
+
// (re)subscribe signal that re-asserts every outstanding op and runs the
|
|
691
|
+
// durability handshake. So an entry whose shape this SDK does not recognise
|
|
692
|
+
// costs the one op it carries, which stays uncovered and is pulled over the
|
|
693
|
+
// HTTP join; refusing the frame over it would cost all of the above.
|
|
694
|
+
//
|
|
695
|
+
// `snapshot_version_vector` is likewise tolerated as absent when it is not a
|
|
696
|
+
// string: the handshake needs a vector it can decode, and skipping the
|
|
697
|
+
// handshake is a smaller loss than dropping the frame that drives it.
|
|
698
|
+
function parseDocInitFrame(frame: unknown): DocInitFrame | null {
|
|
699
|
+
if (typeof frame !== 'object' || frame === null) return null;
|
|
500
700
|
const f = frame as Record<string, unknown>;
|
|
501
|
-
if (f.type !== 'doc_init' || typeof f.topic !== 'string') return
|
|
502
|
-
if (typeof f.current_seq !== 'number') return
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
// than dropping the whole frame and with it the resync trigger.
|
|
518
|
-
if (typeof e.op_blob_b64 !== 'string' && typeof e.op_blob_url !== 'string') {
|
|
519
|
-
return false;
|
|
701
|
+
if (f.type !== 'doc_init' || typeof f.topic !== 'string') return null;
|
|
702
|
+
if (typeof f.current_seq !== 'number') return null;
|
|
703
|
+
const tailOps: DocInitTailOp[] = [];
|
|
704
|
+
if (Array.isArray(f.tail_ops)) {
|
|
705
|
+
for (const entry of f.tail_ops) {
|
|
706
|
+
if (typeof entry !== 'object' || entry === null) continue;
|
|
707
|
+
const e = entry as Record<string, unknown>;
|
|
708
|
+
// The seq is what places an entry in the sequence the cursors and the
|
|
709
|
+
// resync trigger are computed from, so an entry without one is not an
|
|
710
|
+
// entry any reader here can use.
|
|
711
|
+
if (typeof e.seq !== 'number') continue;
|
|
712
|
+
tailOps.push(
|
|
713
|
+
typeof e.op_blob_b64 === 'string'
|
|
714
|
+
? { seq: e.seq, op_blob_b64: e.op_blob_b64 }
|
|
715
|
+
: { seq: e.seq },
|
|
716
|
+
);
|
|
520
717
|
}
|
|
521
718
|
}
|
|
522
|
-
return
|
|
719
|
+
return {
|
|
720
|
+
type: 'doc_init',
|
|
721
|
+
topic: f.topic,
|
|
722
|
+
current_seq: f.current_seq,
|
|
723
|
+
tail_ops: tailOps,
|
|
724
|
+
...(typeof f.snapshot_seq === 'number' ? { snapshot_seq: f.snapshot_seq } : {}),
|
|
725
|
+
...(typeof f.snapshot_version_vector === 'string'
|
|
726
|
+
? { snapshot_version_vector: f.snapshot_version_vector }
|
|
727
|
+
: {}),
|
|
728
|
+
};
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
function isDocCatchUpFrame(frame: unknown): frame is DocCatchUpFrame {
|
|
732
|
+
if (typeof frame !== 'object' || frame === null) return false;
|
|
733
|
+
const f = frame as Record<string, unknown>;
|
|
734
|
+
return (
|
|
735
|
+
f.type === 'doc_catch_up' &&
|
|
736
|
+
typeof f.topic === 'string' &&
|
|
737
|
+
typeof f.current_seq === 'number'
|
|
738
|
+
);
|
|
523
739
|
}
|
|
524
740
|
|
|
525
741
|
function isOpAckFrame(frame: unknown): frame is OpAckFrame {
|
|
@@ -543,6 +759,98 @@ function isOpRejectedFrame(frame: unknown): frame is OpRejectedFrame {
|
|
|
543
759
|
);
|
|
544
760
|
}
|
|
545
761
|
|
|
762
|
+
// `{ type: 'error', topic, reason: 'unavailable' }`: the durable write of an
|
|
763
|
+
// op the server accepted and authorised did not go through, and the same op
|
|
764
|
+
// is expected to succeed on a later attempt. `BrassWsClient` leaves the
|
|
765
|
+
// subscription up and routes the frame here (see `isFrameScopedReason` in
|
|
766
|
+
// `topic-lifecycle.ts`). The frame names no `client_op_id`, so it identifies
|
|
767
|
+
// the shed batch and not one op within it.
|
|
768
|
+
/**
|
|
769
|
+
* What one HTTP attempt at an oversized op resolved to.
|
|
770
|
+
*
|
|
771
|
+
* `retryable` carries the same distinction the WS transport gets from an
|
|
772
|
+
* `unavailable` error frame: the server refused the write on capacity and
|
|
773
|
+
* nothing landed, so the op is still the client's to send. Without it, the
|
|
774
|
+
* two transports disagree about the same refusal, and an op that took the
|
|
775
|
+
* HTTP route because it was too large for the socket is the one that gets
|
|
776
|
+
* dropped.
|
|
777
|
+
*/
|
|
778
|
+
type OversizedOpResult =
|
|
779
|
+
| { ok: true; seq: number }
|
|
780
|
+
// `atCapacity` and `readOnly` ride beside `retryable` rather than
|
|
781
|
+
// replacing it because the three answer different questions: whether to
|
|
782
|
+
// send this op again, whether the document will take a growing op at all,
|
|
783
|
+
// and whether this writer may write to it at all.
|
|
784
|
+
| {
|
|
785
|
+
ok: false;
|
|
786
|
+
message: string;
|
|
787
|
+
retryable: boolean;
|
|
788
|
+
atCapacity?: boolean;
|
|
789
|
+
// Set when the refusal was about the writer rather than the op, which
|
|
790
|
+
// is the same distinction the socket's `op_rejected` reasons draw:
|
|
791
|
+
// every later op meets it too, so the Document turns read-only.
|
|
792
|
+
readOnly?: ReadOnlyReason;
|
|
793
|
+
};
|
|
794
|
+
|
|
795
|
+
// Whether re-sending the same op is the right move. Two statuses refuse to
|
|
796
|
+
// write rather than answer about the op, and both leave the row untouched,
|
|
797
|
+
// so the op is still the client's to send: 503 (the store refused on
|
|
798
|
+
// capacity) and 429 (the caller is over the op-write cap, whose window
|
|
799
|
+
// resets). The WS transport reads its own rate refusal the same way, as an
|
|
800
|
+
// `unavailable` the shed recovery re-asserts, and the two transports carry
|
|
801
|
+
// the same ops, so a refusal cannot mean "later" on one and "never" on the
|
|
802
|
+
// other. Every other status is an answer about the op itself and repeats
|
|
803
|
+
// however many times it is sent.
|
|
804
|
+
function opWriteIsRetryable(status: number): boolean {
|
|
805
|
+
return status === 503 || status === 429;
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
// The HTTP transport's rendering of the same refusal `op_rejected` carries on
|
|
809
|
+
// the socket: the document is full. 507 is the status, and the body's `code`
|
|
810
|
+
// is checked beside it so the classification does not rest on a status a
|
|
811
|
+
// proxy in front of the api could rewrite.
|
|
812
|
+
function isCapacityRefusal(status: number, parsed: unknown): boolean {
|
|
813
|
+
if (status === 507) return true;
|
|
814
|
+
return (parsed as { code?: unknown } | undefined | null)?.code === 'at_capacity';
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
// A failure uploading an oversized op's blob to storage. It carries its own
|
|
818
|
+
// type because staging runs strictly BEFORE the op write, so what a replay
|
|
819
|
+
// means differs from a throw out of the write itself: nothing durable exists
|
|
820
|
+
// yet and the op is still the client's to send, where a throw mid-write
|
|
821
|
+
// leaves it unknown whether the row landed.
|
|
822
|
+
class OpStagingError extends Error {
|
|
823
|
+
override name = 'OpStagingError';
|
|
824
|
+
// The upstream status, or null for a request that drew no answer at all.
|
|
825
|
+
readonly status: number | null;
|
|
826
|
+
constructor(status: number | null, message: string) {
|
|
827
|
+
super(message);
|
|
828
|
+
this.status = status;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
// Whether staging the blob again is the right move: a status the op write
|
|
833
|
+
// itself replays replays here for the same reason, and a request that drew
|
|
834
|
+
// no answer wrote nothing either.
|
|
835
|
+
function opStagingIsRetryable(err: OpStagingError): boolean {
|
|
836
|
+
return err.status === null || opWriteIsRetryable(err.status);
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
function asOpStagingError(err: unknown): OpStagingError {
|
|
840
|
+
if (err instanceof OpStagingError) return err;
|
|
841
|
+
const status = err instanceof BrassError ? err.status ?? null : null;
|
|
842
|
+
return new OpStagingError(
|
|
843
|
+
status,
|
|
844
|
+
err instanceof Error ? err.message : String(err),
|
|
845
|
+
);
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
function isServerUnavailableFrame(frame: unknown): boolean {
|
|
849
|
+
if (typeof frame !== 'object' || frame === null) return false;
|
|
850
|
+
const f = frame as Record<string, unknown>;
|
|
851
|
+
return f.type === 'error' && f.reason === 'unavailable';
|
|
852
|
+
}
|
|
853
|
+
|
|
546
854
|
// The message surfaced through the error reporter for a terminal op rejection.
|
|
547
855
|
// Prefers the server's copy; falls back to a generic, actionable line. The
|
|
548
856
|
// local edit already applied but can never persist, so the app's cue is to
|
|
@@ -554,6 +862,45 @@ function opRejectedMessage(frame: OpRejectedFrame): string {
|
|
|
554
862
|
return 'A change could not be saved and will not be retried. Reload the document to continue.';
|
|
555
863
|
}
|
|
556
864
|
|
|
865
|
+
// The one rejection reason that describes the DOCUMENT rather than the op:
|
|
866
|
+
// every later write meets it too, so the app is told once about the document
|
|
867
|
+
// instead of once per refused edit. Matched on the reason string rather than
|
|
868
|
+
// on the message, which is copy and may be reworded.
|
|
869
|
+
const OP_REJECTED_AT_CAPACITY = 'at_capacity';
|
|
870
|
+
|
|
871
|
+
// Whether the document refused this write because of its size. The server
|
|
872
|
+
// distinguishes a branch that refuses everything from one that refuses only
|
|
873
|
+
// growth, and picks the wording accordingly; the client does not, because both
|
|
874
|
+
// leave refused work on this copy and nowhere else and the answer to that is
|
|
875
|
+
// the same one.
|
|
876
|
+
function isCapacityRejection(frame: OpRejectedFrame): boolean {
|
|
877
|
+
return frame.reason === OP_REJECTED_AT_CAPACITY;
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
// Which refusals describe the WRITER rather than the op. The server reads
|
|
881
|
+
// write authority once, when the connection subscribes, so a refusal on
|
|
882
|
+
// either of these grounds answers every later op the same way: the reader
|
|
883
|
+
// cannot write to this document at all, and the Document says so through
|
|
884
|
+
// `readOnly` instead of letting the app offer edits that keep failing. A
|
|
885
|
+
// refusal about the op itself (too large, a bad topic) leaves the document
|
|
886
|
+
// writable, because the next op may well be accepted.
|
|
887
|
+
//
|
|
888
|
+
// A Map rather than an object literal because the key is a wire string: an
|
|
889
|
+
// object answers for the names it inherits, so a reason reading `constructor`
|
|
890
|
+
// or `toString` would resolve a function and lock the document, which is the
|
|
891
|
+
// opposite of what an unclassified refusal means.
|
|
892
|
+
const READ_ONLY_REJECTION_REASONS = new Map<string, ReadOnlyReason>([
|
|
893
|
+
['role_denied', 'grant'],
|
|
894
|
+
['scope_denied', 'app'],
|
|
895
|
+
]);
|
|
896
|
+
|
|
897
|
+
function readOnlyReasonForRejection(
|
|
898
|
+
frame: OpRejectedFrame,
|
|
899
|
+
): ReadOnlyReason | null {
|
|
900
|
+
if (typeof frame.reason !== 'string') return null;
|
|
901
|
+
return READ_ONLY_REJECTION_REASONS.get(frame.reason) ?? null;
|
|
902
|
+
}
|
|
903
|
+
|
|
557
904
|
function isDocGoneFrame(frame: unknown): frame is DocGoneFrame {
|
|
558
905
|
if (typeof frame !== 'object' || frame === null) return false;
|
|
559
906
|
const f = frame as Record<string, unknown>;
|
|
@@ -593,6 +940,25 @@ function isDocStatusFrame(frame: unknown): frame is DocStatusFrame {
|
|
|
593
940
|
return typeof st.pending === 'boolean';
|
|
594
941
|
}
|
|
595
942
|
|
|
943
|
+
// `message` is typed non-null and rendered verbatim, while the wire gates
|
|
944
|
+
// only on `pending`, so default it here.
|
|
945
|
+
function normalizeWorkStatus(status: DocumentWorkStatus): DocumentWorkStatus {
|
|
946
|
+
return {
|
|
947
|
+
...status,
|
|
948
|
+
message: typeof status.message === 'string' ? status.message : '',
|
|
949
|
+
};
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
// The opaque work status a join response carries, or `null` when it carries
|
|
953
|
+
// none. Gates on `pending` like `isDocStatusFrame`; the rest is display data
|
|
954
|
+
// read defensively at the use site.
|
|
955
|
+
function parseWorkStatus(value: unknown): DocumentWorkStatus | null {
|
|
956
|
+
if (typeof value !== 'object' || value === null) return null;
|
|
957
|
+
const status = value as DocumentWorkStatus;
|
|
958
|
+
if (typeof status.pending !== 'boolean') return null;
|
|
959
|
+
return normalizeWorkStatus(status);
|
|
960
|
+
}
|
|
961
|
+
|
|
596
962
|
function isDocMetaFrame(frame: unknown): frame is DocMetaFrame {
|
|
597
963
|
if (typeof frame !== 'object' || frame === null) return false;
|
|
598
964
|
const f = frame as Record<string, unknown>;
|
|
@@ -625,12 +991,10 @@ function isDocDriveStatusFrame(frame: unknown): frame is DocDriveStatusFrame {
|
|
|
625
991
|
function isDocOpsDroppedFrame(frame: unknown): frame is DocOpsDroppedFrame {
|
|
626
992
|
if (typeof frame !== 'object' || frame === null) return false;
|
|
627
993
|
const f = frame as Record<string, unknown>;
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
typeof f.actor === 'string'
|
|
633
|
-
);
|
|
994
|
+
// Gate on the discriminant alone. The handler reads no other field, so a
|
|
995
|
+
// frame whose informational half changed shape (or went away) must still
|
|
996
|
+
// prompt the durability re-check it is sent for.
|
|
997
|
+
return f.type === 'doc_ops_dropped' && typeof f.topic === 'string';
|
|
634
998
|
}
|
|
635
999
|
|
|
636
1000
|
function defaultClientOpIdMinter(): string {
|
|
@@ -679,6 +1043,18 @@ function isSnapshotResponse(body: unknown): body is DocumentSnapshotResponse {
|
|
|
679
1043
|
return true;
|
|
680
1044
|
}
|
|
681
1045
|
|
|
1046
|
+
// Fold the join's write verdict into the Document options, so a reader the
|
|
1047
|
+
// server will not accept writes from opens read-only instead of learning it
|
|
1048
|
+
// from the first refused op. Only ever ADDS the restriction: a caller that
|
|
1049
|
+
// asked for a read-only Document keeps it whatever the grant says.
|
|
1050
|
+
function readOnlyForGrant(
|
|
1051
|
+
body: DocumentSnapshotResponse,
|
|
1052
|
+
options: DocumentOptions | undefined,
|
|
1053
|
+
): DocumentOptions | undefined {
|
|
1054
|
+
if (body.can_edit !== false) return options;
|
|
1055
|
+
return { ...options, readOnly: true, readOnlyReason: 'grant' };
|
|
1056
|
+
}
|
|
1057
|
+
|
|
682
1058
|
function isAtVersionResponse(
|
|
683
1059
|
body: unknown,
|
|
684
1060
|
): body is DocumentAtVersionResponse {
|
|
@@ -732,14 +1108,30 @@ export function createOpenDocument(
|
|
|
732
1108
|
applyClientVersionHeader(headers);
|
|
733
1109
|
await applyAppAssertionHeader(headers, opts.getAppAssertion);
|
|
734
1110
|
const endFetch = perfStart('open: GET /documents (http + read body)');
|
|
735
|
-
// Idempotent read: ride out a
|
|
736
|
-
//
|
|
737
|
-
//
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
1111
|
+
// Idempotent read: ride out a transient 5xx / 429 instead of failing the
|
|
1112
|
+
// open on something that clears on its own.
|
|
1113
|
+
//
|
|
1114
|
+
// Longer than the default handful of attempts because of what a 503 means
|
|
1115
|
+
// on this route: the document is intact and its storage is busy. Every op
|
|
1116
|
+
// row a document has shares one partition, so the document most likely to
|
|
1117
|
+
// answer 503 is one that was just written hard, which is exactly the
|
|
1118
|
+
// moment a person reloads to see their work. That pressure clears over
|
|
1119
|
+
// tens of seconds, not the one second the default budget spans, and the
|
|
1120
|
+
// cost of giving up early is telling someone their document will not open
|
|
1121
|
+
// when waiting would have opened it.
|
|
1122
|
+
//
|
|
1123
|
+
// A thrown request gets a short budget of its own: it never reached the
|
|
1124
|
+
// API, so the busy partition is not what is wrong, and a blocked origin
|
|
1125
|
+
// or a host that does not resolve answers the seventh attempt as it
|
|
1126
|
+
// answered the first, with the reader watching a loading surface
|
|
1127
|
+
// throughout.
|
|
1128
|
+
const response = await fetchWithRetry(
|
|
1129
|
+
() =>
|
|
1130
|
+
fetch(url, {
|
|
1131
|
+
method: 'GET',
|
|
1132
|
+
headers,
|
|
1133
|
+
}),
|
|
1134
|
+
{ maxAttempts: 7, baseDelayMs: 700, maxNetworkAttempts: 3 },
|
|
743
1135
|
);
|
|
744
1136
|
const text = await response.text();
|
|
745
1137
|
endFetch({ status: response.status, responseChars: text.length });
|
|
@@ -752,7 +1144,7 @@ export function createOpenDocument(
|
|
|
752
1144
|
parseFailed = true;
|
|
753
1145
|
}
|
|
754
1146
|
}
|
|
755
|
-
await resolveParsedTailOps(parsed);
|
|
1147
|
+
await resolveParsedTailOps(parsed, options?.onSnapshotProgress);
|
|
756
1148
|
return { status: response.status, parsed, parseFailed };
|
|
757
1149
|
}
|
|
758
1150
|
|
|
@@ -898,37 +1290,54 @@ export function createOpenDocument(
|
|
|
898
1290
|
opBlobB64: string,
|
|
899
1291
|
token: string,
|
|
900
1292
|
): Promise<string> {
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
const
|
|
913
|
-
|
|
1293
|
+
// Everything in here happens before the op write, so every failure it
|
|
1294
|
+
// can take is tagged as one: what the tag buys the writer is the right
|
|
1295
|
+
// to send the op again, which a failure of the write itself does not
|
|
1296
|
+
// carry.
|
|
1297
|
+
try {
|
|
1298
|
+
const headers: Record<string, string> = {
|
|
1299
|
+
authorization: `Bearer ${token}`,
|
|
1300
|
+
'content-type': 'application/json',
|
|
1301
|
+
};
|
|
1302
|
+
applyClientVersionHeader(headers);
|
|
1303
|
+
await applyAppAssertionHeader(headers, opts.getAppAssertion);
|
|
1304
|
+
const mint = await fetchWithDeadline(
|
|
1305
|
+
`${baseUrl}/documents/${encodeURIComponent(docId)}/ops/upload`,
|
|
1306
|
+
{ method: 'POST', headers, body: '{}' },
|
|
1307
|
+
OP_WRITE_TIMEOUT_MS,
|
|
1308
|
+
);
|
|
1309
|
+
let minted: unknown = undefined;
|
|
914
1310
|
try {
|
|
915
|
-
|
|
1311
|
+
minted = mint.text ? JSON.parse(mint.text) : undefined;
|
|
916
1312
|
} catch {
|
|
917
|
-
|
|
1313
|
+
minted = undefined;
|
|
918
1314
|
}
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
1315
|
+
if (!mint.ok) {
|
|
1316
|
+
throw brassErrorFromBody(mint.status, minted);
|
|
1317
|
+
}
|
|
1318
|
+
const body = (minted ?? {}) as {
|
|
1319
|
+
upload_key?: unknown;
|
|
1320
|
+
url?: unknown;
|
|
1321
|
+
};
|
|
1322
|
+
if (typeof body.upload_key !== 'string' || typeof body.url !== 'string') {
|
|
1323
|
+
throw new BrassError('Malformed op upload response', { status: 500 });
|
|
1324
|
+
}
|
|
1325
|
+
const endUpload = perfStart('op: PUT staged op blob');
|
|
1326
|
+
const put = await fetchWithDeadline(
|
|
1327
|
+
body.url,
|
|
1328
|
+
{ method: 'PUT', body: opBlobB64 },
|
|
1329
|
+
deadlineForUpload(opBlobB64.length),
|
|
1330
|
+
);
|
|
1331
|
+
endUpload({ status: put.status, opChars: opBlobB64.length });
|
|
1332
|
+
if (!put.ok) {
|
|
1333
|
+
throw new BrassError('Could not upload the update', {
|
|
1334
|
+
status: put.status,
|
|
1335
|
+
});
|
|
1336
|
+
}
|
|
1337
|
+
return body.upload_key;
|
|
1338
|
+
} catch (err) {
|
|
1339
|
+
throw asOpStagingError(err);
|
|
930
1340
|
}
|
|
931
|
-
return body.upload_key;
|
|
932
1341
|
}
|
|
933
1342
|
|
|
934
1343
|
async function postDocumentOp(
|
|
@@ -952,7 +1361,7 @@ export function createOpenDocument(
|
|
|
952
1361
|
stagedKey = await stageOpBlob(docId, opBlobB64, token);
|
|
953
1362
|
}
|
|
954
1363
|
const endFetch = perfStart('op: POST /documents/{id}/ops');
|
|
955
|
-
const response = await
|
|
1364
|
+
const response = await fetchWithDeadline(
|
|
956
1365
|
`${baseUrl}/documents/${encodeURIComponent(docId)}/ops`,
|
|
957
1366
|
{
|
|
958
1367
|
method: 'POST',
|
|
@@ -965,12 +1374,14 @@ export function createOpenDocument(
|
|
|
965
1374
|
client_op_id: clientOpId,
|
|
966
1375
|
}),
|
|
967
1376
|
},
|
|
1377
|
+
// An op under the staging threshold travels inline, so this request
|
|
1378
|
+
// carries the blob and its deadline has to allow for sending it.
|
|
1379
|
+
deadlineForUpload(stagedKey === undefined ? opBlobB64.length : 0),
|
|
968
1380
|
);
|
|
969
|
-
const text = await response.text();
|
|
970
1381
|
endFetch({ status: response.status, opChars: opBlobB64.length });
|
|
971
1382
|
let parsed: unknown = undefined;
|
|
972
1383
|
try {
|
|
973
|
-
parsed = text ? JSON.parse(text) : undefined;
|
|
1384
|
+
parsed = response.text ? JSON.parse(response.text) : undefined;
|
|
974
1385
|
} catch {
|
|
975
1386
|
parsed = undefined;
|
|
976
1387
|
}
|
|
@@ -1136,7 +1547,9 @@ export function createOpenDocument(
|
|
|
1136
1547
|
status: 500,
|
|
1137
1548
|
});
|
|
1138
1549
|
}
|
|
1139
|
-
|
|
1550
|
+
// A stream's write authority is the document's, so the line opens
|
|
1551
|
+
// read-only for the same reader the document opens read-only for.
|
|
1552
|
+
const doc = await materialise(parsed, readOnlyForGrant(parsed, undefined));
|
|
1140
1553
|
// A stream line materializes at a historical seq the same way the
|
|
1141
1554
|
// default line does, so the parent's version preview can bring this
|
|
1142
1555
|
// stream to a point in time (`doc.stream(name)._previewAt(seq)`). The
|
|
@@ -1154,8 +1567,15 @@ export function createOpenDocument(
|
|
|
1154
1567
|
),
|
|
1155
1568
|
typeof parsed.current_seq === 'number' ? parsed.current_seq : 0,
|
|
1156
1569
|
);
|
|
1157
|
-
|
|
1158
|
-
|
|
1570
|
+
const lineTailSeqs = parsed.tail_ops.map((entry) => entry.seq);
|
|
1571
|
+
let coveredSeq = joinCoverageFloor(
|
|
1572
|
+
snapshotSeqOf(
|
|
1573
|
+
parsed.snapshot_seq,
|
|
1574
|
+
lineTailSeqs,
|
|
1575
|
+
typeof parsed.current_seq === 'number' ? parsed.current_seq : 0,
|
|
1576
|
+
),
|
|
1577
|
+
lineTailSeqs,
|
|
1578
|
+
);
|
|
1159
1579
|
const coverageParked = new Set<number>();
|
|
1160
1580
|
const advanceCoverage = (seq: number): void => {
|
|
1161
1581
|
if (seq <= coveredSeq) return;
|
|
@@ -1165,45 +1585,77 @@ export function createOpenDocument(
|
|
|
1165
1585
|
coveredSeq += 1;
|
|
1166
1586
|
}
|
|
1167
1587
|
};
|
|
1588
|
+
const raiseCoverage = (floor: number): void => {
|
|
1589
|
+
if (floor > coveredSeq) coveredSeq = floor;
|
|
1590
|
+
for (const seq of coverageParked) {
|
|
1591
|
+
if (seq <= coveredSeq) coverageParked.delete(seq);
|
|
1592
|
+
}
|
|
1593
|
+
while (coverageParked.has(coveredSeq + 1)) {
|
|
1594
|
+
coverageParked.delete(coveredSeq + 1);
|
|
1595
|
+
coveredSeq += 1;
|
|
1596
|
+
}
|
|
1597
|
+
};
|
|
1168
1598
|
for (const entry of parsed.tail_ops) advanceCoverage(entry.seq);
|
|
1169
1599
|
|
|
1170
|
-
//
|
|
1171
|
-
//
|
|
1172
|
-
// clean merge.
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
if (
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1600
|
+
// Re-import the line's join: the reconnect-across-compaction recovery,
|
|
1601
|
+
// and the answer to a catch-up prompt. Loro imports are idempotent, so
|
|
1602
|
+
// re-applying over live state is a clean merge.
|
|
1603
|
+
//
|
|
1604
|
+
// Coalesced and delivery-honest for the same reasons the default line
|
|
1605
|
+
// is: a prompt during a fetch is the only notice of that op the line
|
|
1606
|
+
// gets, and a join's `current_seq` names seqs whose rows the tail query
|
|
1607
|
+
// could not see yet.
|
|
1608
|
+
let lineResyncRun: Promise<void> | null = null;
|
|
1609
|
+
let lineResyncAgain = false;
|
|
1610
|
+
const runLineResync = async (): Promise<void> => {
|
|
1611
|
+
try {
|
|
1612
|
+
do {
|
|
1613
|
+
lineResyncAgain = false;
|
|
1614
|
+
try {
|
|
1615
|
+
const t = await requireToken();
|
|
1616
|
+
const res = await fetchSnapshot(docId, { branch: lineId }, t);
|
|
1617
|
+
if (res.status < 200 || res.status >= 300) continue;
|
|
1618
|
+
if (!isSnapshotResponse(res.parsed)) continue;
|
|
1619
|
+
const snap = res.parsed.snapshot;
|
|
1620
|
+
if (snap !== null) {
|
|
1621
|
+
const bytes =
|
|
1622
|
+
snap.kind === 'inline'
|
|
1623
|
+
? base64ToBytes(snap.data_b64)
|
|
1624
|
+
: await fetchPresignedSnapshot(snap.url);
|
|
1625
|
+
doc._applySnapshot(bytes);
|
|
1626
|
+
}
|
|
1627
|
+
const seqs: number[] = [];
|
|
1628
|
+
for (const entry of res.parsed.tail_ops) {
|
|
1629
|
+
doc.applyRemoteOp(base64ToBytes(entry.op_blob_b64));
|
|
1630
|
+
if (entry.seq > currentSeq) currentSeq = entry.seq;
|
|
1631
|
+
seqs.push(entry.seq);
|
|
1201
1632
|
}
|
|
1633
|
+
const joined = res.parsed.current_seq;
|
|
1634
|
+
if (typeof joined === 'number' && joined > currentSeq) {
|
|
1635
|
+
currentSeq = joined;
|
|
1636
|
+
}
|
|
1637
|
+
raiseCoverage(
|
|
1638
|
+
joinCoverageFloor(
|
|
1639
|
+
snapshotSeqOf(res.parsed.snapshot_seq, seqs, currentSeq),
|
|
1640
|
+
seqs,
|
|
1641
|
+
),
|
|
1642
|
+
);
|
|
1643
|
+
for (const seq of seqs) advanceCoverage(seq);
|
|
1644
|
+
} catch {
|
|
1645
|
+
// Best-effort; the next prompt or doc_init retries.
|
|
1202
1646
|
}
|
|
1203
|
-
}
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
}
|
|
1647
|
+
} while (lineResyncAgain);
|
|
1648
|
+
} finally {
|
|
1649
|
+
lineResyncRun = null;
|
|
1650
|
+
}
|
|
1651
|
+
};
|
|
1652
|
+
const resyncFromSnapshot = (): Promise<void> => {
|
|
1653
|
+
if (lineResyncRun !== null) {
|
|
1654
|
+
lineResyncAgain = true;
|
|
1655
|
+
return lineResyncRun;
|
|
1656
|
+
}
|
|
1657
|
+
lineResyncRun = runLineResync();
|
|
1658
|
+
return lineResyncRun;
|
|
1207
1659
|
};
|
|
1208
1660
|
|
|
1209
1661
|
// Write an op the socket cannot carry. The op stays one commit and
|
|
@@ -1212,7 +1664,7 @@ export function createOpenDocument(
|
|
|
1212
1664
|
const postOversizedOp = async (
|
|
1213
1665
|
opBlobB64: string,
|
|
1214
1666
|
clientOpId: string,
|
|
1215
|
-
): Promise<
|
|
1667
|
+
): Promise<OversizedOpResult> => {
|
|
1216
1668
|
const token = await requireToken();
|
|
1217
1669
|
const { status, parsed: opResult } = await postDocumentOp(
|
|
1218
1670
|
docId,
|
|
@@ -1222,11 +1674,23 @@ export function createOpenDocument(
|
|
|
1222
1674
|
token,
|
|
1223
1675
|
);
|
|
1224
1676
|
if (status < 200 || status >= 300) {
|
|
1225
|
-
return {
|
|
1677
|
+
return {
|
|
1678
|
+
ok: false,
|
|
1679
|
+
message: opWriteFailureMessage(status, opResult),
|
|
1680
|
+
retryable: opWriteIsRetryable(status),
|
|
1681
|
+
atCapacity: isCapacityRefusal(status, opResult),
|
|
1682
|
+
// The route answers 403 when the caller's grant carries no write
|
|
1683
|
+
// access, which is the HTTP face of the socket's `role_denied`.
|
|
1684
|
+
...(status === 403 ? { readOnly: 'grant' as const } : {}),
|
|
1685
|
+
};
|
|
1226
1686
|
}
|
|
1227
1687
|
const seq = (opResult as { seq?: unknown } | undefined)?.seq;
|
|
1228
1688
|
if (typeof seq !== 'number') {
|
|
1229
|
-
return {
|
|
1689
|
+
return {
|
|
1690
|
+
ok: false,
|
|
1691
|
+
message: 'The server did not report a version for this change.',
|
|
1692
|
+
retryable: false,
|
|
1693
|
+
};
|
|
1230
1694
|
}
|
|
1231
1695
|
return { ok: true, seq };
|
|
1232
1696
|
};
|
|
@@ -1267,7 +1731,7 @@ export function createOpenDocument(
|
|
|
1267
1731
|
}
|
|
1268
1732
|
const doc = await materialise(
|
|
1269
1733
|
parsed,
|
|
1270
|
-
options?.documentOptions,
|
|
1734
|
+
readOnlyForGrant(parsed, options?.documentOptions),
|
|
1271
1735
|
options?.onSnapshotProgress,
|
|
1272
1736
|
);
|
|
1273
1737
|
// Populate `doc.status` from the join before the first `doc_status` frame
|
|
@@ -1296,10 +1760,21 @@ export function createOpenDocument(
|
|
|
1296
1760
|
// then can't tell "caught up" from "missed ops since compacted into the
|
|
1297
1761
|
// snapshot". The floor never overestimates, so the `doc_init` resync
|
|
1298
1762
|
// check always detects such a hole (at worst a transient out-of-order
|
|
1299
|
-
// arrival costs one idempotent re-fetch).
|
|
1300
|
-
//
|
|
1301
|
-
|
|
1302
|
-
|
|
1763
|
+
// arrival costs one idempotent re-fetch).
|
|
1764
|
+
//
|
|
1765
|
+
// Seeded from what this join DELIVERED, which is its snapshot plus the
|
|
1766
|
+
// unbroken run of tail seqs above it, never from `current_seq`: a join
|
|
1767
|
+
// read while someone writes in bulk reports a counter above the rows it
|
|
1768
|
+
// could see, and seeding from the counter opens the document already
|
|
1769
|
+
// claiming ops it does not hold.
|
|
1770
|
+
let coveredSeq = joinCoverageFloor(
|
|
1771
|
+
snapshotSeqOf(
|
|
1772
|
+
parsed.snapshot_seq,
|
|
1773
|
+
parsed.tail_ops.map((entry) => entry.seq),
|
|
1774
|
+
typeof parsed.current_seq === 'number' ? parsed.current_seq : 0,
|
|
1775
|
+
),
|
|
1776
|
+
parsed.tail_ops.map((entry) => entry.seq),
|
|
1777
|
+
);
|
|
1303
1778
|
// Seqs applied above the floor, parked until the gap below fills.
|
|
1304
1779
|
const coverageParked = new Set<number>();
|
|
1305
1780
|
const drainCoverage = (): void => {
|
|
@@ -1313,9 +1788,11 @@ export function createOpenDocument(
|
|
|
1313
1788
|
coverageParked.add(seq);
|
|
1314
1789
|
drainCoverage();
|
|
1315
1790
|
};
|
|
1316
|
-
// Mark everything at or below `floor` covered
|
|
1317
|
-
//
|
|
1318
|
-
//
|
|
1791
|
+
// Mark everything at or below `floor` covered. The caller owes the
|
|
1792
|
+
// guarantee that goes with it: every seq under `floor` is applied, or
|
|
1793
|
+
// folded into a snapshot this doc imported. Coverage is what retires a
|
|
1794
|
+
// catch-up prompt, so a floor taken from anything looser than that is a
|
|
1795
|
+
// reader quietly deciding it holds ops nobody sent it.
|
|
1319
1796
|
const raiseCoverage = (floor: number): void => {
|
|
1320
1797
|
if (floor > coveredSeq) coveredSeq = floor;
|
|
1321
1798
|
for (const seq of coverageParked) {
|
|
@@ -1386,14 +1863,38 @@ export function createOpenDocument(
|
|
|
1386
1863
|
if (entry.seq > currentSeq) currentSeq = entry.seq;
|
|
1387
1864
|
if (entry.seq > maxTailSeq) maxTailSeq = entry.seq;
|
|
1388
1865
|
}
|
|
1389
|
-
// Raise both cursors so the floor reflects the import (this clears a
|
|
1390
|
-
// detected missed-compaction hole).
|
|
1391
1866
|
const fetchedSeq =
|
|
1392
1867
|
typeof result.parsed.current_seq === 'number'
|
|
1393
1868
|
? result.parsed.current_seq
|
|
1394
1869
|
: maxTailSeq;
|
|
1395
1870
|
if (fetchedSeq > currentSeq) currentSeq = fetchedSeq;
|
|
1396
|
-
|
|
1871
|
+
// Coverage moves to what this response DELIVERED, not to the head it
|
|
1872
|
+
// reported. The two differ exactly when it matters: the head is the
|
|
1873
|
+
// branch's seq counter, bumped when a seq is handed out rather than
|
|
1874
|
+
// when its row is written, so a join read during a bulk write names
|
|
1875
|
+
// ops the tail query could not see. Crediting the head marks those
|
|
1876
|
+
// seqs covered, and the prompts that would fetch them then short
|
|
1877
|
+
// circuit, which is a reader left permanently short of a write it
|
|
1878
|
+
// watched land.
|
|
1879
|
+
//
|
|
1880
|
+
// The cost is that a seq allocated and never written (a failed op
|
|
1881
|
+
// write leaves a hole rather than a duplicate) now stalls the floor
|
|
1882
|
+
// under it. That resolves at the next compaction, which folds the tail
|
|
1883
|
+
// past the hole and raises `snapshot_seq` over it; until then the gap
|
|
1884
|
+
// pull's own budget is what stops the reader asking forever.
|
|
1885
|
+
const tailSeqs = sortedTail.map((entry) => entry.seq);
|
|
1886
|
+
// A snapshot only counts as a floor once its bytes are in: a shape this
|
|
1887
|
+
// pin does not decode leaves `snapshot_seq` describing content the doc
|
|
1888
|
+
// never received, and the tail alone is then what it holds.
|
|
1889
|
+
if (snapshotBytes !== null || result.parsed.snapshot === null) {
|
|
1890
|
+
raiseCoverage(
|
|
1891
|
+
joinCoverageFloor(
|
|
1892
|
+
snapshotSeqOf(result.parsed.snapshot_seq, tailSeqs, fetchedSeq),
|
|
1893
|
+
tailSeqs,
|
|
1894
|
+
),
|
|
1895
|
+
);
|
|
1896
|
+
}
|
|
1897
|
+
for (const seq of tailSeqs) advanceCoverage(seq);
|
|
1397
1898
|
endPerf({
|
|
1398
1899
|
ok: true,
|
|
1399
1900
|
snapshotBytes: snapshotBytes?.byteLength ?? 0,
|
|
@@ -1429,27 +1930,55 @@ export function createOpenDocument(
|
|
|
1429
1930
|
return out;
|
|
1430
1931
|
};
|
|
1431
1932
|
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1933
|
+
// Coalesced, not skipped. A prompt that lands while a join is in flight
|
|
1934
|
+
// is a statement that the document has moved past what that join can
|
|
1935
|
+
// possibly carry, so dropping it loses the only notice this reader gets:
|
|
1936
|
+
// the server posts each prompt once, and a passive reader has no ack or
|
|
1937
|
+
// op of its own to notice the shortfall from. A bulk write is a run of
|
|
1938
|
+
// prompts (every op past the frame bound takes the HTTP route and
|
|
1939
|
+
// prompts each subscriber), and under a plain skip-if-in-flight the
|
|
1940
|
+
// whole run collapses into the one or two joins that happened not to be
|
|
1941
|
+
// busy, which is a reader stranded partway through a write it watched.
|
|
1942
|
+
//
|
|
1943
|
+
// `resyncAgain` is a flag rather than a queue because a join reads the
|
|
1944
|
+
// whole document: one more pass answers every prompt that arrived during
|
|
1945
|
+
// this one. The loop therefore runs at most one extra fetch after the
|
|
1946
|
+
// last prompt, and settles when the writer stops.
|
|
1947
|
+
//
|
|
1948
|
+
// The returned promise settles when the last pass this call is
|
|
1949
|
+
// responsible for has landed, so a caller deciding whether the reader is
|
|
1950
|
+
// still short reads the floor after the work rather than during it.
|
|
1951
|
+
let resyncRun: Promise<void> | null = null;
|
|
1952
|
+
let resyncAgain = false;
|
|
1953
|
+
const runResync = async (): Promise<void> => {
|
|
1954
|
+
try {
|
|
1955
|
+
do {
|
|
1956
|
+
resyncAgain = false;
|
|
1957
|
+
try {
|
|
1958
|
+
const outcome = await refetchAndImport();
|
|
1959
|
+
if (!outcome.ok) {
|
|
1960
|
+
console.error(
|
|
1961
|
+
'Brass: document resync after reconnect failed; state may lag until the next reconnect or reload',
|
|
1962
|
+
);
|
|
1963
|
+
}
|
|
1964
|
+
} catch (err) {
|
|
1440
1965
|
console.error(
|
|
1441
1966
|
'Brass: document resync after reconnect failed; state may lag until the next reconnect or reload',
|
|
1967
|
+
err,
|
|
1442
1968
|
);
|
|
1443
1969
|
}
|
|
1444
|
-
}
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1970
|
+
} while (resyncAgain);
|
|
1971
|
+
} finally {
|
|
1972
|
+
resyncRun = null;
|
|
1973
|
+
}
|
|
1974
|
+
};
|
|
1975
|
+
const resyncFromSnapshot = (): Promise<void> => {
|
|
1976
|
+
if (resyncRun !== null) {
|
|
1977
|
+
resyncAgain = true;
|
|
1978
|
+
return resyncRun;
|
|
1979
|
+
}
|
|
1980
|
+
resyncRun = runResync();
|
|
1981
|
+
return resyncRun;
|
|
1453
1982
|
};
|
|
1454
1983
|
|
|
1455
1984
|
// When a doc opened mid-load finishes, pull the now-available content
|
|
@@ -1457,90 +1986,84 @@ export function createOpenDocument(
|
|
|
1457
1986
|
// called on the server's authoritative `reload` hint, so it never has to
|
|
1458
1987
|
// tell success from failure: a failure carries no `reload`, so no racy
|
|
1459
1988
|
// read and no loop against a failed hydration.
|
|
1460
|
-
//
|
|
1461
|
-
// A failed fetch here is transient (network, dropped presigned download)
|
|
1462
|
-
// and is retried with backoff: a silent give-up would strand the doc
|
|
1463
|
-
// empty-but-editable, reading as "the model never loaded" and invisible
|
|
1464
|
-
// in every log. On exhaustion the failure is reported and the terminal
|
|
1465
|
-
// status is an honest failure, not the server's success copy.
|
|
1466
1989
|
let reloadInFlight = false;
|
|
1467
|
-
|
|
1468
|
-
|
|
1990
|
+
let reconcileInFlight = false;
|
|
1991
|
+
// A terminal frame that lands while the subscribe-time reconcile is still
|
|
1992
|
+
// fetching waits for it: two concurrent imports each clear the undo stack
|
|
1993
|
+
// and race their terminal emits, and dropping the frame would leave a
|
|
1994
|
+
// reconcile that read the row too early sitting on the pending overlay.
|
|
1995
|
+
let deferredReload: DocumentWorkStatus | null = null;
|
|
1996
|
+
// Armed when the pull below gives up. The work is terminal server-side, so
|
|
1997
|
+
// no further status frame arrives, and later ops are causally dependent on
|
|
1998
|
+
// a snapshot this client never imported (Loro buffers them and the doc
|
|
1999
|
+
// stays empty), so the next frame on the topic drives one more attempt.
|
|
2000
|
+
let retryPullOnNextFrame = false;
|
|
1469
2001
|
const reloadAfterHydration = (finalStatus: DocumentWorkStatus): void => {
|
|
1470
2002
|
if (reloadInFlight) return;
|
|
2003
|
+
if (reconcileInFlight) {
|
|
2004
|
+
deferredReload = finalStatus;
|
|
2005
|
+
return;
|
|
2006
|
+
}
|
|
1471
2007
|
reloadInFlight = true;
|
|
1472
2008
|
void (async () => {
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
// Emit the terminal status AFTER the import settles, so an app
|
|
1480
|
-
// re-reading on `onStatus` sees the content present and (with
|
|
1481
|
-
// `readOnly` now false) clears its loading overlay.
|
|
1482
|
-
doc._emitStatus(finalStatus);
|
|
1483
|
-
return;
|
|
1484
|
-
}
|
|
1485
|
-
lastError = new Error(outcome.reason);
|
|
1486
|
-
} catch (err) {
|
|
1487
|
-
lastError = err instanceof Error ? err : new Error(String(err));
|
|
1488
|
-
}
|
|
1489
|
-
// A retrying attempt is exceptional, so this warn is quiet in
|
|
1490
|
-
// normal operation and decisive in a bug report; without it a
|
|
1491
|
-
// failure here is undiagnosable from any log.
|
|
1492
|
-
console.warn(
|
|
1493
|
-
`[Brass] document content reload attempt ${attempt}/${RELOAD_ATTEMPTS} failed (doc ${docId}): ${lastError.message}`,
|
|
1494
|
-
);
|
|
1495
|
-
if (attempt < RELOAD_ATTEMPTS) {
|
|
1496
|
-
await new Promise((r) => setTimeout(r, RELOAD_BACKOFF_MS * attempt));
|
|
1497
|
-
}
|
|
1498
|
-
}
|
|
1499
|
-
reloadInFlight = false;
|
|
1500
|
-
const error = new BrassError(
|
|
1501
|
-
`Document content reload failed after ${RELOAD_ATTEMPTS} attempts: ${lastError?.message ?? 'unknown error'}`,
|
|
1502
|
-
{ status: 0 },
|
|
1503
|
-
);
|
|
1504
|
-
if (opts.reportError) opts.reportError(error);
|
|
1505
|
-
else console.error('[Brass]', error);
|
|
1506
|
-
doc._emitStatus({
|
|
1507
|
-
message: 'The model could not be loaded. Reload to try again.',
|
|
1508
|
-
pending: false,
|
|
2009
|
+
const imported = await pullContentWithRetry({
|
|
2010
|
+
doc,
|
|
2011
|
+
docId,
|
|
2012
|
+
finalStatus,
|
|
2013
|
+
fetchAndImport: refetchAndImport,
|
|
2014
|
+
...(opts.reportError ? { reportError: opts.reportError } : {}),
|
|
1509
2015
|
});
|
|
2016
|
+
reloadInFlight = false;
|
|
2017
|
+
retryPullOnNextFrame = !imported;
|
|
1510
2018
|
})();
|
|
1511
2019
|
};
|
|
1512
2020
|
|
|
2021
|
+
// Evidence the document moved on the server after a give-up: one more
|
|
2022
|
+
// pull, which on success imports the content and releases the lock the
|
|
2023
|
+
// failed terminal status left in place.
|
|
2024
|
+
const retryContentPull = (): void => {
|
|
2025
|
+
if (!retryPullOnNextFrame) return;
|
|
2026
|
+
retryPullOnNextFrame = false;
|
|
2027
|
+
reloadAfterHydration(WORK_READY_STATUS);
|
|
2028
|
+
};
|
|
2029
|
+
|
|
1513
2030
|
// Reconcile a pending open once the live subscription exists. The
|
|
1514
2031
|
// terminal `doc_status` frame is best-effort: if hydration finishes
|
|
1515
|
-
// before this client's
|
|
2032
|
+
// before this client's subscription record exists, the fan-out finds no subscriber
|
|
1516
2033
|
// and drops the frame, stranding the app on the pending overlay. `doc_init`
|
|
1517
|
-
// is posted only after the
|
|
2034
|
+
// is posted only after the subscription record is written, so one re-fetch here
|
|
1518
2035
|
// closes the gap for a completion that landed before the row existed.
|
|
1519
2036
|
// Only a pending open pays the extra fetch.
|
|
1520
|
-
let reconcileInFlight = false;
|
|
1521
2037
|
const reconcileStatusOnInit = (): void => {
|
|
1522
2038
|
const current = doc.status;
|
|
1523
2039
|
if (!current || !current.pending) return;
|
|
1524
2040
|
if (reconcileInFlight || reloadInFlight) return;
|
|
1525
2041
|
reconcileInFlight = true;
|
|
1526
2042
|
void (async () => {
|
|
2043
|
+
let settled = false;
|
|
1527
2044
|
try {
|
|
1528
2045
|
const outcome = await refetchAndImport();
|
|
1529
2046
|
if (!outcome.ok) return;
|
|
1530
2047
|
// Still pending: leave the overlay up; subscribed now, so the live
|
|
1531
2048
|
// fan-out delivers the terminal frame when the work finishes.
|
|
1532
2049
|
if (outcome.status?.pending) return;
|
|
1533
|
-
|
|
1534
|
-
//
|
|
1535
|
-
// the
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
);
|
|
2050
|
+
settled = true;
|
|
2051
|
+
// Terminal. A status the server still holds is a failure (success
|
|
2052
|
+
// clears the row), and its absence is the success this import just
|
|
2053
|
+
// pulled the content for, reported as the same terminal frame the
|
|
2054
|
+
// platform sends on that transition.
|
|
2055
|
+
if (outcome.status) doc._emitStatus(outcome.status);
|
|
2056
|
+
else doc._emitStatus(WORK_READY_STATUS, { contentLoaded: true });
|
|
1539
2057
|
} catch {
|
|
1540
2058
|
// Best-effort; the live fan-out remains the primary path when
|
|
1541
2059
|
// the work finishes after this subscribe.
|
|
1542
2060
|
} finally {
|
|
1543
2061
|
reconcileInFlight = false;
|
|
2062
|
+
const deferred = deferredReload;
|
|
2063
|
+
deferredReload = null;
|
|
2064
|
+
// A settled reconcile imported the content itself, so the held
|
|
2065
|
+
// frame has nothing left to pull.
|
|
2066
|
+
if (deferred && !settled) reloadAfterHydration(deferred);
|
|
1544
2067
|
}
|
|
1545
2068
|
})();
|
|
1546
2069
|
};
|
|
@@ -1552,7 +2075,7 @@ export function createOpenDocument(
|
|
|
1552
2075
|
const postOversizedOp = async (
|
|
1553
2076
|
opBlobB64: string,
|
|
1554
2077
|
clientOpId: string,
|
|
1555
|
-
): Promise<
|
|
2078
|
+
): Promise<OversizedOpResult> => {
|
|
1556
2079
|
const token = await requireToken();
|
|
1557
2080
|
const { status, parsed: opResult } = await postDocumentOp(
|
|
1558
2081
|
docId,
|
|
@@ -1562,13 +2085,22 @@ export function createOpenDocument(
|
|
|
1562
2085
|
token,
|
|
1563
2086
|
);
|
|
1564
2087
|
if (status < 200 || status >= 300) {
|
|
1565
|
-
return {
|
|
2088
|
+
return {
|
|
2089
|
+
ok: false,
|
|
2090
|
+
message: opWriteFailureMessage(status, opResult),
|
|
2091
|
+
retryable: opWriteIsRetryable(status),
|
|
2092
|
+
atCapacity: isCapacityRefusal(status, opResult),
|
|
2093
|
+
// The route answers 403 when the caller's grant carries no write
|
|
2094
|
+
// access, which is the HTTP face of the socket's `role_denied`.
|
|
2095
|
+
...(status === 403 ? { readOnly: 'grant' as const } : {}),
|
|
2096
|
+
};
|
|
1566
2097
|
}
|
|
1567
2098
|
const seq = (opResult as { seq?: unknown } | undefined)?.seq;
|
|
1568
2099
|
if (typeof seq !== 'number') {
|
|
1569
2100
|
return {
|
|
1570
2101
|
ok: false,
|
|
1571
2102
|
message: 'The server did not report a version for this change.',
|
|
2103
|
+
retryable: false,
|
|
1572
2104
|
};
|
|
1573
2105
|
}
|
|
1574
2106
|
return { ok: true, seq };
|
|
@@ -1587,6 +2119,10 @@ export function createOpenDocument(
|
|
|
1587
2119
|
? { requestedBranchId: options.branch }
|
|
1588
2120
|
: {}),
|
|
1589
2121
|
...(options?.stream !== undefined ? { requestedStream: options.stream } : {}),
|
|
2122
|
+
...(options?.documentOptions !== undefined
|
|
2123
|
+
? { documentOptions: options.documentOptions }
|
|
2124
|
+
: {}),
|
|
2125
|
+
...(opts.identity !== undefined ? { identity: opts.identity } : {}),
|
|
1590
2126
|
wsClient: opts.wsClient,
|
|
1591
2127
|
postOversizedOp,
|
|
1592
2128
|
newClientOpId: opts.newClientOpId ?? defaultClientOpIdMinter,
|
|
@@ -1600,6 +2136,7 @@ export function createOpenDocument(
|
|
|
1600
2136
|
refetchDurableFrontier,
|
|
1601
2137
|
reloadAfterHydration,
|
|
1602
2138
|
reconcileStatusOnInit,
|
|
2139
|
+
onDocumentMoved: retryContentPull,
|
|
1603
2140
|
});
|
|
1604
2141
|
}
|
|
1605
2142
|
// `doc._checkoutVersion(v)` calls back here to materialise historical
|
|
@@ -1746,6 +2283,11 @@ export function createOpenDocument(
|
|
|
1746
2283
|
parseFailed = true;
|
|
1747
2284
|
}
|
|
1748
2285
|
}
|
|
2286
|
+
// A tail op whose bytes spilled to storage arrives as a signed URL, and
|
|
2287
|
+
// a guest join carries them exactly as a signed-in one does. The URL
|
|
2288
|
+
// carries its own signature, so an anonymous reader fetches it with the
|
|
2289
|
+
// same call. Resolving here is what lets `materialise` see one shape.
|
|
2290
|
+
await resolveParsedTailOps(parsed);
|
|
1749
2291
|
return { status: response.status, parsed, parseFailed };
|
|
1750
2292
|
}
|
|
1751
2293
|
|
|
@@ -1812,6 +2354,7 @@ export function createOpenDocument(
|
|
|
1812
2354
|
current_seq?: unknown;
|
|
1813
2355
|
name?: unknown;
|
|
1814
2356
|
link_role?: unknown;
|
|
2357
|
+
status?: unknown;
|
|
1815
2358
|
};
|
|
1816
2359
|
if (
|
|
1817
2360
|
typeof b.doc_id !== 'string' ||
|
|
@@ -1823,7 +2366,7 @@ export function createOpenDocument(
|
|
|
1823
2366
|
// The link's grant decides read vs write. An `editor` link is anonymous
|
|
1824
2367
|
// EDIT: a live Document (not read-only) whose local mutations ship over
|
|
1825
2368
|
// the same token-authenticated socket, with the server admitting the
|
|
1826
|
-
// op-write because the guest's
|
|
2369
|
+
// op-write because the guest's subscription record carries `role: editor`. A
|
|
1827
2370
|
// `viewer` link (the default) is read-only: mutate throws and only
|
|
1828
2371
|
// inbound frames apply.
|
|
1829
2372
|
const canEdit = b.link_role === 'editor';
|
|
@@ -1840,8 +2383,19 @@ export function createOpenDocument(
|
|
|
1840
2383
|
// The view carries the document's display name; set it so the guest
|
|
1841
2384
|
// sees the title (the signed-in open gets this from `finalise`).
|
|
1842
2385
|
if (typeof b.name === 'string') doc._setName(b.name);
|
|
2386
|
+
// A link opened while the platform is still preparing the document
|
|
2387
|
+
// carries the same opaque work status the signed-in join does, and the
|
|
2388
|
+
// guest's own topic delivers the phases after it. Seeded before the
|
|
2389
|
+
// transport so the app renders the wait rather than an empty document.
|
|
2390
|
+
const linkStatus = parseWorkStatus(b.status);
|
|
2391
|
+
if (linkStatus) doc._emitStatus(linkStatus);
|
|
2392
|
+
// An edit link's writes fail the same ways a member's do (a shed the
|
|
2393
|
+
// budget outlived, a link revoked mid-edit), and the app hears those
|
|
2394
|
+
// through the same channel it hears a member's, so the reporter is
|
|
2395
|
+
// wired here as `finalise` wires it for a signed-in open.
|
|
2396
|
+
if (opts.reportError) doc.setErrorReporter(opts.reportError);
|
|
1843
2397
|
|
|
1844
|
-
wireGuestTransport(doc, b.doc_id, b.branch_id, canEdit);
|
|
2398
|
+
wireGuestTransport(doc, token, b.doc_id, b.branch_id, canEdit);
|
|
1845
2399
|
|
|
1846
2400
|
// Streams: a guest resolves `doc.stream(name)` through the token
|
|
1847
2401
|
// routes. `list` hits the anonymous manifest; `openLine` joins the
|
|
@@ -1883,7 +2437,8 @@ export function createOpenDocument(
|
|
|
1883
2437
|
},
|
|
1884
2438
|
{ readOnly: !canEdit },
|
|
1885
2439
|
);
|
|
1886
|
-
|
|
2440
|
+
if (opts.reportError) inner.setErrorReporter(opts.reportError);
|
|
2441
|
+
wireGuestTransport(inner, token, lb.doc_id, lb.branch_id, canEdit);
|
|
1887
2442
|
return inner;
|
|
1888
2443
|
},
|
|
1889
2444
|
create: () => {
|
|
@@ -1895,17 +2450,55 @@ export function createOpenDocument(
|
|
|
1895
2450
|
return doc;
|
|
1896
2451
|
};
|
|
1897
2452
|
|
|
2453
|
+
// Write a guest's op over HTTP: `POST /links/{token}/ops`, the anonymous
|
|
2454
|
+
// counterpart of `POST /documents/{docId}/ops`. Sends NO Authorization
|
|
2455
|
+
// header (the token is the credential) and names the line the guest
|
|
2456
|
+
// opened, which is the branch its socket topic is scoped to.
|
|
2457
|
+
async function postLinkOp(
|
|
2458
|
+
token: string,
|
|
2459
|
+
lineId: string,
|
|
2460
|
+
opBlobB64: string,
|
|
2461
|
+
clientOpId: string,
|
|
2462
|
+
): Promise<{ status: number; parsed: unknown }> {
|
|
2463
|
+
const response = await fetch(
|
|
2464
|
+
`${baseUrl}/links/${encodeURIComponent(token)}/ops`,
|
|
2465
|
+
{
|
|
2466
|
+
method: 'POST',
|
|
2467
|
+
headers: { 'content-type': 'application/json' },
|
|
2468
|
+
body: JSON.stringify({
|
|
2469
|
+
op_blob: opBlobB64,
|
|
2470
|
+
branch_id: lineId,
|
|
2471
|
+
client_op_id: clientOpId,
|
|
2472
|
+
}),
|
|
2473
|
+
},
|
|
2474
|
+
);
|
|
2475
|
+
const text = await response.text();
|
|
2476
|
+
let parsed: unknown = undefined;
|
|
2477
|
+
try {
|
|
2478
|
+
parsed = text ? JSON.parse(text) : undefined;
|
|
2479
|
+
} catch {
|
|
2480
|
+
parsed = undefined;
|
|
2481
|
+
}
|
|
2482
|
+
return { status: response.status, parsed };
|
|
2483
|
+
}
|
|
2484
|
+
|
|
1898
2485
|
// Wire a guest Document (the default stream or a stream line) to the
|
|
1899
2486
|
// token-authenticated socket. Inbound frames (`doc_init` / `doc_op`)
|
|
1900
2487
|
// apply for both link kinds; an edit link ALSO wires the outbound path
|
|
1901
2488
|
// (`op_ack` resolves the outstanding-op tracker, `setLocalOpHook` ships
|
|
1902
|
-
// each local commit
|
|
2489
|
+
// each local commit). Best-effort on the read side (no resync / durability
|
|
1903
2490
|
// handshake, which need a signed-in token); a guest that misses ops
|
|
1904
|
-
// re-opens.
|
|
1905
|
-
//
|
|
1906
|
-
//
|
|
2491
|
+
// re-opens. The WRITE side answers a refusal the way the signed-in
|
|
2492
|
+
// transport does, because a guest editor's edit is the same person's work
|
|
2493
|
+
// as a member's and the same two things can happen to it: a frame the
|
|
2494
|
+
// socket cannot carry takes the HTTP route, and a capacity shed is
|
|
2495
|
+
// re-asserted on the shared backoff and reported once the budget is out.
|
|
2496
|
+
// Patches `close` to detach, mirroring `wireDocTransport`, so `Document`
|
|
2497
|
+
// stays free of WS knowledge. A no-op with no WS client (a snapshot-only
|
|
2498
|
+
// guest).
|
|
1907
2499
|
function wireGuestTransport(
|
|
1908
2500
|
doc: Document,
|
|
2501
|
+
token: string,
|
|
1909
2502
|
docId: string,
|
|
1910
2503
|
lineId: string,
|
|
1911
2504
|
canEdit: boolean,
|
|
@@ -1913,68 +2506,319 @@ export function createOpenDocument(
|
|
|
1913
2506
|
const wsClient = opts.wsClient;
|
|
1914
2507
|
if (!wsClient) return;
|
|
1915
2508
|
const topic = `doc:${docId}:${lineId}`;
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
// `doc.onGone(...)` never fires; for an edit link the next local op
|
|
1930
|
-
// would ship onto the evicted SUB# row and bounce as a misleading
|
|
1931
|
-
// `forbidden`. Mark the doc gone (mapping the reason like the
|
|
1932
|
-
// signed-in transport); the guest path has no resend / reverify
|
|
1933
|
-
// timers to drain.
|
|
1934
|
-
doc._markGone(goneReasonOf(frame));
|
|
1935
|
-
} else if (canEdit && isOpAckFrame(frame)) {
|
|
1936
|
-
doc.resolveOutstandingOp(frame.client_op_id);
|
|
1937
|
-
} else if (canEdit && isOpRejectedFrame(frame)) {
|
|
1938
|
-
// A guest edit the server terminally rejected. No resend tracker here
|
|
1939
|
-
// (guest transport is best-effort), so just fail the op on the doc so
|
|
1940
|
-
// "could not be saved" surfaces instead of a stuck saving count.
|
|
1941
|
-
doc._failOutstandingOp(frame.client_op_id, opRejectedMessage(frame));
|
|
2509
|
+
|
|
2510
|
+
// Ops on the socket awaiting an answer, keyed by `client_op_id`. An
|
|
2511
|
+
// `unavailable` shed names no op, so re-asserting one needs the frames
|
|
2512
|
+
// in hand. Scoped to what the socket carried: an op that took the HTTP
|
|
2513
|
+
// write must never reach the socket, and that write re-asserts itself.
|
|
2514
|
+
const outstanding = new Map<string, OpFrame>();
|
|
2515
|
+
let shedRounds = 0;
|
|
2516
|
+
let shedTimer: ReturnType<typeof setTimeout> | null = null;
|
|
2517
|
+
|
|
2518
|
+
function clearShedRecovery(): void {
|
|
2519
|
+
if (shedTimer !== null) {
|
|
2520
|
+
clearTimeout(shedTimer);
|
|
2521
|
+
shedTimer = null;
|
|
1942
2522
|
}
|
|
1943
|
-
|
|
1944
|
-
if (canEdit) {
|
|
1945
|
-
const minter = opts.newClientOpId ?? defaultClientOpIdMinter;
|
|
1946
|
-
doc.setLocalOpHook((opBlob) => {
|
|
1947
|
-
const clientOpId = minter();
|
|
1948
|
-
wsClient.send({
|
|
1949
|
-
type: 'op',
|
|
1950
|
-
topic,
|
|
1951
|
-
op_blob: bytesToBase64(opBlob),
|
|
1952
|
-
client_op_id: clientOpId,
|
|
1953
|
-
});
|
|
1954
|
-
return clientOpId;
|
|
1955
|
-
});
|
|
2523
|
+
shedRounds = 0;
|
|
1956
2524
|
}
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
2525
|
+
|
|
2526
|
+
// Every outstanding op is dead: drop the tracker entries and tell the
|
|
2527
|
+
// Document, which stops counting them in `savingCount` and reports each
|
|
2528
|
+
// through the error channel the app reads. The ids are taken first
|
|
2529
|
+
// because `_failOutstandingOp` re-enters app code.
|
|
2530
|
+
function failAllOutstanding(message: string): void {
|
|
2531
|
+
const clientOpIds = [...outstanding.keys()];
|
|
2532
|
+
outstanding.clear();
|
|
2533
|
+
for (const clientOpId of clientOpIds) {
|
|
2534
|
+
doc._failOutstandingOp(clientOpId, message);
|
|
1963
2535
|
}
|
|
1964
|
-
|
|
2536
|
+
}
|
|
2537
|
+
|
|
2538
|
+
// Put every op still awaiting an answer back on the wire. The client
|
|
2539
|
+
// DROPS an op frame it cannot send right now rather than buffering one
|
|
2540
|
+
// (`ownsFrameRetry` below), so this is the only thing that carries a
|
|
2541
|
+
// shed or reconnect-orphaned op across the window, and it is also what
|
|
2542
|
+
// keeps a frame from being written twice: a copy the client held would
|
|
2543
|
+
// flush on the `subscribed` ack while this re-asserts on the `doc_init`
|
|
2544
|
+
// behind it, and the server allocates a fresh seq per send.
|
|
2545
|
+
const resendAllOutstanding = (): void => {
|
|
2546
|
+
for (const frame of outstanding.values()) wsClient.send(frame);
|
|
1965
2547
|
};
|
|
1966
|
-
}
|
|
1967
2548
|
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
2549
|
+
const onServerUnavailable = (): void => {
|
|
2550
|
+
if (outstanding.size === 0) return;
|
|
2551
|
+
if (shedTimer !== null) return;
|
|
2552
|
+
shedRounds += 1;
|
|
2553
|
+
if (shedRounds > UNAVAILABLE_MAX_ROUNDS) {
|
|
2554
|
+
clearShedRecovery();
|
|
2555
|
+
failAllOutstanding(UNAVAILABLE_GIVE_UP_MESSAGE);
|
|
2556
|
+
return;
|
|
2557
|
+
}
|
|
2558
|
+
shedTimer = setTimeout(() => {
|
|
2559
|
+
shedTimer = null;
|
|
2560
|
+
resendAllOutstanding();
|
|
2561
|
+
}, unavailableDelayMs(shedRounds));
|
|
2562
|
+
(shedTimer as { unref?: () => void }).unref?.();
|
|
2563
|
+
};
|
|
2564
|
+
|
|
2565
|
+
const unsub = wsClient.subscribe(
|
|
2566
|
+
topic,
|
|
2567
|
+
(frame) => {
|
|
2568
|
+
const init = parseDocInitFrame(frame);
|
|
2569
|
+
if (init) {
|
|
2570
|
+
for (const entry of init.tail_ops) {
|
|
2571
|
+
if (entry.op_blob_b64 === undefined) continue;
|
|
2572
|
+
doc.applyRemoteOp(base64ToBytes(entry.op_blob_b64));
|
|
2573
|
+
}
|
|
2574
|
+
// `doc_init` follows every `subscribed` ack, so this is the point
|
|
2575
|
+
// the topic can carry frames again. A fresh connection also returns
|
|
2576
|
+
// the shed budget to full: this re-assert is the round a pending
|
|
2577
|
+
// one would have run.
|
|
2578
|
+
clearShedRecovery();
|
|
2579
|
+
resendAllOutstanding();
|
|
2580
|
+
return;
|
|
2581
|
+
}
|
|
2582
|
+
if (isDocOpFrame(frame)) {
|
|
2583
|
+
doc.applyRemoteOp(base64ToBytes(frame.op_blob));
|
|
2584
|
+
} else if (isDocGoneFrame(frame)) {
|
|
2585
|
+
// The server fans `doc_gone` to EVERY subscriber on the branched
|
|
2586
|
+
// content topic, guest-link connections included, on a soft-delete /
|
|
2587
|
+
// move of the opened doc.
|
|
2588
|
+
// Without this branch the guest keeps showing removed content and
|
|
2589
|
+
// `doc.onGone(...)` never fires; for an edit link the next local op
|
|
2590
|
+
// would ship onto the evicted subscription record and bounce as a misleading
|
|
2591
|
+
// `forbidden`. Drop the shed recovery first so nothing re-asserts an
|
|
2592
|
+
// op onto an evicted subscription record, then mark the doc gone (mapping the
|
|
2593
|
+
// reason like the signed-in transport).
|
|
2594
|
+
clearShedRecovery();
|
|
2595
|
+
outstanding.clear();
|
|
2596
|
+
doc._markGone(goneReasonOf(frame));
|
|
2597
|
+
} else if (isDocStatusFrame(frame)) {
|
|
2598
|
+
// A guest can open a document whose inbound work is still running,
|
|
2599
|
+
// and hydration writes a SNAPSHOT rather than ops, so nothing on the
|
|
2600
|
+
// op stream would ever fill the view in. Same split as the signed-in
|
|
2601
|
+
// transport: the `reload` hint pulls the content and emits the
|
|
2602
|
+
// terminal status after the import, and anything else renders.
|
|
2603
|
+
const status = normalizeWorkStatus(frame.status);
|
|
2604
|
+
if (frame.reload) pullGuestContent(doc, token, lineId, status);
|
|
2605
|
+
else doc._emitStatus(status);
|
|
2606
|
+
} else if (isDocMetaFrame(frame)) {
|
|
2607
|
+
// `_setName` de-dupes, so a re-emit of the current name is a no-op.
|
|
2608
|
+
doc._setName(frame.name);
|
|
2609
|
+
} else if (canEdit && isOpAckFrame(frame)) {
|
|
2610
|
+
outstanding.delete(frame.client_op_id);
|
|
2611
|
+
doc.resolveOutstandingOp(frame.client_op_id);
|
|
2612
|
+
// The server is durably accepting writes again, so the shed budget
|
|
2613
|
+
// starts over. A round already scheduled still runs: the ops it
|
|
2614
|
+
// re-asserts are the ones this ack did not cover.
|
|
2615
|
+
shedRounds = 0;
|
|
2616
|
+
} else if (canEdit && isServerUnavailableFrame(frame)) {
|
|
2617
|
+
onServerUnavailable();
|
|
2618
|
+
} else if (canEdit && isOpRejectedFrame(frame)) {
|
|
2619
|
+
// Terminal, not transient: stop re-asserting this op (retrying
|
|
2620
|
+
// would only draw the same rejection) and fail it on the Document
|
|
2621
|
+
// so "could not be saved" surfaces instead of a stuck saving count.
|
|
2622
|
+
// A refusal naming the writer (an edit link downgraded or revoked
|
|
2623
|
+
// while the guest had it open) turns the Document read-only, the
|
|
2624
|
+
// same way it does for a signed-in reader.
|
|
2625
|
+
outstanding.delete(frame.client_op_id);
|
|
2626
|
+
if (isCapacityRejection(frame)) doc._markAtCapacity();
|
|
2627
|
+
const readOnly = readOnlyReasonForRejection(frame);
|
|
2628
|
+
if (readOnly !== null) doc._markReadOnly(readOnly);
|
|
2629
|
+
doc._failOutstandingOp(frame.client_op_id, opRejectedMessage(frame));
|
|
2630
|
+
}
|
|
2631
|
+
},
|
|
2632
|
+
// The guest owns its outstanding ops: `resendAllOutstanding` on the
|
|
2633
|
+
// `doc_init` behind every `subscribed` ack IS this topic's
|
|
2634
|
+
// re-assertion, so the client drops an op frame it cannot send now
|
|
2635
|
+
// rather than buffering a second copy of it.
|
|
2636
|
+
{ ownsFrameRetry: true },
|
|
2637
|
+
);
|
|
2638
|
+
|
|
2639
|
+
// An op the socket cannot carry, written over HTTP on the same policy
|
|
2640
|
+
// the signed-in path runs: refuse locally what no write can hold, and
|
|
2641
|
+
// re-assert a capacity refusal on the shared backoff before reporting it.
|
|
2642
|
+
async function writeOversizedGuestOp(
|
|
2643
|
+
opBlobB64: string,
|
|
2644
|
+
clientOpId: string,
|
|
2645
|
+
): Promise<void> {
|
|
2646
|
+
// Yield before any outcome resolves the op. The hook's caller records
|
|
2647
|
+
// the outstanding op AFTER the hook returns, and an async function body
|
|
2648
|
+
// runs synchronously up to its first await, so an exit taken inline
|
|
2649
|
+
// would find nothing tracked and leave the entry behind with
|
|
2650
|
+
// `savingCount` stuck above zero.
|
|
2651
|
+
await Promise.resolve();
|
|
2652
|
+
|
|
2653
|
+
// Both sizes are known here, so refuse without spending a round trip
|
|
2654
|
+
// on a blob that can only come back 400.
|
|
2655
|
+
if (opBlobExceedsDurableLimit(opBlobB64)) {
|
|
2656
|
+
doc._failOutstandingOp(clientOpId, TOO_LARGE_MESSAGE);
|
|
2657
|
+
return;
|
|
2658
|
+
}
|
|
2659
|
+
if (opBlobNeedsStaging(opBlobB64)) {
|
|
2660
|
+
doc._failOutstandingOp(clientOpId, LINK_OP_TOO_LARGE_MESSAGE);
|
|
2661
|
+
return;
|
|
2662
|
+
}
|
|
2663
|
+
|
|
2664
|
+
let status = 0;
|
|
2665
|
+
let parsed: unknown = undefined;
|
|
2666
|
+
for (let round = 0; round <= UNAVAILABLE_MAX_ROUNDS; round += 1) {
|
|
2667
|
+
if (round > 0) {
|
|
2668
|
+
await new Promise((resolve) => {
|
|
2669
|
+
const timer = setTimeout(resolve, unavailableDelayMs(round));
|
|
2670
|
+
(timer as { unref?: () => void }).unref?.();
|
|
2671
|
+
});
|
|
2672
|
+
}
|
|
2673
|
+
try {
|
|
2674
|
+
({ status, parsed } = await postLinkOp(
|
|
2675
|
+
token,
|
|
2676
|
+
lineId,
|
|
2677
|
+
opBlobB64,
|
|
2678
|
+
clientOpId,
|
|
2679
|
+
));
|
|
2680
|
+
} catch (err) {
|
|
2681
|
+
// A throw leaves it unknown whether the handler ran, and this op
|
|
2682
|
+
// carries no idempotency the server dedupes on, so replaying it
|
|
2683
|
+
// could double-apply.
|
|
2684
|
+
doc._failOutstandingOp(
|
|
2685
|
+
clientOpId,
|
|
2686
|
+
err instanceof Error ? err.message : String(err),
|
|
2687
|
+
);
|
|
2688
|
+
return;
|
|
2689
|
+
}
|
|
2690
|
+
if (status >= 200 && status < 300) break;
|
|
2691
|
+
if (!opWriteIsRetryable(status)) break;
|
|
2692
|
+
}
|
|
2693
|
+
if (status < 200 || status >= 300) {
|
|
2694
|
+
doc._failOutstandingOp(
|
|
2695
|
+
clientOpId,
|
|
2696
|
+
opWriteIsRetryable(status)
|
|
2697
|
+
? UNAVAILABLE_GIVE_UP_MESSAGE
|
|
2698
|
+
: opWriteFailureMessage(status, parsed),
|
|
2699
|
+
);
|
|
2700
|
+
return;
|
|
2701
|
+
}
|
|
2702
|
+
doc.resolveOutstandingOp(clientOpId);
|
|
2703
|
+
}
|
|
2704
|
+
|
|
2705
|
+
if (canEdit) {
|
|
2706
|
+
const minter = opts.newClientOpId ?? defaultClientOpIdMinter;
|
|
2707
|
+
doc.setLocalOpHook((opBlob) => {
|
|
2708
|
+
const clientOpId = minter();
|
|
2709
|
+
const opBlobB64 = bytesToBase64(opBlob);
|
|
2710
|
+
const sendFrame: OpFrame = {
|
|
2711
|
+
type: 'op',
|
|
2712
|
+
topic,
|
|
2713
|
+
op_blob: opBlobB64,
|
|
2714
|
+
client_op_id: clientOpId,
|
|
2715
|
+
};
|
|
2716
|
+
// A frame past the transport's cap is not delivered and takes the
|
|
2717
|
+
// connection down with it, so it must never reach the socket: a
|
|
2718
|
+
// guest's document, presence and every other op ride that one
|
|
2719
|
+
// connection. `mutate` batches to a budget well above what a frame
|
|
2720
|
+
// carries, so an ordinary bulk edit arrives here.
|
|
2721
|
+
if (opFrameExceedsWsLimit(sendFrame)) {
|
|
2722
|
+
void writeOversizedGuestOp(opBlobB64, clientOpId);
|
|
2723
|
+
return clientOpId;
|
|
2724
|
+
}
|
|
2725
|
+
outstanding.set(clientOpId, sendFrame);
|
|
2726
|
+
wsClient.send(sendFrame);
|
|
2727
|
+
return clientOpId;
|
|
2728
|
+
});
|
|
2729
|
+
}
|
|
2730
|
+
const originalClose = doc.close.bind(doc);
|
|
2731
|
+
(doc as Document & { close: () => void }).close = () => {
|
|
2732
|
+
clearShedRecovery();
|
|
2733
|
+
try {
|
|
2734
|
+
unsub();
|
|
2735
|
+
} catch {
|
|
2736
|
+
// Already detached; closing the Document is what matters.
|
|
2737
|
+
}
|
|
2738
|
+
originalClose();
|
|
2739
|
+
};
|
|
2740
|
+
}
|
|
2741
|
+
|
|
2742
|
+
// Guest documents with a content pull running, so a repeated terminal frame
|
|
2743
|
+
// does not stack a second pull on one document.
|
|
2744
|
+
const guestPullInFlight = new WeakSet<Document>();
|
|
2745
|
+
|
|
2746
|
+
function pullGuestContent(
|
|
2747
|
+
doc: Document,
|
|
2748
|
+
token: string,
|
|
2749
|
+
lineId: string,
|
|
2750
|
+
finalStatus: DocumentWorkStatus,
|
|
2751
|
+
): void {
|
|
2752
|
+
if (guestPullInFlight.has(doc)) return;
|
|
2753
|
+
guestPullInFlight.add(doc);
|
|
2754
|
+
void (async () => {
|
|
2755
|
+
await pullContentWithRetry({
|
|
2756
|
+
doc,
|
|
2757
|
+
docId: doc.id ?? '',
|
|
2758
|
+
finalStatus,
|
|
2759
|
+
fetchAndImport: () => importLinkView(doc, token, lineId),
|
|
2760
|
+
...(opts.reportError ? { reportError: opts.reportError } : {}),
|
|
2761
|
+
});
|
|
2762
|
+
guestPullInFlight.delete(doc);
|
|
2763
|
+
})();
|
|
2764
|
+
}
|
|
2765
|
+
|
|
2766
|
+
// Re-read the link view and import it into the live guest Document: the
|
|
2767
|
+
// anonymous counterpart of the signed-in `refetchAndImport`, over the same
|
|
2768
|
+
// envelope. Named on the line the guest opened, which is the line its topic
|
|
2769
|
+
// is scoped to.
|
|
2770
|
+
async function importLinkView(
|
|
2771
|
+
doc: Document,
|
|
2772
|
+
token: string,
|
|
2773
|
+
lineId: string,
|
|
2774
|
+
): Promise<{ ok: true } | { ok: false; reason: string }> {
|
|
2775
|
+
const result = await fetchLinkView(token, lineId);
|
|
2776
|
+
if (result.status < 200 || result.status >= 300) {
|
|
2777
|
+
return { ok: false, reason: `link view returned ${result.status}` };
|
|
2778
|
+
}
|
|
2779
|
+
const b = result.parsed as {
|
|
2780
|
+
snapshot?: unknown;
|
|
2781
|
+
tail_ops?: unknown;
|
|
2782
|
+
};
|
|
2783
|
+
if (!Array.isArray(b.tail_ops)) {
|
|
2784
|
+
return { ok: false, reason: 'malformed link view response' };
|
|
2785
|
+
}
|
|
2786
|
+
const snap = b.snapshot as DocumentSnapshotResponse['snapshot'];
|
|
2787
|
+
let snapshotBytes: Uint8Array | null = null;
|
|
2788
|
+
if (snap !== null && snap !== undefined && snap.kind === 'inline') {
|
|
2789
|
+
snapshotBytes = base64ToBytes(snap.data_b64);
|
|
2790
|
+
} else if (snap !== null && snap !== undefined && snap.kind === 'presigned') {
|
|
2791
|
+
try {
|
|
2792
|
+
snapshotBytes = await fetchPresignedSnapshot(snap.url);
|
|
2793
|
+
} catch (err) {
|
|
2794
|
+
const status = err instanceof BrassError ? err.status ?? 0 : 0;
|
|
2795
|
+
return {
|
|
2796
|
+
ok: false,
|
|
2797
|
+
reason:
|
|
2798
|
+
status > 0
|
|
2799
|
+
? `presigned snapshot fetch returned ${status}`
|
|
2800
|
+
: `presigned snapshot fetch failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
2801
|
+
};
|
|
2802
|
+
}
|
|
2803
|
+
}
|
|
2804
|
+
if (snapshotBytes) doc._applySnapshot(snapshotBytes);
|
|
2805
|
+
const tail = b.tail_ops as DocumentSnapshotResponse['tail_ops'];
|
|
2806
|
+
for (const entry of [...tail].sort((x, y) => x.seq - y.seq)) {
|
|
2807
|
+
doc.applyRemoteOp(base64ToBytes(entry.op_blob_b64));
|
|
2808
|
+
}
|
|
2809
|
+
return { ok: true };
|
|
2810
|
+
}
|
|
2811
|
+
|
|
2812
|
+
// Strict open: a 404 propagates as `BrassError`. A `version` option
|
|
2813
|
+
// diverts to the read-only at-version path.
|
|
2814
|
+
const open: OpenDocumentByIdFn = async (docId, options) => {
|
|
2815
|
+
// Cache hit: return the parked Document. Its subscription stayed alive
|
|
2816
|
+
// while parked, and the UndoController + outstanding-op map survived, so
|
|
2817
|
+
// undo works as if the user never switched away.
|
|
2818
|
+
const parked = takeParkedDocument(docId, options, opts.identity?.() ?? null);
|
|
2819
|
+
if (parked) {
|
|
2820
|
+
parked._resume();
|
|
2821
|
+
return parked as OpenedDocument;
|
|
1978
2822
|
}
|
|
1979
2823
|
const endTotal = perfStart('open: openDocumentById total');
|
|
1980
2824
|
const token = await requireToken();
|
|
@@ -2049,10 +2893,15 @@ interface WireDocTransportArgs {
|
|
|
2049
2893
|
getCoveredSeq: () => number;
|
|
2050
2894
|
advanceCoveredSeq: (seq: number) => void;
|
|
2051
2895
|
// Re-import the HTTP join. Invoked when a `doc_init` tail starts above the
|
|
2052
|
-
// coverage floor
|
|
2053
|
-
// snapshot and the tail cannot bridge the hole
|
|
2054
|
-
// transport-less paths.
|
|
2055
|
-
|
|
2896
|
+
// coverage floor (ops missed while disconnected were compacted into the
|
|
2897
|
+
// snapshot and the tail cannot bridge the hole) and on every catch-up
|
|
2898
|
+
// prompt. Optional only for transport-less paths.
|
|
2899
|
+
//
|
|
2900
|
+
// A returned promise lets the caller re-read the coverage floor once the
|
|
2901
|
+
// import has landed, which is how a reader that is still short asks
|
|
2902
|
+
// again. A void return keeps that caller correct, just later: it falls
|
|
2903
|
+
// back to whatever the next frame arms.
|
|
2904
|
+
resyncFromSnapshot?: () => void | Promise<void>;
|
|
2056
2905
|
// Re-fetch the durable frontier (version vector + tail blobs) without
|
|
2057
2906
|
// importing. Invoked by the `doc_ops_dropped` handler so the handshake
|
|
2058
2907
|
// runs against fresh ground truth the instant the server discards ops, not
|
|
@@ -2063,22 +2912,26 @@ interface WireDocTransportArgs {
|
|
|
2063
2912
|
tailBlobs: Uint8Array[];
|
|
2064
2913
|
} | null>;
|
|
2065
2914
|
// Write an op the socket cannot carry, over HTTP. Resolves to the
|
|
2066
|
-
// server-allocated seq, or
|
|
2067
|
-
//
|
|
2068
|
-
//
|
|
2915
|
+
// server-allocated seq, or a failure the Document surfaces on the op.
|
|
2916
|
+
// Absent in transport-only tests, where an oversized op is failed rather
|
|
2917
|
+
// than rerouted.
|
|
2069
2918
|
postOversizedOp?: (
|
|
2070
2919
|
opBlobB64: string,
|
|
2071
2920
|
clientOpId: string,
|
|
2072
|
-
) => Promise<
|
|
2921
|
+
) => Promise<OversizedOpResult>;
|
|
2073
2922
|
// Invoked on a terminal `doc_status` frame: re-fetches once and, on
|
|
2074
2923
|
// success, imports the now-available snapshot so content appears without
|
|
2075
2924
|
// the app reopening; on failure leaves the status and does not loop.
|
|
2076
2925
|
// Absent in transport-only tests.
|
|
2077
2926
|
reloadAfterHydration?: (finalStatus: DocumentWorkStatus) => void;
|
|
2078
|
-
// Invoked on `doc_init`. Posted only after the
|
|
2927
|
+
// Invoked on `doc_init`. Posted only after the subscription record is written, so it
|
|
2079
2928
|
// is the safe point to reconcile a pending open whose terminal
|
|
2080
2929
|
// `doc_status` frame may have been dropped. Absent in transport-only tests.
|
|
2081
2930
|
reconcileStatusOnInit?: () => void;
|
|
2931
|
+
// Invoked on every frame carrying server-side movement (`doc_init`, a
|
|
2932
|
+
// remote op). Evidence the document has content to pull for a client whose
|
|
2933
|
+
// content pull gave up. Absent in transport-only tests.
|
|
2934
|
+
onDocumentMoved?: () => void;
|
|
2082
2935
|
// True for a stream line's internal Document: presence is a
|
|
2083
2936
|
// document-level surface (one roster per open doc, on the default
|
|
2084
2937
|
// line's presence topic), so per-stream transports skip attaching a
|
|
@@ -2103,6 +2956,13 @@ interface WireDocTransportArgs {
|
|
|
2103
2956
|
// open that named none. Part of the park key for the same reason the
|
|
2104
2957
|
// branch is: it says which state the parked Document holds.
|
|
2105
2958
|
requestedStream?: string;
|
|
2959
|
+
// The `documentOptions` this Document was built from, recorded on the park
|
|
2960
|
+
// entry so a reopen asking for different ones gets its own Document.
|
|
2961
|
+
documentOptions?: DocumentOptions;
|
|
2962
|
+
// Who this Document was opened for, recorded on the park entry (see
|
|
2963
|
+
// `ParkedDocEntry.identity`). Read at close rather than captured at open so
|
|
2964
|
+
// the stamp is the identity the page holds when the Document is parked.
|
|
2965
|
+
identity?: () => string | null;
|
|
2106
2966
|
}
|
|
2107
2967
|
|
|
2108
2968
|
// Per-process parking cache for opened Documents. `doc.close()` does NOT
|
|
@@ -2124,6 +2984,29 @@ interface ParkedDocEntry {
|
|
|
2124
2984
|
// local-op hook, runs the underlying `Document.close()`.
|
|
2125
2985
|
teardown: () => void;
|
|
2126
2986
|
key: string;
|
|
2987
|
+
// The `documentOptions` this Document was built from. A reopen asking for
|
|
2988
|
+
// different ones (another root key, another undo depth) describes a
|
|
2989
|
+
// different Document, so it gets a fresh one rather than this state.
|
|
2990
|
+
optionsSig: string;
|
|
2991
|
+
// Who this Document was opened for (`identity`, the signed-in subject), so
|
|
2992
|
+
// a take by anyone else misses.
|
|
2993
|
+
//
|
|
2994
|
+
// A hit skips the join entirely, which is the whole point of parking and
|
|
2995
|
+
// also the reason the entry has to name its owner: the SDK reads the
|
|
2996
|
+
// person from the token set, and a `/refresh` resolves whichever platform
|
|
2997
|
+
// session the browser now holds, so the identity behind an open page can
|
|
2998
|
+
// become someone else's between a close and the next open. Without this,
|
|
2999
|
+
// the second person's open is answered out of memory, with the first
|
|
3000
|
+
// person's content and no request the server could refuse.
|
|
3001
|
+
identity: string | null;
|
|
3002
|
+
}
|
|
3003
|
+
|
|
3004
|
+
function documentOptionsSig(options: DocumentOptions | undefined): string {
|
|
3005
|
+
return JSON.stringify([
|
|
3006
|
+
options?.rootKey ?? null,
|
|
3007
|
+
options?.readOnly ?? null,
|
|
3008
|
+
options?.undo ?? null,
|
|
3009
|
+
]);
|
|
2127
3010
|
}
|
|
2128
3011
|
|
|
2129
3012
|
const parkedDocs = new Map<string, ParkedDocEntry>();
|
|
@@ -2144,7 +3027,22 @@ function parkCacheKey(
|
|
|
2144
3027
|
return `${docId}/${branchId}/${streamName ?? DEFAULT_STREAM_NAME}`;
|
|
2145
3028
|
}
|
|
2146
3029
|
|
|
3030
|
+
function evictParked(entry: ParkedDocEntry | undefined): void {
|
|
3031
|
+
if (!entry) return;
|
|
3032
|
+
parkedDocs.delete(entry.key);
|
|
3033
|
+
try {
|
|
3034
|
+
entry.teardown();
|
|
3035
|
+
} catch {
|
|
3036
|
+
// Teardown errors must not strand the cache; the evicted Doc is gone
|
|
3037
|
+
// from the user's perspective either way.
|
|
3038
|
+
}
|
|
3039
|
+
}
|
|
3040
|
+
|
|
2147
3041
|
function parkDocument(entry: ParkedDocEntry): void {
|
|
3042
|
+
// One Document per key: whatever sat here is unreachable the moment this
|
|
3043
|
+
// entry replaces it, so it is torn down rather than left holding a live
|
|
3044
|
+
// subscription nothing can close.
|
|
3045
|
+
evictParked(parkedDocs.get(entry.key));
|
|
2148
3046
|
parkedDocs.set(entry.key, entry);
|
|
2149
3047
|
// Map iteration is insertion order, so evicting the front entry is LRU
|
|
2150
3048
|
// (a newly parked entry is the most-recent; a hit removes the entry
|
|
@@ -2152,43 +3050,55 @@ function parkDocument(entry: ParkedDocEntry): void {
|
|
|
2152
3050
|
while (parkedDocs.size > PARK_CACHE_CAPACITY) {
|
|
2153
3051
|
const oldestKey = parkedDocs.keys().next().value;
|
|
2154
3052
|
if (oldestKey === undefined) break;
|
|
2155
|
-
|
|
2156
|
-
parkedDocs.delete(oldestKey);
|
|
2157
|
-
try {
|
|
2158
|
-
oldest?.teardown();
|
|
2159
|
-
} catch {
|
|
2160
|
-
// Teardown errors must not strand the cache; the evicted Doc is gone
|
|
2161
|
-
// from the user's perspective either way.
|
|
2162
|
-
}
|
|
3053
|
+
evictParked(parkedDocs.get(oldestKey));
|
|
2163
3054
|
}
|
|
2164
3055
|
}
|
|
2165
3056
|
|
|
2166
3057
|
function takeParkedDocument(
|
|
2167
3058
|
docId: string,
|
|
2168
3059
|
options: OpenDocumentByIdOptions | undefined,
|
|
3060
|
+
identity: string | null,
|
|
2169
3061
|
): Document | null {
|
|
2170
3062
|
if (options?.version !== undefined) return null;
|
|
2171
3063
|
const key = parkCacheKey(docId, options?.branch, options?.stream);
|
|
2172
3064
|
if (!key) return null;
|
|
2173
3065
|
const entry = parkedDocs.get(key);
|
|
2174
3066
|
if (!entry) return null;
|
|
3067
|
+
// A Document the server evicted while it was parked describes a document
|
|
3068
|
+
// that is no longer there, one built from different `documentOptions` is
|
|
3069
|
+
// not the Document this caller asked for, and one opened for a different
|
|
3070
|
+
// person is not this caller's to read. All three take the fresh open, which
|
|
3071
|
+
// for the last is the request that establishes the new caller may have it;
|
|
3072
|
+
// the parked copy goes with its subscription.
|
|
3073
|
+
if (
|
|
3074
|
+
entry.doc._goneReason !== null ||
|
|
3075
|
+
entry.optionsSig !== documentOptionsSig(options?.documentOptions) ||
|
|
3076
|
+
entry.identity !== identity
|
|
3077
|
+
) {
|
|
3078
|
+
evictParked(entry);
|
|
3079
|
+
return null;
|
|
3080
|
+
}
|
|
2175
3081
|
parkedDocs.delete(key);
|
|
2176
3082
|
return entry.doc;
|
|
2177
3083
|
}
|
|
2178
3084
|
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
3085
|
+
/** Retire every parked Document. The client calls this as an identity leaves
|
|
3086
|
+
* the page, so what the previous person opened is torn down rather than left
|
|
3087
|
+
* resident for the rest of the tab's life.
|
|
3088
|
+
* @internal */
|
|
3089
|
+
export function clearParkedDocuments(): void {
|
|
2182
3090
|
for (const entry of parkedDocs.values()) {
|
|
2183
|
-
|
|
2184
|
-
entry.teardown();
|
|
2185
|
-
} catch {
|
|
2186
|
-
// ignore
|
|
2187
|
-
}
|
|
3091
|
+
evictParked(entry);
|
|
2188
3092
|
}
|
|
2189
3093
|
parkedDocs.clear();
|
|
2190
3094
|
}
|
|
2191
3095
|
|
|
3096
|
+
// Test-only: drop the parking cache. Used by unit tests that need
|
|
3097
|
+
// a clean slate between cases.
|
|
3098
|
+
export function _resetParkCacheForTesting(): void {
|
|
3099
|
+
clearParkedDocuments();
|
|
3100
|
+
}
|
|
3101
|
+
|
|
2192
3102
|
// Wire the WS transport onto a materialised Document. Three pieces:
|
|
2193
3103
|
// 1. WS subscribe on the doc topic; the server sends `doc_op` (other
|
|
2194
3104
|
// clients' edits) and `op_ack` (our ops), topic-scoped so the
|
|
@@ -2215,6 +3125,71 @@ export const PERIODIC_REVERIFY_INTERVAL_MS = 5 * 60_000;
|
|
|
2215
3125
|
// fake-timer test.
|
|
2216
3126
|
export const MISSED_OP_RESYNC_DELAY_MS = 2_000;
|
|
2217
3127
|
|
|
3128
|
+
// Per-op resend backoff: 1.5s, ~1.6x per attempt, to a 30s ceiling. The
|
|
3129
|
+
// ceiling keeps a doc on a dead connection from hammering; the
|
|
3130
|
+
// reconnect-driven resend (`resendAllOutstanding`) fires the instant
|
|
3131
|
+
// `doc_init` arrives, so the ceiling never delays recovery past the next
|
|
3132
|
+
// reconnect. Exported for the fake-timer tests.
|
|
3133
|
+
export const RESEND_BASE_DELAY_MS = 1_500;
|
|
3134
|
+
export const RESEND_MAX_DELAY_MS = 30_000;
|
|
3135
|
+
|
|
3136
|
+
// How long the oldest op on the socket may go unacked before its resend
|
|
3137
|
+
// timer spends an attempt even though the connection is answering other
|
|
3138
|
+
// ops. See `scheduleResend` for what this bounds and why only the oldest op
|
|
3139
|
+
// reaches it.
|
|
3140
|
+
//
|
|
3141
|
+
// The value sits above every latency a working connection produces. Acks
|
|
3142
|
+
// measured on a deployed environment run 175 ms at p99 and 1.6 s at their
|
|
3143
|
+
// worst, and an op whose own backoff has grown to `RESEND_MAX_DELAY_MS` is
|
|
3144
|
+
// still well inside this, so a queue that is merely slow never reaches it and
|
|
3145
|
+
// the timer keeps re-arming free. Sitting above the backoff ceiling is also
|
|
3146
|
+
// what makes this value, rather than the backoff, the interval a forced
|
|
3147
|
+
// resend is paced at: the timer fires several times inside one window, and
|
|
3148
|
+
// the first expiry past the window is the one that spends.
|
|
3149
|
+
//
|
|
3150
|
+
// What a false positive costs bounds the risk of setting it at all: one
|
|
3151
|
+
// duplicate op per document per window, which is a Loro no-op on import. The
|
|
3152
|
+
// failure this replaces is unbounded in the other direction, an op the
|
|
3153
|
+
// server never received and nothing ever sends again.
|
|
3154
|
+
// Exported for the fake-timer tests.
|
|
3155
|
+
export const RESEND_FORCE_CEILING_MS = 60_000;
|
|
3156
|
+
|
|
3157
|
+
// Server-shed recovery policy, driven by `unavailable` error frames. The
|
|
3158
|
+
// frame names no op, so a round re-asserts every outstanding op at once and
|
|
3159
|
+
// any further frame arriving before that round runs is part of the same shed
|
|
3160
|
+
// batch: coalescing them is what keeps N shed ops from fanning out into N
|
|
3161
|
+
// rounds of N resends. Backoff is 1s doubling to a 30s ceiling, so the budget
|
|
3162
|
+
// spans roughly a minute and a half of sustained shedding. An `op_ack` (the
|
|
3163
|
+
// server taking writes again) or a reconnect returns the budget to full.
|
|
3164
|
+
//
|
|
3165
|
+
// Module-scoped because both transports answer a shed on it: the signed-in
|
|
3166
|
+
// one, the HTTP write for an op past the frame cap, and the guest-link one.
|
|
3167
|
+
// A shed costs an edit the same nothing whoever wrote it and whichever
|
|
3168
|
+
// transport carried it.
|
|
3169
|
+
const UNAVAILABLE_MAX_ROUNDS = 6;
|
|
3170
|
+
const UNAVAILABLE_BASE_DELAY_MS = 1000;
|
|
3171
|
+
const UNAVAILABLE_MAX_DELAY_MS = 30_000;
|
|
3172
|
+
const UNAVAILABLE_GIVE_UP_MESSAGE =
|
|
3173
|
+
'The server could not store this change, and re-sending it did not ' +
|
|
3174
|
+
'help. Reload the document to continue from the last saved state.';
|
|
3175
|
+
|
|
3176
|
+
// Reported for every op left behind when the server ends this document's
|
|
3177
|
+
// subscription (it refused the subscribe, or evicted it again after the one
|
|
3178
|
+
// silent re-subscribe a connection grants). The ops can never be acked on a
|
|
3179
|
+
// subscription that is over, and a fresh one arrives only with a fresh open,
|
|
3180
|
+
// so the app's cue is to reload.
|
|
3181
|
+
const SUBSCRIPTION_ENDED_MESSAGE =
|
|
3182
|
+
'The live connection to this document ended, so this change was not ' +
|
|
3183
|
+
'saved. Reload the document to continue from the last saved state.';
|
|
3184
|
+
|
|
3185
|
+
/** How long the Nth (1-based) recovery round waits before re-asserting. */
|
|
3186
|
+
function unavailableDelayMs(round: number): number {
|
|
3187
|
+
return Math.min(
|
|
3188
|
+
UNAVAILABLE_BASE_DELAY_MS * Math.pow(2, round - 1),
|
|
3189
|
+
UNAVAILABLE_MAX_DELAY_MS,
|
|
3190
|
+
);
|
|
3191
|
+
}
|
|
3192
|
+
|
|
2218
3193
|
// A mutation carrying many records commits as it goes, so the only write
|
|
2219
3194
|
// that reaches this is one record nothing can split.
|
|
2220
3195
|
const TOO_LARGE_MESSAGE =
|
|
@@ -2222,13 +3197,19 @@ const TOO_LARGE_MESSAGE =
|
|
|
2222
3197
|
'update; store the bulk of it as a file instead, or split it across ' +
|
|
2223
3198
|
'several records.';
|
|
2224
3199
|
|
|
3200
|
+
// What a guest hears for an op no request body can carry. A signed-in
|
|
3201
|
+
// editor stages such an op to storage first; that route is behind the
|
|
3202
|
+
// sign-in gate, so the anonymous tier writes ops that fit one request.
|
|
3203
|
+
const LINK_OP_TOO_LARGE_MESSAGE =
|
|
3204
|
+
'This change is too large to save over a shared link. Sign in to the ' +
|
|
3205
|
+
'document to make a change this size.';
|
|
3206
|
+
|
|
2225
3207
|
function wireDocTransport(args: WireDocTransportArgs): void {
|
|
2226
3208
|
const topic = `doc:${args.docId}:${args.branchId}`;
|
|
2227
3209
|
|
|
2228
|
-
// Per-op retry tracker. The op-write endpoint
|
|
2229
|
-
//
|
|
2230
|
-
//
|
|
2231
|
-
// That frame has no `type`, so the multiplexer drops it,
|
|
3210
|
+
// Per-op retry tracker. The op-write endpoint can answer with an untyped
|
|
3211
|
+
// error frame instead of reaching the handler. That frame has no `type`, so
|
|
3212
|
+
// the multiplexer drops it,
|
|
2232
3213
|
// leaving the op stuck in `Document._outstandingByClientId` and the
|
|
2233
3214
|
// actor's lamport stream with a gap that blocks other subscribers' Loro
|
|
2234
3215
|
// imports from converging (Loro requires per-actor causal contiguity).
|
|
@@ -2240,14 +3221,157 @@ function wireDocTransport(args: WireDocTransportArgs): void {
|
|
|
2240
3221
|
// retry lands the dependent chain converges everywhere. `doc.savingCount`
|
|
2241
3222
|
// stays above zero until the `op_ack` lands (the "Saving…" surface); there
|
|
2242
3223
|
// is no give-up to surface because the SDK keeps trying.
|
|
3224
|
+
// When each op this transport shipped was first committed, keyed the same
|
|
3225
|
+
// way the tracker is. The tracker itself carries no commit time (its
|
|
3226
|
+
// `dispatchedAt` restarts on every resend, which is the right basis for
|
|
3227
|
+
// deciding a resend and the wrong one for reporting how long a reader has
|
|
3228
|
+
// been waiting), and an op past the frame cap is never in the tracker at
|
|
3229
|
+
// all. Entries are dropped as their op settles, so the map's size is the
|
|
3230
|
+
// work outstanding on either transport.
|
|
3231
|
+
const opStartedAt = new Map<string, number>();
|
|
3232
|
+
|
|
3233
|
+
// Reconnects the socket had made when this transport last saw a
|
|
3234
|
+
// `doc_init`. The frame itself is not a reconnect signal: it rides every
|
|
3235
|
+
// `subscribed` ack, including the opening one and the silent re-subscribe
|
|
3236
|
+
// a connection grants after a mid-stream eviction, neither of which
|
|
3237
|
+
// involved the socket dropping. A report keyed off the frame therefore
|
|
3238
|
+
// fires when nothing is wrong, which is the fault this latch exists to
|
|
3239
|
+
// avoid, so it is keyed off the socket instead.
|
|
3240
|
+
let docInitReconnectCount: number | null = null;
|
|
3241
|
+
|
|
2243
3242
|
const outstandingResends = new Map<
|
|
2244
3243
|
string,
|
|
2245
3244
|
{
|
|
2246
3245
|
frame: { type: 'op'; topic: string; op_blob: string; client_op_id: string };
|
|
2247
3246
|
attempts: number;
|
|
2248
3247
|
timer: ReturnType<typeof setTimeout>;
|
|
3248
|
+
// When this op's current timer was armed, compared against the last
|
|
3249
|
+
// ack to tell a slow queue from a silent server. See `scheduleResend`.
|
|
3250
|
+
armedAt: number;
|
|
3251
|
+
// When this op was last put on the socket, or 0 while it is queued.
|
|
3252
|
+
// Time on the wire rather than time outstanding, so an op held back by
|
|
3253
|
+
// the in-flight budget is not charged for the wait. See
|
|
3254
|
+
// `scheduleResend`.
|
|
3255
|
+
dispatchedAt: number;
|
|
3256
|
+
// Bytes this op is charging to the in-flight budget, or 0 while it is
|
|
3257
|
+
// still queued. See `dispatchOp`.
|
|
3258
|
+
inFlight: number;
|
|
3259
|
+
// Whether this op is waiting in `sendQueue`. Held on the entry so
|
|
3260
|
+
// neither queueing nor forgetting an op has to scan the queue, which
|
|
3261
|
+
// would be quadratic over a mutation that queues thousands.
|
|
3262
|
+
queued: boolean;
|
|
2249
3263
|
}
|
|
2250
3264
|
>();
|
|
3265
|
+
|
|
3266
|
+
// How many bytes of op frames may be on the socket unacked at once.
|
|
3267
|
+
//
|
|
3268
|
+
// The transport drops the connection when a client pushes at it hard
|
|
3269
|
+
// enough, and it takes everything multiplexed onto that socket with it;
|
|
3270
|
+
// every op then goes unacked, the reconnect re-asserts them, and that
|
|
3271
|
+
// burst drops the connection again, so nothing is ever persisted.
|
|
3272
|
+
// Measured against one write, a client pushing 26 frames in 320 ms lost
|
|
3273
|
+
// the socket where one spreading 90 frames over 5.3 s did not, so what
|
|
3274
|
+
// this bounds is the RATE rather than any one frame's size.
|
|
3275
|
+
//
|
|
3276
|
+
// Which shape reaches it is worth knowing before tuning it. A bulk write
|
|
3277
|
+
// does not: `mutate` batches to a budget far above what a frame carries,
|
|
3278
|
+
// so its ops take the HTTP write and never enter this queue. What does is
|
|
3279
|
+
// an app issuing many separate mutations that each land under the frame
|
|
3280
|
+
// cap, a drag loop or a per-row import, where the op count is set by how
|
|
3281
|
+
// often the app calls rather than by how much it writes.
|
|
3282
|
+
//
|
|
3283
|
+
// A byte budget rather than a rate is what makes this self-tuning: sends
|
|
3284
|
+
// resume as acks land, so the client goes exactly as fast as the server
|
|
3285
|
+
// is answering and no faster, whatever that speed turns out to be. The
|
|
3286
|
+
// budget is generous enough to keep several ops in flight, so an ack's
|
|
3287
|
+
// round trip is not serialised into the write.
|
|
3288
|
+
const IN_FLIGHT_BUDGET_BYTES = 256 * 1024;
|
|
3289
|
+
// Ops committed but not yet on the socket, oldest first.
|
|
3290
|
+
const sendQueue: string[] = [];
|
|
3291
|
+
let inFlightBytes = 0;
|
|
3292
|
+
|
|
3293
|
+
/** Put an op on the socket if the budget allows, else queue it. */
|
|
3294
|
+
function dispatchOp(clientOpId: string): void {
|
|
3295
|
+
const entry = outstandingResends.get(clientOpId);
|
|
3296
|
+
if (!entry || entry.inFlight > 0) return;
|
|
3297
|
+
const size = entry.frame.op_blob.length;
|
|
3298
|
+
// An op larger than the whole budget goes on its own rather than
|
|
3299
|
+
// waiting for room that can never appear.
|
|
3300
|
+
if (inFlightBytes > 0 && inFlightBytes + size > IN_FLIGHT_BUDGET_BYTES) {
|
|
3301
|
+
if (!entry.queued) {
|
|
3302
|
+
entry.queued = true;
|
|
3303
|
+
entry.dispatchedAt = 0;
|
|
3304
|
+
sendQueue.push(clientOpId);
|
|
3305
|
+
}
|
|
3306
|
+
return;
|
|
3307
|
+
}
|
|
3308
|
+
entry.queued = false;
|
|
3309
|
+
entry.inFlight = size;
|
|
3310
|
+
entry.dispatchedAt = Date.now();
|
|
3311
|
+
inFlightBytes += size;
|
|
3312
|
+
args.wsClient.send(entry.frame);
|
|
3313
|
+
scheduleResend(clientOpId);
|
|
3314
|
+
}
|
|
3315
|
+
|
|
3316
|
+
/** The oldest op currently on the socket, which is the one a gap blocks
|
|
3317
|
+
* the rest of this actor's stream behind. Map order is commit order, and
|
|
3318
|
+
* an op waiting on the in-flight budget carries no timer, so the first
|
|
3319
|
+
* dispatched entry is that op. */
|
|
3320
|
+
function isOldestDispatchedOp(clientOpId: string): boolean {
|
|
3321
|
+
for (const [id, entry] of outstandingResends) {
|
|
3322
|
+
if (entry.dispatchedAt === 0) continue;
|
|
3323
|
+
return id === clientOpId;
|
|
3324
|
+
}
|
|
3325
|
+
return false;
|
|
3326
|
+
}
|
|
3327
|
+
|
|
3328
|
+
/** Send whatever the freed budget now has room for. */
|
|
3329
|
+
function drainSendQueue(): void {
|
|
3330
|
+
while (sendQueue.length > 0) {
|
|
3331
|
+
const next = sendQueue[0]!;
|
|
3332
|
+
const entry = outstandingResends.get(next);
|
|
3333
|
+
// An op acked, rejected or abandoned while queued leaves its id behind;
|
|
3334
|
+
// dropping it here is what keeps `forgetOp` off a queue scan.
|
|
3335
|
+
if (!entry || !entry.queued) {
|
|
3336
|
+
sendQueue.shift();
|
|
3337
|
+
continue;
|
|
3338
|
+
}
|
|
3339
|
+
const size = entry.frame.op_blob.length;
|
|
3340
|
+
if (inFlightBytes > 0 && inFlightBytes + size > IN_FLIGHT_BUDGET_BYTES) {
|
|
3341
|
+
return;
|
|
3342
|
+
}
|
|
3343
|
+
sendQueue.shift();
|
|
3344
|
+
entry.queued = false;
|
|
3345
|
+
dispatchOp(next);
|
|
3346
|
+
}
|
|
3347
|
+
}
|
|
3348
|
+
|
|
3349
|
+
/** Drop one op from the tracker, returning its budget. */
|
|
3350
|
+
function forgetOp(clientOpId: string): void {
|
|
3351
|
+
const entry = outstandingResends.get(clientOpId);
|
|
3352
|
+
if (!entry) return;
|
|
3353
|
+
clearTimeout(entry.timer);
|
|
3354
|
+
inFlightBytes -= entry.inFlight;
|
|
3355
|
+
entry.inFlight = 0;
|
|
3356
|
+
entry.queued = false;
|
|
3357
|
+
outstandingResends.delete(clientOpId);
|
|
3358
|
+
opStartedAt.delete(clientOpId);
|
|
3359
|
+
drainSendQueue();
|
|
3360
|
+
}
|
|
3361
|
+
|
|
3362
|
+
/** Drop every tracked op and reset the budget with them. */
|
|
3363
|
+
function forgetAllOps(): void {
|
|
3364
|
+
for (const [clientOpId, entry] of outstandingResends) {
|
|
3365
|
+
clearTimeout(entry.timer);
|
|
3366
|
+
// Only the ops this tracker holds. An op past the frame cap is
|
|
3367
|
+
// travelling over HTTP and is not in here, so clearing the whole
|
|
3368
|
+
// timing map would report it settled while its write is still going.
|
|
3369
|
+
opStartedAt.delete(clientOpId);
|
|
3370
|
+
}
|
|
3371
|
+
outstandingResends.clear();
|
|
3372
|
+
sendQueue.length = 0;
|
|
3373
|
+
inFlightBytes = 0;
|
|
3374
|
+
}
|
|
2251
3375
|
// Ops this client is writing over HTTP, and the highest seq a catch-up
|
|
2252
3376
|
// prompt named while one was in flight.
|
|
2253
3377
|
//
|
|
@@ -2269,45 +3393,399 @@ function wireDocTransport(args: WireDocTransportArgs): void {
|
|
|
2269
3393
|
if (httpOpWritesInFlight > 0) return;
|
|
2270
3394
|
const seq = heldCatchUpSeq;
|
|
2271
3395
|
heldCatchUpSeq = 0;
|
|
2272
|
-
if (seq > 0 && args.getCoveredSeq() < seq)
|
|
3396
|
+
if (seq > 0 && args.getCoveredSeq() < seq) pullJoin();
|
|
2273
3397
|
}
|
|
2274
3398
|
|
|
2275
|
-
//
|
|
2276
|
-
//
|
|
2277
|
-
//
|
|
2278
|
-
//
|
|
2279
|
-
|
|
2280
|
-
|
|
3399
|
+
// Pull the join, and ask again if it left the reader short.
|
|
3400
|
+
//
|
|
3401
|
+
// A join read while a peer writes in bulk is a torn cut: the branch's seq
|
|
3402
|
+
// counter has run ahead of the op rows, so the response reports a head it
|
|
3403
|
+
// could not deliver the tail for. The reader applies what arrived and its
|
|
3404
|
+
// floor lands below that head, and nothing re-sends the difference. So
|
|
3405
|
+
// whatever the reader still lacks after an import is a hole it has to ask
|
|
3406
|
+
// about itself, which is what a passive reader has instead of the `doc_op`
|
|
3407
|
+
// and `op_ack` frames a writing client arms its checks from.
|
|
3408
|
+
function pullJoin(): void {
|
|
3409
|
+
void Promise.resolve(args.resyncFromSnapshot?.()).then(
|
|
3410
|
+
() => {
|
|
3411
|
+
scheduleGapCheck();
|
|
3412
|
+
},
|
|
3413
|
+
() => {
|
|
3414
|
+
// `resyncFromSnapshot` reports its own failures and resolves; a
|
|
3415
|
+
// rejection is a caller that does not, and asking again is still the
|
|
3416
|
+
// right move.
|
|
3417
|
+
scheduleGapCheck();
|
|
3418
|
+
},
|
|
3419
|
+
);
|
|
3420
|
+
}
|
|
3421
|
+
|
|
3422
|
+
// Re-import the join for ops this client cannot bridge to from what it
|
|
3423
|
+
// holds, deferring the judgement while one of its own HTTP writes is still
|
|
3424
|
+
// in flight. Shared by the catch-up prompt and by the reconnect handshake,
|
|
3425
|
+
// which reaches the same conclusion off its tail.
|
|
3426
|
+
// Re-import when the durable frontier names an author whose changes never
|
|
3427
|
+
// came through the op stream. Separate from the coverage check because it
|
|
3428
|
+
// answers a different question: not "did I miss an op" but "did what I hold
|
|
3429
|
+
// stop being how this document is stored".
|
|
3430
|
+
function maybeResyncPastRewrite(vectorB64: string | undefined): void {
|
|
3431
|
+
if (!vectorB64) return;
|
|
3432
|
+
let unseen = false;
|
|
3433
|
+
try {
|
|
3434
|
+
unseen = args.doc._snapshotHasUnseenAuthor(base64ToBytes(vectorB64));
|
|
3435
|
+
} catch {
|
|
3436
|
+
return;
|
|
3437
|
+
}
|
|
3438
|
+
if (unseen) pullJoin();
|
|
3439
|
+
}
|
|
3440
|
+
|
|
3441
|
+
function resyncPastSeq(seq: number): void {
|
|
3442
|
+
if (httpOpWritesInFlight > 0) {
|
|
3443
|
+
if (seq > heldCatchUpSeq) heldCatchUpSeq = seq;
|
|
3444
|
+
return;
|
|
3445
|
+
}
|
|
3446
|
+
pullJoin();
|
|
3447
|
+
}
|
|
2281
3448
|
|
|
2282
3449
|
function resendDelayMs(attempts: number): number {
|
|
2283
3450
|
const grown = RESEND_BASE_DELAY_MS * Math.pow(1.6, attempts - 1);
|
|
2284
3451
|
return Math.min(grown, RESEND_MAX_DELAY_MS);
|
|
2285
3452
|
}
|
|
2286
3453
|
|
|
3454
|
+
// When this connection last proved it was working: any `op_ack`, from any
|
|
3455
|
+
// op on this topic.
|
|
3456
|
+
//
|
|
3457
|
+
// What this timer is for is a server that answered NOTHING, which is a
|
|
3458
|
+
// real case (an op frame can draw the opaque `{"message":"Internal server
|
|
3459
|
+
// error"}` reply the multiplexer drops). What it must not read as silence
|
|
3460
|
+
// is an ack that is merely behind, because with many ops in flight one
|
|
3461
|
+
// op's latency is the queue's depth times its service time, not evidence
|
|
3462
|
+
// about that op. Resending on that reading writes the op a second time,
|
|
3463
|
+
// which deepens the queue, which slows the next ack, which fires more
|
|
3464
|
+
// resends.
|
|
3465
|
+
//
|
|
3466
|
+
// Measured on a 500,000-element write: the same content went through in
|
|
3467
|
+
// 1,320 op writes on a run whose acks stayed under 375 ms, and in 3,600 on
|
|
3468
|
+
// a run whose acks reached 1,615 ms, just past this backoff's first step.
|
|
3469
|
+
// The extra writes are durable duplicates, so they inflate the document's
|
|
3470
|
+
// history, and every later compaction pays for them: the run that tripled
|
|
3471
|
+
// its ops left a tail compaction could not absorb inside its memory, and
|
|
3472
|
+
// the document stopped opening.
|
|
3473
|
+
//
|
|
3474
|
+
// So the timer fires on silence rather than on slowness: an ack that
|
|
3475
|
+
// arrived after this op's timer was armed re-arms it without spending an
|
|
3476
|
+
// attempt, within the bound `forceResendDue` puts on how long one op may
|
|
3477
|
+
// ride that evidence.
|
|
3478
|
+
let lastAckAt = 0;
|
|
3479
|
+
|
|
2287
3480
|
function scheduleResend(clientOpId: string): void {
|
|
2288
3481
|
const entry = outstandingResends.get(clientOpId);
|
|
2289
3482
|
if (!entry) return;
|
|
3483
|
+
entry.armedAt = Date.now();
|
|
2290
3484
|
entry.timer = setTimeout(() => {
|
|
2291
3485
|
const current = outstandingResends.get(clientOpId);
|
|
2292
3486
|
if (!current) return;
|
|
3487
|
+
const forced = forceResendDue(clientOpId, current);
|
|
3488
|
+
if (lastAckAt >= current.armedAt && !forced) {
|
|
3489
|
+
scheduleResend(clientOpId);
|
|
3490
|
+
return;
|
|
3491
|
+
}
|
|
3492
|
+
// A forced resend is the transport concluding this op was lost, which
|
|
3493
|
+
// is the one condition here the server cannot infer, so it is what the
|
|
3494
|
+
// report is pinned to.
|
|
3495
|
+
if (forced) reportSyncHealth('op_stuck');
|
|
2293
3496
|
current.attempts += 1;
|
|
3497
|
+
current.dispatchedAt = Date.now();
|
|
2294
3498
|
args.wsClient.send(current.frame);
|
|
2295
3499
|
scheduleResend(clientOpId);
|
|
2296
3500
|
}, resendDelayMs(entry.attempts));
|
|
2297
3501
|
}
|
|
2298
3502
|
|
|
3503
|
+
// The one case where an answering connection is still evidence this op was
|
|
3504
|
+
// lost: it is the oldest op on the socket and it has been there longer than
|
|
3505
|
+
// any working connection takes to answer.
|
|
3506
|
+
//
|
|
3507
|
+
// The evidence `lastAckAt` carries is that the connection is working, not
|
|
3508
|
+
// that it received this op, so an app committing faster than the backoff
|
|
3509
|
+
// supplies that evidence continuously and a dropped op would re-arm free
|
|
3510
|
+
// forever. The server's copy of the document then stops at the gap (Loro
|
|
3511
|
+
// requires per-actor causal contiguity, so every later op is stored and
|
|
3512
|
+
// never applied), the tracker entry never reaches a terminal state, and the
|
|
3513
|
+
// app is told nothing.
|
|
3514
|
+
//
|
|
3515
|
+
// Ack ORDER cannot stand in for that evidence. Acks overtake heavily: 24
|
|
3516
|
+
// ops sent back to back on a deployed environment came back with 10 of 23
|
|
3517
|
+
// adjacent pairs out of send order, and the op sent first was acked 23rd. A
|
|
3518
|
+
// rule reading a later op's ack as proof this one was skipped re-sends ops
|
|
3519
|
+
// that are merely in flight, which is the write-it-twice failure the gate
|
|
3520
|
+
// above exists to prevent.
|
|
3521
|
+
//
|
|
3522
|
+
// Restricting this to the oldest dispatched op is what keeps the traffic
|
|
3523
|
+
// bounded and is sufficient. Bounded: one op per ceiling window however
|
|
3524
|
+
// many ops are outstanding, which matters because this resend hands the
|
|
3525
|
+
// frame straight to `wsClient.send` and so spends none of the in-flight
|
|
3526
|
+
// budget, and a simultaneous round of them is the burst that drops the
|
|
3527
|
+
// socket. Sufficient: the gap at the oldest op is what holds the rest of
|
|
3528
|
+
// this actor's stream unapplied, and an op behind it becomes the oldest in
|
|
3529
|
+
// its turn as the ops in front retire.
|
|
3530
|
+
function forceResendDue(
|
|
3531
|
+
clientOpId: string,
|
|
3532
|
+
entry: { dispatchedAt: number },
|
|
3533
|
+
): boolean {
|
|
3534
|
+
if (entry.dispatchedAt === 0) return false;
|
|
3535
|
+
if (Date.now() - entry.dispatchedAt <= RESEND_FORCE_CEILING_MS) return false;
|
|
3536
|
+
return isOldestDispatchedOp(clientOpId);
|
|
3537
|
+
}
|
|
3538
|
+
|
|
3539
|
+
// What this transport knows about its own sync, for a reader whose save
|
|
3540
|
+
// is taking longer than it should. Everything here is closure state the
|
|
3541
|
+
// transport already keeps; collecting it costs a walk of the outstanding
|
|
3542
|
+
// ops, so it is safe to call from a render.
|
|
3543
|
+
function collectDiagnostics(): DocSyncDiagnostics {
|
|
3544
|
+
const now = Date.now();
|
|
3545
|
+
let oldestClientOpId: string | null = null;
|
|
3546
|
+
let oldestStartedAt = 0;
|
|
3547
|
+
// Insertion order is commit order, so the first entry is the oldest.
|
|
3548
|
+
for (const [clientOpId, startedAt] of opStartedAt) {
|
|
3549
|
+
oldestClientOpId = clientOpId;
|
|
3550
|
+
oldestStartedAt = startedAt;
|
|
3551
|
+
break;
|
|
3552
|
+
}
|
|
3553
|
+
return {
|
|
3554
|
+
unackedOps: opStartedAt.size,
|
|
3555
|
+
oldestUnackedMs:
|
|
3556
|
+
oldestClientOpId === null ? 0 : Math.max(0, now - oldestStartedAt),
|
|
3557
|
+
oldestClientOpId,
|
|
3558
|
+
reconnects: args.wsClient.reconnectCount ?? 0,
|
|
3559
|
+
lastAckAgeMs: lastAckAt === 0 ? null : Math.max(0, now - lastAckAt),
|
|
3560
|
+
inFlightBytes,
|
|
3561
|
+
httpOpsInFlight: httpOpWritesInFlight,
|
|
3562
|
+
healRounds: divergenceHealRound,
|
|
3563
|
+
hidden: args.wsClient.pageHidden ?? false,
|
|
3564
|
+
};
|
|
3565
|
+
}
|
|
3566
|
+
|
|
3567
|
+
// Push the same picture to the server, so an incident on a device with no
|
|
3568
|
+
// console attached leaves a record an operator can find. Sent on the
|
|
3569
|
+
// conditions the client can recognise and the server cannot, each of
|
|
3570
|
+
// which is already bounded by the machinery that detects it: a forced
|
|
3571
|
+
// resend fires at most once per ceiling window, a reconnect is one
|
|
3572
|
+
// socket, and a heal round is one push. The server drops the excess
|
|
3573
|
+
// anyway.
|
|
3574
|
+
//
|
|
3575
|
+
// Fire-and-forget by design. The frame carries no `client_op_id` of its
|
|
3576
|
+
// own and draws no reply, so it never joins the outstanding-op tracker
|
|
3577
|
+
// and a report cannot become one more thing a stuck document is waiting
|
|
3578
|
+
// on.
|
|
3579
|
+
function reportSyncHealth(reason: ClientReportReason): void {
|
|
3580
|
+
// Only worth sending while the socket can carry it. Held through an
|
|
3581
|
+
// outage, a report flushes on the re-subscribe describing state from
|
|
3582
|
+
// before the outage, and it arrives on the new connection ahead of the
|
|
3583
|
+
// accurate one the reconnect itself produces, so the stale figures are
|
|
3584
|
+
// what the server's one-per-connection budget accepts.
|
|
3585
|
+
if (args.wsClient.connected === false) return;
|
|
3586
|
+
const d = collectDiagnostics();
|
|
3587
|
+
args.wsClient.send({
|
|
3588
|
+
type: 'client_report',
|
|
3589
|
+
topic,
|
|
3590
|
+
client_report: {
|
|
3591
|
+
reason,
|
|
3592
|
+
unacked_ops: d.unackedOps,
|
|
3593
|
+
oldest_unacked_ms: d.oldestUnackedMs,
|
|
3594
|
+
...(d.oldestClientOpId === null
|
|
3595
|
+
? {}
|
|
3596
|
+
: { oldest_client_op_id: d.oldestClientOpId }),
|
|
3597
|
+
reconnects: d.reconnects,
|
|
3598
|
+
last_ack_age_ms: d.lastAckAgeMs,
|
|
3599
|
+
in_flight_bytes: d.inFlightBytes,
|
|
3600
|
+
http_ops_in_flight: d.httpOpsInFlight,
|
|
3601
|
+
heal_rounds: d.healRounds,
|
|
3602
|
+
hidden: d.hidden,
|
|
3603
|
+
},
|
|
3604
|
+
});
|
|
3605
|
+
}
|
|
3606
|
+
|
|
3607
|
+
args.doc._setDiagnosticsProvider(collectDiagnostics);
|
|
3608
|
+
|
|
2299
3609
|
// Resend every outstanding op now and restart its backoff. Called on
|
|
2300
3610
|
// reconnect (`doc_init`): a dropped-and-restored connection is the most
|
|
2301
3611
|
// common reason an op went unacked, and the backoff may have grown to its
|
|
2302
3612
|
// ceiling while the socket was down, so re-asserting immediately is the
|
|
2303
3613
|
// fast recovery path.
|
|
2304
3614
|
function resendAllOutstanding(): void {
|
|
3615
|
+
// Nothing is on the wire any more, whether this follows a reconnect or a
|
|
3616
|
+
// shed, so the budget starts empty and every op re-enters through it.
|
|
3617
|
+
// Sending them all at once is what turns one dropped connection into a
|
|
3618
|
+
// loop that never persists anything: the burst that lost the socket is
|
|
3619
|
+
// exactly the burst the reconnect would repeat.
|
|
3620
|
+
inFlightBytes = 0;
|
|
3621
|
+
sendQueue.length = 0;
|
|
3622
|
+
// Insertion order is the order the ops were committed in, so a round
|
|
3623
|
+
// re-asserts them oldest-first.
|
|
2305
3624
|
for (const [clientOpId, entry] of outstandingResends) {
|
|
2306
3625
|
clearTimeout(entry.timer);
|
|
2307
3626
|
entry.attempts += 1;
|
|
2308
|
-
|
|
2309
|
-
|
|
3627
|
+
entry.inFlight = 0;
|
|
3628
|
+
entry.queued = false;
|
|
3629
|
+
dispatchOp(clientOpId);
|
|
3630
|
+
}
|
|
3631
|
+
}
|
|
3632
|
+
|
|
3633
|
+
let unavailableRounds = 0;
|
|
3634
|
+
let unavailableTimer: ReturnType<typeof setTimeout> | null = null;
|
|
3635
|
+
|
|
3636
|
+
function clearUnavailableRecovery(): void {
|
|
3637
|
+
if (unavailableTimer !== null) {
|
|
3638
|
+
clearTimeout(unavailableTimer);
|
|
3639
|
+
unavailableTimer = null;
|
|
3640
|
+
}
|
|
3641
|
+
unavailableRounds = 0;
|
|
3642
|
+
}
|
|
3643
|
+
|
|
3644
|
+
// Every outstanding op is dead: drop the tracker entries and tell the
|
|
3645
|
+
// Document, which stops counting them in `savingCount` and reports each
|
|
3646
|
+
// through the error channel the app reads (`Brass.onError`). The ids are
|
|
3647
|
+
// taken first because `_failOutstandingOp` re-enters app code.
|
|
3648
|
+
//
|
|
3649
|
+
// Dropping the ops is what opens the divergence, so the heal is armed here
|
|
3650
|
+
// rather than at the call site: the local doc keeps the abandoned ops and
|
|
3651
|
+
// keeps accepting edits that depend on them, and only the handshake can
|
|
3652
|
+
// hand the server the slice it needs to apply any of it.
|
|
3653
|
+
function failAllOutstanding(message: string): void {
|
|
3654
|
+
const clientOpIds = [...outstandingResends.keys()];
|
|
3655
|
+
forgetAllOps();
|
|
3656
|
+
for (const clientOpId of clientOpIds) {
|
|
3657
|
+
args.doc._failOutstandingOp(clientOpId, message);
|
|
3658
|
+
}
|
|
3659
|
+
if (clientOpIds.length > 0) {
|
|
3660
|
+
divergenceHealRound = 0;
|
|
3661
|
+
scheduleDivergenceHeal();
|
|
3662
|
+
}
|
|
3663
|
+
}
|
|
3664
|
+
|
|
3665
|
+
// One op is dead: report it and arm the heal, for the same reason
|
|
3666
|
+
// `failAllOutstanding` does. The op stays applied locally and every edit
|
|
3667
|
+
// made after it depends on it, so the server holds a stream it can store
|
|
3668
|
+
// and never apply, and its own escape for an op nothing can apply is to
|
|
3669
|
+
// discard it. Most refusals do not change that: the handshake pushes
|
|
3670
|
+
// whatever slice the durable frontier is missing and splits it under the
|
|
3671
|
+
// frame cap, so it heals the dependents even where the op itself is
|
|
3672
|
+
// irreducibly too large to re-send.
|
|
3673
|
+
//
|
|
3674
|
+
// `healable: false` is for the refusal where pushing again is asking the
|
|
3675
|
+
// same question. A full document refuses every op whatever its size, so
|
|
3676
|
+
// the handshake's own chunks come back refused, and the counter reset
|
|
3677
|
+
// below turns that into a loop rather than a bounded retry. What recovers
|
|
3678
|
+
// that document is the periodic re-verify, once the server can take a
|
|
3679
|
+
// write at all.
|
|
3680
|
+
// A refusal the heal cannot work around leaves the writer holding a change
|
|
3681
|
+
// that reached no database and no other reader, so the writer is the only
|
|
3682
|
+
// copy of the document that shows it. Left editable it reads as applied: the
|
|
3683
|
+
// element is on screen, the count includes it, every later edit builds on
|
|
3684
|
+
// it, and a reader who opens the document sees none of it. Nothing local
|
|
3685
|
+
// removes it either, because a CRDT merges rather than replaces and an undo
|
|
3686
|
+
// inverts an operation instead of discarding it, so the only state holding
|
|
3687
|
+
// exactly what is durable is one built from the server.
|
|
3688
|
+
//
|
|
3689
|
+
// Stopping the divergence and resolving it are separated, because they cost
|
|
3690
|
+
// different things and doing them in one silent step is what made a refusal
|
|
3691
|
+
// read as the platform losing work. Stopping is free and immediate:
|
|
3692
|
+
// `readOnly` turns true, which closes an app's existing edit gate, so the
|
|
3693
|
+
// copy cannot drift further. Resolving discards the refused work and swaps
|
|
3694
|
+
// the document under whoever was working in it, so the user is told first
|
|
3695
|
+
// and the reload follows their acknowledgement.
|
|
3696
|
+
//
|
|
3697
|
+
// The telling is the SDK's job rather than each app's. A copy left showing a
|
|
3698
|
+
// change nothing else has is the failure being closed, so an app that
|
|
3699
|
+
// renders no surface for this cannot be the reason a user stays on one.
|
|
3700
|
+
//
|
|
3701
|
+
// Once per document. The first refusal is what diverged this copy, so the
|
|
3702
|
+
// later ones describe the same state and have nothing to add.
|
|
3703
|
+
let lockedForRefusal = false;
|
|
3704
|
+
function lockForUnhealableRefusal(message: string): void {
|
|
3705
|
+
if (lockedForRefusal) return;
|
|
3706
|
+
lockedForRefusal = true;
|
|
3707
|
+
// Torn down in the order the `doc_gone` handler uses, and for the same
|
|
3708
|
+
// reason: every one of these keeps working against a copy that is now
|
|
3709
|
+
// read-only, and the resync push in particular would re-offer the refused
|
|
3710
|
+
// op this exists to stop.
|
|
3711
|
+
forgetAllOps();
|
|
3712
|
+
clearUnavailableRecovery();
|
|
3713
|
+
if (reverifyTimer !== null) {
|
|
3714
|
+
clearInterval(reverifyTimer);
|
|
3715
|
+
reverifyTimer = null;
|
|
3716
|
+
}
|
|
3717
|
+
clearDivergenceHeal();
|
|
3718
|
+
if (gapTimer !== null) {
|
|
3719
|
+
clearTimeout(gapTimer);
|
|
3720
|
+
gapTimer = null;
|
|
2310
3721
|
}
|
|
3722
|
+
args.doc._markReadOnly('unsaved');
|
|
3723
|
+
// The user is told by the SDK, not by each app, because an app that
|
|
3724
|
+
// renders nothing is exactly the case this has to cover: a document that
|
|
3725
|
+
// reloaded and a document that reloaded for a reason look the same
|
|
3726
|
+
// otherwise. `alert` holds the main thread until it is acknowledged, so
|
|
3727
|
+
// the reason cannot be missed and the reload lands on a page the user
|
|
3728
|
+
// is looking at. A browser that suppresses the dialog returns from it
|
|
3729
|
+
// immediately, which reloads rather than skipping, and a caller outside
|
|
3730
|
+
// a browser (a worker, a test) has nobody to tell and does the same.
|
|
3731
|
+
//
|
|
3732
|
+
// The reload survives a dialog that throws, which is the one way telling
|
|
3733
|
+
// the user could cost them the fix: `alert` is a page-global an app is
|
|
3734
|
+
// free to replace, and the copy this leaves behind is the diverged one
|
|
3735
|
+
// the reload exists to discard.
|
|
3736
|
+
try {
|
|
3737
|
+
if (typeof alert === 'function') alert(message);
|
|
3738
|
+
} catch {
|
|
3739
|
+
// Told or not, the copy is resolved.
|
|
3740
|
+
}
|
|
3741
|
+
args.doc.reload();
|
|
3742
|
+
}
|
|
3743
|
+
|
|
3744
|
+
function failOpTerminally(
|
|
3745
|
+
clientOpId: string,
|
|
3746
|
+
message: string,
|
|
3747
|
+
opts: { healable?: boolean; report?: boolean } = {},
|
|
3748
|
+
): void {
|
|
3749
|
+
// Read before `_failOutstandingOp`, which clears the id from the
|
|
3750
|
+
// resync-push set on its way out.
|
|
3751
|
+
const wasHealPush = args.doc._isResyncPushOp(clientOpId);
|
|
3752
|
+
opStartedAt.delete(clientOpId);
|
|
3753
|
+
// A refusal the heal cannot answer is about the document, so it answers
|
|
3754
|
+
// every op in flight the same way and is reported once, by the lock. Per
|
|
3755
|
+
// op, a bulk write's outstanding ops each produce an identical error, so
|
|
3756
|
+
// the reader watches the same message arrive dozens of times before the
|
|
3757
|
+
// one dialog that explains it.
|
|
3758
|
+
args.doc._failOutstandingOp(clientOpId, message, {
|
|
3759
|
+
report: opts.report ?? true,
|
|
3760
|
+
});
|
|
3761
|
+
if (opts.healable === false) {
|
|
3762
|
+
lockForUnhealableRefusal(message);
|
|
3763
|
+
return;
|
|
3764
|
+
}
|
|
3765
|
+
// Restart the backoff only for a failure the heal did not itself
|
|
3766
|
+
// produce. A refused heal chunk that zeroed the round count would keep
|
|
3767
|
+
// the bound from ever binding: each round pushes, each push is refused,
|
|
3768
|
+
// and each refusal buys another round. Leaving the count alone lets the
|
|
3769
|
+
// rounds run out the way an unanswered handshake already does.
|
|
3770
|
+
if (!wasHealPush) divergenceHealRound = 0;
|
|
3771
|
+
scheduleDivergenceHeal();
|
|
3772
|
+
}
|
|
3773
|
+
|
|
3774
|
+
function onServerUnavailable(): void {
|
|
3775
|
+
if (outstandingResends.size === 0) return;
|
|
3776
|
+
if (unavailableTimer !== null) return;
|
|
3777
|
+
unavailableRounds += 1;
|
|
3778
|
+
if (unavailableRounds > UNAVAILABLE_MAX_ROUNDS) {
|
|
3779
|
+
clearUnavailableRecovery();
|
|
3780
|
+
failAllOutstanding(UNAVAILABLE_GIVE_UP_MESSAGE);
|
|
3781
|
+
return;
|
|
3782
|
+
}
|
|
3783
|
+
const delay = unavailableDelayMs(unavailableRounds);
|
|
3784
|
+
unavailableTimer = setTimeout(() => {
|
|
3785
|
+
unavailableTimer = null;
|
|
3786
|
+
resendAllOutstanding();
|
|
3787
|
+
}, delay);
|
|
3788
|
+
(unavailableTimer as { unref?: () => void }).unref?.();
|
|
2311
3789
|
}
|
|
2312
3790
|
|
|
2313
3791
|
// Durability handshake, run on every `doc_init`. The persisted version
|
|
@@ -2323,10 +3801,14 @@ function wireDocTransport(args: WireDocTransportArgs): void {
|
|
|
2323
3801
|
durableTailBlobs: Uint8Array[],
|
|
2324
3802
|
): void {
|
|
2325
3803
|
if (!snapshotVersionVectorB64) return;
|
|
2326
|
-
args.doc.maybeResyncPush(
|
|
3804
|
+
const outcome = args.doc.maybeResyncPush(
|
|
2327
3805
|
base64ToBytes(snapshotVersionVectorB64),
|
|
2328
3806
|
durableTailBlobs,
|
|
2329
3807
|
);
|
|
3808
|
+
// Reported here as well as from the re-verify, or the record holds only
|
|
3809
|
+
// the heals that took the slower route and an operator reading it cannot
|
|
3810
|
+
// tell that the ordinary reconnect handshake is missing from the set.
|
|
3811
|
+
if (outcome === 'pushed') reportSyncHealth('divergence_healed');
|
|
2330
3812
|
}
|
|
2331
3813
|
|
|
2332
3814
|
// Re-fetch the durable frontier from the HTTP join and run the handshake.
|
|
@@ -2344,225 +3826,500 @@ function wireDocTransport(args: WireDocTransportArgs): void {
|
|
|
2344
3826
|
let reverifyAgain = false;
|
|
2345
3827
|
// Armed after the subscription is wired, cleared on `doc_gone` and teardown.
|
|
2346
3828
|
let reverifyTimer: ReturnType<typeof setInterval> | null = null;
|
|
2347
|
-
|
|
3829
|
+
// The last run's verdict, so a caller healing a known divergence can tell a
|
|
3830
|
+
// finished handshake from one that fetched nothing.
|
|
3831
|
+
async function reverifyDurability(): Promise<ResyncPushOutcome | 'no-fetch'> {
|
|
2348
3832
|
if (reverifyInFlight) {
|
|
2349
3833
|
reverifyAgain = true;
|
|
2350
|
-
return;
|
|
3834
|
+
return 'in-flight';
|
|
2351
3835
|
}
|
|
2352
3836
|
reverifyInFlight = true;
|
|
3837
|
+
let outcome: ResyncPushOutcome | 'no-fetch' = 'no-fetch';
|
|
2353
3838
|
try {
|
|
2354
3839
|
do {
|
|
2355
3840
|
reverifyAgain = false;
|
|
2356
3841
|
try {
|
|
2357
3842
|
const frontier = await args.refetchDurableFrontier?.();
|
|
2358
3843
|
if (frontier && frontier.snapshotVersionVector) {
|
|
2359
|
-
args.doc.maybeResyncPush(
|
|
3844
|
+
outcome = args.doc.maybeResyncPush(
|
|
2360
3845
|
base64ToBytes(frontier.snapshotVersionVector),
|
|
2361
3846
|
frontier.tailBlobs,
|
|
2362
3847
|
);
|
|
3848
|
+
// The handshake having work to do means the server's copy was
|
|
3849
|
+
// behind this client's, which is the state a lost op leaves and
|
|
3850
|
+
// which nothing on the server side reports.
|
|
3851
|
+
if (outcome === 'pushed') reportSyncHealth('divergence_healed');
|
|
2363
3852
|
}
|
|
2364
3853
|
} catch {
|
|
2365
3854
|
// Best-effort: a failed re-fetch defers the check to the next tick
|
|
2366
3855
|
// or reconnect's `doc_init` handshake. Swallowed so the timer's
|
|
2367
3856
|
// fire-and-forget never surfaces an unhandled rejection and one
|
|
2368
|
-
// failed tick does not wedge the loop guard.
|
|
3857
|
+
// failed tick does not wedge the loop guard. The `no-fetch` verdict
|
|
3858
|
+
// is what keeps a divergence heal asking.
|
|
3859
|
+
outcome = 'no-fetch';
|
|
2369
3860
|
}
|
|
2370
3861
|
} while (reverifyAgain);
|
|
2371
3862
|
} finally {
|
|
2372
3863
|
reverifyInFlight = false;
|
|
2373
3864
|
}
|
|
3865
|
+
return outcome;
|
|
3866
|
+
}
|
|
3867
|
+
|
|
3868
|
+
// Divergence heal. Giving up on an op leaves the local doc holding CRDT
|
|
3869
|
+
// knowledge the server does not, and Loro requires per-actor causal
|
|
3870
|
+
// contiguity, so every op the app writes afterwards reaches the server as
|
|
3871
|
+
// causally unapplyable and stays that way until the missing slice arrives.
|
|
3872
|
+
// The server's own escape for an op nothing can apply is to discard it, so
|
|
3873
|
+
// the window between the give-up and the next handshake is a window where
|
|
3874
|
+
// the app writes into a document that keeps none of it.
|
|
3875
|
+
//
|
|
3876
|
+
// The periodic re-verify would eventually close that window, but on its own
|
|
3877
|
+
// schedule: a bulk write finishes and the reader reloads well inside one
|
|
3878
|
+
// tick. A give-up is the moment the client knows it has diverged, so it
|
|
3879
|
+
// drives the handshake itself, and keeps driving it on a backoff until one
|
|
3880
|
+
// run reports the durable frontier covering the local doc. The first delay
|
|
3881
|
+
// is not zero because the give-up follows sustained shedding, so the fetch
|
|
3882
|
+
// this schedules would join the burst it is recovering from.
|
|
3883
|
+
const DIVERGENCE_HEAL_BASE_DELAY_MS = 2_000;
|
|
3884
|
+
const DIVERGENCE_HEAL_MAX_DELAY_MS = 30_000;
|
|
3885
|
+
// Roughly four minutes of attempts, after which the periodic re-verify is
|
|
3886
|
+
// the remaining backstop and the tab has long since reported the failure.
|
|
3887
|
+
const DIVERGENCE_HEAL_MAX_ROUNDS = 12;
|
|
3888
|
+
let divergenceHealRound = 0;
|
|
3889
|
+
let divergenceHealTimer: ReturnType<typeof setTimeout> | null = null;
|
|
3890
|
+
|
|
3891
|
+
function scheduleDivergenceHeal(): void {
|
|
3892
|
+
if (divergenceHealTimer !== null) return;
|
|
3893
|
+
if (divergenceHealRound >= DIVERGENCE_HEAL_MAX_ROUNDS) return;
|
|
3894
|
+
divergenceHealRound += 1;
|
|
3895
|
+
const delay = Math.min(
|
|
3896
|
+
DIVERGENCE_HEAL_BASE_DELAY_MS * Math.pow(2, divergenceHealRound - 1),
|
|
3897
|
+
DIVERGENCE_HEAL_MAX_DELAY_MS,
|
|
3898
|
+
);
|
|
3899
|
+
divergenceHealTimer = setTimeout(() => {
|
|
3900
|
+
divergenceHealTimer = null;
|
|
3901
|
+
void reverifyDurability().then((outcome) => {
|
|
3902
|
+
// `covered` is the only verdict that proves the hole is closed. A
|
|
3903
|
+
// push that just went out is not yet durable, so the next round
|
|
3904
|
+
// re-checks it rather than assuming its acks land, and the backoff
|
|
3905
|
+
// it comes back on is what keeps a push settling from being polled.
|
|
3906
|
+
if (outcome !== 'covered') scheduleDivergenceHeal();
|
|
3907
|
+
else divergenceHealRound = 0;
|
|
3908
|
+
});
|
|
3909
|
+
}, delay);
|
|
3910
|
+
(divergenceHealTimer as { unref?: () => void }).unref?.();
|
|
3911
|
+
}
|
|
3912
|
+
|
|
3913
|
+
function clearDivergenceHeal(): void {
|
|
3914
|
+
if (divergenceHealTimer !== null) {
|
|
3915
|
+
clearTimeout(divergenceHealTimer);
|
|
3916
|
+
divergenceHealTimer = null;
|
|
3917
|
+
}
|
|
3918
|
+
divergenceHealRound = 0;
|
|
2374
3919
|
}
|
|
2375
3920
|
|
|
2376
3921
|
// A coverage floor below the highest seq seen records ops this client
|
|
2377
3922
|
// applied nothing for, and the server posts each op once, so nothing
|
|
2378
3923
|
// re-sends them. Pull the join instead, after a delay long enough for an
|
|
2379
3924
|
// out-of-order arrival to close the hole itself.
|
|
3925
|
+
//
|
|
3926
|
+
// The delay is only meaningful against the hole this check was ARMED on,
|
|
3927
|
+
// which is why the seq it waits to reach is captured here rather than read
|
|
3928
|
+
// again when it fires. Holes are the ordinary state of a client that is
|
|
3929
|
+
// writing: acks overtake heavily, so the newest acked seq sits above the
|
|
3930
|
+
// contiguous floor almost continuously, and each such hole closes as the ack
|
|
3931
|
+
// below it lands. Asking on expiry whether ANY hole exists reads a
|
|
3932
|
+
// brand-new one, opened by ops sent since, as the original persisting, and a
|
|
3933
|
+
// client that keeps writing always has one. That answer costs a join
|
|
3934
|
+
// re-import, which discards the undo and redo stacks, so the reader loses
|
|
3935
|
+
// their history every couple of seconds while nothing is wrong.
|
|
3936
|
+
//
|
|
3937
|
+
// Rounds that move nothing back off and then stop, and any advance in the
|
|
3938
|
+
// floor starts the budget over. What separates the two cases is exactly
|
|
3939
|
+
// that: a hole left by a write still in flight fills as the rows land, so
|
|
3940
|
+
// each pull delivers more and earns the next one, while a seq the server
|
|
3941
|
+
// allocated and never wrote (a failed op write leaves a hole rather than a
|
|
3942
|
+
// duplicate) delivers nothing however often it is asked for. Without the
|
|
3943
|
+
// budget that second hole is a join re-import every couple of seconds for
|
|
3944
|
+
// the life of the tab, each one discarding the reader's undo history. It
|
|
3945
|
+
// clears at the next compaction, which folds the tail past the hole.
|
|
3946
|
+
const GAP_PULL_MAX_ROUNDS = 6;
|
|
3947
|
+
const GAP_PULL_MAX_DELAY_MS = 30_000;
|
|
2380
3948
|
let gapTimer: ReturnType<typeof setTimeout> | null = null;
|
|
3949
|
+
let gapPullRounds = 0;
|
|
3950
|
+
let gapPullFloor = -1;
|
|
2381
3951
|
function scheduleGapCheck(): void {
|
|
2382
3952
|
if (gapTimer !== null) return;
|
|
2383
|
-
|
|
3953
|
+
const awaited = args.getCurrentSeq();
|
|
3954
|
+
const floor = args.getCoveredSeq();
|
|
3955
|
+
if (floor >= awaited) {
|
|
3956
|
+
gapPullRounds = 0;
|
|
3957
|
+
return;
|
|
3958
|
+
}
|
|
3959
|
+
if (floor > gapPullFloor) gapPullRounds = 0;
|
|
3960
|
+
if (gapPullRounds >= GAP_PULL_MAX_ROUNDS) return;
|
|
3961
|
+
gapPullRounds += 1;
|
|
3962
|
+
gapPullFloor = floor;
|
|
3963
|
+
const delay = Math.min(
|
|
3964
|
+
MISSED_OP_RESYNC_DELAY_MS * Math.pow(2, gapPullRounds - 1),
|
|
3965
|
+
GAP_PULL_MAX_DELAY_MS,
|
|
3966
|
+
);
|
|
2384
3967
|
gapTimer = setTimeout(() => {
|
|
2385
3968
|
gapTimer = null;
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
3969
|
+
// Covered up to where the hole was: it closed on its own, whatever has
|
|
3970
|
+
// been sent since. Anything newer arms its own check.
|
|
3971
|
+
if (args.getCoveredSeq() >= awaited) {
|
|
3972
|
+
gapPullRounds = 0;
|
|
3973
|
+
return;
|
|
3974
|
+
}
|
|
3975
|
+
pullJoin();
|
|
3976
|
+
}, delay);
|
|
2389
3977
|
(gapTimer as { unref?: () => void }).unref?.();
|
|
2390
3978
|
}
|
|
2391
3979
|
|
|
3980
|
+
// The server ended this subscription: it refused the subscribe (a revoked
|
|
3981
|
+
// grant), or evicted it again after the one silent re-subscribe a
|
|
3982
|
+
// connection grants. The client detaches this transport from the topic as
|
|
3983
|
+
// it reports that, so no ack, no fanout and no reconnect reaches it again.
|
|
3984
|
+
//
|
|
3985
|
+
// Everything the transport is waiting on is therefore dead. The ops it
|
|
3986
|
+
// holds can never be acked, and re-sending them puts frames on a
|
|
3987
|
+
// subscription the server has refused while `savingCount` climbs and the
|
|
3988
|
+
// app hears nothing, so they are failed here and every later edit fails
|
|
3989
|
+
// the same way: the app reads one actionable error per unsaved change and
|
|
3990
|
+
// its cue to reload, rather than a document that looks like it is saving
|
|
3991
|
+
// forever. The ids are taken before the reports because
|
|
3992
|
+
// `_failOutstandingOp` re-enters app code.
|
|
3993
|
+
let subscriptionEnded = false;
|
|
3994
|
+
function endSubscription(): void {
|
|
3995
|
+
if (subscriptionEnded) return;
|
|
3996
|
+
subscriptionEnded = true;
|
|
3997
|
+
const clientOpIds = [...outstandingResends.keys()];
|
|
3998
|
+
forgetAllOps();
|
|
3999
|
+
clearUnavailableRecovery();
|
|
4000
|
+
// The heal and the gap pull both work through this transport, so they
|
|
4001
|
+
// are stopped rather than armed: what they would push or fetch has no
|
|
4002
|
+
// subscription left to travel on.
|
|
4003
|
+
clearDivergenceHeal();
|
|
4004
|
+
if (reverifyTimer !== null) {
|
|
4005
|
+
clearInterval(reverifyTimer);
|
|
4006
|
+
reverifyTimer = null;
|
|
4007
|
+
}
|
|
4008
|
+
if (gapTimer !== null) {
|
|
4009
|
+
clearTimeout(gapTimer);
|
|
4010
|
+
gapTimer = null;
|
|
4011
|
+
}
|
|
4012
|
+
for (const clientOpId of clientOpIds) {
|
|
4013
|
+
args.doc._failOutstandingOp(clientOpId, SUBSCRIPTION_ENDED_MESSAGE);
|
|
4014
|
+
}
|
|
4015
|
+
}
|
|
4016
|
+
|
|
2392
4017
|
// 1. Subscribe to the doc topic. Frame routing:
|
|
2393
4018
|
// - `doc_init`: bridge the join-vs-fanout race. Ops committed between
|
|
2394
4019
|
// the HTTP join (T0) and the WS subscribe (T1) miss the HTTP tail and
|
|
2395
|
-
// can race the
|
|
4020
|
+
// can race the subscription record so the first `doc_op` fanout may arrive BEFORE
|
|
2396
4021
|
// doc_init (separate server invocations, no wire ordering). tail_ops
|
|
2397
4022
|
// apply unconditionally: filtering on `seq > currentSeq` would mask
|
|
2398
4023
|
// tail_ops a racing `doc_op` inflated the cursor past, leaving lamport
|
|
2399
4024
|
// gaps Loro cannot converge across (re-applying is an idempotent
|
|
2400
4025
|
// no-op).
|
|
4026
|
+
// - `doc_catch_up`: the document moved past this client's coverage on
|
|
4027
|
+
// an op the socket could not carry; re-import the join and nothing
|
|
4028
|
+
// more (see the branch).
|
|
2401
4029
|
// - `doc_op`: apply, advance cursor.
|
|
2402
4030
|
// - `op_ack`: resolve the outstanding-op tracker, advance cursor, cancel
|
|
2403
4031
|
// that op's resend timer.
|
|
2404
4032
|
// - everything else: handled by `BrassWsClient`.
|
|
2405
|
-
const unsub = args.wsClient.subscribe(
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
//
|
|
2424
|
-
//
|
|
2425
|
-
//
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
args.
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
4033
|
+
const unsub = args.wsClient.subscribe(
|
|
4034
|
+
topic,
|
|
4035
|
+
(frame) => {
|
|
4036
|
+
const init = parseDocInitFrame(frame);
|
|
4037
|
+
if (init) {
|
|
4038
|
+
// Recorded as the frame arrives rather than after the tail applies.
|
|
4039
|
+
// Importing a tail op can throw, and the client swallows it, so a
|
|
4040
|
+
// latch written further down would stay behind and read the NEXT
|
|
4041
|
+
// reconnect as the opening subscribe, suppressing the report this
|
|
4042
|
+
// exists to make.
|
|
4043
|
+
const reconnects = args.wsClient.reconnectCount ?? 0;
|
|
4044
|
+
const socketDropped =
|
|
4045
|
+
docInitReconnectCount !== null && reconnects > docInitReconnectCount;
|
|
4046
|
+
docInitReconnectCount = reconnects;
|
|
4047
|
+
// Resync check BEFORE applying. The tail covers (snapshot_seq,
|
|
4048
|
+
// current_seq]; if the coverage floor sits below the tail's lower edge
|
|
4049
|
+
// (or below current_seq when the tail is empty), ops in between were
|
|
4050
|
+
// compacted into the snapshot while disconnected and exist nowhere in
|
|
4051
|
+
// this tail or future fanout, so re-import the join's snapshot. The
|
|
4052
|
+
// initial subscribe never trips this (the floor was just seeded), so
|
|
4053
|
+
// the cost is paid only on a reconnect that crossed a compaction.
|
|
4054
|
+
const snapshotFloor = snapshotSeqOf(
|
|
4055
|
+
init.snapshot_seq,
|
|
4056
|
+
init.tail_ops.map((entry) => entry.seq),
|
|
4057
|
+
init.current_seq,
|
|
4058
|
+
);
|
|
4059
|
+
const needsResync = args.getCoveredSeq() < snapshotFloor;
|
|
4060
|
+
const tailBlobs: Uint8Array[] = [];
|
|
4061
|
+
// Whether the frame's own entries can describe the durable frontier.
|
|
4062
|
+
// An entry the server withheld the bytes of still NAMES an op the
|
|
4063
|
+
// server holds, so a frontier folded from the inline entries alone
|
|
4064
|
+
// reports every withheld op as missing from the server. See the
|
|
4065
|
+
// handshake call below.
|
|
4066
|
+
let tailComplete = true;
|
|
4067
|
+
for (const entry of init.tail_ops) {
|
|
4068
|
+
// An op with no inline bytes is deliberately left uncovered: it
|
|
4069
|
+
// has not been applied, so marking it covered would retire the
|
|
4070
|
+
// very resync that fetches it.
|
|
4071
|
+
if (entry.op_blob_b64 === undefined) {
|
|
4072
|
+
tailComplete = false;
|
|
4073
|
+
continue;
|
|
4074
|
+
}
|
|
4075
|
+
const blob = base64ToBytes(entry.op_blob_b64);
|
|
4076
|
+
tailBlobs.push(blob);
|
|
4077
|
+
args.doc.applyRemoteOp(blob);
|
|
4078
|
+
if (entry.seq > args.getCurrentSeq()) args.setCurrentSeq(entry.seq);
|
|
4079
|
+
args.advanceCoveredSeq(entry.seq);
|
|
4080
|
+
}
|
|
4081
|
+
// The frame also has to ACCOUNT for every seq the head names, not
|
|
4082
|
+
// merely carry bytes for the entries it lists. The server drops an
|
|
4083
|
+
// oversized or spilled tail from this frame ENTIRELY rather than
|
|
4084
|
+
// listing it byte-less (`document.tail.dropped`), so an empty tail on
|
|
4085
|
+
// a branch whose head sits above its snapshot is not a branch with
|
|
4086
|
+
// nothing in flight: it is the whole tail withheld. Measured on a
|
|
4087
|
+
// real document: snapshot_seq 565, current_seq 1089, 524 ops dropped
|
|
4088
|
+
// from the frame and every one of them already imported by the HTTP
|
|
4089
|
+
// join, so a frontier folded from this frame alone sat 524 ops behind
|
|
4090
|
+
// the local doc and pushed them back on every open.
|
|
4091
|
+
if (
|
|
4092
|
+
typeof init.snapshot_seq === 'number' &&
|
|
4093
|
+
tailBlobs.length < init.current_seq - init.snapshot_seq
|
|
4094
|
+
) {
|
|
4095
|
+
tailComplete = false;
|
|
4096
|
+
}
|
|
4097
|
+
if (init.current_seq > args.getCurrentSeq()) {
|
|
4098
|
+
args.setCurrentSeq(init.current_seq);
|
|
4099
|
+
}
|
|
4100
|
+
if (needsResync) {
|
|
4101
|
+
resyncPastSeq(init.current_seq);
|
|
4102
|
+
} else {
|
|
4103
|
+
// A frame that named a head it did not carry the tail for leaves
|
|
4104
|
+
// the reader short with nothing further coming: the server drops an
|
|
4105
|
+
// oversized or spilled tail from this frame and posts each op once.
|
|
4106
|
+
// For a reader that never writes, this and the catch-up prompt are
|
|
4107
|
+
// the only two places a gap gets noticed at all.
|
|
4108
|
+
scheduleGapCheck();
|
|
4109
|
+
}
|
|
4110
|
+
// `doc_init` is the reconnect signal the retry-forever recovery hangs
|
|
4111
|
+
// off: re-assert every outstanding op and run the durability handshake
|
|
4112
|
+
// now, so a client holding work the server lost re-asserts it on
|
|
4113
|
+
// reconnect rather than waiting out the per-op backoff. A fresh
|
|
4114
|
+
// connection also returns the shed budget to full: this re-assert is
|
|
4115
|
+
// the round the pending one would have run.
|
|
4116
|
+
clearUnavailableRecovery();
|
|
4117
|
+
// A reconnect, rather than the initial subscribe: the socket dropped
|
|
4118
|
+
// while this document held work, which is the condition the server
|
|
4119
|
+
// has no way to see. The first `doc_init` arrives on the opening
|
|
4120
|
+
// subscribe, and an app that writes as soon as it opens already has
|
|
4121
|
+
// ops outstanding by then, so reporting on every `doc_init` would
|
|
4122
|
+
// fire routinely on a healthy first open and train a reader of the
|
|
4123
|
+
// log to discount the event.
|
|
4124
|
+
//
|
|
4125
|
+
// Before the round rather than after, so the counters describe the
|
|
4126
|
+
// work the reconnect found rather than the work it has just put back
|
|
4127
|
+
// on the wire.
|
|
4128
|
+
if (socketDropped && outstandingResends.size > 0) {
|
|
4129
|
+
reportSyncHealth('reconnected_unsaved');
|
|
4130
|
+
}
|
|
4131
|
+
resendAllOutstanding();
|
|
4132
|
+
// The handshake needs the WHOLE durable tail, and this frame carries
|
|
4133
|
+
// it only when every entry came inline. A spilled op is named here
|
|
4134
|
+
// with its bytes withheld, and the HTTP join the reader already ran
|
|
4135
|
+
// resolves and imports it, so a frontier built from the inline
|
|
4136
|
+
// entries alone sits behind the local doc by exactly the ops that
|
|
4137
|
+
// spilled, and the handshake pushes the reader's copy of the server's
|
|
4138
|
+
// own tail back at it. On a branch at capacity every chunk is refused
|
|
4139
|
+
// and reported, which is an error on screen for someone who only
|
|
4140
|
+
// reopened the document; on any other branch it lands, writing a
|
|
4141
|
+
// second durable copy of every spilled op and growing the document
|
|
4142
|
+
// toward the ceiling on each open. `reverifyDurability` fetches
|
|
4143
|
+
// through `fetchSnapshot`, which resolves spilled entries, so the
|
|
4144
|
+
// frontier it diffs against is complete.
|
|
4145
|
+
if (tailComplete) {
|
|
4146
|
+
maybeResyncPush(init.snapshot_version_vector, tailBlobs);
|
|
2439
4147
|
} else {
|
|
2440
|
-
|
|
4148
|
+
void reverifyDurability();
|
|
2441
4149
|
}
|
|
4150
|
+
// The tail has applied, so every author this client can hear from is
|
|
4151
|
+
// in its doc. An author left in the durable frontier is one whose
|
|
4152
|
+
// changes exist only inside the snapshot, which is the platform having
|
|
4153
|
+
// rewritten how the document stores what it stores. A client cannot
|
|
4154
|
+
// undo across that: its stack inverts the operations it made, and the
|
|
4155
|
+
// values those operations wrote have moved, so an undo would remove
|
|
4156
|
+
// nothing and report success. Re-import instead, which drops both
|
|
4157
|
+
// stacks and leaves the reader on fresh state with undo honestly
|
|
4158
|
+
// unavailable.
|
|
4159
|
+
//
|
|
4160
|
+
// Coverage does not catch this on its own, and the case it misses is
|
|
4161
|
+
// the ordinary one: the fold runs on a branch nobody has open, so the
|
|
4162
|
+
// client that returns to it missed no ops and has no gap to notice.
|
|
4163
|
+
if (!needsResync) maybeResyncPastRewrite(init.snapshot_version_vector);
|
|
4164
|
+
// Posted after the subscription record exists, so this is the safe point to
|
|
4165
|
+
// reconcile a pending open whose terminal status frame was dropped.
|
|
4166
|
+
args.reconcileStatusOnInit?.();
|
|
4167
|
+
args.onDocumentMoved?.();
|
|
4168
|
+
return;
|
|
2442
4169
|
}
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
return;
|
|
2460
|
-
}
|
|
2461
|
-
if (isDocOpFrame(frame)) {
|
|
2462
|
-
// Apply unconditionally. A `seq <= currentSeq` short-circuit looks safe
|
|
2463
|
-
// (import is idempotent) but permanently drops ops on out-of-order
|
|
2464
|
-
// fanout: seq=14 before seq=13 inflates the cursor past 13, after which
|
|
2465
|
-
// seq=13 is skipped forever, leaving a lamport gap Loro cannot converge
|
|
2466
|
-
// across.
|
|
2467
|
-
const bytes = base64ToBytes(frame.op_blob);
|
|
2468
|
-
args.doc.applyRemoteOp(bytes);
|
|
2469
|
-
if (frame.seq > args.getCurrentSeq()) args.setCurrentSeq(frame.seq);
|
|
2470
|
-
args.advanceCoveredSeq(frame.seq);
|
|
2471
|
-
scheduleGapCheck();
|
|
2472
|
-
return;
|
|
2473
|
-
}
|
|
2474
|
-
if (isOpAckFrame(frame)) {
|
|
2475
|
-
args.doc.resolveOutstandingOp(frame.client_op_id);
|
|
2476
|
-
args.setCurrentSeq(frame.seq);
|
|
2477
|
-
args.advanceCoveredSeq(frame.seq);
|
|
2478
|
-
// An ack for a local op sits above any remote op this client never
|
|
2479
|
-
// received, so the user's own next edit is a second chance to notice.
|
|
2480
|
-
scheduleGapCheck();
|
|
2481
|
-
const pending = outstandingResends.get(frame.client_op_id);
|
|
2482
|
-
if (pending) {
|
|
2483
|
-
clearTimeout(pending.timer);
|
|
2484
|
-
outstandingResends.delete(frame.client_op_id);
|
|
4170
|
+
if (isDocCatchUpFrame(frame)) {
|
|
4171
|
+
// The document moved past what this client can bridge to, so pull the
|
|
4172
|
+
// join. Nothing else: the prompt describes the document rather than
|
|
4173
|
+
// this subscription, and every subscriber gets one on each op the
|
|
4174
|
+
// socket could not carry. Re-asserting outstanding ops here writes a
|
|
4175
|
+
// second durable copy of each of them (the server allocates a fresh
|
|
4176
|
+
// seq per send) for every such op any writer makes, and cancelling
|
|
4177
|
+
// the shed recovery here parks a shed op on its per-op backoff
|
|
4178
|
+
// instead of the schedule the shed armed.
|
|
4179
|
+
if (frame.current_seq > args.getCurrentSeq()) {
|
|
4180
|
+
args.setCurrentSeq(frame.current_seq);
|
|
4181
|
+
}
|
|
4182
|
+
if (args.getCoveredSeq() < frame.current_seq) {
|
|
4183
|
+
resyncPastSeq(frame.current_seq);
|
|
4184
|
+
}
|
|
4185
|
+
return;
|
|
2485
4186
|
}
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
return;
|
|
2500
|
-
}
|
|
2501
|
-
if (isDocGoneFrame(frame)) {
|
|
2502
|
-
// Cancel every resend timer FIRST so a doc that was saving when delete
|
|
2503
|
-
// landed stops re-trying ops the server will only reject as `forbidden`
|
|
2504
|
-
// (the SUB# row was evicted at fanout). Draining before the `_markGone`
|
|
2505
|
-
// listeners keeps the lifecycle observably clean.
|
|
2506
|
-
for (const pending of outstandingResends.values()) {
|
|
2507
|
-
clearTimeout(pending.timer);
|
|
2508
|
-
}
|
|
2509
|
-
outstandingResends.clear();
|
|
2510
|
-
// Stop re-verifying durability against an evicted doc: a resync push
|
|
2511
|
-
// would only bounce as `forbidden`, and a gap pull would only 404.
|
|
2512
|
-
if (reverifyTimer !== null) {
|
|
2513
|
-
clearInterval(reverifyTimer);
|
|
2514
|
-
reverifyTimer = null;
|
|
2515
|
-
}
|
|
2516
|
-
if (gapTimer !== null) {
|
|
2517
|
-
clearTimeout(gapTimer);
|
|
2518
|
-
gapTimer = null;
|
|
2519
|
-
}
|
|
2520
|
-
args.doc._markGone(goneReasonOf(frame));
|
|
2521
|
-
return;
|
|
2522
|
-
}
|
|
2523
|
-
if (isDocStatusFrame(frame)) {
|
|
2524
|
-
// The validator gates only on `pending`; default `message` here so a
|
|
2525
|
-
// message-less status still carries a valid `DocumentWorkStatus`
|
|
2526
|
-
// (`message` is typed non-null and rendered verbatim).
|
|
2527
|
-
const status: DocumentWorkStatus = {
|
|
2528
|
-
...frame.status,
|
|
2529
|
-
message:
|
|
2530
|
-
typeof frame.status.message === 'string' ? frame.status.message : '',
|
|
2531
|
-
};
|
|
2532
|
-
// `reload` is the server's authoritative "succeeded, content ready"
|
|
2533
|
-
// hint: pull the content in and emit the terminal status only AFTER the
|
|
2534
|
-
// import lands, so an app re-reading on `onStatus` sees content present.
|
|
2535
|
-
// A failure frame has no `reload`, so emit immediately and fetch
|
|
2536
|
-
// nothing (no racy read, no loop).
|
|
2537
|
-
if (frame.reload && args.reloadAfterHydration) {
|
|
2538
|
-
args.reloadAfterHydration(status);
|
|
2539
|
-
} else {
|
|
2540
|
-
args.doc._emitStatus(status);
|
|
4187
|
+
if (isDocOpFrame(frame)) {
|
|
4188
|
+
// Apply unconditionally. A `seq <= currentSeq` short-circuit looks safe
|
|
4189
|
+
// (import is idempotent) but permanently drops ops on out-of-order
|
|
4190
|
+
// fanout: seq=14 before seq=13 inflates the cursor past 13, after which
|
|
4191
|
+
// seq=13 is skipped forever, leaving a lamport gap Loro cannot converge
|
|
4192
|
+
// across.
|
|
4193
|
+
const bytes = base64ToBytes(frame.op_blob);
|
|
4194
|
+
args.doc.applyRemoteOp(bytes);
|
|
4195
|
+
if (frame.seq > args.getCurrentSeq()) args.setCurrentSeq(frame.seq);
|
|
4196
|
+
args.advanceCoveredSeq(frame.seq);
|
|
4197
|
+
scheduleGapCheck();
|
|
4198
|
+
args.onDocumentMoved?.();
|
|
4199
|
+
return;
|
|
2541
4200
|
}
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
4201
|
+
if (isOpAckFrame(frame)) {
|
|
4202
|
+
// Evidence the connection is working through its queue, which is what
|
|
4203
|
+
// the resend timers read to tell a slow ack from a silent server.
|
|
4204
|
+
lastAckAt = Date.now();
|
|
4205
|
+
args.doc.resolveOutstandingOp(frame.client_op_id);
|
|
4206
|
+
args.setCurrentSeq(frame.seq);
|
|
4207
|
+
args.advanceCoveredSeq(frame.seq);
|
|
4208
|
+
// An ack for a local op sits above any remote op this client never
|
|
4209
|
+
// received, so the user's own next edit is a second chance to notice.
|
|
4210
|
+
scheduleGapCheck();
|
|
4211
|
+
forgetOp(frame.client_op_id);
|
|
4212
|
+
// The server is durably accepting writes again, so the shed budget
|
|
4213
|
+
// starts over. A round already scheduled still runs: the ops it
|
|
4214
|
+
// re-asserts are the ones this ack did not cover.
|
|
4215
|
+
unavailableRounds = 0;
|
|
4216
|
+
return;
|
|
4217
|
+
}
|
|
4218
|
+
if (isServerUnavailableFrame(frame)) {
|
|
4219
|
+
onServerUnavailable();
|
|
4220
|
+
return;
|
|
4221
|
+
}
|
|
4222
|
+
if (isOpRejectedFrame(frame)) {
|
|
4223
|
+
// Terminal, not transient: cancel this op's resend (retrying would only
|
|
4224
|
+
// draw the same rejection forever and pin `savingCount`) and fail it on
|
|
4225
|
+
// the Document so the app hears "could not be saved" instead of a silent
|
|
4226
|
+
// wedge. Scoped to this one op; the topic subscription stays live.
|
|
4227
|
+
forgetOp(frame.client_op_id);
|
|
4228
|
+
// The flip is the one moment the refusal is worth reporting: every op
|
|
4229
|
+
// in flight draws the same sentence, and a bulk write has as many of
|
|
4230
|
+
// those as it has unacked writes.
|
|
4231
|
+
const atCapacity = isCapacityRejection(frame);
|
|
4232
|
+
const firstCapacityRefusal = atCapacity && args.doc._markAtCapacity();
|
|
4233
|
+
// A refusal that names the writer rather than the op applies to
|
|
4234
|
+
// every op this reader will send, so it turns the whole Document
|
|
4235
|
+
// read-only before the failure is reported: the app's own
|
|
4236
|
+
// `doc.readOnly` gate then disables the editor in the same tick the
|
|
4237
|
+
// reader hears why, rather than letting them keep typing into a
|
|
4238
|
+
// surface nothing will save.
|
|
4239
|
+
const readOnly = readOnlyReasonForRejection(frame);
|
|
4240
|
+
if (readOnly !== null) args.doc._markReadOnly(readOnly);
|
|
4241
|
+
// NO capacity refusal is healable, whichever kind it is. The refused
|
|
4242
|
+
// work sits on this copy and nowhere else, and the heal is the only
|
|
4243
|
+
// thing that could place it: when it runs out of rounds it stops
|
|
4244
|
+
// silently, so a copy the handshake could not settle is one showing
|
|
4245
|
+
// work that a reload will not, with nothing said. Both kinds therefore
|
|
4246
|
+
// lock, tell and reload, which is what the server's own wording for
|
|
4247
|
+
// each of them already promises.
|
|
4248
|
+
//
|
|
4249
|
+
// The other refusals here stay healable: the handshake splits the
|
|
4250
|
+
// slice under the frame cap, which is what reaches the dependents of
|
|
4251
|
+
// an op that is itself too large to re-send.
|
|
4252
|
+
failOpTerminally(frame.client_op_id, opRejectedMessage(frame), {
|
|
4253
|
+
healable: !atCapacity,
|
|
4254
|
+
report: !atCapacity || firstCapacityRefusal,
|
|
4255
|
+
});
|
|
4256
|
+
return;
|
|
4257
|
+
}
|
|
4258
|
+
if (isDocGoneFrame(frame)) {
|
|
4259
|
+
// Cancel every resend timer FIRST so a doc that was saving when delete
|
|
4260
|
+
// landed stops re-trying ops the server will only reject as `forbidden`
|
|
4261
|
+
// (the subscription record was evicted at fanout). Draining before the `_markGone`
|
|
4262
|
+
// listeners keeps the lifecycle observably clean.
|
|
4263
|
+
forgetAllOps();
|
|
4264
|
+
clearUnavailableRecovery();
|
|
4265
|
+
// Stop re-verifying durability against an evicted doc: a resync push
|
|
4266
|
+
// would only bounce as `forbidden`, and a gap pull would only 404.
|
|
4267
|
+
if (reverifyTimer !== null) {
|
|
4268
|
+
clearInterval(reverifyTimer);
|
|
4269
|
+
reverifyTimer = null;
|
|
4270
|
+
}
|
|
4271
|
+
clearDivergenceHeal();
|
|
4272
|
+
if (gapTimer !== null) {
|
|
4273
|
+
clearTimeout(gapTimer);
|
|
4274
|
+
gapTimer = null;
|
|
4275
|
+
}
|
|
4276
|
+
args.doc._markGone(goneReasonOf(frame));
|
|
4277
|
+
return;
|
|
4278
|
+
}
|
|
4279
|
+
if (isDocStatusFrame(frame)) {
|
|
4280
|
+
const status = normalizeWorkStatus(frame.status);
|
|
4281
|
+
// `reload` is the server's authoritative "succeeded, content ready"
|
|
4282
|
+
// hint: pull the content in and emit the terminal status only AFTER the
|
|
4283
|
+
// import lands, so an app re-reading on `onStatus` sees content present.
|
|
4284
|
+
// A failure frame has no `reload`, so emit immediately and fetch
|
|
4285
|
+
// nothing (no racy read, no loop).
|
|
4286
|
+
if (frame.reload && args.reloadAfterHydration) {
|
|
4287
|
+
args.reloadAfterHydration(status);
|
|
4288
|
+
} else {
|
|
4289
|
+
args.doc._emitStatus(status);
|
|
4290
|
+
}
|
|
4291
|
+
return;
|
|
4292
|
+
}
|
|
4293
|
+
if (isDocMetaFrame(frame)) {
|
|
4294
|
+
// `_setName` de-dupes, so a re-emit of the current name is a no-op.
|
|
4295
|
+
args.doc._setName(frame.name);
|
|
4296
|
+
return;
|
|
4297
|
+
}
|
|
4298
|
+
if (isDocDriveStatusFrame(frame)) {
|
|
4299
|
+
// The signal de-dupes a no-op re-emit, so forwarding every frame is safe.
|
|
4300
|
+
args.doc._emitDriveStatus(toDriveStatus(frame.drive));
|
|
4301
|
+
return;
|
|
4302
|
+
}
|
|
4303
|
+
if (isDocOpsDroppedFrame(frame)) {
|
|
4304
|
+
// Prompt durability re-check: the discard sealed a new snapshot, so
|
|
4305
|
+
// re-fetch the moved frontier and run the handshake now instead of at
|
|
4306
|
+
// the author's next reconnect. The handshake self-selects: the author
|
|
4307
|
+
// finds local ahead and resync-pushes under a new seq, others no-op.
|
|
4308
|
+
// Covered by the `doc_init` handshake / periodic re-verify if missed.
|
|
4309
|
+
void reverifyDurability();
|
|
4310
|
+
return;
|
|
4311
|
+
}
|
|
4312
|
+
// Unknown frame: ignore, so server-side schema growth doesn't break
|
|
4313
|
+
// older SDK builds.
|
|
4314
|
+
},
|
|
4315
|
+
{
|
|
4316
|
+
// `resendAllOutstanding` on the `doc_init` that follows every
|
|
4317
|
+
// `subscribed` ack IS this topic's re-assertion, so the client drops an
|
|
4318
|
+
// op frame it cannot send now rather than buffering a second copy.
|
|
4319
|
+
ownsFrameRetry: true,
|
|
4320
|
+
onEnded: endSubscription,
|
|
4321
|
+
},
|
|
4322
|
+
);
|
|
2566
4323
|
|
|
2567
4324
|
// Arm the periodic durability re-verify (the long-lived-tab backstop).
|
|
2568
4325
|
// `unref` so the interval never keeps a Node host alive (no-op in the
|
|
@@ -2577,6 +4334,22 @@ function wireDocTransport(args: WireDocTransportArgs): void {
|
|
|
2577
4334
|
// tracker so a silently-dropped server response triggers a retry.
|
|
2578
4335
|
args.doc.setLocalOpHook((opBlob) => {
|
|
2579
4336
|
const clientOpId = args.newClientOpId();
|
|
4337
|
+
// Nothing carries an op once the subscription is over, so it is failed
|
|
4338
|
+
// instead of tracked: an entry nothing can ack pins `savingCount` and
|
|
4339
|
+
// its resends reach a topic the server refused. The caller records the
|
|
4340
|
+
// entry AFTER this hook returns, so the failure waits a microtask for
|
|
4341
|
+
// there to be one to fail.
|
|
4342
|
+
//
|
|
4343
|
+
// Taken before the op is timed, because this path fails the op through
|
|
4344
|
+
// the Document directly rather than through `failOpTerminally`, so a
|
|
4345
|
+
// timing entry written here is one nothing ever drops.
|
|
4346
|
+
if (subscriptionEnded) {
|
|
4347
|
+
void Promise.resolve().then(() => {
|
|
4348
|
+
args.doc._failOutstandingOp(clientOpId, SUBSCRIPTION_ENDED_MESSAGE);
|
|
4349
|
+
});
|
|
4350
|
+
return clientOpId;
|
|
4351
|
+
}
|
|
4352
|
+
opStartedAt.set(clientOpId, Date.now());
|
|
2580
4353
|
const opBlobB64 = bytesToBase64(opBlob);
|
|
2581
4354
|
const sendFrame: {
|
|
2582
4355
|
type: 'op';
|
|
@@ -2601,14 +4374,18 @@ function wireDocTransport(args: WireDocTransportArgs): void {
|
|
|
2601
4374
|
return clientOpId;
|
|
2602
4375
|
}
|
|
2603
4376
|
|
|
2604
|
-
args.wsClient.send(sendFrame);
|
|
2605
4377
|
outstandingResends.set(clientOpId, {
|
|
2606
4378
|
frame: sendFrame,
|
|
2607
4379
|
attempts: 1,
|
|
2608
|
-
//
|
|
4380
|
+
// Placeholders, overwritten by `dispatchOp` and `scheduleResend` once
|
|
4381
|
+
// the op is actually on the socket.
|
|
2609
4382
|
timer: setTimeout(() => undefined, 0),
|
|
4383
|
+
armedAt: 0,
|
|
4384
|
+
dispatchedAt: 0,
|
|
4385
|
+
inFlight: 0,
|
|
4386
|
+
queued: false,
|
|
2610
4387
|
});
|
|
2611
|
-
|
|
4388
|
+
dispatchOp(clientOpId);
|
|
2612
4389
|
return clientOpId;
|
|
2613
4390
|
});
|
|
2614
4391
|
|
|
@@ -2640,32 +4417,72 @@ function wireDocTransport(args: WireDocTransportArgs): void {
|
|
|
2640
4417
|
// The size is known here, so refuse without spending a round trip on a
|
|
2641
4418
|
// blob that can only come back 400.
|
|
2642
4419
|
if (opBlobExceedsDurableLimit(opBlobB64)) {
|
|
2643
|
-
|
|
4420
|
+
failOpTerminally(clientOpId, TOO_LARGE_MESSAGE);
|
|
2644
4421
|
return;
|
|
2645
4422
|
}
|
|
2646
4423
|
|
|
2647
4424
|
if (!args.postOversizedOp) {
|
|
2648
|
-
|
|
4425
|
+
failOpTerminally(clientOpId, TOO_LARGE_MESSAGE);
|
|
2649
4426
|
return;
|
|
2650
4427
|
}
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
4428
|
+
// An oversized op is deliberately kept out of `outstandingResends` (a
|
|
4429
|
+
// frame past the cap must never reach the socket), so the shed recovery
|
|
4430
|
+
// that re-asserts the tracker cannot reach this one. It runs its own
|
|
4431
|
+
// round of the same policy instead, on the same budget and backoff, so a
|
|
4432
|
+
// capacity refusal costs an op the same nothing whichever transport
|
|
4433
|
+
// carried it.
|
|
4434
|
+
let result: OversizedOpResult | undefined;
|
|
4435
|
+
for (let round = 0; round <= UNAVAILABLE_MAX_ROUNDS; round += 1) {
|
|
4436
|
+
if (round > 0) {
|
|
4437
|
+
await new Promise((resolve) => {
|
|
4438
|
+
const timer = setTimeout(resolve, unavailableDelayMs(round));
|
|
4439
|
+
(timer as { unref?: () => void }).unref?.();
|
|
4440
|
+
});
|
|
4441
|
+
}
|
|
4442
|
+
try {
|
|
4443
|
+
result = await args.postOversizedOp(opBlobB64, clientOpId);
|
|
4444
|
+
} catch (err) {
|
|
4445
|
+
result = {
|
|
4446
|
+
ok: false,
|
|
4447
|
+
message: err instanceof Error ? err.message : String(err),
|
|
4448
|
+
// Staging the blob runs strictly before the op write, so a failure
|
|
4449
|
+
// there wrote nothing and the op is still the client's to send: it
|
|
4450
|
+
// is priced the way the write prices its own refusals. Any other
|
|
4451
|
+
// throw leaves it unknown whether the handler ran, and this op
|
|
4452
|
+
// carries no idempotency the server dedupes on, so replaying it
|
|
4453
|
+
// could double-apply.
|
|
4454
|
+
retryable:
|
|
4455
|
+
err instanceof OpStagingError && opStagingIsRetryable(err),
|
|
4456
|
+
};
|
|
4457
|
+
}
|
|
4458
|
+
if (result.ok || !result.retryable) break;
|
|
4459
|
+
}
|
|
4460
|
+
if (result === undefined) {
|
|
4461
|
+
opStartedAt.delete(clientOpId);
|
|
4462
|
+
return;
|
|
2659
4463
|
}
|
|
2660
4464
|
if (!result.ok) {
|
|
2661
4465
|
// Terminal for this op, the same way `op_rejected` is: the local
|
|
2662
4466
|
// commit stays applied but never persists, so the app hears about it
|
|
2663
|
-
// instead of watching `savingCount` sit above zero forever.
|
|
2664
|
-
|
|
4467
|
+
// instead of watching `savingCount` sit above zero forever. A refusal
|
|
4468
|
+
// about the writer turns the Document read-only first, so the app's
|
|
4469
|
+
// own gate closes the editor as the reader is told why.
|
|
4470
|
+
const firstCapacityRefusal =
|
|
4471
|
+
result.atCapacity === true && args.doc._markAtCapacity();
|
|
4472
|
+
if (result.readOnly) args.doc._markReadOnly(result.readOnly);
|
|
4473
|
+
failOpTerminally(
|
|
4474
|
+
clientOpId,
|
|
4475
|
+
result.retryable ? UNAVAILABLE_GIVE_UP_MESSAGE : result.message,
|
|
4476
|
+
{
|
|
4477
|
+
healable: result.atCapacity !== true,
|
|
4478
|
+
report: result.atCapacity !== true || firstCapacityRefusal,
|
|
4479
|
+
},
|
|
4480
|
+
);
|
|
2665
4481
|
return;
|
|
2666
4482
|
}
|
|
2667
4483
|
// Same bookkeeping an `op_ack` does: resolve the outstanding entry and
|
|
2668
4484
|
// advance both cursors past the seq the server allocated.
|
|
4485
|
+
opStartedAt.delete(clientOpId);
|
|
2669
4486
|
args.doc.resolveOutstandingOp(clientOpId);
|
|
2670
4487
|
args.setCurrentSeq(result.seq);
|
|
2671
4488
|
args.advanceCoveredSeq(result.seq);
|
|
@@ -2722,14 +4539,13 @@ function wireDocTransport(args: WireDocTransportArgs): void {
|
|
|
2722
4539
|
clearInterval(reverifyTimer);
|
|
2723
4540
|
reverifyTimer = null;
|
|
2724
4541
|
}
|
|
4542
|
+
clearDivergenceHeal();
|
|
2725
4543
|
if (gapTimer !== null) {
|
|
2726
4544
|
clearTimeout(gapTimer);
|
|
2727
4545
|
gapTimer = null;
|
|
2728
4546
|
}
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
}
|
|
2732
|
-
outstandingResends.clear();
|
|
4547
|
+
forgetAllOps();
|
|
4548
|
+
clearUnavailableRecovery();
|
|
2733
4549
|
originalClose();
|
|
2734
4550
|
};
|
|
2735
4551
|
(args.doc as Document & { close: () => void }).close = () => {
|
|
@@ -2742,16 +4558,40 @@ function wireDocTransport(args: WireDocTransportArgs): void {
|
|
|
2742
4558
|
// (its lifecycle is the parent Document's). Keying on the REQUESTED branch
|
|
2743
4559
|
// (not the resolved one) keeps park and take symmetric; a default-branch
|
|
2744
4560
|
// open has no requested branch and so skips the cache.
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
4561
|
+
//
|
|
4562
|
+
// A doc with work still running skips it too: parking clears the status
|
|
4563
|
+
// listeners, and the SDK's own waiters on that work (the import runner
|
|
4564
|
+
// waits for the phase that produces the file it imports) are keyed to
|
|
4565
|
+
// this instance, so a reopen would hand back a Document whose wait can
|
|
4566
|
+
// never fire. A fresh open re-attaches everything.
|
|
4567
|
+
//
|
|
4568
|
+
// So does one the server has evicted. `takeParkedDocument` refuses such an
|
|
4569
|
+
// entry, so parking it buys nothing and costs everything parking keeps
|
|
4570
|
+
// alive: the topic subscription (which goes on delivering every op of a
|
|
4571
|
+
// document nobody is reading) and the LoroDoc's heap, until some later
|
|
4572
|
+
// open or eviction reaches that key. A move is the case that makes this
|
|
4573
|
+
// routine rather than terminal, because the session follows it by
|
|
4574
|
+
// reopening the same id, so the evicted copy is retired on every move.
|
|
4575
|
+
const parkable =
|
|
4576
|
+
!args.doc.readOnly &&
|
|
4577
|
+
args.parkable !== false &&
|
|
4578
|
+
args.doc.status?.pending !== true &&
|
|
4579
|
+
args.doc._goneReason === null;
|
|
4580
|
+
const cacheKey = parkable
|
|
4581
|
+
? parkCacheKey(args.docId, args.requestedBranchId, args.requestedStream)
|
|
4582
|
+
: null;
|
|
2749
4583
|
if (cacheKey === null) {
|
|
2750
4584
|
fullTeardown();
|
|
2751
4585
|
return;
|
|
2752
4586
|
}
|
|
2753
4587
|
args.doc._suspend();
|
|
2754
|
-
parkDocument({
|
|
4588
|
+
parkDocument({
|
|
4589
|
+
doc: args.doc,
|
|
4590
|
+
teardown: fullTeardown,
|
|
4591
|
+
key: cacheKey,
|
|
4592
|
+
optionsSig: documentOptionsSig(args.documentOptions),
|
|
4593
|
+
identity: args.identity?.() ?? null,
|
|
4594
|
+
});
|
|
2755
4595
|
};
|
|
2756
4596
|
}
|
|
2757
4597
|
|