@loro-dev/streams-crdt 0.16.0 → 0.16.2

package/dist/{stream-id-DwQujw_H.js → stream-id-Dlnc1PNH.js}

@@ -1,4 +1,4 @@
-import { StreamsClient, decodeSsePayload, readSseEvents } from "@loro-dev/streams-client";
+import { StreamsClient, decodeSsePayload, readSseEvents, thrownToStreamError } from "@loro-dev/streams-client";
 //#region src/deque.ts
 const DEFAULT_COMPACT_THRESHOLD = 64;
 const DEFAULT_MIN_CAPACITY = 16;
@@ -171,6 +171,51 @@ function computeRetryDelay(attempt, config = DEFAULT_RECONNECT_CONFIG, minDelayM
 	return applyJitter(typeof minDelayMs === "number" && minDelayMs > scheduled ? minDelayMs : scheduled, jitterFraction);
 }
 //#endregion
+//#region src/error-context.ts
+const contexts = /* @__PURE__ */ new WeakMap();
+var ThrownValue = class extends Error {
+	constructor(original) {
+		super("Local operation failed");
+		this.original = original;
+	}
+};
+function errorContext(error) {
+	return typeof error === "object" && error != null ? contexts.get(error) : void 0;
+}
+function originalThrown(error) {
+	return error instanceof ThrownValue ? error.original : error;
+}
+function recordErrorContext(error, context, replace = false) {
+	const tagged = typeof error === "object" && error != null ? error : new ThrownValue(error);
+	contexts.set(tagged, replace ? context : {
+		...context,
+		...contexts.get(tagged)
+	});
+	return tagged;
+}
+async function atErrorBoundary(context, task) {
+	const startedAt = Date.now();
+	try {
+		return await task();
+	} catch (error) {
+		throw recordErrorContext(error, {
+			...context,
+			elapsedMs: Math.max(0, Date.now() - startedAt)
+		}, context.source != null);
+	}
+}
+function atSyncErrorBoundary(context, task) {
+	const startedAt = Date.now();
+	try {
+		return task();
+	} catch (error) {
+		throw recordErrorContext(error, {
+			...context,
+			elapsedMs: Math.max(0, Date.now() - startedAt)
+		}, context.source != null);
+	}
+}
+//#endregion
 //#region src/items.ts
 const BINARY_CONTENT_TYPE = "application/octet-stream";
 function encodeItems(items) {
@@ -698,7 +743,7 @@ function classifyFetchError(err) {
 		retriable: true,
 		atMs: Date.now()
 	};
-	if (err instanceof TypeError) return {
+	if (errorContext(err)?.originalCode === "network_error") return {
 		code: "network",
 		message,
 		retriable: true,
@@ -720,7 +765,33 @@ function classifyFetchError(err) {
 function isStreamNotFoundError(error) {
 	return error instanceof HttpError && error.status === 404;
 }
-function toTransportError(error) {
+function toTransportError(error, operation = "unknown") {
+	const evidence = errorContext(error);
+	const result = classifyTransportError(originalThrown(error), evidence?.originalCode === "network_error");
+	const source = isPayloadProtectionError(error) ? "payload_protection" : evidence?.source ?? (error instanceof AuthProviderError ? "auth" : error instanceof HttpError ? "http" : "unknown");
+	return {
+		...result,
+		context: {
+			stage: "unknown",
+			originalCode: result.code,
+			...error instanceof HttpError && { status: error.status },
+			...error instanceof TimeoutError && {
+				phase: error.timeoutType,
+				timeoutMs: error.timeoutMs
+			},
+			...error instanceof InitialSyncTimeoutError && {
+				stage: "initial_sync",
+				timeoutMs: error.timeoutMs,
+				elapsedMs: error.elapsedMs
+			},
+			...evidence,
+			source,
+			retryable: result.retryable,
+			operation
+		}
+	};
+}
+function classifyTransportError(error, fromFetch) {
 	if (isCrdtApplyIncompleteError(error)) return {
 		code: "apply_incomplete",
 		retryable: true,
@@ -810,10 +881,10 @@ function toTransportError(error) {
 		reason: error.reason,
 		message: payloadProtectionFailureMessage(error.reason)
 	};
-	if (error instanceof TypeError) return {
+	if (fromFetch) return {
 		code: "network_error",
 		retryable: true,
-		message: error.message
+		message: error instanceof Error ? error.message : "Fetch failed"
 	};
 	if (error instanceof Error) return {
 		code: "internal_error",
@@ -1057,7 +1128,10 @@ var StreamsTransportClient = class {
 		return "large-post";
 	}
 	async createStreamRequest() {
-		const adapterHeaders = this.adapter.getCreateStreamHeaders ? await this.adapter.getCreateStreamHeaders() : void 0;
+		const adapterHeaders = this.adapter.getCreateStreamHeaders ? await atErrorBoundary({
+			source: "crdt",
+			stage: "crdt_export"
+		}, this.adapter.getCreateStreamHeaders.bind(this.adapter)) : void 0;
 		const result = await this.client.create({
 			headers: adapterHeaders,
 			ttlSeconds: this.streamTtlSeconds
@@ -1246,7 +1320,7 @@ var StreamsTransportClient = class {
 		const dataEncoding = response.headers.get("Stream-SSE-Data-Encoding") ?? void 0;
 		const consume = async () => {
 			let sawBootstrap = false;
-			for await (const event of readSseEvents(response.body, { onActivity })) {
+			for await (const event of this.translateSseEvents(readSseEvents(response.body, { onActivity }))) {
 				if (!sawBootstrap) {
 					if (event.event !== "bootstrap") throw new ProtocolError(`ephemeral sse must start with bootstrap, got '${event.event}'`);
 					await onBootstrap(decodeSsePayload(event.data, dataEncoding));
@@ -1302,10 +1376,13 @@ var StreamsTransportClient = class {
 			controller.abort(new TimeoutError(phase, timeoutMs));
 		}, timeoutMs);
 		try {
-			return await this.fetchImpl(input, {
+			return await atErrorBoundary({
+				source: "fetch",
+				originalCode: "network_error"
+			}, () => this.fetchImpl(input, {
 				...init,
 				signal
-			});
+			}));
 		} catch (error) {
 			if (controller.signal.aborted && controller.signal.reason instanceof TimeoutError) throw controller.signal.reason;
 			throw error;
@@ -1358,7 +1435,30 @@ var StreamsTransportClient = class {
 		target.searchParams.set("live", "sse");
 		return target.toString();
 	}
+	async *translateSseEvents(events) {
+		try {
+			yield* events;
+		} catch (error) {
+			throw this.toTransportFailure(thrownToStreamError(error, "live-sse"));
+		}
+	}
 	toTransportFailure(error) {
+		const failure = this.convertTransportFailure(error);
+		if (!(failure instanceof InitialSyncTimeoutError)) {
+			const { operation, ...context } = error.context ?? {
+				source: "unknown",
+				operation: "unknown",
+				originalCode: error.code
+			};
+			recordErrorContext(failure, {
+				...context,
+				requestOperation: operation,
+				stage: operation === "bootstrap" ? "bootstrap" : operation === "read-once" ? "catch_up" : operation === "append" || operation === "append-cas" ? "upload" : "unknown"
+			}, true);
+		}
+		return failure;
+	}
+	convertTransportFailure(error) {
 		switch (error.code) {
 			case "bad_request":
 			case "unauthorized":
@@ -1366,13 +1466,16 @@ var StreamsTransportClient = class {
 			case "not_found":
 			case "conflict":
 			case "gone":
-			case "rate_limited": return new HttpError(error.message, error.status, { body: error.bodyText });
-			case "timeout": return new TimeoutError(error.phase, error.phase === "poll" ? this.reconnectConfig.pollTimeoutMs : this.reconnectConfig.connectTimeoutMs);
-			case "network_error": return error.cause instanceof TypeError ? error.cause : new TypeError(error.message);
+			case "rate_limited": return new HttpError(error.message, error.status, {
+				body: error.bodyText,
+				requestId: error.context?.requestId
+			});
+			case "timeout": return new TimeoutError(error.phase, error.context?.timeoutMs ?? (error.phase === "poll" ? this.reconnectConfig.pollTimeoutMs : this.reconnectConfig.connectTimeoutMs));
+			case "network_error": return error.cause instanceof Error ? error.cause : new TypeError(error.message);
 			case "protocol_error": return new ProtocolError(error.message, error.status);
 			case "unknown":
 				if (typeof error.status === "number") return new HttpError(error.message, error.status, {
-					requestId: extractRequestIdFromHeaders(error.headers),
+					requestId: error.context?.requestId ?? extractRequestIdFromHeaders(error.headers),
 					retryAfterMs: parseRetryAfterMs(error.headers?.["retry-after"]),
 					body: error.bodyText
 				});
@@ -1441,7 +1544,7 @@ var StreamsTransportClient = class {
 						latestCursor,
 						paused: false
 					};
-					for await (const event of session.events) {
+					for await (const event of this.translateSseEvents(session.events)) {
 						if (event.event === "data") {
 							pendingDataEvents.push(event.data);
 							continue;
@@ -2036,11 +2139,17 @@ async function persistRemoteCursor(input) {
 		...input.cursor,
 		updatedAtMs: (input.now ?? Date.now)()
 	};
-	if (input.beforeRemoteCursorSave != null) await input.beforeRemoteCursorSave({
+	if (input.beforeRemoteCursorSave != null) await atErrorBoundary({
+		source: "durability_hook",
+		stage: "durability_hook"
+	}, input.beforeRemoteCursorSave.bind(void 0, {
 		cursor: next,
 		source: input.source
-	});
-	await input.remoteCursorStore.save(next);
+	}));
+	await atErrorBoundary({
+		source: "cursor_store",
+		stage: "cursor_save"
+	}, () => input.remoteCursorStore.save(next));
 	return next;
 }
 function key(streamUrl) {
@@ -2270,14 +2379,24 @@ var TransportCursorManager = class {
 		let unresolved = [];
 		if (snapshot.byteLength > 0) {
 			if (typeof snapshotOffset !== "string" || snapshotOffset.length === 0 || snapshotOffset.length > 1024 || snapshotOffset === "now" || snapshotOffset === "-1") throw new Error("bootstrap snapshot offset does not match an admitted continuation");
-			const outcome = normalizeApplyReturn(await this.adapter.applySnapshot(this.decodeSnapshot == null ? snapshot : await this.decodeSnapshot(snapshot, snapshotOffset)));
+			const clearSnapshot = this.decodeSnapshot == null ? snapshot : await atErrorBoundary({
+				source: "local",
+				stage: "payload_decode"
+			}, this.decodeSnapshot.bind(this, snapshot, snapshotOffset));
+			const outcome = normalizeApplyReturn(await atErrorBoundary({
+				source: "crdt",
+				stage: "crdt_apply"
+			}, () => this.adapter.applySnapshot(clearSnapshot)));
 			nextVersion = outcome.version;
 			complete = outcome.complete;
 			unresolved = outcome.unresolved ?? [];
 		}
 		const clearUpdates = await this.decodeRemoteUpdateItems(updates);
 		if (clearUpdates.length > 0) {
-			const outcome = normalizeApplyReturn(await this.adapter.applyRemoteUpdates(clearUpdates, nextVersion));
+			const outcome = normalizeApplyReturn(await atErrorBoundary({
+				source: "crdt",
+				stage: "crdt_apply"
+			}, async () => this.adapter.applyRemoteUpdates(clearUpdates, nextVersion)));
 			nextVersion = outcome.version;
 			complete = outcome.complete;
 			unresolved = outcome.unresolved ?? [];
@@ -2305,7 +2424,10 @@ var TransportCursorManager = class {
 		let unresolved = this.incompleteCursors.get(cursor) ?? [];
 		const clearUpdates = await this.decodeRemoteUpdateItems(result.updates);
 		if (clearUpdates.length > 0) {
-			const outcome = normalizeApplyReturn(await this.adapter.applyRemoteUpdates(clearUpdates, nextVersion));
+			const outcome = normalizeApplyReturn(await atErrorBoundary({
+				source: "crdt",
+				stage: "crdt_apply"
+			}, async () => this.adapter.applyRemoteUpdates(clearUpdates, nextVersion)));
 			nextVersion = outcome.version;
 			complete = outcome.complete;
 			unresolved = outcome.unresolved ?? [];
@@ -2324,7 +2446,10 @@ var TransportCursorManager = class {
 		let unresolved = this.incompleteCursors.get(cursor) ?? [];
 		const clearUpdates = await this.decodeRemoteUpdateItems(updates);
 		if (clearUpdates.length > 0) {
-			const outcome = normalizeApplyReturn(await this.adapter.applyRemoteUpdates(clearUpdates, nextVersion));
+			const outcome = normalizeApplyReturn(await atErrorBoundary({
+				source: "crdt",
+				stage: "crdt_apply"
+			}, async () => this.adapter.applyRemoteUpdates(clearUpdates, nextVersion)));
 			nextVersion = outcome.version;
 			complete = outcome.complete;
 			unresolved = outcome.unresolved ?? [];
@@ -2346,7 +2471,10 @@ var TransportCursorManager = class {
 	}
 	async decodeRemoteUpdateItems(items) {
 		if (items.length === 0) return [];
-		return this.decodeUpdateItems == null ? [...items] : await this.decodeUpdateItems(items);
+		return this.decodeUpdateItems == null ? [...items] : await atErrorBoundary({
+			source: "local",
+			stage: "payload_decode"
+		}, this.decodeUpdateItems.bind(this, items));
 	}
 };
 //#endregion
@@ -2453,8 +2581,14 @@ var TransportSnapshotManager = class {
 			}
 			if (tailBytes <= snapshotBytes) return;
 			if (tailBytes - snapshotBytes < config.minBytesSinceRemoteSnapshot) return;
-			const snapshot = this.adapter.exportSnapshot();
-			await this.putSnapshotAt(state.cursor.nextOffset, this.encodeSnapshot == null ? snapshot : await this.encodeSnapshot(snapshot, state.cursor.nextOffset));
+			const snapshot = atSyncErrorBoundary({
+				source: "crdt",
+				stage: "crdt_export"
+			}, () => this.adapter.exportSnapshot());
+			await this.putSnapshotAt(state.cursor.nextOffset, this.encodeSnapshot == null ? snapshot : await atErrorBoundary({
+				source: "local",
+				stage: "payload_encode"
+			}, this.encodeSnapshot.bind(this, snapshot, state.cursor.nextOffset)));
 		} catch (error) {
 			this.logError("snapshot upload failed", error);
 		}
@@ -2681,7 +2815,7 @@ var StreamsCrdt = class StreamsCrdt {
 			state.started = this.startJoin(state);
 			state.started.catch((error) => {
 				if (state.closed) return;
-				const transportError = toTransportError(error);
+				const transportError = toTransportError(error, "live");
 				if (transportError.retryable) {
 					state.initialSyncFailed = true;
 					state.lastError = transportError;
@@ -2739,7 +2873,7 @@ var StreamsCrdt = class StreamsCrdt {
 		} catch (error) {
 			return {
 				ok: false,
-				error: toTransportError(error)
+				error: toTransportError(error, "snapshot")
 			};
 		}
 	}
@@ -2751,10 +2885,19 @@ var StreamsCrdt = class StreamsCrdt {
 	*/
 	async uploadSnapshotForTesting() {
 		try {
-			await this.adapter.flushLocalExports?.();
+			await atErrorBoundary({
+				source: "crdt",
+				stage: "crdt_export"
+			}, () => this.adapter.flushLocalExports?.());
 			const cursor = await this.syncOnce("sync");
-			const rawSnapshot = this.adapter.exportSnapshot();
-			const snapshot = this.encodeSnapshot == null ? rawSnapshot : await this.encodeSnapshot(rawSnapshot, cursor.nextOffset);
+			const rawSnapshot = atSyncErrorBoundary({
+				source: "crdt",
+				stage: "crdt_export"
+			}, () => this.adapter.exportSnapshot());
+			const snapshot = this.encodeSnapshot == null ? rawSnapshot : await atErrorBoundary({
+				source: "local",
+				stage: "payload_encode"
+			}, this.encodeSnapshot.bind(this, rawSnapshot, cursor.nextOffset));
 			await this.client.putSnapshotAt(cursor.nextOffset, snapshot);
 			return {
 				ok: true,
@@ -2766,7 +2909,7 @@ var StreamsCrdt = class StreamsCrdt {
 		} catch (error) {
 			return {
 				ok: false,
-				error: toTransportError(error)
+				error: toTransportError(error, "snapshot")
 			};
 		}
 	}
@@ -2782,7 +2925,7 @@ var StreamsCrdt = class StreamsCrdt {
 		} catch (error) {
 			return {
 				ok: false,
-				error: toTransportError(error)
+				error: toTransportError(error, "create")
 			};
 		}
 	}
@@ -2796,7 +2939,10 @@ var StreamsCrdt = class StreamsCrdt {
 				ok: true,
 				value: { deleted: await this.enqueueExclusive(async () => {
 					const deleted = await this.client.deleteStreamRequest();
-					await this.remoteCursorStore.delete?.(this.streamUrl);
+					await atErrorBoundary({
+						source: "cursor_store",
+						stage: "cursor_delete"
+					}, () => this.remoteCursorStore.delete?.(this.streamUrl));
 					this.producerEpoch = 0;
 					this.nextProducerSeq = 0;
 					this.frozenLocalAppend = void 0;
@@ -2808,7 +2954,7 @@ var StreamsCrdt = class StreamsCrdt {
 		} catch (error) {
 			return {
 				ok: false,
-				error: toTransportError(error)
+				error: toTransportError(error, "delete")
 			};
 		}
 	}
@@ -2825,7 +2971,7 @@ var StreamsCrdt = class StreamsCrdt {
 		} catch (error) {
 			return {
 				ok: false,
-				error: toTransportError(error)
+				error: toTransportError(error, "sync")
 			};
 		}
 	}
@@ -2853,7 +2999,7 @@ var StreamsCrdt = class StreamsCrdt {
 		} catch (error) {
 			return {
 				ok: false,
-				error: toTransportError(error)
+				error: toTransportError(error, "catchup")
 			};
 		}
 	}
@@ -2868,7 +3014,10 @@ var StreamsCrdt = class StreamsCrdt {
 		return await this.enqueueExclusive(async () => {
 			try {
 				if (this.writeOnlyDisabledReason != null) throw new Error(this.writeOnlyDisabledReason);
-				await this.adapter.flushLocalExports?.();
+				await atErrorBoundary({
+					source: "crdt",
+					stage: "crdt_export"
+				}, () => this.adapter.flushLocalExports?.());
 				const drained = await this.drainPendingWriteOnlyBatch();
 				if (drained == null) return {
 					ok: true,
@@ -2893,7 +3042,7 @@ var StreamsCrdt = class StreamsCrdt {
 			} catch (error) {
 				return {
 					ok: false,
-					error: toTransportError(error)
+					error: toTransportError(error, "upload")
 				};
 			}
 		});
@@ -2918,14 +3067,17 @@ var StreamsCrdt = class StreamsCrdt {
 			} catch (error) {
 				return {
 					ok: false,
-					error: toTransportError(error)
+					error: toTransportError(error, "join")
 				};
 			}
 		}
 		const state = createJoinState(createInitialRemoteCursor({
 			streamUrl: this.streamUrl,
 			nextOffset: "-1",
-			serverLowerBoundVersion: this.adapter.emptyVersion()
+			serverLowerBoundVersion: atSyncErrorBoundary({
+				source: "crdt",
+				stage: "crdt_export"
+			}, () => this.adapter.emptyVersion())
 		}));
 		const statusListener = params?.onStatusChange;
 		if (statusListener != null) state.statusListeners.add(statusListener);
@@ -2938,7 +3090,7 @@ var StreamsCrdt = class StreamsCrdt {
 				value: this.createSubscription(state, params)
 			};
 		} catch (error) {
-			const transportError = toTransportError(error);
+			const transportError = toTransportError(error, "join");
 			if (transportError.retryable && !state.closed) {
 				state.initialSyncFailed = true;
 				state.lastError = transportError;
@@ -3008,7 +3160,7 @@ var StreamsCrdt = class StreamsCrdt {
 				if (state.readSubStatus === "connecting") try {
 					await state.started;
 				} catch (error) {
-					const te = toTransportError(error);
+					const te = toTransportError(error, "join");
 					throw new Error(te.code ? `initial sync failed: ${te.code}` : "initial sync failed");
 				}
 				if (state.writeSubStatus === "error") {
@@ -3016,7 +3168,10 @@ var StreamsCrdt = class StreamsCrdt {
 					const msg = state.lastError?.code ? `local append failed: ${state.lastError.code}` : "local append failed";
 					throw new Error(msg);
 				}
-				await this.adapter.flushLocalExports?.();
+				await atErrorBoundary({
+					source: "crdt",
+					stage: "crdt_export"
+				}, () => this.adapter.flushLocalExports?.());
 				if (state.closed) throw new Error("subscription closed");
 				const remaining = state.pendingLocal.length;
 				if (remaining === 0) return;
@@ -3059,6 +3214,11 @@ var StreamsCrdt = class StreamsCrdt {
 		if (typeof hardTimeoutMs === "number" && Number.isFinite(hardTimeoutMs) && hardTimeoutMs > 0) hardTimeoutPromise = new Promise((_, reject) => {
 			hardTimer = setTimeout(() => {
 				hardTimeoutError = new InitialSyncTimeoutError(operation, hardTimeoutMs, Date.now() - startedAtMs);
+				recordErrorContext(hardTimeoutError, {
+					source: "deadline",
+					stage: "initial_sync",
+					originalCode: "initial_sync_timeout"
+				});
 				hardController.abort(hardTimeoutError);
 				reject(hardTimeoutError);
 			}, hardTimeoutMs);
@@ -3107,7 +3267,7 @@ var StreamsCrdt = class StreamsCrdt {
 		} catch (error) {
 			if (state.closed) return;
 			this.snapshotManager.dispose(state);
-			state.lastError = toTransportError(error);
+			state.lastError = toTransportError(error, "join");
 			this.logError("join first sync failed", error, { streamUrl: this.streamUrl });
 			state.readSubStatus = "error";
 			state.writeSubStatus = "error";
@@ -3155,7 +3315,10 @@ var StreamsCrdt = class StreamsCrdt {
 					this.commitProducerAck(ack.producerAck);
 					didAppend = true;
 				}
-				await this.adapter.flushLocalExports?.();
+				await atErrorBoundary({
+					source: "crdt",
+					stage: "crdt_export"
+				}, () => this.adapter.flushLocalExports?.());
 				const localBatch = this.exportUpdates(appliedVersion);
 				if (localBatch.batch != null) {
 					const appendedBatch = await this.appendLocalBatchRemote({
@@ -3175,12 +3338,49 @@ var StreamsCrdt = class StreamsCrdt {
 		}
 	}
 	async syncOnce(operation, parentSignal) {
-		let cursor = (await this.finalizeDirectLocalAppendIfNeeded(await this.resolveInitialRemoteState(operation, parentSignal))).cursor;
+		return this.enqueueExclusive(async () => {
+			await atErrorBoundary({
+				source: "crdt",
+				stage: "crdt_export"
+			}, () => this.adapter.flushLocalExports?.());
+			return this.syncOnceExclusive(operation, parentSignal);
+		});
+	}
+	/** Caller owns opQueue; never wait for the background flush from here. */
+	async syncOnceExclusive(operation, parentSignal) {
+		const state = this.joinState;
+		if (state != null && !state.closed) while (!state.closed && state.pendingLocal.length > 0) await this.flushPendingLocalBatchExclusive(state);
+		if ((state == null || state.closed) && this.frozenLocalAppend?.source === "pending") {
+			const frozen = this.frozenLocalAppend;
+			const appended = await this.appendFrozenLocalBatchRemote(frozen);
+			await this.catchupLocalAppend(frozen);
+			this.commitProducerAck(appended.producerAck);
+		}
+		const initial = await this.finalizeDirectLocalAppendIfNeeded(await this.resolveInitialRemoteState(operation, parentSignal));
+		let cursor = initial.cursor;
+		if (state != null && !state.closed) {
+			state.cursor = cursor;
+			state.streamCursor = initial.streamCursor;
+			while (!state.closed && state.pendingLocal.length > 0) {
+				await this.flushPendingLocalBatchExclusive(state);
+				cursor = state.cursor;
+			}
+		}
 		const localBatch = this.exportUpdates(cursor.serverLowerBoundVersion);
-		if (localBatch.batch != null) cursor = (await this.appendLocalBatchDurably(cursor, localBatch.batch)).cursor;
+		if (localBatch.batch != null) {
+			const appended = await this.appendLocalBatchDurably(cursor, localBatch.batch);
+			cursor = appended.cursor;
+			if (state != null && !state.closed) {
+				state.cursor = cursor;
+				state.streamCursor = appended.streamCursor;
+			}
+		}
 		return cursor;
 	}
 	async performInitialJoinSync(preferredLiveMode, parentSignal) {
+		return this.enqueueExclusive(() => this.performInitialJoinSyncExclusive(preferredLiveMode, parentSignal));
+	}
+	async performInitialJoinSyncExclusive(preferredLiveMode, parentSignal) {
 		const initial = await this.finalizeDirectLocalAppendIfNeeded(await this.resolveInitialRemoteState("join", parentSignal));
 		let cursor = initial.cursor;
 		let localExportRefVersion = cursor.serverLowerBoundVersion;
@@ -3190,9 +3390,7 @@ var StreamsCrdt = class StreamsCrdt {
 		localExportRefVersion = exportedLocal.nextRefVersion;
 		const exportedBatch = exportedLocal.batch;
 		if (exportedBatch != null) {
-			const appendCursor = cursor;
-			const appendStreamCursor = streamCursor;
-			const committed = await this.enqueueExclusive(() => this.appendLocalBatchDurably(appendCursor, exportedBatch, void 0, "direct", appendStreamCursor));
+			const committed = await this.appendLocalBatchDurably(cursor, exportedBatch, void 0, "direct", streamCursor);
 			cursor = committed.cursor;
 			streamCursor = committed.streamCursor;
 		}
@@ -3208,11 +3406,17 @@ var StreamsCrdt = class StreamsCrdt {
 		return createInitialRemoteCursor({
 			streamUrl: this.streamUrl,
 			nextOffset: "-1",
-			serverLowerBoundVersion: this.adapter.emptyVersion()
+			serverLowerBoundVersion: atSyncErrorBoundary({
+				source: "crdt",
+				stage: "crdt_export"
+			}, () => this.adapter.emptyVersion())
 		});
 	}
 	async resolveInitialRemoteState(operation, parentSignal) {
-		const loaded = await this.remoteCursorStore.load(this.streamUrl);
+		const loaded = await atErrorBoundary({
+			source: "cursor_store",
+			stage: "cursor_load"
+		}, () => this.remoteCursorStore.load(this.streamUrl));
 		const cursor = loaded ?? this.createInitialCursor();
 		const runInitial = (task) => this.runInitialSyncWithDeadline(operation, task, parentSignal);
 		try {
@@ -3237,7 +3441,10 @@ var StreamsCrdt = class StreamsCrdt {
 		}
 	}
 	async resolvePullOnlyRemoteState(operation, parentSignal) {
-		const loaded = await this.remoteCursorStore.load(this.streamUrl);
+		const loaded = await atErrorBoundary({
+			source: "cursor_store",
+			stage: "cursor_load"
+		}, () => this.remoteCursorStore.load(this.streamUrl));
 		const cursor = loaded ?? this.createInitialCursor();
 		const runInitial = (task) => this.runInitialSyncWithDeadline(operation, task, parentSignal);
 		try {
@@ -3279,7 +3486,7 @@ var StreamsCrdt = class StreamsCrdt {
 	}
 	async bootstrapState(cursor, options = {}) {
 		const cursorManager = options.cursorManager ?? this.cursorManager;
-		const result = await this.client.bootstrap(options.signal, options.timeoutMode);
+		const result = await atErrorBoundary({ stage: "bootstrap" }, () => this.client.bootstrap(options.signal, options.timeoutMode));
 		let saved = await cursorManager.applyBootstrapPayload(cursor, result.snapshot, result.updates, result.nextOffset, result.upToDate, result.snapshotOffset);
 		let bootstrapCursor = result.streamCursor;
 		if (!result.upToDate) {
@@ -3316,13 +3523,16 @@ var StreamsCrdt = class StreamsCrdt {
 		const cursorManager = options.cursorManager ?? this.cursorManager;
 		while (true) {
 			const requestOffset = current.nextOffset;
-			const response = await this.client.readFromOffset(requestOffset, currentStreamCursor, options.live, options.signal, options.timeoutMode);
+			const response = await atErrorBoundary({ stage: "catch_up" }, () => this.client.readFromOffset(requestOffset, currentStreamCursor, options.live, options.signal, options.timeoutMode));
 			if (response.kind === "gone") {
 				if (goneStrategy === "return-gone") return StreamsCrdt.GONE_CATCHUP;
 				return await this.bootstrapState({
 					...current,
 					nextOffset: "-1",
-					serverLowerBoundVersion: this.adapter.emptyVersion()
+					serverLowerBoundVersion: atSyncErrorBoundary({
+						source: "crdt",
+						stage: "crdt_export"
+					}, () => this.adapter.emptyVersion())
 				}, {
 					cursorManager,
 					signal: options.signal,
@@ -3337,7 +3547,10 @@ var StreamsCrdt = class StreamsCrdt {
 				if (error instanceof CrdtApplyIncompleteError && result.upToDate) return await this.bootstrapState({
 					...current,
 					nextOffset: "-1",
-					serverLowerBoundVersion: this.adapter.emptyVersion()
+					serverLowerBoundVersion: atSyncErrorBoundary({
+						source: "crdt",
+						stage: "crdt_export"
+					}, () => this.adapter.emptyVersion())
 				}, {
 					cursorManager,
 					signal: options.signal,
@@ -3358,7 +3571,10 @@ var StreamsCrdt = class StreamsCrdt {
 			const bootstrapped = await this.bootstrapState({
 				...state.cursor,
 				nextOffset: "-1",
-				serverLowerBoundVersion: this.adapter.emptyVersion()
+				serverLowerBoundVersion: atSyncErrorBoundary({
+					source: "crdt",
+					stage: "crdt_export"
+				}, () => this.adapter.emptyVersion())
 			}, { signal: state.requestController?.signal });
 			state.cursor = bootstrapped.cursor;
 			state.streamCursor = bootstrapped.streamCursor;
@@ -3374,7 +3590,7 @@ var StreamsCrdt = class StreamsCrdt {
 				streamUrl: this.streamUrl,
 				code: classified.code
 			});
-			state.lastError = toTransportError(error);
+			state.lastError = toTransportError(error, "live");
 			if (state.pendingLocal.length > 0) this.flushPendingLocal(state);
 			if (classified.retriable && !isAuthOrProtocolError(error)) {
 				state.incompleteRecoveryAttempt += 1;
@@ -3407,7 +3623,10 @@ var StreamsCrdt = class StreamsCrdt {
 	localAppendReadCursor(frozen) {
 		return {
 			...frozen.baseCursor,
-			serverLowerBoundVersion: this.adapter.mergeVersions(frozen.baseCursor.serverLowerBoundVersion, frozen.batch.version)
+			serverLowerBoundVersion: atSyncErrorBoundary({
+				source: "crdt",
+				stage: "crdt_export"
+			}, () => this.adapter.mergeVersions(frozen.baseCursor.serverLowerBoundVersion, frozen.batch.version))
 		};
 	}
 	async catchupLocalAppend(frozen, streamCursor) {
@@ -3439,7 +3658,10 @@ var StreamsCrdt = class StreamsCrdt {
 			baseCursor: cursor,
 			batch,
 			pendingCount,
-			wireUpdates: this.encodeUpdateItems == null ? batch.updates : await this.encodeUpdateItems(batch.updates),
+			wireUpdates: this.encodeUpdateItems == null ? batch.updates : await atErrorBoundary({
+				source: "local",
+				stage: "payload_encode"
+			}, this.encodeUpdateItems.bind(this, batch.updates)),
 			producerSession: this.createCurrentProducerSession()
 		};
 		this.frozenLocalAppend = frozen;
@@ -3450,7 +3672,7 @@ var StreamsCrdt = class StreamsCrdt {
 		for (let attempt = 0; attempt < 3; attempt += 1) {
 			let result;
 			try {
-				result = await this.client.appendUpdates(frozen.wireUpdates, frozen.producerSession);
+				result = await atErrorBoundary({ stage: "upload" }, () => this.client.appendUpdates(frozen.wireUpdates, frozen.producerSession));
 			} catch (error) {
 				if (isStreamNotFoundError(error) && this.createStreamIfMissing) {
 					await this.ensureStreamExists();
@@ -3463,7 +3685,10 @@ var StreamsCrdt = class StreamsCrdt {
 					cursor: {
 						...frozen.baseCursor,
 						nextOffset: result.nextOffset,
-						serverLowerBoundVersion: this.adapter.mergeVersions(frozen.baseCursor.serverLowerBoundVersion, frozen.batch.version)
+						serverLowerBoundVersion: atSyncErrorBoundary({
+							source: "crdt",
+							stage: "crdt_export"
+						}, () => this.adapter.mergeVersions(frozen.baseCursor.serverLowerBoundVersion, frozen.batch.version))
 					},
 					producerAck: {
 						epoch: result.ackEpoch,
@@ -3538,9 +3763,15 @@ var StreamsCrdt = class StreamsCrdt {
 		}
 	}
 	exportUpdates(fromVersion) {
-		const exported = this.adapter.exportUpdates(fromVersion);
+		const exported = atSyncErrorBoundary({
+			source: "crdt",
+			stage: "crdt_export"
+		}, () => this.adapter.exportUpdates(fromVersion));
 		if (exported == null) return { nextRefVersion: fromVersion };
-		const nextRefVersion = this.adapter.mergeVersions(fromVersion, exported.version);
+		const nextRefVersion = atSyncErrorBoundary({
+			source: "crdt",
+			stage: "crdt_export"
+		}, () => this.adapter.mergeVersions(fromVersion, exported.version));
 		if (exported.updates.length === 0) return { nextRefVersion };
 		return {
 			batch: exported,
@@ -3597,10 +3828,16 @@ var StreamsCrdt = class StreamsCrdt {
 		const bootstrapCursor = {
 			...cursor,
 			nextOffset: "-1",
-			serverLowerBoundVersion: this.adapter.emptyVersion()
+			serverLowerBoundVersion: atSyncErrorBoundary({
+				source: "crdt",
+				stage: "crdt_export"
+			}, () => this.adapter.emptyVersion())
 		};
 		if (isolatedCursorManager != null) return await this.inferRemoteAndAppendLocal(bootstrapCursor, isolatedCursorManager, operation, parentSignal);
-		const preservedLocalSnapshot = this.adapter.exportSnapshot();
+		const preservedLocalSnapshot = atSyncErrorBoundary({
+			source: "crdt",
+			stage: "crdt_export"
+		}, () => this.adapter.exportSnapshot());
 		let restoredLocalSnapshot = false;
 		try {
 			const inferredRemote = await this.runInitialSyncWithDeadline(operation, (signal) => this.bootstrapState(bootstrapCursor, {
@@ -3608,7 +3845,10 @@ var StreamsCrdt = class StreamsCrdt {
 				signal,
 				timeoutMode: "initial"
 			}), parentSignal);
-			await this.adapter.applySnapshot(preservedLocalSnapshot);
+			await atErrorBoundary({
+				source: "crdt",
+				stage: "crdt_apply"
+			}, async () => this.adapter.applySnapshot(preservedLocalSnapshot));
 			restoredLocalSnapshot = true;
 			let pendingAppend;
 			if (this.frozenLocalAppend != null) {
@@ -3625,7 +3865,10 @@ var StreamsCrdt = class StreamsCrdt {
 			if (pendingAppend != null) this.commitProducerAck(pendingAppend.producerAck);
 			return result;
 		} catch (error) {
-			if (!restoredLocalSnapshot) await this.adapter.applySnapshot(preservedLocalSnapshot);
+			if (!restoredLocalSnapshot) await atErrorBoundary({
+				source: "crdt",
+				stage: "crdt_apply"
+			}, async () => this.adapter.applySnapshot(preservedLocalSnapshot));
 			throw error;
 		}
 	}
@@ -3712,7 +3955,7 @@ var StreamsCrdt = class StreamsCrdt {
 				streamUrl: this.streamUrl,
 				attempts: state.retryAttempt
 			});
-			state.lastError = toTransportError(error);
+			state.lastError = toTransportError(error, "live");
 			this.setReadSubStatus(state, "disconnected");
 			return false;
 		}
@@ -3745,12 +3988,15 @@ var StreamsCrdt = class StreamsCrdt {
 			try {
 				if (state.liveMode === "long-poll") {
 					const requestOffset = state.cursor.nextOffset;
-					const response = await this.client.readFromOffset(requestOffset, state.streamCursor, "long-poll", capturedController?.signal);
+					const response = await atErrorBoundary({ stage: "catch_up" }, () => this.client.readFromOffset(requestOffset, state.streamCursor, "long-poll", capturedController?.signal));
 					if (response.kind === "gone") {
 						const bootstrapped = await this.bootstrapState({
 							...state.cursor,
 							nextOffset: "-1",
-							serverLowerBoundVersion: this.adapter.emptyVersion()
+							serverLowerBoundVersion: atSyncErrorBoundary({
+								source: "crdt",
+								stage: "crdt_export"
+							}, () => this.adapter.emptyVersion())
 						}, { signal: capturedController?.signal });
 						state.cursor = bootstrapped.cursor;
 						state.streamCursor = bootstrapped.streamCursor;
@@ -3837,14 +4083,17 @@ var StreamsCrdt = class StreamsCrdt {
 						streamUrl: this.streamUrl,
 						code: "stream_not_found"
 					});
-					state.lastError = toTransportError(notFoundError);
+					state.lastError = toTransportError(notFoundError, "live");
 					this.setReadSubStatus(state, "error");
 					return;
 				}
 				const bootstrapped = await this.bootstrapState({
 					...state.cursor,
 					nextOffset: "-1",
-					serverLowerBoundVersion: this.adapter.emptyVersion()
+					serverLowerBoundVersion: atSyncErrorBoundary({
+						source: "crdt",
+						stage: "crdt_export"
+					}, () => this.adapter.emptyVersion())
 				}, { signal: capturedController?.signal });
 				state.cursor = bootstrapped.cursor;
 				state.streamCursor = bootstrapped.streamCursor;
@@ -3901,7 +4150,7 @@ var StreamsCrdt = class StreamsCrdt {
 								streamUrl: this.streamUrl,
 								code: flushClassified.code
 							});
-							state.lastError = toTransportError(flushError);
+							state.lastError = toTransportError(flushError, "live");
 							this.setReadSubStatus(state, "error");
 							return;
 						}
@@ -3916,7 +4165,7 @@ var StreamsCrdt = class StreamsCrdt {
 						streamUrl: this.streamUrl,
 						code: classified.code
 					});
-					state.lastError = toTransportError(error);
+					state.lastError = toTransportError(error, "live");
 					this.setReadSubStatus(state, "error");
 					return;
 				}
@@ -3949,7 +4198,10 @@ var StreamsCrdt = class StreamsCrdt {
 			const candidateUpdates = [...allUpdates, ...next.updates];
 			if (await this.measureAppendBodyByteLength(candidateUpdates) > maxBytes) break;
 			allUpdates.push(...next.updates);
-			mergedVersion = this.adapter.mergeVersions(mergedVersion, next.version);
+			mergedVersion = atSyncErrorBoundary({
+				source: "crdt",
+				stage: "crdt_export"
+			}, () => this.adapter.mergeVersions(mergedVersion, next.version));
 			count += 1;
 		}
 		if (count === 1) return {
@@ -3971,6 +4223,50 @@ var StreamsCrdt = class StreamsCrdt {
 		return this.drainQueuedBatch(this.pendingWriteOnly);
 	}
 	/**
+	* One FIFO transaction, called only while holding opQueue. Selecting the
+	* head and committing its ACK/count must share the lock: sync may drain
+	* the queue while a background writer is queued or sleeping between retries.
+	*/
+	async flushPendingLocalBatchExclusive(state) {
+		while (!state.closed) {
+			const drained = await this.drainPendingBatch(state);
+			if (drained == null || state.closed) return false;
+			try {
+				let committedCount;
+				const initial = await this.finalizeDirectLocalAppendIfNeeded({
+					cursor: state.cursor,
+					streamCursor: state.streamCursor
+				});
+				state.cursor = initial.cursor;
+				state.streamCursor = initial.streamCursor;
+				if (this.cursorManager.isCursorComplete(state.cursor)) {
+					state.cursor = await this.appendLocalBatch(state.cursor, drained.batch, drained.count, state.streamCursor);
+					const streamCursor = this.consumeCommittedLocalStreamCursor();
+					if (streamCursor != null) state.streamCursor = streamCursor;
+					committedCount = this.consumeCommittedPendingCount() ?? drained.count;
+					if (!state.closed) this.snapshotManager.schedule(state);
+				} else {
+					const appended = await this.appendLocalBatchRemote(state.cursor, drained.batch, drained.count, "pending");
+					this.commitProducerAck(appended.producerAck);
+					committedCount = appended.pendingCount ?? drained.count;
+				}
+				for (let i = 0; i < committedCount; i += 1) state.pendingLocal.popFront();
+				this.notifySyncWaiters(state, committedCount);
+				state.flushHeadBatchOnly = false;
+				state.lastWriteError = void 0;
+				this.setWriteSubStatus(state, "ok");
+				return true;
+			} catch (error) {
+				if (isPayloadTooLargeError(error) && drained.count > 1 && this.abandonPendingFrozenLocalAppend()) {
+					state.flushHeadBatchOnly = true;
+					continue;
+				}
+				throw error;
+			}
+		}
+		return false;
+	}
+	/**
 	* Drains `pendingLocal` to the server in strict FIFO order.
 	*
 	* FIFO invariant: batches are only popped after a durable append, and a
@@ -3995,36 +4291,14 @@ var StreamsCrdt = class StreamsCrdt {
 		this.clearWriteRetryTimer(state);
 		state.flushingLocal = true;
 		try {
-			outer: while (!state.closed) {
-				const drained = await this.drainPendingBatch(state);
-				if (drained == null) return;
+			while (!state.closed) {
 				let attempts = 0;
 				while (!state.closed) try {
-					let committedCount;
-					if (this.cursorManager.isCursorComplete(state.cursor)) {
-						state.cursor = await this.enqueueExclusive(async () => this.appendLocalBatch(state.cursor, drained.batch, drained.count, state.streamCursor));
-						const committedStreamCursor = this.consumeCommittedLocalStreamCursor();
-						if (committedStreamCursor != null) state.streamCursor = committedStreamCursor;
-						committedCount = this.consumeCommittedPendingCount() ?? drained.count;
-						this.snapshotManager.schedule(state);
-					} else {
-						const appended = await this.enqueueExclusive(async () => this.appendLocalBatchRemote(state.cursor, drained.batch, drained.count, "pending"));
-						this.commitProducerAck(appended.producerAck);
-						committedCount = appended.pendingCount ?? drained.count;
-					}
-					for (let i = 0; i < committedCount; i += 1) state.pendingLocal.popFront();
-					this.notifySyncWaiters(state, committedCount);
-					state.flushHeadBatchOnly = false;
-					state.lastWriteError = void 0;
-					this.setWriteSubStatus(state, "ok");
+					if (!await this.enqueueExclusive(() => this.flushPendingLocalBatchExclusive(state))) return;
 					break;
 				} catch (error) {
-					if (isPayloadTooLargeError(error) && drained.count > 1 && this.abandonPendingFrozenLocalAppend()) {
-						state.flushHeadBatchOnly = true;
-						continue outer;
-					}
 					const classified = classifyFetchError(error);
-					const transportError = toTransportError(error);
+					const transportError = toTransportError(error, "upload");
 					if (!classified.retriable || isAuthOrProtocolError(error)) {
 						if (isPayloadTooLargeError(error)) state.flushHeadBatchOnly = true;
 						this.logError("local append failed", error, {
@@ -4201,4 +4475,4 @@ function isValidRillId(id) {
 //#endregion
 export { createInitialRemoteCursor as a, PayloadProtectionError as c, IndexedDbRemoteCursorStore as i, StreamsCrdt as n, EphemeralStreamCrdt as o, InMemoryRemoteCursorStore as r, LocalAppendFailedError as s, isValidRillId as t };
 
-//# sourceMappingURL=stream-id-DwQujw_H.js.map
+//# sourceMappingURL=stream-id-Dlnc1PNH.js.map