@mengine/sync 1.1.0 → 1.2.1-alpha.0

package/dist/index.d.ts

@@ -106,6 +106,43 @@ declare class ClientServerSynchronizer implements Synchronizer {
   private consumeJobs;
   private updateStatus;
   private waitConnectionReady;
+  /**
+   * Send everything the remote is missing as ONE update derived from the doc,
+   * rather than forwarding each queued update as its own request.
+   *
+   * A single `export({ mode: 'update', from })` carries every commit the remote
+   * lacks as one causally complete blob, so no second request exists to arrive out
+   * of order and there is no gap for a validating peer to reject —
+   * `mengine-server` answers 409 `missing_dependency` for such a gap and discards
+   * the bytes, since its reads (snapshot replay, log folding, version-vector
+   * diffs) all assume every stored update is applicable. `syncWithRemote` already
+   * worked this way for catch-up; the per-commit fan-out this replaces was N
+   * requests in which a later update could overtake the one it depended on.
+   *
+   * The batch's bytes are still imported first, because local storage is a peer
+   * in its own right: another writer (a second tab, a foreign process) can append
+   * an update this doc has never seen, and relaying local writes upstream is part
+   * of the contract. Import is idempotent, so ops the doc already holds cost
+   * nothing, and folding them in is what lets the export cover them.
+   *
+   * The watermark advances to what the remote reports it now holds, and only
+   * failing that to the version this blob actually carried — read BEFORE the
+   * export, so a commit landing during the await is not marked sent. A failure
+   * leaves it untouched, so the next push re-derives the same ops. Rethrowing is
+   * what makes that next push happen: `consumeJobs` marks the doc `retrying` with
+   * the message and the cycle retries. Swallowing it made a refusal or a dead
+   * network look exactly like a successful sync — the doc went `synced: true`
+   * having pushed nothing.
+   *
+   * A readonly server is the one case that returns without pushing rather than
+   * failing. It is not a transient error, so routing it through the retry path
+   * wedges the doc: every cycle would refuse, mark `retrying`, wait 5s and refuse
+   * again, forever, never reaching `synced`. Nothing is queued by that loop either —
+   * `consumeJobs` removes each job before awaiting it, so the retry only works
+   * because the next push re-derives the ops from the doc. Skipping keeps the
+   * watermark untouched, so if the peer ever becomes writable the very next export
+   * still carries everything it never received.
+   */
   private pushUpdatesToRemote;
   private pullUpdatesFromRemote;
   private saveDocUpdates;

package/dist/index.js

@@ -288,6 +288,37 @@ var SynchronizingDoc = class extends Disposable {
 	doc;
 	state;
 	jobs = [];
+	/**
+	* Encoded version vector the remote is known to hold, or null before anything
+	* has been confirmed.
+	*
+	* Strictly a LOWER BOUND, and the asymmetry is what makes it safe: too low only
+	* means the next push re-sends ops the remote already has, which a CRDT import
+	* dedups and a validating peer answers `duplicate`. Too high would mean ops are
+	* never sent at all, so it may only advance to a version some request actually
+	* carried — never to the doc's current version at an arbitrary moment.
+	*
+	* Two kinds of evidence may set it. A complete statement by the remote about
+	* itself — `getDocDiff`'s version, or a push receipt's — REPLACES it, including
+	* downward: a remote that lost an op it once acked says so by reporting less,
+	* and adopting that lower version is what makes the next push re-send it. Absent
+	* such a statement, a push that resolved raises it to the version that request
+	* carried, the only bound provable without the remote's cooperation. A relayed
+	* update is neither — a broadcast is not proof of a durable write, and crediting
+	* one drops that op and every local op behind it while the push still reports
+	* success.
+	*
+	* A statement is believed about OTHER peers and never about our own. It may LEAD
+	* this doc on peers we have not seen, and adopting that is the point — those
+	* concurrent ops are then never re-sent. On our OWN peer the same lead is silent
+	* loss, because we author that history: a claim beyond what we actually sent makes
+	* `export({ from })` skip exactly the ops it names while the push still reports
+	* success, so the entry is clamped to what a request provably carried. Against a
+	* peer that reports nothing the bound instead lags on other peers' ops between
+	* syncs, so a push re-sends ops the remote already has — a duplicate row in its
+	* update log until the next sync corrects it, the deliberate side of the asymmetry.
+	*/
+	remoteVersion = null;
 	eventBus = new EventBus();
 	disposables = new DisposableSet();
 	constructor(docId, doc) {
@@ -330,6 +361,54 @@ const SYNCHRONIZER_ORIGIN_PREFIX = "ClientServerSynchronizer:";
 const isSynchronizerOrigin = (origin) => {
 	return typeof origin === "string" && origin.startsWith(SYNCHRONIZER_ORIGIN_PREFIX);
 };
+/**
+* Turn a remote's claimed version into one it is safe to diff against, or null
+* when the bytes are unusable.
+*
+* A remote's report is trusted about OTHER peers — that is the whole value of it,
+* and where a lead legitimately covers ops we have never seen. It is not trusted
+* about `doc`'s own peer, because there the same lead is silent data loss:
+* `export({ from })` skips every local op at or below the counter in `from`, so a
+* version claiming more of our peer than we actually sent makes those ops
+* unsendable while the push still reports success. Measured: exporting from a VV
+* leading on the doc's own peer yields a 22-byte envelope carrying nothing, while
+* the same lead on a foreign peer still carries the local op.
+*
+* `carried`, not the doc's current version, is the ceiling. The doc may have
+* committed more while the push was in flight, and those ops were never sent, so
+* clamping to the doc would strand exactly them.
+*
+* Only the own-peer lead is checked, because it is the only direction that loses
+* data. `VersionVector.decode` is lenient — `[1, 2, 3]` yields a peer with counter
+* `-2` rather than throwing — but such garbage exports MORE, not less: a version
+* that undershoots or names peers we do not have re-sends ops the remote already
+* holds, which its import dedups. Rejecting it would buy nothing, so it is left
+* alone; null is returned only when the bytes will not decode at all.
+*/
+const usableRemoteVersion = (reported, doc, carried) => {
+	let version;
+	try {
+		version = VersionVector.decode(reported);
+	} catch {
+		return null;
+	}
+	const ownPeer = doc.peerIdStr;
+	const provenOwnCounter = carried.get(ownPeer);
+	const claimedOwnCounter = version.get(ownPeer);
+	if (claimedOwnCounter != null && (provenOwnCounter == null || claimedOwnCounter > provenOwnCounter)) if (provenOwnCounter == null) version.remove(ownPeer);
+	else version.setEnd({
+		peer: ownPeer,
+		counter: provenOwnCounter
+	});
+	return version;
+};
+/**
+* The watermark to keep after a push resolved: the remote's own vetted report when
+* it gave a usable one, and otherwise the version this request provably carried.
+*/
+const confirmedRemoteVersion = (reported, doc, carried) => {
+	return ((reported == null ? null : usableRemoteVersion(reported, doc, carried)) ?? carried).encode();
+};
 var ClientServerSynchronizer = class {
 	local;
 	server;
@@ -566,15 +645,59 @@ var ClientServerSynchronizer = class {
 			await Task.all([localConnected, serverConnected]).timeout(30 * 1e3).abortOn(signal);
 		});
 	}
+	/**
+	* Send everything the remote is missing as ONE update derived from the doc,
+	* rather than forwarding each queued update as its own request.
+	*
+	* A single `export({ mode: 'update', from })` carries every commit the remote
+	* lacks as one causally complete blob, so no second request exists to arrive out
+	* of order and there is no gap for a validating peer to reject —
+	* `mengine-server` answers 409 `missing_dependency` for such a gap and discards
+	* the bytes, since its reads (snapshot replay, log folding, version-vector
+	* diffs) all assume every stored update is applicable. `syncWithRemote` already
+	* worked this way for catch-up; the per-commit fan-out this replaces was N
+	* requests in which a later update could overtake the one it depended on.
+	*
+	* The batch's bytes are still imported first, because local storage is a peer
+	* in its own right: another writer (a second tab, a foreign process) can append
+	* an update this doc has never seen, and relaying local writes upstream is part
+	* of the contract. Import is idempotent, so ops the doc already holds cost
+	* nothing, and folding them in is what lets the export cover them.
+	*
+	* The watermark advances to what the remote reports it now holds, and only
+	* failing that to the version this blob actually carried — read BEFORE the
+	* export, so a commit landing during the await is not marked sent. A failure
+	* leaves it untouched, so the next push re-derives the same ops. Rethrowing is
+	* what makes that next push happen: `consumeJobs` marks the doc `retrying` with
+	* the message and the cycle retries. Swallowing it made a refusal or a dead
+	* network look exactly like a successful sync — the doc went `synced: true`
+	* having pushed nothing.
+	*
+	* A readonly server is the one case that returns without pushing rather than
+	* failing. It is not a transient error, so routing it through the retry path
+	* wedges the doc: every cycle would refuse, mark `retrying`, wait 5s and refuse
+	* again, forever, never reaching `synced`. Nothing is queued by that loop either —
+	* `consumeJobs` removes each job before awaiting it, so the retry only works
+	* because the next push re-derives the ops from the doc. Skipping keeps the
+	* watermark untouched, so if the peer ever becomes writable the very next export
+	* still carries everything it never received.
+	*/
 	async pushUpdatesToRemote(docState, updates, signal) {
 		return runWithCheckpoint(signal, async () => {
-			const failure = (await Promise.allSettled(updates.filter(hasUpdateData).map((update) => {
-				return this.server.pushDocUpdate({
-					docId: docState.docId,
-					data: update
-				}, this.id);
-			}))).find((result) => result.status === "rejected");
-			if (failure != null) throw failure.reason;
+			for (const update of updates.filter(hasUpdateData)) docState.doc.import(update);
+			if (this.server.isReadonly) return;
+			const from = docState.remoteVersion;
+			if (from == null) return;
+			const carried = docState.doc.oplogVersion();
+			const missing = docState.doc.export({
+				mode: "update",
+				from: VersionVector.decode(from)
+			});
+			if (!hasUpdateData(missing)) return;
+			docState.remoteVersion = confirmedRemoteVersion((await this.server.pushDocUpdate({
+				docId: docState.docId,
+				data: missing
+			}, this.id))?.version, docState.doc, carried);
 		});
 	}
 	async pullUpdatesFromRemote(docState, signal) {
@@ -582,7 +705,8 @@ var ClientServerSynchronizer = class {
 			const doc = docState.doc;
 			const serverDoc = await this.server.getDocDiff(docState.docId, doc.version().encode());
 			if (!serverDoc) return;
-			const { missing } = serverDoc;
+			const { missing, version } = serverDoc;
+			docState.remoteVersion = usableRemoteVersion(version, doc, doc.oplogVersion())?.encode() ?? docState.remoteVersion;
 			if (hasUpdateData(missing)) {
 				doc.import(missing);
 				await this.local.pushDocUpdate({
@@ -605,6 +729,7 @@ var ClientServerSynchronizer = class {
 			const localDoc = docState.doc;
 			const response = await this.server.getDocDiff(docState.docId, localDoc.version().encode());
 			let serverMissing = null;
+			let carried = null;
 			if (response) {
 				const { missing, version } = response;
 				if (hasUpdateData(missing)) {
@@ -614,15 +739,21 @@ var ClientServerSynchronizer = class {
 						data: missing
 					}, this.id);
 				}
-				serverMissing = localDoc.export({
+				carried = localDoc.oplogVersion();
+				const vetted = usableRemoteVersion(version, localDoc, carried);
+				docState.remoteVersion = vetted?.encode() ?? docState.remoteVersion;
+				serverMissing = vetted ? localDoc.export({
 					mode: "update",
-					from: VersionVector.decode(version)
-				});
-			} else serverMissing = localDoc.export({ mode: "update" });
-			if (serverMissing && hasUpdateData(serverMissing)) await this.server.pushDocUpdate({
+					from: vetted
+				}) : localDoc.export({ mode: "update" });
+			} else {
+				carried = localDoc.oplogVersion();
+				serverMissing = localDoc.export({ mode: "update" });
+			}
+			if (serverMissing && hasUpdateData(serverMissing) && !this.server.isReadonly) docState.remoteVersion = confirmedRemoteVersion((await this.server.pushDocUpdate({
 				docId: docState.docId,
 				data: serverMissing
-			}, this.id);
+			}, this.id))?.version, localDoc, carried);
 		});
 	}
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mengine/sync",
-  "version": "1.1.0",
+  "version": "1.2.1-alpha.0",
   "license": "UNLICENSED",
   "repository": {
     "type": "git",
@@ -22,8 +22,8 @@
   "dependencies": {
     "lodash-es": "^4.18.1",
     "nanoid": "^5.1.11",
-    "@mengine/utils": "1.1.0",
-    "@mengine/storage": "1.1.0"
+    "@mengine/storage": "1.2.1-alpha.0",
+    "@mengine/utils": "1.2.1-alpha.0"
   },
   "devDependencies": {
     "@types/lodash-es": "^4.17.12",