@fluidframework/merge-tree 2.41.0 → 2.42.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/merge-tree",
-  "version": "2.41.0",
+  "version": "2.42.0",
   "description": "Merge tree",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -81,30 +81,30 @@
     "temp-directory": "nyc/.nyc_output"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": "~2.41.0",
-    "@fluidframework/container-definitions": "~2.41.0",
-    "@fluidframework/core-interfaces": "~2.41.0",
-    "@fluidframework/core-utils": "~2.41.0",
-    "@fluidframework/datastore-definitions": "~2.41.0",
-    "@fluidframework/driver-definitions": "~2.41.0",
-    "@fluidframework/runtime-definitions": "~2.41.0",
-    "@fluidframework/runtime-utils": "~2.41.0",
-    "@fluidframework/shared-object-base": "~2.41.0",
-    "@fluidframework/telemetry-utils": "~2.41.0"
+    "@fluid-internal/client-utils": "~2.42.0",
+    "@fluidframework/container-definitions": "~2.42.0",
+    "@fluidframework/core-interfaces": "~2.42.0",
+    "@fluidframework/core-utils": "~2.42.0",
+    "@fluidframework/datastore-definitions": "~2.42.0",
+    "@fluidframework/driver-definitions": "~2.42.0",
+    "@fluidframework/runtime-definitions": "~2.42.0",
+    "@fluidframework/runtime-utils": "~2.42.0",
+    "@fluidframework/shared-object-base": "~2.42.0",
+    "@fluidframework/telemetry-utils": "~2.42.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.17.1",
     "@biomejs/biome": "~1.9.3",
-    "@fluid-internal/mocha-test-setup": "~2.41.0",
-    "@fluid-private/stochastic-test-utils": "~2.41.0",
-    "@fluid-private/test-pairwise-generator": "~2.41.0",
+    "@fluid-internal/mocha-test-setup": "~2.42.0",
+    "@fluid-private/stochastic-test-utils": "~2.42.0",
+    "@fluid-private/test-pairwise-generator": "~2.42.0",
     "@fluid-tools/benchmark": "^0.51.0",
     "@fluid-tools/build-cli": "^0.55.0",
     "@fluidframework/build-common": "^2.0.3",
     "@fluidframework/build-tools": "^0.55.0",
     "@fluidframework/eslint-config-fluid": "^5.7.4",
-    "@fluidframework/merge-tree-previous": "npm:@fluidframework/merge-tree@2.40.0",
-    "@fluidframework/test-runtime-utils": "~2.41.0",
+    "@fluidframework/merge-tree-previous": "npm:@fluidframework/merge-tree@2.41.0",
+    "@fluidframework/test-runtime-utils": "~2.42.0",
     "@microsoft/api-extractor": "7.52.8",
     "@types/diff": "^3.5.1",
     "@types/mocha": "^10.0.10",

package/src/client.ts

@@ -27,7 +27,7 @@ import {
 
 import { MergeTreeTextHelper, type IMergeTreeTextHelper } from "./MergeTreeTextHelper.js";
 import { DoublyLinkedList, RedBlackTree } from "./collections/index.js";
-import { NonCollabClient, UniversalSequenceNumber } from "./constants.js";
+import { NonCollabClient, SquashClient, UniversalSequenceNumber } from "./constants.js";
 import { LocalReferencePosition, SlidingPreference } from "./localReference.js";
 import {
 	MergeTree,
@@ -84,6 +84,7 @@ import {
 } from "./ops.js";
 import {
 	LocalReconnectingPerspective,
+	LocalSquashPerspective,
 	PriorPerspective,
 	type Perspective,
 } from "./perspective.js";
@@ -95,6 +96,7 @@ import {
 	overwriteInfo,
 	toRemovalInfo,
 	type IHasInsertionInfo,
+	type IHasRemovalInfo,
 } from "./segmentInfos.js";
 import { Side, type InteriorSequencePlace } from "./sequencePlace.js";
 import { SnapshotLoader } from "./snapshotLoader.js";
@@ -363,6 +365,7 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 		end: number | undefined,
 		accum: TClientData,
 		splitRange?: boolean,
+		perspective?: Pick<ISequencedDocumentMessage, "clientId" | "referenceSequenceNumber">,
 	): void;
 	public walkSegments(
 		handler: ISegmentAction<undefined>,
@@ -370,6 +373,7 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 		end?: number,
 		accum?: undefined,
 		splitRange?: boolean,
+		perspective?: Pick<ISequencedDocumentMessage, "clientId" | "referenceSequenceNumber">,
 	): void;
 	public walkSegments<TClientData>(
 		handler: ISegmentAction<TClientData>,
@@ -377,10 +381,13 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 		end: number | undefined,
 		accum: TClientData,
 		splitRange: boolean = false,
+		perspective?: Pick<ISequencedDocumentMessage, "clientId" | "referenceSequenceNumber">,
 	): void {
 		this._mergeTree.mapRange(
 			handler,
-			this._mergeTree.localPerspective,
+			perspective === undefined
+				? this.getCollabWindow().localPerspective
+				: this.getOperationPerspective(perspective),
 			accum,
 			start,
 			end,
@@ -556,7 +563,9 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 	}
 
 	private getOperationPerspective(
-		sequencedMessage: ISequencedDocumentMessage | undefined,
+		sequencedMessage:
+			| Pick<ISequencedDocumentMessage, "clientId" | "referenceSequenceNumber">
+			| undefined,
 	): Perspective {
 		if (!sequencedMessage) {
 			return this._mergeTree.localPerspective;
@@ -898,16 +907,17 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 		return { segment: newSegment, offset: newOffset, side: newSide };
 	}
 
-	private computeNewObliterateEndpoints(obliterateInfo: ObliterateInfo): {
+	private computeNewObliterateEndpoints(
+		obliterateInfo: ObliterateInfo,
+		squash: boolean,
+	): {
 		start: RebasedObliterateEndpoint;
 		end: RebasedObliterateEndpoint;
 	} {
 		const { currentSeq, clientId } = this.getCollabWindow();
-		const reconnectingPerspective = new LocalReconnectingPerspective(
-			currentSeq,
-			clientId,
-			obliterateInfo.stamp.localSeq! - 1,
-		);
+		const reconnectingPerspective = new (
+			squash ? LocalSquashPerspective : LocalReconnectingPerspective
+		)(currentSeq, clientId, obliterateInfo.stamp.localSeq! - 1);
 
 		const newStart = this.rebaseSidedLocalReference(
 			obliterateInfo.start,
@@ -932,6 +942,7 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 		resetOp: IMergeTreeDeltaOp,
 
 		segmentGroup: SegmentGroup,
+		squash: boolean,
 	): IMergeTreeDeltaOp[] {
 		assert(!!segmentGroup, 0x033 /* "Segment group undefined" */);
 		const NACKedSegmentGroup = this.pendingRebase?.shift()?.data;
@@ -991,13 +1002,20 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 					);
 					const lastRemove = segment.removes[segment.removes.length - 1];
 					assert(
-						lastRemove.type === "sliceRemove" && lastRemove.localSeq === segmentGroup.localSeq,
-						0xb67 /* Last remove should be the obliterate that is being resubmitted. */,
+						(lastRemove.type === "sliceRemove" &&
+							lastRemove.localSeq === segmentGroup.localSeq) ||
+							opstampUtils.isSquashedOp(lastRemove),
+						0xbad /* Last remove should be the obliterate that is being resubmitted. */,
 					);
-					// The original obliterate affected this segment, but it has since been removed and overlapping removes
-					// are only possible when they are concurrent. We adjust the metadata on that segment now to reflect
-					// the fact that the obliterate no longer affects it.
-					segment.removes.pop();
+
+					// The original obliterate affected this segment, but it has since been removed.
+					// This can happen when a concurrent obliterate also removed the segment, as well as when the segment was
+					// only locally inserted and its insertion was squashed upon reconnecting.
+					// In the concurrent removal case (where we didn't avoid sending the segment's insertion in the first place due
+					// to squashing), we adjust the metadata on that segment to reflect the fact that this obliterate no longer removes it.
+					if (!opstampUtils.isSquashedOp(lastRemove)) {
+						segment.removes.pop();
+					}
 				}
 
 				this._mergeTree.rebaseObliterateTo(obliterateInfo, undefined);
@@ -1047,10 +1065,11 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 					0x035 /* "Segment group not in segment pending queue" */,
 				);
 				if (
-					(segment.ordinal > newStartSegment.ordinal &&
+					!isRemovedAndAcked(segment) &&
+					((segment.ordinal > newStartSegment.ordinal &&
 						segment.ordinal < newEndSegment.ordinal) ||
-					(segment === newStartSegment && newStartSide === Side.Before) ||
-					(segment === newEndSegment && newEndSide === Side.After)
+						(segment === newStartSegment && newStartSide === Side.Before) ||
+						(segment === newEndSegment && newEndSide === Side.After))
 				) {
 					segment.segmentGroups.enqueue(newObliterate.segmentGroup);
 				} else {
@@ -1060,12 +1079,17 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 					);
 					const lastRemove = segment.removes[segment.removes.length - 1];
 					assert(
-						lastRemove.type === "sliceRemove" && lastRemove.localSeq === segmentGroup.localSeq,
-						0xb6a /* Last remove should be the obliterate that is being resubmitted. */,
+						(lastRemove.type === "sliceRemove" &&
+							lastRemove.localSeq === segmentGroup.localSeq) ||
+							opstampUtils.isSquashedOp(lastRemove),
+						0xbae /* Last remove should be the obliterate that is being resubmitted. */,
 					);
-					// The original obliterate affected this segment, but it has since been removed and it's impossible to apply the
-					// local obliterate so that is so. We adjust the metadata on that segment now.
-					segment.removes.pop();
+
+					if (!opstampUtils.isSquashedOp(lastRemove)) {
+						// The original obliterate affected this segment, but it has since been removed and it's impossible to apply the
+						// local obliterate so that is so. We adjust the metadata on that segment now.
+						segment.removes.pop();
+					}
 				}
 			}
 
@@ -1154,13 +1178,25 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 				}
 
 				case MergeTreeDeltaType.INSERT: {
+					if (isInserted(segment) && opstampUtils.isSquashedOp(segment.insert)) {
+						break;
+					}
 					assert(
 						isInserted(segment) && opstampUtils.isLocal(segment.insert),
 						0x037 /* "Segment already has assigned sequence number" */,
 					);
 					const removeInfo = toRemovalInfo(segment);
 
-					if (removeInfo !== undefined && opstampUtils.isAcked(removeInfo.removes[0])) {
+					const unusedStamp: OperationStamp = { seq: 0, clientId: 0 };
+					if (removeInfo !== undefined && squash) {
+						assert(
+							removeInfo.removes.length === 1 ||
+								opstampUtils.isAcked(removeInfo.removes[removeInfo.removes.length - 2]),
+							0xbaf /* Expected only one local remove */,
+						);
+						this.squashInsertion(segment);
+						break;
+					} else if (removeInfo !== undefined && opstampUtils.isAcked(removeInfo.removes[0])) {
 						assert(
 							removeInfo.removes[0].type === "sliceRemove",
 							0xb5c /* Remove on insertion must be caused by obliterate. */,
@@ -1181,6 +1217,7 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 								clientId: NonCollabClient,
 							},
 						});
+						this._mergeTree.blockUpdatePathLengths(segment.parent, unusedStamp, true);
 						break;
 					}
 
@@ -1360,13 +1397,40 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 		{ start: RebasedObliterateEndpoint; end: RebasedObliterateEndpoint }
 	> = new Map();
 
+	private squashInsertion(segment: ISegmentLeaf): void {
+		overwriteInfo<IHasInsertionInfo & IHasRemovalInfo>(segment, {
+			insert: {
+				type: "insert",
+				seq: UniversalSequenceNumber,
+				localSeq: undefined,
+				clientId: SquashClient,
+			},
+			removes: [
+				{
+					type: "setRemove",
+					seq: UniversalSequenceNumber,
+					localSeq: undefined,
+					clientId: SquashClient,
+				},
+			],
+		});
+
+		this._mergeTree.blockUpdatePathLengths(segment.parent, { seq: 0, clientId: 0 }, true);
+	}
+
 	/**
 	 * Given a pending operation and segment group, regenerate the op, so it
 	 * can be resubmitted
 	 * @param resetOp - The op to reset
 	 * @param segmentGroup - The segment group associated with the op
+	 * @param squash - whether intermediate states should be squashed. See `IDeltaHandler.reSubmit`'s squash parameter
+	 * documentation for more details.
 	 */
-	public regeneratePendingOp(resetOp: IMergeTreeOp, localOpMetadata: unknown): IMergeTreeOp {
+	public regeneratePendingOp(
+		resetOp: IMergeTreeOp,
+		localOpMetadata: unknown,
+		squash: boolean,
+	): IMergeTreeOp {
 		const segmentGroup = localOpMetadata as SegmentGroup | SegmentGroup[];
 		if (this.pendingRebase === undefined || this.pendingRebase.empty) {
 			let firstGroup: SegmentGroup;
@@ -1396,13 +1460,14 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 			collabWindow.currentSeq !== this.lastNormalization.refSeq ||
 			collabWindow.localSeq !== this.lastNormalization.localRefSeq
 		) {
+			const allPendingSegments = [...this._mergeTree.pendingSegments, ...this.pendingRebase];
 			// Compute obliterate endpoint destinations before segments are normalized.
 			// Segment normalization can affect what should be the semantically correct segments for the endpoints to be placed on.
 			this.cachedObliterateRebases.clear();
-			for (const group of [...this._mergeTree.pendingSegments, ...this.pendingRebase]) {
+			for (const group of allPendingSegments) {
 				const { obliterateInfo } = group.data;
 				if (obliterateInfo !== undefined) {
-					const { start, end } = this.computeNewObliterateEndpoints(obliterateInfo);
+					const { start, end } = this.computeNewObliterateEndpoints(obliterateInfo, squash);
 					const { localSeq } = obliterateInfo.stamp;
 					assert(localSeq !== undefined, 0xb6d /* Local seq must be defined */);
 					this.cachedObliterateRebases.set(localSeq, { start, end });
@@ -1426,7 +1491,7 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 				);
 
 				for (let i = 0; i < resetOp.ops.length; i++) {
-					opList.push(...this.resetPendingDeltaToOps(resetOp.ops[i], segmentGroup[i]));
+					opList.push(...this.resetPendingDeltaToOps(resetOp.ops[i], segmentGroup[i], squash));
 				}
 			} else {
 				// A group op containing a single op will pass a direct reference to 'segmentGroup'
@@ -1435,7 +1500,7 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 					resetOp.ops.length === 1,
 					0x03b /* "Number of ops in 'resetOp' must match the number of segment groups provided." */,
 				);
-				opList.push(...this.resetPendingDeltaToOps(resetOp.ops[0], segmentGroup));
+				opList.push(...this.resetPendingDeltaToOps(resetOp.ops[0], segmentGroup, squash));
 			}
 		} else {
 			assert(
@@ -1446,7 +1511,7 @@ export class Client extends TypedEventEmitter<IClientEvents> {
 				!Array.isArray(segmentGroup),
 				0x03d /* "segmentGroup is array rather than singleton!" */,
 			);
-			opList.push(...this.resetPendingDeltaToOps(resetOp, segmentGroup));
+			opList.push(...this.resetPendingDeltaToOps(resetOp, segmentGroup, squash));
 		}
 
 		return opList.length === 1 ? opList[0] : createGroupOp(...opList);

package/src/constants.ts

@@ -34,3 +34,9 @@ export const LocalClientId = -1;
  * @internal
  */
 export const NonCollabClient = -2;
+
+/**
+ * Used as the client id for operations that were squashed upon resubmission and should therefore
+ * never be seen by other clients.
+ */
+export const SquashClient = -3;

package/src/mergeTree.ts

@@ -1270,7 +1270,7 @@ export class MergeTree {
 			segment,
 			(node) => {
 				if (node.isLeaf()) {
-					if (Marker.is(node) && refHasTileLabel(node, markerLabel)) {
+					if (!isRemoved(node) && Marker.is(node) && refHasTileLabel(node, markerLabel)) {
 						foundMarker = node;
 					}
 				} else {

package/src/perspective.ts

@@ -117,6 +117,32 @@ export class LocalReconnectingPerspective extends PerspectiveBase implements Per
 	}
 }
 
+/**
+ * This perspective is used when rebasing obliterate endpoints to find the segment to slide to when squash is enabled.
+ *
+ * TODO:AB#39357: This class would not be necessary if obliterate rebasing occurred as resubmit was called rather than
+ * precomputed before segment normalization. It also adds more dependencies on all ops being resubmitted (the squash
+ * parameter coming from rebasing an obliterate does not necessarily align with an inserted segment), which is not
+ * fully correct.
+ */
+export class LocalSquashPerspective extends LocalReconnectingPerspective {
+	public constructor(
+		readonly refSeq: number,
+		readonly clientId: number,
+		readonly localSeq: number,
+	) {
+		super(refSeq, clientId, localSeq);
+	}
+
+	public override isSegmentPresent(seg: ISegment): boolean {
+		// Avoid sliding to segments whose insertion will be squashed.
+		if (isInserted(seg) && opstampUtils.isLocal(seg.insert) && isRemoved(seg)) {
+			return false;
+		}
+		return super.isSegmentPresent(seg);
+	}
+}
+
 /**
  * A perspective which includes edits which were either:
  * - acked and at or before some reference sequence number

package/src/stamps.ts

@@ -3,7 +3,11 @@
  * Licensed under the MIT License.
  */
 
-import { UnassignedSequenceNumber } from "./constants.js";
+import {
+	SquashClient,
+	UnassignedSequenceNumber,
+	UniversalSequenceNumber,
+} from "./constants.js";
 
 /**
  * A stamp that identifies provenance of an operation performed on the MergeTree.
@@ -122,6 +126,10 @@ export function isLocal(a: OperationStamp): boolean {
 	return a.seq === UnassignedSequenceNumber;
 }
 
+export function isSquashedOp(a: OperationStamp): boolean {
+	return a.clientId === SquashClient && a.seq === UniversalSequenceNumber;
+}
+
 export function isAcked(a: OperationStamp): boolean {
 	return a.seq !== UnassignedSequenceNumber;
 }