@fluidframework/matrix 2.41.0 → 2.42.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/matrix",
-  "version": "2.41.0",
+  "version": "2.42.0",
   "description": "Distributed matrix",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -81,17 +81,17 @@
     "temp-directory": "nyc/.nyc_output"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": "~2.41.0",
-    "@fluidframework/core-interfaces": "~2.41.0",
-    "@fluidframework/core-utils": "~2.41.0",
-    "@fluidframework/datastore-definitions": "~2.41.0",
-    "@fluidframework/driver-definitions": "~2.41.0",
-    "@fluidframework/driver-utils": "~2.41.0",
-    "@fluidframework/merge-tree": "~2.41.0",
-    "@fluidframework/runtime-definitions": "~2.41.0",
-    "@fluidframework/runtime-utils": "~2.41.0",
-    "@fluidframework/shared-object-base": "~2.41.0",
-    "@fluidframework/telemetry-utils": "~2.41.0",
+    "@fluid-internal/client-utils": "~2.42.0",
+    "@fluidframework/core-interfaces": "~2.42.0",
+    "@fluidframework/core-utils": "~2.42.0",
+    "@fluidframework/datastore-definitions": "~2.42.0",
+    "@fluidframework/driver-definitions": "~2.42.0",
+    "@fluidframework/driver-utils": "~2.42.0",
+    "@fluidframework/merge-tree": "~2.42.0",
+    "@fluidframework/runtime-definitions": "~2.42.0",
+    "@fluidframework/runtime-utils": "~2.42.0",
+    "@fluidframework/shared-object-base": "~2.42.0",
+    "@fluidframework/telemetry-utils": "~2.42.0",
     "@tiny-calc/nano": "0.0.0-alpha.5",
     "double-ended-queue": "^2.1.0-0",
     "tslib": "^1.10.0"
@@ -99,17 +99,17 @@
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.17.1",
     "@biomejs/biome": "~1.9.3",
-    "@fluid-internal/mocha-test-setup": "~2.41.0",
-    "@fluid-private/stochastic-test-utils": "~2.41.0",
-    "@fluid-private/test-dds-utils": "~2.41.0",
+    "@fluid-internal/mocha-test-setup": "~2.42.0",
+    "@fluid-private/stochastic-test-utils": "~2.42.0",
+    "@fluid-private/test-dds-utils": "~2.42.0",
     "@fluid-tools/benchmark": "^0.51.0",
     "@fluid-tools/build-cli": "^0.55.0",
     "@fluidframework/build-common": "^2.0.3",
     "@fluidframework/build-tools": "^0.55.0",
-    "@fluidframework/container-definitions": "~2.41.0",
+    "@fluidframework/container-definitions": "~2.42.0",
     "@fluidframework/eslint-config-fluid": "^5.7.4",
-    "@fluidframework/matrix-previous": "npm:@fluidframework/matrix@2.40.0",
-    "@fluidframework/test-runtime-utils": "~2.41.0",
+    "@fluidframework/matrix-previous": "npm:@fluidframework/matrix@2.41.0",
+    "@fluidframework/test-runtime-utils": "~2.42.0",
     "@microsoft/api-extractor": "7.52.8",
     "@tiny-calc/micro": "0.0.0-alpha.5",
     "@types/double-ended-queue": "^2.1.0",
@@ -172,6 +172,7 @@
     "lint:fix": "fluid-build . --task eslint:fix --task format",
     "pack:tests": "tar -cf ./matrix.test-files.tar ./src/test ./dist/test ./lib/test",
     "test": "npm run test:mocha",
+    "test:benchmark:report": "mocha --config src/test/time/.mocharc.cjs",
     "test:coverage": "c8 npm test",
     "test:memory": "mocha --config src/test/memory/.mocharc.cjs",
     "test:memory-profiling:report": "mocha --config src/test/memory/.mocharc.cjs",

package/src/matrix.ts

@@ -213,6 +213,22 @@ type FirstWriterWinsPolicy =
 			cellLastWriteTracker: SparseArray2D<CellLastWriteTrackerItem>;
 	  };
 
+/**
+ * Tracks pending local changes for a cell.
+ */
+interface PendingCellChanges<T> {
+	/**
+	 * The local changes including the local seq, and the value set at that local seq.
+	 */
+	local: { localSeq: number; value: MatrixItem<T> }[];
+	/**
+	 * The latest consensus value across all clients.
+	 * this will either be a remote value or ack'd local
+	 * value.
+	 */
+	consensus?: MatrixItem<T>;
+}
+
 /**
  * A SharedMatrix holds a rectangular 2D array of values.  Supported operations
  * include setting values and inserting/removing rows and columns.
@@ -252,7 +268,7 @@ export class SharedMatrix<T = any>
 	private readonly cols: PermutationVector; // Map logical col to storage handle (if any)
 
 	private cells = new SparseArray2D<MatrixItem<T>>(); // Stores cell values.
-	private readonly pending = new SparseArray2D<number>(); // Tracks pending writes.
+	private readonly pending = new SparseArray2D<PendingCellChanges<T>>(); // Tracks pending writes.
 
 	private fwwPolicy: FirstWriterWinsPolicy = {
 		state: "off",
@@ -418,21 +434,22 @@ export class SharedMatrix<T = any>
 		value: MatrixItem<T>,
 		rowHandle = this.rows.getAllocatedHandle(row),
 		colHandle = this.cols.getAllocatedHandle(col),
+		rollback?: boolean,
 	): void {
 		this.protectAgainstReentrancy(() => {
-			if (this.undo !== undefined) {
-				let oldValue = this.cells.getCell(rowHandle, colHandle);
-				if (oldValue === null) {
-					oldValue = undefined;
-				}
+			const oldValue = this.cells.getCell(rowHandle, colHandle) ?? undefined;
 
+			if (this.undo !== undefined) {
 				this.undo.cellSet(rowHandle, colHandle, oldValue);
 			}
 
 			this.cells.setCell(rowHandle, colHandle, value);
 
-			if (this.isAttached()) {
-				this.sendSetCellOp(row, col, value, rowHandle, colHandle);
+			if (this.isAttached() && rollback !== true) {
+				const pending = this.sendSetCellOp(row, col, value, rowHandle, colHandle);
+				if (pending.local.length === 1) {
+					pending.consensus ??= oldValue;
+				}
 			}
 
 			// Avoid reentrancy by raising change notifications after the op is queued.
@@ -467,7 +484,7 @@ export class SharedMatrix<T = any>
 		rowHandle: Handle,
 		colHandle: Handle,
 		localSeq = this.nextLocalSeq(),
-	): void {
+	): PendingCellChanges<T> {
 		assert(
 			this.isAttached(),
 			0x1e2 /* "Caller must ensure 'isAttached()' before calling 'sendSetCellOp'." */,
@@ -493,7 +510,12 @@ export class SharedMatrix<T = any>
 		};
 
 		this.submitLocalMessage(op, metadata);
-		this.pending.setCell(rowHandle, colHandle, localSeq);
+		const pendingCell: PendingCellChanges<T> = this.pending.getCell(rowHandle, colHandle) ?? {
+			local: [],
+		};
+		pendingCell.local.push({ localSeq, value });
+		this.pending.setCell(rowHandle, colHandle, pendingCell);
+		return pendingCell;
 	}
 
 	/**
@@ -679,7 +701,16 @@ export class SharedMatrix<T = any>
 			| undefined
 			| number
 			| ReturnType<SparseArray2D<MatrixItem<T> | number>["snapshot"]>
-		)[] = [this.cells.snapshot(), this.pending.snapshot()];
+		)[] = [
+			this.cells.snapshot(),
+			/**
+			 * we used to write this.pending.snapshot(). this should have never been done, as pending is only for local
+			 * changes, and there should never be local changes in the summarizer. This was also never used on load
+			 * as there is no way to understand a previous clients pending changes. so we just set this to a constant
+			 * which matches an empty this.pending.snapshot() for back-compat in terms of the array length
+			 */
+			[undefined],
+		];
 
 		// Only need to store it in the snapshot if we have switched the policy already.
 		if (this.fwwPolicy.state === "on") {
@@ -806,31 +837,43 @@ export class SharedMatrix<T = any>
 			const col = this.rebasePosition(this.cols, colsRef, localSeq);
 			this.rows.removeLocalReferencePosition(rowsRef);
 			this.cols.removeLocalReferencePosition(colsRef);
-			if (row !== undefined && col !== undefined && row >= 0 && col >= 0) {
-				// If the mode is LWW, then send the op.
+
+			const pendingCell = this.pending.getCell(rowHandle, colHandle);
+			assert(pendingCell !== undefined, 0xba4 /* local operation must have a pending array */);
+			const { local } = pendingCell;
+			assert(local !== undefined, 0xba5 /* local operation must have a pending array */);
+			const localSeqIndex = local.findIndex((p) => p.localSeq === localSeq);
+			assert(localSeqIndex >= 0, 0xba6 /* local operation must have a pending entry */);
+			const [change] = local.splice(localSeqIndex, 1);
+			assert(change.localSeq === localSeq, 0xba7 /* must match */);
+
+			if (
+				row !== undefined &&
+				col !== undefined &&
+				row >= 0 &&
+				col >= 0 && // If the mode is LWW, then send the op.
 				// Otherwise if the current mode is FWW and if we generated this op, after seeing the
 				// last set op, or it is the first set op for the cell, then regenerate the op,
 				// otherwise raise conflict. We want to check the current mode here and not that
 				// whether op was made in FWW or not.
-				if (
-					this.fwwPolicy.state !== "on" ||
+				(this.fwwPolicy.state !== "on" ||
 					referenceSeqNumber >=
-						(this.fwwPolicy.cellLastWriteTracker.getCell(rowHandle, colHandle)?.seqNum ?? 0)
-				) {
-					this.sendSetCellOp(row, col, setOp.value, rowHandle, colHandle, localSeq);
-				} else if (this.pending.getCell(rowHandle, colHandle) !== undefined) {
-					// Clear the pending changes if any as we are not sending the op.
-					this.pending.setCell(rowHandle, colHandle, undefined);
-				}
+						(this.fwwPolicy.cellLastWriteTracker.getCell(rowHandle, colHandle)?.seqNum ?? 0))
+			) {
+				this.sendSetCellOp(row, col, setOp.value, rowHandle, colHandle, localSeq);
 			}
 		} else {
 			switch (content.target) {
 				case SnapshotPath.cols: {
-					this.submitColMessage(this.cols.regeneratePendingOp(content, localOpMetadata));
+					this.submitColMessage(
+						this.cols.regeneratePendingOp(content, localOpMetadata, false),
+					);
 					break;
 				}
 				case SnapshotPath.rows: {
-					this.submitRowMessage(this.rows.regeneratePendingOp(content, localOpMetadata));
+					this.submitRowMessage(
+						this.rows.regeneratePendingOp(content, localOpMetadata, false),
+					);
 					break;
 				}
 				default: {
@@ -840,6 +883,47 @@ export class SharedMatrix<T = any>
 		}
 	}
 
+	protected rollback(content: unknown, localOpMetadata: unknown): void {
+		const contents = content as MatrixSetOrVectorOp<T>;
+		const target = contents.target;
+
+		switch (target) {
+			case SnapshotPath.cols: {
+				this.cols.rollback(content, localOpMetadata);
+				break;
+			}
+			case SnapshotPath.rows: {
+				this.rows.rollback(content, localOpMetadata);
+				break;
+			}
+			case undefined: {
+				assert(contents.type === MatrixOp.set, 0xba8 /* only sets supported */);
+				const setMetadata = localOpMetadata as ISetOpMetadata;
+
+				const pendingCell = this.pending.getCell(setMetadata.rowHandle, setMetadata.colHandle);
+				assert(pendingCell !== undefined, 0xba9 /* must have pending */);
+
+				const change = pendingCell.local.pop();
+				assert(change?.localSeq === setMetadata.localSeq, 0xbaa /* must have change */);
+
+				const previous =
+					pendingCell.local.length > 0
+						? pendingCell.local[pendingCell.local.length - 1].value
+						: pendingCell.consensus;
+
+				this.setCellCore(
+					contents.row,
+					contents.col,
+					previous,
+					setMetadata.rowHandle,
+					setMetadata.colHandle,
+					true,
+				);
+			}
+			default:
+		}
+	}
+
 	protected onDisconnect(): void {}
 
 	/**
@@ -968,73 +1052,86 @@ export class SharedMatrix<T = any>
 					// We are receiving the ACK for a local pending set operation.
 					const { rowHandle, colHandle, localSeq, rowsRef, colsRef } =
 						localOpMetadata as ISetOpMetadata;
-					const isLatestPendingOp = this.isLatestPendingWrite(rowHandle, colHandle, localSeq);
 					this.rows.removeLocalReferencePosition(rowsRef);
 					this.cols.removeLocalReferencePosition(colsRef);
+
+					const pendingCell = this.pending.getCell(rowHandle, colHandle);
+					const ackedChange = pendingCell?.local.shift();
+					assert(ackedChange?.localSeq === localSeq, 0xbab /* must match */);
+					if (pendingCell?.local.length === 0) {
+						this.pending.setCell(rowHandle, colHandle, undefined);
+					}
+
 					// If policy is switched and cell should be modified too based on policy, then update the tracker.
 					// If policy is not switched, then also update the tracker in case it is the latest.
 					if (
 						this.fwwPolicy.state === "on" &&
 						this.shouldSetCellBasedOnFWW(rowHandle, colHandle, msg)
 					) {
+						// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+						pendingCell!.consensus = ackedChange.value;
 						this.fwwPolicy.cellLastWriteTracker.setCell(rowHandle, colHandle, {
 							seqNum: msg.sequenceNumber,
 							clientId: msg.clientId,
 						});
 					}
-
-					if (isLatestPendingOp) {
-						this.pending.setCell(rowHandle, colHandle, undefined);
-					}
 				} else {
 					const adjustedRow = this.rows.adjustPosition(row, msg);
-					if (adjustedRow !== undefined) {
-						const adjustedCol = this.cols.adjustPosition(col, msg);
-
-						if (adjustedCol !== undefined) {
-							const rowHandle = this.rows.getAllocatedHandle(adjustedRow);
-							const colHandle = this.cols.getAllocatedHandle(adjustedCol);
-
-							assert(
-								isHandleValid(rowHandle) && isHandleValid(colHandle),
-								0x022 /* "SharedMatrix row and/or col handles are invalid!" */,
-							);
-							if (this.fwwPolicy.state === "on") {
-								// If someone tried to Overwrite the cell value or first write on this cell or
-								// same client tried to modify the cell or if the previous mode was LWW, then we need to still
-								// overwrite the cell and raise conflict if we have pending changes as our change is going to be lost.
-								if (this.shouldSetCellBasedOnFWW(rowHandle, colHandle, msg)) {
-									const previousValue = this.cells.getCell(rowHandle, colHandle);
-									this.cells.setCell(rowHandle, colHandle, value);
-									this.fwwPolicy.cellLastWriteTracker.setCell(rowHandle, colHandle, {
-										seqNum: msg.sequenceNumber,
-										clientId: msg.clientId,
-									});
-									for (const consumer of this.consumers.values()) {
-										consumer.cellsChanged(adjustedRow, adjustedCol, 1, 1, this);
-									}
-									// Check is there are any pending changes, which will be rejected. If so raise conflict.
-									if (this.pending.getCell(rowHandle, colHandle) !== undefined) {
-										// Don't reset the pending value yet, as there maybe more fww op from same client, so we want
-										// to raise conflict event for that op also.
-										this.emit(
-											"conflict",
-											row,
-											col,
-											value, // Current value
-											previousValue, // Ignored local value
-											this,
-										);
-									}
+					const adjustedCol = this.cols.adjustPosition(col, msg);
+
+					const rowHandle = adjustedRow.handle;
+					const colHandle = adjustedCol.handle;
+
+					assert(
+						isHandleValid(rowHandle) && isHandleValid(colHandle),
+						0x022 /* "SharedMatrix row and/or col handles are invalid!" */,
+					);
+					const pendingCell = this.pending.getCell(rowHandle, colHandle);
+					if (this.fwwPolicy.state === "on") {
+						// If someone tried to Overwrite the cell value or first write on this cell or
+						// same client tried to modify the cell or if the previous mode was LWW, then we need to still
+						// overwrite the cell and raise conflict if we have pending changes as our change is going to be lost.
+						if (this.shouldSetCellBasedOnFWW(rowHandle, colHandle, msg)) {
+							const previousValue = this.cells.getCell(rowHandle, colHandle);
+							this.cells.setCell(rowHandle, colHandle, value);
+							this.fwwPolicy.cellLastWriteTracker.setCell(rowHandle, colHandle, {
+								seqNum: msg.sequenceNumber,
+								clientId: msg.clientId,
+							});
+							if (pendingCell !== undefined) {
+								pendingCell.consensus = value;
+							}
+							if (adjustedRow.pos !== undefined && adjustedCol.pos !== undefined) {
+								for (const consumer of this.consumers.values()) {
+									consumer.cellsChanged(adjustedRow.pos, adjustedCol.pos, 1, 1, this);
+								}
+								// Check is there are any pending changes, which will be rejected. If so raise conflict.
+								if (pendingCell !== undefined && pendingCell.local.length > 0) {
+									// Don't reset the pending value yet, as there maybe more fww op from same client, so we want
+									// to raise conflict event for that op also.
+									this.emit(
+										"conflict",
+										row,
+										col,
+										value, // Current value
+										previousValue, // Ignored local value
+										this,
+									);
 								}
-							} else if (this.pending.getCell(rowHandle, colHandle) === undefined) {
-								// If there is a pending (unACKed) local write to the same cell, skip the current op
-								// since it "happened before" the pending write.
-								this.cells.setCell(rowHandle, colHandle, value);
+							}
+						}
+					} else {
+						if (pendingCell === undefined || pendingCell.local.length === 0) {
+							// If there is a pending (unACKed) local write to the same cell, skip the current op
+							// since it "happened before" the pending write.
+							this.cells.setCell(rowHandle, colHandle, value);
+							if (adjustedRow.pos !== undefined && adjustedCol.pos !== undefined) {
 								for (const consumer of this.consumers.values()) {
-									consumer.cellsChanged(adjustedRow, adjustedCol, 1, 1, this);
+									consumer.cellsChanged(adjustedRow.pos, adjustedCol.pos, 1, 1, this);
 								}
 							}
+						} else {
+							pendingCell.consensus = value;
 						}
 					}
 				}
@@ -1106,35 +1203,6 @@ export class SharedMatrix<T = any>
 		}
 	}
 
-	/**
-	 * Returns true if the latest pending write to the cell indicated by the given row/col handles
-	 * matches the given 'localSeq'.
-	 *
-	 * A return value of `true` indicates that there are no later local operations queued that will
-	 * clobber the write op at the given 'localSeq'.  This includes later ops that overwrite the cell
-	 * with a different value as well as row/col removals that might recycled the given row/col handles.
-	 */
-	private isLatestPendingWrite(
-		rowHandle: Handle,
-		colHandle: Handle,
-		localSeq: number,
-	): boolean {
-		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		const pendingLocalSeq = this.pending.getCell(rowHandle, colHandle)!;
-
-		// Note while we're awaiting the ACK for a local set, it's possible for the row/col to be
-		// locally removed and the row/col handles recycled.  If this happens, the pendingLocalSeq will
-		// be 'undefined' or > 'localSeq'.
-		assert(
-			!(pendingLocalSeq < localSeq),
-			0x023 /* "The 'localSeq' of pending write (if any) must be <= the localSeq of the currently processed op." */,
-		);
-
-		// If this is the most recent write to the cell by the local client, the stored localSeq
-		// will be an exact match for the given 'localSeq'.
-		return pendingLocalSeq === localSeq;
-	}
-
 	public toString(): string {
 		let s = `client:${
 			this.runtime.clientId

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/matrix";
-export const pkgVersion = "2.41.0";
+export const pkgVersion = "2.42.0";

package/src/permutationvector.ts

@@ -18,7 +18,6 @@ import {
 	IMergeTreeDeltaOpArgs,
 	IMergeTreeMaintenanceCallbackArgs,
 	ISegment,
-	ISegmentInternal,
 	MergeTreeDeltaType,
 	MergeTreeMaintenanceType,
 	segmentIsRemoved,
@@ -199,23 +198,50 @@ export class PermutationVector extends Client {
 	}
 
 	public adjustPosition(
-		pos: number,
-		op: Pick<ISequencedDocumentMessage, "referenceSequenceNumber" | "clientId">,
-	): number | undefined {
-		const { segment, offset } = this.getContainingSegment<ISegmentInternal>(pos, {
+		posToAdjust: number,
+		op: ISequencedDocumentMessage,
+	): { pos: number | undefined; handle: Handle } {
+		const { segment, offset } = this.getContainingSegment<PermutationSegment>(posToAdjust, {
 			referenceSequenceNumber: op.referenceSequenceNumber,
 			clientId: op.clientId,
 		});
 
-		// Note that until the MergeTree GCs, the segment is still reachable via `getContainingSegment()` with
-		// a `refSeq` in the past.  Prevent remote ops from accidentally allocating or using recycled handles
-		// by checking for the presence of 'removedSeq'.
-		if (segment === undefined || segmentIsRemoved(segment)) {
-			return undefined;
-		}
+		assert(
+			segment !== undefined && offset !== undefined,
+			0xbac /* segment must be available for operations in the collab window */,
+		);
 
-		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		return this.getPosition(segment) + offset!;
+		if (segmentIsRemoved(segment)) {
+			// this case is tricky. the segment which the row or column data is remove
+			// but an op before that remove references a cell. we still want to apply
+			// the op, as the row/col could become active again in the case where
+			// the remove was local and it get's rolled back. so we allocate a handle
+			// for the row/col if not allocated, but don't put it in the cache
+			// as the cache can only contain live positions.
+			let handle = segment.start;
+			if (!isHandleValid(handle)) {
+				this.walkSegments(
+					(s) => {
+						const asPerm = s as PermutationSegment;
+						asPerm.start = handle = this.handleTable.allocate();
+						return true;
+					},
+					posToAdjust,
+					posToAdjust + 1,
+					/* accum: */ undefined,
+					/* splitRange: */ true,
+					op,
+				);
+			}
+
+			return { handle, pos: undefined };
+		} else {
+			const pos = this.getPosition(segment) + offset;
+			return {
+				pos,
+				handle: this.getAllocatedHandle(pos),
+			};
+		}
 	}
 
 	public handleToPosition(handle: Handle, localSeq = this.getCollabWindow().localSeq): number {
@@ -342,10 +368,12 @@ export class PermutationVector extends Client {
 			case MergeTreeDeltaType.INSERT: {
 				// Pass 1: Perform any internal maintenance first to avoid reentrancy.
 				for (const { segment, position } of ranges) {
-					// HACK: We need to include the allocated handle in the segment's JSON representation
-					//       for snapshots, but need to ignore the remote client's handle allocations when
-					//       processing remote ops.
-					segment.reset();
+					if (opArgs.rollback !== true) {
+						// HACK: We need to include the allocated handle in the segment's JSON representation
+						//       for snapshots, but need to ignore the remote client's handle allocations when
+						//       processing remote ops.
+						segment.reset();
+					}
 
 					this.handleCache.itemsChanged(
 						position,
@@ -364,7 +392,6 @@ export class PermutationVector extends Client {
 				}
 				break;
 			}
-
 			case MergeTreeDeltaType.REMOVE: {
 				// Pass 1: Perform any internal maintenance first to avoid reentrancy.
 				for (const { segment, position } of ranges) {
@@ -385,7 +412,6 @@ export class PermutationVector extends Client {
 				}
 				break;
 			}
-
 			default: {
 				throw new Error("Unhandled MergeTreeDeltaType");
 			}