@fluidframework/map 2.51.0 → 2.52.0

package/src/mapKernel.ts

@@ -5,22 +5,14 @@
 
 import type { TypedEventEmitter } from "@fluid-internal/client-utils";
 import type { IFluidHandle } from "@fluidframework/core-interfaces";
-import {
-	assert,
-	DoublyLinkedList,
-	type ListNode,
-	unreachableCase,
-} from "@fluidframework/core-utils/internal";
+import { assert, unreachableCase } from "@fluidframework/core-utils/internal";
 import type { IFluidSerializer } from "@fluidframework/shared-object-base/internal";
 import { ValueType } from "@fluidframework/shared-object-base/internal";
 
 import type { ISharedMapEvents } from "./interfaces.js";
 import type {
-	IMapClearLocalOpMetadata,
 	IMapClearOperation,
 	IMapDeleteOperation,
-	IMapKeyAddLocalOpMetadata,
-	IMapKeyEditLocalOpMetadata,
 	IMapSetOperation,
 	// eslint-disable-next-line import/no-deprecated
 	ISerializableValue,
@@ -46,7 +38,7 @@ interface IMapMessageHandler {
 	process(
 		op: IMapOperation,
 		local: boolean,
-		localOpMetadata: ListNode<MapLocalOpMetadata> | undefined,
+		localOpMetadata: PendingLocalOpMetadata | undefined,
 	): void;
 
 	/**
@@ -54,18 +46,13 @@ interface IMapMessageHandler {
 	 * @param op - The map operation to resubmit
 	 * @param localOpMetadata - The metadata that was originally submitted with the message.
 	 */
-	resubmit(op: IMapOperation, localOpMetadata: ListNode<MapLocalOpMetadata>): void;
+	resubmit(op: IMapOperation, localOpMetadata: PendingLocalOpMetadata): void;
 }
 
 /**
- * Map key operations are one of several types.
- */
-export type IMapKeyOperation = IMapSetOperation | IMapDeleteOperation;
-
-/**
- * Description of a map delta operation
+ * Union of all possible map operations.
  */
-export type IMapOperation = IMapKeyOperation | IMapClearOperation;
+export type IMapOperation = IMapSetOperation | IMapDeleteOperation | IMapClearOperation;
 
 /**
  * Defines the in-memory object structure to be used for the conversion to/from serialized.
@@ -83,53 +70,61 @@ export type IMapDataObjectSerializable = Record<string, ISerializableValue>;
  */
 export type IMapDataObjectSerialized = Record<string, ISerializedValue>;
 
-type MapKeyLocalOpMetadata = IMapKeyEditLocalOpMetadata | IMapKeyAddLocalOpMetadata;
-type MapLocalOpMetadata = IMapClearLocalOpMetadata | MapKeyLocalOpMetadata;
-
-/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access */
-
-function isMapKeyLocalOpMetadata(metadata: any): metadata is MapKeyLocalOpMetadata {
-	return (
-		metadata !== undefined &&
-		typeof metadata.pendingMessageId === "number" &&
-		(metadata.type === "add" || metadata.type === "edit")
-	);
+interface PendingKeySet {
+	type: "set";
+	value: ILocalValue;
 }
 
-function isClearLocalOpMetadata(metadata: any): metadata is IMapClearLocalOpMetadata {
-	return (
-		metadata !== undefined &&
-		metadata.type === "clear" &&
-		typeof metadata.pendingMessageId === "number"
-	);
+interface PendingKeyDelete {
+	type: "delete";
+	key: string;
 }
 
-/* eslint-enable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access */
-
-function createClearLocalOpMetadata(
-	op: IMapClearOperation,
-	pendingClearMessageId: number,
-	previousMap?: Map<string, ILocalValue>,
-): IMapClearLocalOpMetadata {
-	const localMetadata: IMapClearLocalOpMetadata = {
-		type: "clear",
-		pendingMessageId: pendingClearMessageId,
-		previousMap,
-	};
-	return localMetadata;
+interface PendingClear {
+	type: "clear";
 }
 
-function createKeyLocalOpMetadata(
-	op: IMapKeyOperation,
-	pendingMessageId: number,
-	previousValue?: ILocalValue,
-): MapKeyLocalOpMetadata {
-	const localMetadata: MapKeyLocalOpMetadata = previousValue
-		? { type: "edit", pendingMessageId, previousValue }
-		: { type: "add", pendingMessageId };
-	return localMetadata;
+interface PendingKeyLifetime {
+	type: "lifetime";
+	key: string;
+	/**
+	 * A non-empty array of pending key sets that occurred during this lifetime.  If the list
+	 * becomes empty (e.g. during processing or rollback), the lifetime no longer exists and
+	 * must be removed from the pending data.
+	 */
+	keySets: PendingKeySet[];
 }
 
+/**
+ * A member of the pendingData array, which tracks outstanding changes and can be used to
+ * compute optimistic values. Local sets are aggregated into lifetimes.
+ */
+type PendingDataEntry = PendingKeyLifetime | PendingKeyDelete | PendingClear;
+/**
+ * An individual outstanding change, which will also be found in the pendingData array
+ * (though the PendingKeySets will be contained within a PendingKeyLifetime there).
+ */
+type PendingLocalOpMetadata = PendingKeySet | PendingKeyDelete | PendingClear;
+
+/**
+ * Rough polyfill for Array.findLastIndex until we target ES2023 or greater.
+ */
+const findLastIndex = <T>(array: T[], callbackFn: (value: T) => boolean): number => {
+	for (let i = array.length - 1; i >= 0; i--) {
+		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+		if (callbackFn(array[i]!)) {
+			return i;
+		}
+	}
+	return -1;
+};
+
+/**
+ * Rough polyfill for Array.findLast until we target ES2023 or greater.
+ */
+const findLast = <T>(array: T[], callbackFn: (value: T) => boolean): T | undefined =>
+	array[findLastIndex(array, callbackFn)];
+
 /**
  * A SharedMap is a map-like distributed data structure.
  */
@@ -138,7 +133,8 @@ export class MapKernel {
 	 * The number of key/value pairs stored in the map.
 	 */
 	public get size(): number {
-		return this.data.size;
+		const iterableItems = [...this.internalIterator()];
+		return iterableItems.length;
 	}
 
 	/**
@@ -147,30 +143,18 @@ export class MapKernel {
 	private readonly messageHandlers: ReadonlyMap<string, IMapMessageHandler> = new Map();
 
 	/**
-	 * The in-memory data the map is storing.
+	 * The data the map is storing, but only including sequenced values (no local pending
+	 * modifications are included).
 	 */
-	private readonly data = new Map<string, ILocalValue>();
-
+	private readonly sequencedData = new Map<string, ILocalValue>();
 	/**
-	 * Keys that have been modified locally but not yet ack'd from the server.
-	 */
-	private readonly pendingKeys = new Map<string, number[]>();
-
-	/**
-	 * This is used to assign a unique id to every outgoing operation and helps in tracking unack'd ops.
-	 */
-	private nextPendingMessageId: number = 0;
-
-	/**
-	 * The pending metadata for any local operations that have not yet been ack'd from the server, in order.
-	 */
-	private readonly pendingMapLocalOpMetadata: DoublyLinkedList<MapLocalOpMetadata> =
-		new DoublyLinkedList<MapLocalOpMetadata>();
-
-	/**
-	 * The pending ids of any clears that have been performed locally but not yet ack'd from the server
+	 * A data structure containing all local pending modifications, which is used in combination
+	 * with the sequencedData to compute optimistic values.
+	 *
+	 * Pending sets are aggregated into "lifetimes", which permit correct relative iteration order
+	 * even across remote operations and rollbacks.
 	 */
-	private readonly pendingClearMessageIds: number[] = [];
+	private readonly pendingData: PendingDataEntry[] = [];
 
 	/**
 	 * Create a new shared map kernel.
@@ -192,27 +176,112 @@ export class MapKernel {
 	}
 
 	/**
-	 * Get an iterator over the keys in this map.
-	 * @returns The iterator
-	 */
-	public keys(): IterableIterator<string> {
-		return this.data.keys();
-	}
+	 * Get an iterator over the optimistically observable ILocalValue entries in the map. For example, excluding
+	 * sequenced entries that have pending deletes/clears.
+	 *
+	 * @remarks
+	 * There is no perfect solution here, particularly when the iterator is retained over time and the map is
+	 * modified or new ack's are received. The pendingData portion of the iteration is the most susceptible to
+	 * this problem. The implementation prioritizes (in roughly this order):
+	 * 1. Correct immediate iteration (i.e. when the map is not modified before iteration completes)
+	 * 2. Consistent iteration order before/after sequencing of pending ops; acks don't change order
+	 * 3. Consistent iteration order between synchronized clients, even if they each modified the map concurrently
+	 * 4. Remaining as close as possible to the native Map iterator behavior, e.g. live-ish view rather than snapshot
+	 *
+	 * For this reason, it's important not to internally snapshot the output of the iterator for any purpose that
+	 * does not immediately (synchronously) consume that output and dispose of it.
+	 */
+	private readonly internalIterator = (): IterableIterator<[string, ILocalValue]> => {
+		// We perform iteration in two steps - first by iterating over members of the sequenced data that are not
+		// optimistically deleted or cleared, and then over the pending data lifetimes that have not subsequently
+		// been deleted or cleared.  In total, this give an ordering of members based on when they were initially
+		// added to the map (even if they were later modified), similar to the native Map.
+		const sequencedDataIterator = this.sequencedData.keys();
+		const pendingDataIterator = this.pendingData.values();
+		const next = (): IteratorResult<[string, ILocalValue]> => {
+			let nextSequencedKey = sequencedDataIterator.next();
+			while (!nextSequencedKey.done) {
+				const key = nextSequencedKey.value;
+				// If we have any pending deletes or clears, then we won't iterate to this key yet (if at all).
+				// Either it is optimistically deleted and will not be part of the iteration, or it was
+				// re-added later and we'll iterate to it when we get to the pending data.
+				if (
+					!this.pendingData.some(
+						(entry) =>
+							entry.type === "clear" || (entry.type === "delete" && entry.key === key),
+					)
+				) {
+					const optimisticValue = this.getOptimisticLocalValue(key);
+					assert(
+						optimisticValue !== undefined,
+						0xbf1 /* Should never iterate to a key with undefined optimisticValue */,
+					);
+					return { value: [key, optimisticValue], done: false };
+				}
+				nextSequencedKey = sequencedDataIterator.next();
+			}
+
+			let nextPending = pendingDataIterator.next();
+			while (!nextPending.done) {
+				const nextPendingEntry = nextPending.value;
+				// A lifetime entry may need to be iterated.
+				if (nextPendingEntry.type === "lifetime") {
+					const nextPendingEntryIndex = this.pendingData.indexOf(nextPendingEntry);
+					const mostRecentDeleteOrClearIndex = findLastIndex(
+						this.pendingData,
+						(entry) =>
+							entry.type === "clear" ||
+							(entry.type === "delete" && entry.key === nextPendingEntry.key),
+					);
+					// Only iterate the pending entry now if it hasn't been deleted or cleared.
+					if (nextPendingEntryIndex > mostRecentDeleteOrClearIndex) {
+						const latestPendingValue =
+							// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+							nextPendingEntry.keySets[nextPendingEntry.keySets.length - 1]!;
+						// Skip iterating if we would have would have already iterated it as part of the sequenced data.
+						// This is not a perfect check in the case the map has changed since the iterator was created
+						// (e.g. if a remote client added the same key in the meantime).
+						if (
+							!this.sequencedData.has(nextPendingEntry.key) ||
+							mostRecentDeleteOrClearIndex !== -1
+						) {
+							return { value: [nextPendingEntry.key, latestPendingValue.value], done: false };
+						}
+					}
+				}
+				nextPending = pendingDataIterator.next();
+			}
+
+			return { value: undefined, done: true };
+		};
+
+		const iterator = {
+			next,
+			[Symbol.iterator](): IterableIterator<[string, ILocalValue]> {
+				return this;
+			},
+		};
+		return iterator;
+	};
 
 	/**
 	 * Get an iterator over the entries in this map.
 	 * @returns The iterator
 	 */
 	public entries(): IterableIterator<[string, unknown]> {
-		const localEntriesIterator = this.data.entries();
+		const internalIterator = this.internalIterator();
+		const next = (): IteratorResult<[string, unknown]> => {
+			const nextResult = internalIterator.next();
+			if (nextResult.done) {
+				return { value: undefined, done: true };
+			}
+			// Unpack the stored value
+			const [key, localValue] = nextResult.value;
+			return { value: [key, localValue.value], done: false };
+		};
+
 		const iterator = {
-			next(): IteratorResult<[string, unknown]> {
-				const nextVal = localEntriesIterator.next();
-				return nextVal.done
-					? { value: undefined, done: true }
-					: // Unpack the stored value
-						{ value: [nextVal.value[0], nextVal.value[1].value], done: false };
-			},
+			next,
 			[Symbol.iterator](): IterableIterator<[string, unknown]> {
 				return this;
 			},
@@ -220,20 +289,45 @@ export class MapKernel {
 		return iterator;
 	}
 
+	/**
+	 * Get an iterator over the keys in this map.
+	 * @returns The iterator
+	 */
+	public keys(): IterableIterator<string> {
+		const internalIterator = this.internalIterator();
+		const next = (): IteratorResult<string> => {
+			const nextResult = internalIterator.next();
+			if (nextResult.done) {
+				return { value: undefined, done: true };
+			}
+			const [key] = nextResult.value;
+			return { value: key, done: false };
+		};
+		const iterator = {
+			next,
+			[Symbol.iterator](): IterableIterator<string> {
+				return this;
+			},
+		};
+		return iterator;
+	}
+
 	/**
 	 * Get an iterator over the values in this map.
 	 * @returns The iterator
 	 */
 	public values(): IterableIterator<unknown> {
-		const localValuesIterator = this.data.values();
+		const internalIterator = this.internalIterator();
+		const next = (): IteratorResult<unknown> => {
+			const nextResult = internalIterator.next();
+			if (nextResult.done) {
+				return { value: undefined, done: true };
+			}
+			const [, localValue] = nextResult.value;
+			return { value: localValue.value, done: false };
+		};
 		const iterator = {
-			next(): IteratorResult<unknown> {
-				const nextVal = localValuesIterator.next();
-				return nextVal.done
-					? { value: undefined, done: true }
-					: // Unpack the stored value
-						{ value: nextVal.value.value, done: false };
-			},
+			next,
 			[Symbol.iterator](): IterableIterator<unknown> {
 				return this;
 			},
@@ -256,17 +350,44 @@ export class MapKernel {
 	public forEach(
 		callbackFn: (value: unknown, key: string, map: Map<string, unknown>) => void,
 	): void {
+		// It would be better to iterate over the data without a temp map.  However, we don't have a valid
+		// map to pass for the third argument here (really, it should probably should be a reference to the
+		// SharedMap and not the MapKernel).
+		const tempMap = new Map(this.internalIterator());
 		// eslint-disable-next-line unicorn/no-array-for-each
-		this.data.forEach((localValue, key, m) => {
+		tempMap.forEach((localValue, key, m) => {
 			callbackFn(localValue.value, key, m);
 		});
 	}
 
+	/**
+	 * Compute the optimistic local value for a given key. This combines the sequenced data with
+	 * any pending changes that have not yet been sequenced.
+	 */
+	private readonly getOptimisticLocalValue = (key: string): ILocalValue | undefined => {
+		const latestPendingEntry = findLast(
+			this.pendingData,
+			(entry) => entry.type === "clear" || entry.key === key,
+		);
+
+		if (latestPendingEntry === undefined) {
+			return this.sequencedData.get(key);
+		} else if (latestPendingEntry.type === "lifetime") {
+			const latestPendingSet =
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				latestPendingEntry.keySets[latestPendingEntry.keySets.length - 1]!;
+			return latestPendingSet.value;
+		} else {
+			// Delete or clear
+			return undefined;
+		}
+	};
+
 	/**
 	 * {@inheritDoc ISharedMap.get}
 	 */
 	public get<T = unknown>(key: string): T | undefined {
-		const localValue = this.data.get(key);
+		const localValue = this.getOptimisticLocalValue(key);
 		return localValue === undefined ? undefined : (localValue.value as T);
 	}
 
@@ -276,7 +397,7 @@ export class MapKernel {
 	 * @returns True if the key exists, false otherwise
 	 */
 	public has(key: string): boolean {
-		return this.data.has(key);
+		return this.getOptimisticLocalValue(key) !== undefined;
 	}
 
 	/**
@@ -288,20 +409,55 @@ export class MapKernel {
 			throw new Error("Undefined and null keys are not supported");
 		}
 
-		// Set the value locally.
-		const previousValue = this.setCore(key, { value }, true);
+		const localValue: ILocalValue = { value };
+		const previousOptimisticLocalValue = this.getOptimisticLocalValue(key);
 
 		// If we are not attached, don't submit the op.
 		if (!this.isAttached()) {
+			this.sequencedData.set(key, localValue);
+			this.eventEmitter.emit(
+				"valueChanged",
+				{ key, previousValue: previousOptimisticLocalValue?.value },
+				true,
+				this.eventEmitter,
+			);
 			return;
 		}
 
+		// A new pending key lifetime is created if:
+		// 1. There isn't any pending entry for the key yet
+		// 2. The most recent pending entry for the key was a deletion (as this terminates the prior lifetime)
+		// 3. A clear was sent after the last pending entry for the key (which also terminates the prior lifetime)
+		let latestPendingEntry = findLast(
+			this.pendingData,
+			(entry) => entry.type === "clear" || entry.key === key,
+		);
+		if (
+			latestPendingEntry === undefined ||
+			latestPendingEntry.type === "delete" ||
+			latestPendingEntry.type === "clear"
+		) {
+			latestPendingEntry = { type: "lifetime", key, keySets: [] };
+			this.pendingData.push(latestPendingEntry);
+		}
+		const pendingKeySet: PendingKeySet = {
+			type: "set",
+			value: localValue,
+		};
+		latestPendingEntry.keySets.push(pendingKeySet);
+
 		const op: IMapSetOperation = {
 			key,
 			type: "set",
-			value: { type: ValueType[ValueType.Plain], value },
+			value: { type: ValueType[ValueType.Plain], value: localValue.value },
 		};
-		this.submitMapKeyMessage(op, previousValue);
+		this.submitMessage(op, pendingKeySet);
+		this.eventEmitter.emit(
+			"valueChanged",
+			{ key, previousValue: previousOptimisticLocalValue?.value },
+			true,
+			this.eventEmitter,
+		);
 	}
 
 	/**
@@ -310,44 +466,68 @@ export class MapKernel {
 	 * @returns True if the key existed and was deleted, false if it did not exist
 	 */
 	public delete(key: string): boolean {
-		// Delete the key locally first.
-		const previousValue = this.deleteCore(key, true);
+		const previousOptimisticLocalValue = this.getOptimisticLocalValue(key);
 
-		// If we are not attached, don't submit the op.
 		if (!this.isAttached()) {
-			return previousValue !== undefined;
+			const successfullyRemoved = this.sequencedData.delete(key);
+			// Only emit if we actually deleted something.
+			if (previousOptimisticLocalValue !== undefined) {
+				this.eventEmitter.emit(
+					"valueChanged",
+					{ key, previousValue: previousOptimisticLocalValue.value },
+					true,
+					this.eventEmitter,
+				);
+			}
+			return successfullyRemoved;
 		}
 
+		const pendingKeyDelete: PendingKeyDelete = {
+			type: "delete",
+			key,
+		};
+		this.pendingData.push(pendingKeyDelete);
+
 		const op: IMapDeleteOperation = {
 			key,
 			type: "delete",
 		};
-		this.submitMapKeyMessage(op, previousValue);
+		this.submitMessage(op, pendingKeyDelete);
+		// Only emit if we locally believe we deleted something.  Otherwise we still send the op
+		// (permitting speculative deletion even if we don't see anything locally) but don't emit
+		// a valueChanged since we in fact did not locally observe a value change.
+		if (previousOptimisticLocalValue !== undefined) {
+			this.eventEmitter.emit(
+				"valueChanged",
+				{ key, previousValue: previousOptimisticLocalValue.value },
+				true,
+				this.eventEmitter,
+			);
+		}
 
-		return previousValue !== undefined;
+		return true;
 	}
 
 	/**
 	 * Clear all data from the map.
 	 */
 	public clear(): void {
-		const copy = this.isAttached() ? new Map<string, ILocalValue>(this.data) : undefined;
-
-		// Clear the data locally first.
-		this.clearCore(true);
-
-		// Clear the pendingKeys immediately, the local unack'd operations are aborted
-		this.pendingKeys.clear();
-
-		// If we are not attached, don't submit the op.
 		if (!this.isAttached()) {
+			this.sequencedData.clear();
+			this.eventEmitter.emit("clear", true, this.eventEmitter);
 			return;
 		}
 
+		const pendingClear: PendingClear = {
+			type: "clear",
+		};
+		this.pendingData.push(pendingClear);
+
 		const op: IMapClearOperation = {
 			type: "clear",
 		};
-		this.submitMapClearMessage(op, copy);
+		this.submitMessage(op, pendingClear);
+		this.eventEmitter.emit("clear", true, this.eventEmitter);
 	}
 
 	/**
@@ -357,7 +537,7 @@ export class MapKernel {
 	 */
 	public getSerializedStorage(serializer: IFluidSerializer): IMapDataObjectSerialized {
 		const serializedMapData: IMapDataObjectSerialized = {};
-		for (const [key, localValue] of this.data.entries()) {
+		for (const [key, localValue] of this.sequencedData.entries()) {
 			serializedMapData[key] = serializeValue(localValue.value, serializer, this.handle);
 		}
 		return serializedMapData;
@@ -372,7 +552,7 @@ export class MapKernel {
 			this.serializer.decode(json) as IMapDataObjectSerializable,
 		)) {
 			migrateIfSharedSerializable(serializable, this.serializer, this.handle);
-			this.data.set(key, { value: serializable.value });
+			this.sequencedData.set(key, { value: serializable.value });
 		}
 	}
 
@@ -389,7 +569,7 @@ export class MapKernel {
 		if (handler === undefined) {
 			return false;
 		}
-		handler.resubmit(op, localOpMetadata as ListNode<MapLocalOpMetadata>);
+		handler.resubmit(op, localOpMetadata as PendingLocalOpMetadata);
 		return true;
 	}
 
@@ -438,7 +618,7 @@ export class MapKernel {
 		if (handler === undefined) {
 			return false;
 		}
-		handler.process(op, local, localOpMetadata as ListNode<MapLocalOpMetadata> | undefined);
+		handler.process(op, local, localOpMetadata as PendingLocalOpMetadata | undefined);
 		return true;
 	}
 
@@ -449,167 +629,66 @@ export class MapKernel {
 	 */
 	public rollback(op: unknown, localOpMetadata: unknown): void {
 		const mapOp: IMapOperation = op as IMapOperation;
-		const listNodeLocalOpMetadata = localOpMetadata as ListNode<MapLocalOpMetadata>;
-		const removedLocalOpMetadata = this.pendingMapLocalOpMetadata.pop();
-		assert(
-			removedLocalOpMetadata !== undefined &&
-				removedLocalOpMetadata === listNodeLocalOpMetadata,
-			0xbcb /* Rolling back unexpected op */,
-		);
-
-		if (mapOp.type === "clear" && listNodeLocalOpMetadata.data.type === "clear") {
-			if (listNodeLocalOpMetadata.data.previousMap === undefined) {
-				throw new Error("Cannot rollback without previous map");
-			}
-			for (const [key, localValue] of listNodeLocalOpMetadata.data.previousMap.entries()) {
-				this.setCore(key, localValue, true);
-			}
-
-			const lastPendingClearId = this.pendingClearMessageIds.pop();
-			if (
-				lastPendingClearId === undefined ||
-				lastPendingClearId !== listNodeLocalOpMetadata.data.pendingMessageId
-			) {
-				throw new Error("Rollback op does match last clear");
-			}
-		} else if (mapOp.type === "delete" || mapOp.type === "set") {
-			if (listNodeLocalOpMetadata.data.type === "add") {
-				this.deleteCore(mapOp.key, true);
-			} else if (
-				listNodeLocalOpMetadata.data.type === "edit" &&
-				listNodeLocalOpMetadata.data.previousValue !== undefined
-			) {
-				this.setCore(mapOp.key, listNodeLocalOpMetadata.data.previousValue, true);
-			} else {
-				throw new Error("Cannot rollback without previous value");
-			}
-
-			const pendingMessageIds = this.pendingKeys.get(mapOp.key);
-			const lastPendingMessageId = pendingMessageIds?.pop();
-			if (
-				!pendingMessageIds ||
-				lastPendingMessageId !== listNodeLocalOpMetadata.data.pendingMessageId
-			) {
-				throw new Error("Rollback op does not match last pending");
-			}
-			if (pendingMessageIds.length === 0) {
-				this.pendingKeys.delete(mapOp.key);
-			}
-		} else {
-			throw new Error("Unsupported op for rollback");
-		}
-	}
-
-	/**
-	 * Set implementation used for both locally sourced sets as well as incoming remote sets.
-	 * @param key - The key being set
-	 * @param value - The value being set
-	 * @param local - Whether the message originated from the local client
-	 * @returns Previous local value of the key, if any
-	 */
-	private setCore(key: string, value: ILocalValue, local: boolean): ILocalValue | undefined {
-		const previousLocalValue = this.data.get(key);
-		const previousValue: unknown = previousLocalValue?.value;
-		this.data.set(key, value);
-		this.eventEmitter.emit("valueChanged", { key, previousValue }, local, this.eventEmitter);
-		return previousLocalValue;
-	}
-
-	/**
-	 * Clear implementation used for both locally sourced clears as well as incoming remote clears.
-	 * @param local - Whether the message originated from the local client
-	 */
-	private clearCore(local: boolean): void {
-		this.data.clear();
-		this.eventEmitter.emit("clear", local, this.eventEmitter);
-	}
-
-	/**
-	 * Delete implementation used for both locally sourced deletes as well as incoming remote deletes.
-	 * @param key - The key being deleted
-	 * @param local - Whether the message originated from the local client
-	 * @returns Previous local value of the key if it existed, undefined if it did not exist
-	 */
-	private deleteCore(key: string, local: boolean): ILocalValue | undefined {
-		const previousLocalValue = this.data.get(key);
-		const previousValue: unknown = previousLocalValue?.value;
-		const successfullyRemoved = this.data.delete(key);
-		if (successfullyRemoved) {
-			this.eventEmitter.emit("valueChanged", { key, previousValue }, local, this.eventEmitter);
-		}
-		return previousLocalValue;
-	}
-
-	/**
-	 * Clear all keys in memory in response to a remote clear, but retain keys we have modified but not yet been ack'd.
-	 */
-	private clearExceptPendingKeys(): void {
-		// Assuming the pendingKeys is small and the map is large
-		// we will get the value for the pendingKeys and clear the map
-		const temp = new Map<string, ILocalValue>();
-		for (const key of this.pendingKeys.keys()) {
-			// Verify if the most recent pending operation is a delete op, no need to retain it if so.
-			// This ensures the map size remains consistent.
-			if (this.data.has(key)) {
-				temp.set(key, this.data.get(key) as ILocalValue);
-			}
-		}
-		this.clearCore(false);
-		for (const [key, value] of temp.entries()) {
-			this.setCore(key, value, true);
-		}
-	}
-
-	/**
-	 * If our local operations that have not yet been ack'd will eventually overwrite an incoming operation, we should
-	 * not process the incoming operation.
-	 * @param op - Operation to check
-	 * @param local - Whether the message originated from the local client
-	 * @param localOpMetadata - For local client messages, this is the metadata that was submitted with the message.
-	 * For messages from a remote client, this will be undefined.
-	 * @returns True if the operation should be processed, false otherwise
-	 */
-	private needProcessKeyOperation(
-		op: IMapKeyOperation,
-		local: boolean,
-		localOpMetadata: MapLocalOpMetadata | undefined,
-	): boolean {
-		if (this.pendingClearMessageIds[0] !== undefined) {
-			if (local) {
-				assert(
-					localOpMetadata !== undefined &&
-						isMapKeyLocalOpMetadata(localOpMetadata) &&
-						localOpMetadata.pendingMessageId < this.pendingClearMessageIds[0],
-					0x013 /* "Received out of order op when there is an unackd clear message" */,
+		const typedLocalOpMetadata = localOpMetadata as PendingLocalOpMetadata;
+		if (mapOp.type === "clear") {
+			// A pending clear will be last in the list, since it terminates all prior lifetimes.
+			const pendingClear = this.pendingData.pop();
+			assert(
+				pendingClear !== undefined &&
+					pendingClear.type === "clear" &&
+					pendingClear === typedLocalOpMetadata,
+				0xbf2 /* Unexpected clear rollback */,
+			);
+			for (const [key] of this.internalIterator()) {
+				this.eventEmitter.emit(
+					"valueChanged",
+					{ key, previousValue: undefined },
+					true,
+					this.eventEmitter,
 				);
 			}
-			// If we have an unack'd clear, we can ignore all ops.
-			return false;
-		}
-
-		const pendingKeyMessageIds = this.pendingKeys.get(op.key);
-		if (pendingKeyMessageIds !== undefined) {
-			// Found an unack'd op. Clear it from the map if the pendingMessageId in the map matches this message's
-			// and don't process the op.
-			if (local) {
-				assert(
-					localOpMetadata !== undefined && isMapKeyLocalOpMetadata(localOpMetadata),
-					0x014 /* pendingMessageId is missing from the local client's operation */,
-				);
+		} else {
+			// A pending set/delete may not be last in the list, as the lifetimes' order is based on when
+			// they were created, not when they were last modified.
+			const pendingEntryIndex = findLastIndex(
+				this.pendingData,
+				(entry) => entry.type !== "clear" && entry.key === mapOp.key,
+			);
+			const pendingEntry = this.pendingData[pendingEntryIndex];
+			assert(
+				pendingEntry !== undefined &&
+					(pendingEntry.type === "delete" || pendingEntry.type === "lifetime"),
+				0xbf3 /* Unexpected pending data for set/delete op */,
+			);
+			if (pendingEntry.type === "delete") {
+				assert(pendingEntry === typedLocalOpMetadata, 0xbf4 /* Unexpected delete rollback */);
+				this.pendingData.splice(pendingEntryIndex, 1);
+				// Only emit if rolling back the delete actually results in a value becoming visible.
+				if (this.getOptimisticLocalValue(mapOp.key) !== undefined) {
+					this.eventEmitter.emit(
+						"valueChanged",
+						{ key: mapOp.key, previousValue: undefined },
+						true,
+						this.eventEmitter,
+					);
+				}
+			} else if (pendingEntry.type === "lifetime") {
+				const pendingKeySet = pendingEntry.keySets.pop();
 				assert(
-					pendingKeyMessageIds[0] === localOpMetadata.pendingMessageId,
-					0x2fa /* Unexpected pending message received */,
+					pendingKeySet !== undefined && pendingKeySet === typedLocalOpMetadata,
+					0xbf5 /* Unexpected set rollback */,
 				);
-				pendingKeyMessageIds.shift();
-				if (pendingKeyMessageIds.length === 0) {
-					this.pendingKeys.delete(op.key);
+				if (pendingEntry.keySets.length === 0) {
+					this.pendingData.splice(pendingEntryIndex, 1);
 				}
+				this.eventEmitter.emit(
+					"valueChanged",
+					{ key: mapOp.key, previousValue: pendingKeySet.value.value },
+					true,
+					this.eventEmitter,
+				);
 			}
-			return false;
 		}
-
-		// If we don't have a NACK op on the key, we need to process the remote ops.
-		return !local;
 	}
 
 	/**
@@ -622,191 +701,118 @@ export class MapKernel {
 			process: (
 				op: IMapClearOperation,
 				local: boolean,
-				localOpMetadata: ListNode<MapLocalOpMetadata> | undefined,
+				localOpMetadata: PendingLocalOpMetadata | undefined,
 			) => {
+				this.sequencedData.clear();
 				if (local) {
-					const removedLocalOpMetadata = this.pendingMapLocalOpMetadata.shift();
+					const pendingClear = this.pendingData.shift();
 					assert(
-						removedLocalOpMetadata !== undefined && removedLocalOpMetadata === localOpMetadata,
-						0xbcc /* Processing unexpected local clear op */,
+						pendingClear !== undefined &&
+							pendingClear.type === "clear" &&
+							pendingClear === localOpMetadata,
+						0xbf6 /* Got a local clear message we weren't expecting */,
 					);
-					assert(
-						isClearLocalOpMetadata(localOpMetadata.data),
-						0x015 /* "pendingMessageId is missing from the local client's clear operation" */,
-					);
-					const pendingClearMessageId = this.pendingClearMessageIds.shift();
-					assert(
-						pendingClearMessageId === localOpMetadata.data.pendingMessageId,
-						0x2fb /* pendingMessageId does not match */,
-					);
-					return;
+				} else {
+					// Only emit for remote ops, we would have already emitted for local ops. Only emit if there
+					// is no optimistically-applied local pending clear that would supersede this remote clear.
+					if (!this.pendingData.some((entry) => entry.type === "clear")) {
+						this.eventEmitter.emit("clear", local, this.eventEmitter);
+					}
 				}
-				if (this.pendingKeys.size > 0) {
-					this.clearExceptPendingKeys();
-					return;
-				}
-				this.clearCore(local);
 			},
-			resubmit: (op: IMapClearOperation, localOpMetadata: ListNode<MapLocalOpMetadata>) => {
-				const removedLocalOpMetadata = localOpMetadata.remove()?.data;
-				assert(
-					removedLocalOpMetadata !== undefined,
-					0xbcd /* Resubmitting unexpected local clear op */,
-				);
-				assert(
-					isClearLocalOpMetadata(localOpMetadata.data),
-					0x2fc /* Invalid localOpMetadata for clear */,
-				);
-				// We don't reuse the metadata pendingMessageId but send a new one on each submit.
-				const pendingClearMessageId = this.pendingClearMessageIds.shift();
-				assert(
-					pendingClearMessageId === localOpMetadata.data.pendingMessageId,
-					0x2fd /* pendingMessageId does not match */,
-				);
-				this.submitMapClearMessage(op, localOpMetadata.data.previousMap);
+			resubmit: (op: IMapClearOperation, localOpMetadata: PendingLocalOpMetadata) => {
+				this.submitMessage(op, localOpMetadata);
 			},
 		});
 		messageHandlers.set("delete", {
 			process: (
 				op: IMapDeleteOperation,
 				local: boolean,
-				localOpMetadata: ListNode<MapLocalOpMetadata> | undefined,
+				localOpMetadata: PendingLocalOpMetadata | undefined,
 			) => {
+				const { key } = op;
+
 				if (local) {
-					const removedLocalOpMetadata = this.pendingMapLocalOpMetadata.shift();
+					const pendingEntryIndex = this.pendingData.findIndex(
+						(entry) => entry.type !== "clear" && entry.key === key,
+					);
+					const pendingEntry = this.pendingData[pendingEntryIndex];
 					assert(
-						removedLocalOpMetadata !== undefined && removedLocalOpMetadata === localOpMetadata,
-						0xbce /* Processing unexpected local delete op */,
+						pendingEntry !== undefined &&
+							pendingEntry.type === "delete" &&
+							pendingEntry === localOpMetadata,
+						0xbf7 /* Got a local delete message we weren't expecting */,
 					);
+					this.pendingData.splice(pendingEntryIndex, 1);
+
+					this.sequencedData.delete(key);
+				} else {
+					const previousValue: unknown = this.sequencedData.get(key)?.value;
+					this.sequencedData.delete(key);
+					// Suppress the event if local changes would cause the incoming change to be invisible optimistically.
+					if (!this.pendingData.some((entry) => entry.type === "clear" || entry.key === key)) {
+						this.eventEmitter.emit(
+							"valueChanged",
+							{ key, previousValue },
+							local,
+							this.eventEmitter,
+						);
+					}
 				}
-				if (!this.needProcessKeyOperation(op, local, localOpMetadata?.data)) {
-					return;
-				}
-				this.deleteCore(op.key, local);
 			},
-			resubmit: (op: IMapDeleteOperation, localOpMetadata: ListNode<MapLocalOpMetadata>) => {
-				const removedLocalOpMetadata = localOpMetadata.remove()?.data;
-				assert(
-					removedLocalOpMetadata !== undefined,
-					0xbcf /* Resubmitting unexpected local delete op */,
-				);
-				this.resubmitMapKeyMessage(op, localOpMetadata.data);
+			resubmit: (op: IMapDeleteOperation, localOpMetadata: PendingLocalOpMetadata) => {
+				this.submitMessage(op, localOpMetadata);
 			},
 		});
 		messageHandlers.set("set", {
 			process: (
 				op: IMapSetOperation,
 				local: boolean,
-				localOpMetadata: ListNode<MapLocalOpMetadata> | undefined,
+				localOpMetadata: PendingLocalOpMetadata | undefined,
 			) => {
+				const { key, value } = op;
+
 				if (local) {
-					const removedLocalOpMetadata = this.pendingMapLocalOpMetadata.shift();
+					const pendingEntryIndex = this.pendingData.findIndex(
+						(entry) => entry.type !== "clear" && entry.key === key,
+					);
+					const pendingEntry = this.pendingData[pendingEntryIndex];
 					assert(
-						removedLocalOpMetadata !== undefined && removedLocalOpMetadata === localOpMetadata,
-						0xbd0 /* Processing unexpected local set op */,
+						pendingEntry !== undefined && pendingEntry.type === "lifetime",
+						0xbf8 /* Couldn't match local set message to pending lifetime */,
 					);
+					const pendingKeySet = pendingEntry.keySets.shift();
+					assert(
+						pendingKeySet !== undefined && pendingKeySet === localOpMetadata,
+						0xbf9 /* Got a local set message we weren't expecting */,
+					);
+					if (pendingEntry.keySets.length === 0) {
+						this.pendingData.splice(pendingEntryIndex, 1);
+					}
+
+					this.sequencedData.set(key, pendingKeySet.value);
+				} else {
+					migrateIfSharedSerializable(value, this.serializer, this.handle);
+					const localValue: ILocalValue = { value: value.value };
+					const previousValue: unknown = this.sequencedData.get(key)?.value;
+					this.sequencedData.set(key, localValue);
+
+					// Suppress the event if local changes would cause the incoming change to be invisible optimistically.
+					if (!this.pendingData.some((entry) => entry.type === "clear" || entry.key === key)) {
+						this.eventEmitter.emit(
+							"valueChanged",
+							{ key, previousValue },
+							local,
+							this.eventEmitter,
+						);
+					}
 				}
-				if (!this.needProcessKeyOperation(op, local, localOpMetadata?.data)) {
-					return;
-				}
-
-				// needProcessKeyOperation should have returned false if local is true
-				migrateIfSharedSerializable(op.value, this.serializer, this.handle);
-				this.setCore(op.key, { value: op.value.value }, local);
 			},
-			resubmit: (op: IMapSetOperation, localOpMetadata: ListNode<MapLocalOpMetadata>) => {
-				const removedLocalOpMetadata = localOpMetadata.remove()?.data;
-				assert(
-					removedLocalOpMetadata !== undefined,
-					0xbd1 /* Resubmitting unexpected local set op */,
-				);
-				this.resubmitMapKeyMessage(op, localOpMetadata.data);
+			resubmit: (op: IMapSetOperation, localOpMetadata: PendingLocalOpMetadata) => {
+				this.submitMessage(op, localOpMetadata);
 			},
 		});
 
 		return messageHandlers;
 	}
-
-	private getMapClearMessageId(): number {
-		const pendingMessageId = this.nextPendingMessageId++;
-		this.pendingClearMessageIds.push(pendingMessageId);
-		return pendingMessageId;
-	}
-
-	/**
-	 * Submit a clear message to remote clients.
-	 * @param op - The clear message
-	 */
-	private submitMapClearMessage(
-		op: IMapClearOperation,
-		previousMap?: Map<string, ILocalValue>,
-	): void {
-		const pendingMessageId = this.getMapClearMessageId();
-		const localMetadata = createClearLocalOpMetadata(op, pendingMessageId, previousMap);
-		const listNode = this.pendingMapLocalOpMetadata.push(localMetadata).first;
-		this.submitMessage(op, listNode);
-	}
-
-	private getMapKeyMessageId(op: IMapKeyOperation): number {
-		const pendingMessageId = this.nextPendingMessageId++;
-		const pendingMessageIds = this.pendingKeys.get(op.key);
-		if (pendingMessageIds === undefined) {
-			this.pendingKeys.set(op.key, [pendingMessageId]);
-		} else {
-			pendingMessageIds.push(pendingMessageId);
-		}
-		return pendingMessageId;
-	}
-
-	/**
-	 * Submit a map key message to remote clients.
-	 * @param op - The map key message
-	 * @param previousValue - The value of the key before this op
-	 */
-	private submitMapKeyMessage(op: IMapKeyOperation, previousValue?: ILocalValue): void {
-		const pendingMessageId = this.getMapKeyMessageId(op);
-		const localMetadata = createKeyLocalOpMetadata(op, pendingMessageId, previousValue);
-		const listNode = this.pendingMapLocalOpMetadata.push(localMetadata).first;
-		this.submitMessage(op, listNode);
-	}
-
-	/**
-	 * Submit a map key message to remote clients based on a previous submit.
-	 * @param op - The map key message
-	 * @param localOpMetadata - Metadata from the previous submit
-	 */
-	private resubmitMapKeyMessage(
-		op: IMapKeyOperation,
-		localOpMetadata: MapLocalOpMetadata,
-	): void {
-		assert(
-			isMapKeyLocalOpMetadata(localOpMetadata),
-			0x2fe /* Invalid localOpMetadata in submit */,
-		);
-
-		// no need to submit messages for op's that have been aborted
-		const pendingMessageIds = this.pendingKeys.get(op.key);
-		if (pendingMessageIds === undefined) {
-			return;
-		}
-
-		const index = pendingMessageIds.indexOf(localOpMetadata.pendingMessageId);
-		if (index === -1) {
-			return;
-		}
-
-		pendingMessageIds.splice(index, 1);
-		if (pendingMessageIds.length === 0) {
-			this.pendingKeys.delete(op.key);
-		}
-
-		// We don't reuse the metadata pendingMessageId but send a new one on each submit.
-		const pendingMessageId = this.getMapKeyMessageId(op);
-		const localMetadata: MapKeyLocalOpMetadata =
-			localOpMetadata.type === "edit"
-				? { type: "edit", pendingMessageId, previousValue: localOpMetadata.previousValue }
-				: { type: "add", pendingMessageId };
-		const listNode = this.pendingMapLocalOpMetadata.push(localMetadata).first;
-		this.submitMessage(op, listNode);
-	}
 }