@fluidframework/map 2.51.0 → 2.52.0

package/dist/mapKernel.js

@@ -8,32 +8,22 @@ exports.MapKernel = void 0;
 const internal_1 = require("@fluidframework/core-utils/internal");
 const internal_2 = require("@fluidframework/shared-object-base/internal");
 const localValues_js_1 = require("./localValues.js");
-/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access */
-function isMapKeyLocalOpMetadata(metadata) {
-    return (metadata !== undefined &&
-        typeof metadata.pendingMessageId === "number" &&
-        (metadata.type === "add" || metadata.type === "edit"));
-}
-function isClearLocalOpMetadata(metadata) {
-    return (metadata !== undefined &&
-        metadata.type === "clear" &&
-        typeof metadata.pendingMessageId === "number");
-}
-/* eslint-enable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access */
-function createClearLocalOpMetadata(op, pendingClearMessageId, previousMap) {
-    const localMetadata = {
-        type: "clear",
-        pendingMessageId: pendingClearMessageId,
-        previousMap,
-    };
-    return localMetadata;
-}
-function createKeyLocalOpMetadata(op, pendingMessageId, previousValue) {
-    const localMetadata = previousValue
-        ? { type: "edit", pendingMessageId, previousValue }
-        : { type: "add", pendingMessageId };
-    return localMetadata;
-}
+/**
+ * Rough polyfill for Array.findLastIndex until we target ES2023 or greater.
+ */
+const findLastIndex = (array, callbackFn) => {
+    for (let i = array.length - 1; i >= 0; i--) {
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        if (callbackFn(array[i])) {
+            return i;
+        }
+    }
+    return -1;
+};
+/**
+ * Rough polyfill for Array.findLast until we target ES2023 or greater.
+ */
+const findLast = (array, callbackFn) => array[findLastIndex(array, callbackFn)];
 /**
  * A SharedMap is a map-like distributed data structure.
  */
@@ -42,7 +32,8 @@ class MapKernel {
      * The number of key/value pairs stored in the map.
      */
     get size() {
-        return this.data.size;
+        const iterableItems = [...this.internalIterator()];
+        return iterableItems.length;
     }
     /**
      * Create a new shared map kernel.
@@ -64,48 +55,150 @@ class MapKernel {
          */
         this.messageHandlers = new Map();
         /**
-         * The in-memory data the map is storing.
-         */
-        this.data = new Map();
-        /**
-         * Keys that have been modified locally but not yet ack'd from the server.
+         * The data the map is storing, but only including sequenced values (no local pending
+         * modifications are included).
          */
-        this.pendingKeys = new Map();
+        this.sequencedData = new Map();
         /**
-         * This is used to assign a unique id to every outgoing operation and helps in tracking unack'd ops.
+         * A data structure containing all local pending modifications, which is used in combination
+         * with the sequencedData to compute optimistic values.
+         *
+         * Pending sets are aggregated into "lifetimes", which permit correct relative iteration order
+         * even across remote operations and rollbacks.
          */
-        this.nextPendingMessageId = 0;
+        this.pendingData = [];
         /**
-         * The pending metadata for any local operations that have not yet been ack'd from the server, in order.
+         * Get an iterator over the optimistically observable ILocalValue entries in the map. For example, excluding
+         * sequenced entries that have pending deletes/clears.
+         *
+         * @remarks
+         * There is no perfect solution here, particularly when the iterator is retained over time and the map is
+         * modified or new ack's are received. The pendingData portion of the iteration is the most susceptible to
+         * this problem. The implementation prioritizes (in roughly this order):
+         * 1. Correct immediate iteration (i.e. when the map is not modified before iteration completes)
+         * 2. Consistent iteration order before/after sequencing of pending ops; acks don't change order
+         * 3. Consistent iteration order between synchronized clients, even if they each modified the map concurrently
+         * 4. Remaining as close as possible to the native Map iterator behavior, e.g. live-ish view rather than snapshot
+         *
+         * For this reason, it's important not to internally snapshot the output of the iterator for any purpose that
+         * does not immediately (synchronously) consume that output and dispose of it.
          */
-        this.pendingMapLocalOpMetadata = new internal_1.DoublyLinkedList();
+        this.internalIterator = () => {
+            // We perform iteration in two steps - first by iterating over members of the sequenced data that are not
+            // optimistically deleted or cleared, and then over the pending data lifetimes that have not subsequently
+            // been deleted or cleared.  In total, this give an ordering of members based on when they were initially
+            // added to the map (even if they were later modified), similar to the native Map.
+            const sequencedDataIterator = this.sequencedData.keys();
+            const pendingDataIterator = this.pendingData.values();
+            const next = () => {
+                let nextSequencedKey = sequencedDataIterator.next();
+                while (!nextSequencedKey.done) {
+                    const key = nextSequencedKey.value;
+                    // If we have any pending deletes or clears, then we won't iterate to this key yet (if at all).
+                    // Either it is optimistically deleted and will not be part of the iteration, or it was
+                    // re-added later and we'll iterate to it when we get to the pending data.
+                    if (!this.pendingData.some((entry) => entry.type === "clear" || (entry.type === "delete" && entry.key === key))) {
+                        const optimisticValue = this.getOptimisticLocalValue(key);
+                        (0, internal_1.assert)(optimisticValue !== undefined, 0xbf1 /* Should never iterate to a key with undefined optimisticValue */);
+                        return { value: [key, optimisticValue], done: false };
+                    }
+                    nextSequencedKey = sequencedDataIterator.next();
+                }
+                let nextPending = pendingDataIterator.next();
+                while (!nextPending.done) {
+                    const nextPendingEntry = nextPending.value;
+                    // A lifetime entry may need to be iterated.
+                    if (nextPendingEntry.type === "lifetime") {
+                        const nextPendingEntryIndex = this.pendingData.indexOf(nextPendingEntry);
+                        const mostRecentDeleteOrClearIndex = findLastIndex(this.pendingData, (entry) => entry.type === "clear" ||
+                            (entry.type === "delete" && entry.key === nextPendingEntry.key));
+                        // Only iterate the pending entry now if it hasn't been deleted or cleared.
+                        if (nextPendingEntryIndex > mostRecentDeleteOrClearIndex) {
+                            const latestPendingValue = 
+                            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                            nextPendingEntry.keySets[nextPendingEntry.keySets.length - 1];
+                            // Skip iterating if we would have would have already iterated it as part of the sequenced data.
+                            // This is not a perfect check in the case the map has changed since the iterator was created
+                            // (e.g. if a remote client added the same key in the meantime).
+                            if (!this.sequencedData.has(nextPendingEntry.key) ||
+                                mostRecentDeleteOrClearIndex !== -1) {
+                                return { value: [nextPendingEntry.key, latestPendingValue.value], done: false };
+                            }
+                        }
+                    }
+                    nextPending = pendingDataIterator.next();
+                }
+                return { value: undefined, done: true };
+            };
+            const iterator = {
+                next,
+                [Symbol.iterator]() {
+                    return this;
+                },
+            };
+            return iterator;
+        };
         /**
-         * The pending ids of any clears that have been performed locally but not yet ack'd from the server
+         * Compute the optimistic local value for a given key. This combines the sequenced data with
+         * any pending changes that have not yet been sequenced.
          */
-        this.pendingClearMessageIds = [];
+        this.getOptimisticLocalValue = (key) => {
+            const latestPendingEntry = findLast(this.pendingData, (entry) => entry.type === "clear" || entry.key === key);
+            if (latestPendingEntry === undefined) {
+                return this.sequencedData.get(key);
+            }
+            else if (latestPendingEntry.type === "lifetime") {
+                const latestPendingSet = 
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                latestPendingEntry.keySets[latestPendingEntry.keySets.length - 1];
+                return latestPendingSet.value;
+            }
+            else {
+                // Delete or clear
+                return undefined;
+            }
+        };
         this.messageHandlers = this.getMessageHandlers();
     }
     /**
-     * Get an iterator over the keys in this map.
+     * Get an iterator over the entries in this map.
      * @returns The iterator
      */
-    keys() {
-        return this.data.keys();
+    entries() {
+        const internalIterator = this.internalIterator();
+        const next = () => {
+            const nextResult = internalIterator.next();
+            if (nextResult.done) {
+                return { value: undefined, done: true };
+            }
+            // Unpack the stored value
+            const [key, localValue] = nextResult.value;
+            return { value: [key, localValue.value], done: false };
+        };
+        const iterator = {
+            next,
+            [Symbol.iterator]() {
+                return this;
+            },
+        };
+        return iterator;
     }
     /**
-     * Get an iterator over the entries in this map.
+     * Get an iterator over the keys in this map.
      * @returns The iterator
      */
-    entries() {
-        const localEntriesIterator = this.data.entries();
+    keys() {
+        const internalIterator = this.internalIterator();
+        const next = () => {
+            const nextResult = internalIterator.next();
+            if (nextResult.done) {
+                return { value: undefined, done: true };
+            }
+            const [key] = nextResult.value;
+            return { value: key, done: false };
+        };
         const iterator = {
-            next() {
-                const nextVal = localEntriesIterator.next();
-                return nextVal.done
-                    ? { value: undefined, done: true }
-                    : // Unpack the stored value
-                        { value: [nextVal.value[0], nextVal.value[1].value], done: false };
-            },
+            next,
             [Symbol.iterator]() {
                 return this;
             },
@@ -117,15 +210,17 @@ class MapKernel {
      * @returns The iterator
      */
     values() {
-        const localValuesIterator = this.data.values();
+        const internalIterator = this.internalIterator();
+        const next = () => {
+            const nextResult = internalIterator.next();
+            if (nextResult.done) {
+                return { value: undefined, done: true };
+            }
+            const [, localValue] = nextResult.value;
+            return { value: localValue.value, done: false };
+        };
         const iterator = {
-            next() {
-                const nextVal = localValuesIterator.next();
-                return nextVal.done
-                    ? { value: undefined, done: true }
-                    : // Unpack the stored value
-                        { value: nextVal.value.value, done: false };
-            },
+            next,
             [Symbol.iterator]() {
                 return this;
             },
@@ -144,8 +239,12 @@ class MapKernel {
      * @param callbackFn - Callback function
      */
     forEach(callbackFn) {
+        // It would be better to iterate over the data without a temp map.  However, we don't have a valid
+        // map to pass for the third argument here (really, it should probably should be a reference to the
+        // SharedMap and not the MapKernel).
+        const tempMap = new Map(this.internalIterator());
         // eslint-disable-next-line unicorn/no-array-for-each
-        this.data.forEach((localValue, key, m) => {
+        tempMap.forEach((localValue, key, m) => {
             callbackFn(localValue.value, key, m);
         });
     }
@@ -153,7 +252,7 @@ class MapKernel {
      * {@inheritDoc ISharedMap.get}
      */
     get(key) {
-        const localValue = this.data.get(key);
+        const localValue = this.getOptimisticLocalValue(key);
         return localValue === undefined ? undefined : localValue.value;
     }
     /**
@@ -162,7 +261,7 @@ class MapKernel {
      * @returns True if the key exists, false otherwise
      */
     has(key) {
-        return this.data.has(key);
+        return this.getOptimisticLocalValue(key) !== undefined;
     }
     /**
      * {@inheritDoc ISharedMap.set}
@@ -172,18 +271,37 @@ class MapKernel {
         if (key === undefined || key === null) {
             throw new Error("Undefined and null keys are not supported");
         }
-        // Set the value locally.
-        const previousValue = this.setCore(key, { value }, true);
+        const localValue = { value };
+        const previousOptimisticLocalValue = this.getOptimisticLocalValue(key);
         // If we are not attached, don't submit the op.
         if (!this.isAttached()) {
+            this.sequencedData.set(key, localValue);
+            this.eventEmitter.emit("valueChanged", { key, previousValue: previousOptimisticLocalValue?.value }, true, this.eventEmitter);
             return;
         }
+        // A new pending key lifetime is created if:
+        // 1. There isn't any pending entry for the key yet
+        // 2. The most recent pending entry for the key was a deletion (as this terminates the prior lifetime)
+        // 3. A clear was sent after the last pending entry for the key (which also terminates the prior lifetime)
+        let latestPendingEntry = findLast(this.pendingData, (entry) => entry.type === "clear" || entry.key === key);
+        if (latestPendingEntry === undefined ||
+            latestPendingEntry.type === "delete" ||
+            latestPendingEntry.type === "clear") {
+            latestPendingEntry = { type: "lifetime", key, keySets: [] };
+            this.pendingData.push(latestPendingEntry);
+        }
+        const pendingKeySet = {
+            type: "set",
+            value: localValue,
+        };
+        latestPendingEntry.keySets.push(pendingKeySet);
         const op = {
             key,
             type: "set",
-            value: { type: internal_2.ValueType[internal_2.ValueType.Plain], value },
+            value: { type: internal_2.ValueType[internal_2.ValueType.Plain], value: localValue.value },
         };
-        this.submitMapKeyMessage(op, previousValue);
+        this.submitMessage(op, pendingKeySet);
+        this.eventEmitter.emit("valueChanged", { key, previousValue: previousOptimisticLocalValue?.value }, true, this.eventEmitter);
     }
     /**
      * Delete a key from the map.
@@ -191,36 +309,51 @@ class MapKernel {
      * @returns True if the key existed and was deleted, false if it did not exist
      */
     delete(key) {
-        // Delete the key locally first.
-        const previousValue = this.deleteCore(key, true);
-        // If we are not attached, don't submit the op.
+        const previousOptimisticLocalValue = this.getOptimisticLocalValue(key);
         if (!this.isAttached()) {
-            return previousValue !== undefined;
+            const successfullyRemoved = this.sequencedData.delete(key);
+            // Only emit if we actually deleted something.
+            if (previousOptimisticLocalValue !== undefined) {
+                this.eventEmitter.emit("valueChanged", { key, previousValue: previousOptimisticLocalValue.value }, true, this.eventEmitter);
+            }
+            return successfullyRemoved;
         }
+        const pendingKeyDelete = {
+            type: "delete",
+            key,
+        };
+        this.pendingData.push(pendingKeyDelete);
         const op = {
             key,
             type: "delete",
         };
-        this.submitMapKeyMessage(op, previousValue);
-        return previousValue !== undefined;
+        this.submitMessage(op, pendingKeyDelete);
+        // Only emit if we locally believe we deleted something.  Otherwise we still send the op
+        // (permitting speculative deletion even if we don't see anything locally) but don't emit
+        // a valueChanged since we in fact did not locally observe a value change.
+        if (previousOptimisticLocalValue !== undefined) {
+            this.eventEmitter.emit("valueChanged", { key, previousValue: previousOptimisticLocalValue.value }, true, this.eventEmitter);
+        }
+        return true;
     }
     /**
      * Clear all data from the map.
      */
     clear() {
-        const copy = this.isAttached() ? new Map(this.data) : undefined;
-        // Clear the data locally first.
-        this.clearCore(true);
-        // Clear the pendingKeys immediately, the local unack'd operations are aborted
-        this.pendingKeys.clear();
-        // If we are not attached, don't submit the op.
         if (!this.isAttached()) {
+            this.sequencedData.clear();
+            this.eventEmitter.emit("clear", true, this.eventEmitter);
             return;
         }
+        const pendingClear = {
+            type: "clear",
+        };
+        this.pendingData.push(pendingClear);
         const op = {
             type: "clear",
         };
-        this.submitMapClearMessage(op, copy);
+        this.submitMessage(op, pendingClear);
+        this.eventEmitter.emit("clear", true, this.eventEmitter);
     }
     /**
      * Serializes the data stored in the shared map to a JSON string
@@ -229,7 +362,7 @@ class MapKernel {
      */
     getSerializedStorage(serializer) {
         const serializedMapData = {};
-        for (const [key, localValue] of this.data.entries()) {
+        for (const [key, localValue] of this.sequencedData.entries()) {
             serializedMapData[key] = (0, localValues_js_1.serializeValue)(localValue.value, serializer, this.handle);
         }
         return serializedMapData;
@@ -241,7 +374,7 @@ class MapKernel {
     populateFromSerializable(json) {
         for (const [key, serializable] of Object.entries(this.serializer.decode(json))) {
             (0, localValues_js_1.migrateIfSharedSerializable)(serializable, this.serializer, this.handle);
-            this.data.set(key, { value: serializable.value });
+            this.sequencedData.set(key, { value: serializable.value });
         }
     }
     /**
@@ -310,139 +443,41 @@ class MapKernel {
      */
     rollback(op, localOpMetadata) {
         const mapOp = op;
-        const listNodeLocalOpMetadata = localOpMetadata;
-        const removedLocalOpMetadata = this.pendingMapLocalOpMetadata.pop();
-        (0, internal_1.assert)(removedLocalOpMetadata !== undefined &&
-            removedLocalOpMetadata === listNodeLocalOpMetadata, 0xbcb /* Rolling back unexpected op */);
-        if (mapOp.type === "clear" && listNodeLocalOpMetadata.data.type === "clear") {
-            if (listNodeLocalOpMetadata.data.previousMap === undefined) {
-                throw new Error("Cannot rollback without previous map");
-            }
-            for (const [key, localValue] of listNodeLocalOpMetadata.data.previousMap.entries()) {
-                this.setCore(key, localValue, true);
-            }
-            const lastPendingClearId = this.pendingClearMessageIds.pop();
-            if (lastPendingClearId === undefined ||
-                lastPendingClearId !== listNodeLocalOpMetadata.data.pendingMessageId) {
-                throw new Error("Rollback op does match last clear");
-            }
-        }
-        else if (mapOp.type === "delete" || mapOp.type === "set") {
-            if (listNodeLocalOpMetadata.data.type === "add") {
-                this.deleteCore(mapOp.key, true);
-            }
-            else if (listNodeLocalOpMetadata.data.type === "edit" &&
-                listNodeLocalOpMetadata.data.previousValue !== undefined) {
-                this.setCore(mapOp.key, listNodeLocalOpMetadata.data.previousValue, true);
-            }
-            else {
-                throw new Error("Cannot rollback without previous value");
-            }
-            const pendingMessageIds = this.pendingKeys.get(mapOp.key);
-            const lastPendingMessageId = pendingMessageIds?.pop();
-            if (!pendingMessageIds ||
-                lastPendingMessageId !== listNodeLocalOpMetadata.data.pendingMessageId) {
-                throw new Error("Rollback op does not match last pending");
-            }
-            if (pendingMessageIds.length === 0) {
-                this.pendingKeys.delete(mapOp.key);
+        const typedLocalOpMetadata = localOpMetadata;
+        if (mapOp.type === "clear") {
+            // A pending clear will be last in the list, since it terminates all prior lifetimes.
+            const pendingClear = this.pendingData.pop();
+            (0, internal_1.assert)(pendingClear !== undefined &&
+                pendingClear.type === "clear" &&
+                pendingClear === typedLocalOpMetadata, 0xbf2 /* Unexpected clear rollback */);
+            for (const [key] of this.internalIterator()) {
+                this.eventEmitter.emit("valueChanged", { key, previousValue: undefined }, true, this.eventEmitter);
             }
         }
         else {
-            throw new Error("Unsupported op for rollback");
-        }
-    }
-    /**
-     * Set implementation used for both locally sourced sets as well as incoming remote sets.
-     * @param key - The key being set
-     * @param value - The value being set
-     * @param local - Whether the message originated from the local client
-     * @returns Previous local value of the key, if any
-     */
-    setCore(key, value, local) {
-        const previousLocalValue = this.data.get(key);
-        const previousValue = previousLocalValue?.value;
-        this.data.set(key, value);
-        this.eventEmitter.emit("valueChanged", { key, previousValue }, local, this.eventEmitter);
-        return previousLocalValue;
-    }
-    /**
-     * Clear implementation used for both locally sourced clears as well as incoming remote clears.
-     * @param local - Whether the message originated from the local client
-     */
-    clearCore(local) {
-        this.data.clear();
-        this.eventEmitter.emit("clear", local, this.eventEmitter);
-    }
-    /**
-     * Delete implementation used for both locally sourced deletes as well as incoming remote deletes.
-     * @param key - The key being deleted
-     * @param local - Whether the message originated from the local client
-     * @returns Previous local value of the key if it existed, undefined if it did not exist
-     */
-    deleteCore(key, local) {
-        const previousLocalValue = this.data.get(key);
-        const previousValue = previousLocalValue?.value;
-        const successfullyRemoved = this.data.delete(key);
-        if (successfullyRemoved) {
-            this.eventEmitter.emit("valueChanged", { key, previousValue }, local, this.eventEmitter);
-        }
-        return previousLocalValue;
-    }
-    /**
-     * Clear all keys in memory in response to a remote clear, but retain keys we have modified but not yet been ack'd.
-     */
-    clearExceptPendingKeys() {
-        // Assuming the pendingKeys is small and the map is large
-        // we will get the value for the pendingKeys and clear the map
-        const temp = new Map();
-        for (const key of this.pendingKeys.keys()) {
-            // Verify if the most recent pending operation is a delete op, no need to retain it if so.
-            // This ensures the map size remains consistent.
-            if (this.data.has(key)) {
-                temp.set(key, this.data.get(key));
-            }
-        }
-        this.clearCore(false);
-        for (const [key, value] of temp.entries()) {
-            this.setCore(key, value, true);
-        }
-    }
-    /**
-     * If our local operations that have not yet been ack'd will eventually overwrite an incoming operation, we should
-     * not process the incoming operation.
-     * @param op - Operation to check
-     * @param local - Whether the message originated from the local client
-     * @param localOpMetadata - For local client messages, this is the metadata that was submitted with the message.
-     * For messages from a remote client, this will be undefined.
-     * @returns True if the operation should be processed, false otherwise
-     */
-    needProcessKeyOperation(op, local, localOpMetadata) {
-        if (this.pendingClearMessageIds[0] !== undefined) {
-            if (local) {
-                (0, internal_1.assert)(localOpMetadata !== undefined &&
-                    isMapKeyLocalOpMetadata(localOpMetadata) &&
-                    localOpMetadata.pendingMessageId < this.pendingClearMessageIds[0], 0x013 /* "Received out of order op when there is an unackd clear message" */);
+            // A pending set/delete may not be last in the list, as the lifetimes' order is based on when
+            // they were created, not when they were last modified.
+            const pendingEntryIndex = findLastIndex(this.pendingData, (entry) => entry.type !== "clear" && entry.key === mapOp.key);
+            const pendingEntry = this.pendingData[pendingEntryIndex];
+            (0, internal_1.assert)(pendingEntry !== undefined &&
+                (pendingEntry.type === "delete" || pendingEntry.type === "lifetime"), 0xbf3 /* Unexpected pending data for set/delete op */);
+            if (pendingEntry.type === "delete") {
+                (0, internal_1.assert)(pendingEntry === typedLocalOpMetadata, 0xbf4 /* Unexpected delete rollback */);
+                this.pendingData.splice(pendingEntryIndex, 1);
+                // Only emit if rolling back the delete actually results in a value becoming visible.
+                if (this.getOptimisticLocalValue(mapOp.key) !== undefined) {
+                    this.eventEmitter.emit("valueChanged", { key: mapOp.key, previousValue: undefined }, true, this.eventEmitter);
+                }
             }
-            // If we have an unack'd clear, we can ignore all ops.
-            return false;
-        }
-        const pendingKeyMessageIds = this.pendingKeys.get(op.key);
-        if (pendingKeyMessageIds !== undefined) {
-            // Found an unack'd op. Clear it from the map if the pendingMessageId in the map matches this message's
-            // and don't process the op.
-            if (local) {
-                (0, internal_1.assert)(localOpMetadata !== undefined && isMapKeyLocalOpMetadata(localOpMetadata), 0x014 /* pendingMessageId is missing from the local client's operation */);
-                (0, internal_1.assert)(pendingKeyMessageIds[0] === localOpMetadata.pendingMessageId, 0x2fa /* Unexpected pending message received */);
-                pendingKeyMessageIds.shift();
-                if (pendingKeyMessageIds.length === 0) {
-                    this.pendingKeys.delete(op.key);
+            else if (pendingEntry.type === "lifetime") {
+                const pendingKeySet = pendingEntry.keySets.pop();
+                (0, internal_1.assert)(pendingKeySet !== undefined && pendingKeySet === typedLocalOpMetadata, 0xbf5 /* Unexpected set rollback */);
+                if (pendingEntry.keySets.length === 0) {
+                    this.pendingData.splice(pendingEntryIndex, 1);
                 }
+                this.eventEmitter.emit("valueChanged", { key: mapOp.key, previousValue: pendingKeySet.value.value }, true, this.eventEmitter);
             }
-            return false;
         }
-        // If we don't have a NACK op on the key, we need to process the remote ops.
-        return !local;
     }
     /**
      * Get the message handlers for the map.
@@ -452,133 +487,81 @@ class MapKernel {
         const messageHandlers = new Map();
         messageHandlers.set("clear", {
             process: (op, local, localOpMetadata) => {
+                this.sequencedData.clear();
                 if (local) {
-                    const removedLocalOpMetadata = this.pendingMapLocalOpMetadata.shift();
-                    (0, internal_1.assert)(removedLocalOpMetadata !== undefined && removedLocalOpMetadata === localOpMetadata, 0xbcc /* Processing unexpected local clear op */);
-                    (0, internal_1.assert)(isClearLocalOpMetadata(localOpMetadata.data), 0x015 /* "pendingMessageId is missing from the local client's clear operation" */);
-                    const pendingClearMessageId = this.pendingClearMessageIds.shift();
-                    (0, internal_1.assert)(pendingClearMessageId === localOpMetadata.data.pendingMessageId, 0x2fb /* pendingMessageId does not match */);
-                    return;
+                    const pendingClear = this.pendingData.shift();
+                    (0, internal_1.assert)(pendingClear !== undefined &&
+                        pendingClear.type === "clear" &&
+                        pendingClear === localOpMetadata, 0xbf6 /* Got a local clear message we weren't expecting */);
                 }
-                if (this.pendingKeys.size > 0) {
-                    this.clearExceptPendingKeys();
-                    return;
+                else {
+                    // Only emit for remote ops, we would have already emitted for local ops. Only emit if there
+                    // is no optimistically-applied local pending clear that would supersede this remote clear.
+                    if (!this.pendingData.some((entry) => entry.type === "clear")) {
+                        this.eventEmitter.emit("clear", local, this.eventEmitter);
+                    }
                 }
-                this.clearCore(local);
             },
             resubmit: (op, localOpMetadata) => {
-                const removedLocalOpMetadata = localOpMetadata.remove()?.data;
-                (0, internal_1.assert)(removedLocalOpMetadata !== undefined, 0xbcd /* Resubmitting unexpected local clear op */);
-                (0, internal_1.assert)(isClearLocalOpMetadata(localOpMetadata.data), 0x2fc /* Invalid localOpMetadata for clear */);
-                // We don't reuse the metadata pendingMessageId but send a new one on each submit.
-                const pendingClearMessageId = this.pendingClearMessageIds.shift();
-                (0, internal_1.assert)(pendingClearMessageId === localOpMetadata.data.pendingMessageId, 0x2fd /* pendingMessageId does not match */);
-                this.submitMapClearMessage(op, localOpMetadata.data.previousMap);
+                this.submitMessage(op, localOpMetadata);
             },
         });
         messageHandlers.set("delete", {
             process: (op, local, localOpMetadata) => {
+                const { key } = op;
                 if (local) {
-                    const removedLocalOpMetadata = this.pendingMapLocalOpMetadata.shift();
-                    (0, internal_1.assert)(removedLocalOpMetadata !== undefined && removedLocalOpMetadata === localOpMetadata, 0xbce /* Processing unexpected local delete op */);
+                    const pendingEntryIndex = this.pendingData.findIndex((entry) => entry.type !== "clear" && entry.key === key);
+                    const pendingEntry = this.pendingData[pendingEntryIndex];
+                    (0, internal_1.assert)(pendingEntry !== undefined &&
+                        pendingEntry.type === "delete" &&
+                        pendingEntry === localOpMetadata, 0xbf7 /* Got a local delete message we weren't expecting */);
+                    this.pendingData.splice(pendingEntryIndex, 1);
+                    this.sequencedData.delete(key);
                 }
-                if (!this.needProcessKeyOperation(op, local, localOpMetadata?.data)) {
-                    return;
+                else {
+                    const previousValue = this.sequencedData.get(key)?.value;
+                    this.sequencedData.delete(key);
+                    // Suppress the event if local changes would cause the incoming change to be invisible optimistically.
+                    if (!this.pendingData.some((entry) => entry.type === "clear" || entry.key === key)) {
+                        this.eventEmitter.emit("valueChanged", { key, previousValue }, local, this.eventEmitter);
+                    }
                 }
-                this.deleteCore(op.key, local);
             },
             resubmit: (op, localOpMetadata) => {
-                const removedLocalOpMetadata = localOpMetadata.remove()?.data;
-                (0, internal_1.assert)(removedLocalOpMetadata !== undefined, 0xbcf /* Resubmitting unexpected local delete op */);
-                this.resubmitMapKeyMessage(op, localOpMetadata.data);
+                this.submitMessage(op, localOpMetadata);
             },
         });
         messageHandlers.set("set", {
             process: (op, local, localOpMetadata) => {
+                const { key, value } = op;
                 if (local) {
-                    const removedLocalOpMetadata = this.pendingMapLocalOpMetadata.shift();
-                    (0, internal_1.assert)(removedLocalOpMetadata !== undefined && removedLocalOpMetadata === localOpMetadata, 0xbd0 /* Processing unexpected local set op */);
+                    const pendingEntryIndex = this.pendingData.findIndex((entry) => entry.type !== "clear" && entry.key === key);
+                    const pendingEntry = this.pendingData[pendingEntryIndex];
+                    (0, internal_1.assert)(pendingEntry !== undefined && pendingEntry.type === "lifetime", 0xbf8 /* Couldn't match local set message to pending lifetime */);
+                    const pendingKeySet = pendingEntry.keySets.shift();
+                    (0, internal_1.assert)(pendingKeySet !== undefined && pendingKeySet === localOpMetadata, 0xbf9 /* Got a local set message we weren't expecting */);
+                    if (pendingEntry.keySets.length === 0) {
+                        this.pendingData.splice(pendingEntryIndex, 1);
+                    }
+                    this.sequencedData.set(key, pendingKeySet.value);
                 }
-                if (!this.needProcessKeyOperation(op, local, localOpMetadata?.data)) {
-                    return;
+                else {
+                    (0, localValues_js_1.migrateIfSharedSerializable)(value, this.serializer, this.handle);
+                    const localValue = { value: value.value };
+                    const previousValue = this.sequencedData.get(key)?.value;
+                    this.sequencedData.set(key, localValue);
+                    // Suppress the event if local changes would cause the incoming change to be invisible optimistically.
+                    if (!this.pendingData.some((entry) => entry.type === "clear" || entry.key === key)) {
+                        this.eventEmitter.emit("valueChanged", { key, previousValue }, local, this.eventEmitter);
+                    }
                 }
-                // needProcessKeyOperation should have returned false if local is true
-                (0, localValues_js_1.migrateIfSharedSerializable)(op.value, this.serializer, this.handle);
-                this.setCore(op.key, { value: op.value.value }, local);
             },
             resubmit: (op, localOpMetadata) => {
-                const removedLocalOpMetadata = localOpMetadata.remove()?.data;
-                (0, internal_1.assert)(removedLocalOpMetadata !== undefined, 0xbd1 /* Resubmitting unexpected local set op */);
-                this.resubmitMapKeyMessage(op, localOpMetadata.data);
+                this.submitMessage(op, localOpMetadata);
             },
         });
         return messageHandlers;
     }
-    getMapClearMessageId() {
-        const pendingMessageId = this.nextPendingMessageId++;
-        this.pendingClearMessageIds.push(pendingMessageId);
-        return pendingMessageId;
-    }
-    /**
-     * Submit a clear message to remote clients.
-     * @param op - The clear message
-     */
-    submitMapClearMessage(op, previousMap) {
-        const pendingMessageId = this.getMapClearMessageId();
-        const localMetadata = createClearLocalOpMetadata(op, pendingMessageId, previousMap);
-        const listNode = this.pendingMapLocalOpMetadata.push(localMetadata).first;
-        this.submitMessage(op, listNode);
-    }
-    getMapKeyMessageId(op) {
-        const pendingMessageId = this.nextPendingMessageId++;
-        const pendingMessageIds = this.pendingKeys.get(op.key);
-        if (pendingMessageIds === undefined) {
-            this.pendingKeys.set(op.key, [pendingMessageId]);
-        }
-        else {
-            pendingMessageIds.push(pendingMessageId);
-        }
-        return pendingMessageId;
-    }
-    /**
-     * Submit a map key message to remote clients.
-     * @param op - The map key message
-     * @param previousValue - The value of the key before this op
-     */
-    submitMapKeyMessage(op, previousValue) {
-        const pendingMessageId = this.getMapKeyMessageId(op);
-        const localMetadata = createKeyLocalOpMetadata(op, pendingMessageId, previousValue);
-        const listNode = this.pendingMapLocalOpMetadata.push(localMetadata).first;
-        this.submitMessage(op, listNode);
-    }
-    /**
-     * Submit a map key message to remote clients based on a previous submit.
-     * @param op - The map key message
-     * @param localOpMetadata - Metadata from the previous submit
-     */
-    resubmitMapKeyMessage(op, localOpMetadata) {
-        (0, internal_1.assert)(isMapKeyLocalOpMetadata(localOpMetadata), 0x2fe /* Invalid localOpMetadata in submit */);
-        // no need to submit messages for op's that have been aborted
-        const pendingMessageIds = this.pendingKeys.get(op.key);
-        if (pendingMessageIds === undefined) {
-            return;
-        }
-        const index = pendingMessageIds.indexOf(localOpMetadata.pendingMessageId);
-        if (index === -1) {
-            return;
-        }
-        pendingMessageIds.splice(index, 1);
-        if (pendingMessageIds.length === 0) {
-            this.pendingKeys.delete(op.key);
-        }
-        // We don't reuse the metadata pendingMessageId but send a new one on each submit.
-        const pendingMessageId = this.getMapKeyMessageId(op);
-        const localMetadata = localOpMetadata.type === "edit"
-            ? { type: "edit", pendingMessageId, previousValue: localOpMetadata.previousValue }
-            : { type: "add", pendingMessageId };
-        const listNode = this.pendingMapLocalOpMetadata.push(localMetadata).first;
-        this.submitMessage(op, listNode);
-    }
 }
 exports.MapKernel = MapKernel;
 //# sourceMappingURL=mapKernel.js.map