@fluidframework/map 2.52.0 → 2.53.0

package/lib/directory.js

@@ -13,6 +13,7 @@ import { SharedObject, ValueType, bindHandles, parseHandles, } from "@fluidframe
 import { UsageError, } from "@fluidframework/telemetry-utils/internal";
 import path from "path-browserify";
 import { serializeValue, migrateIfSharedSerializable } from "./localValues.js";
+import { findLast, findLastIndex } from "./utils.js";
 // We use path-browserify since this code can run safely on the server or the browser.
 // We standardize on using posix slashes everywhere.
 const posix = path.posix;
@@ -340,7 +341,7 @@ export class SharedDirectory extends SharedObject {
         const message = content;
         const handler = this.messageHandlers.get(message.type);
         assert(handler !== undefined, 0x00d /* Missing message handler for message type */);
-        handler.submit(message, localOpMetadata);
+        handler.resubmit(message, localOpMetadata);
     }
     /**
      * {@inheritDoc @fluidframework/shared-object-base#SharedObject.loadCore}
@@ -488,7 +489,7 @@ export class SharedDirectory extends SharedObject {
                     subdir.processClearMessage(msg, op, local, localOpMetadata);
                 }
             },
-            submit: (op, localOpMetadata) => {
+            resubmit: (op, localOpMetadata) => {
                 const subdir = this.getWorkingDirectory(op.path);
                 if (subdir) {
                     subdir.resubmitClearMessage(op, localOpMetadata);
@@ -504,7 +505,7 @@ export class SharedDirectory extends SharedObject {
                     subdir.processDeleteMessage(msg, op, local, localOpMetadata);
                 }
             },
-            submit: (op, localOpMetadata) => {
+            resubmit: (op, localOpMetadata) => {
                 const subdir = this.getWorkingDirectory(op.path);
                 if (subdir) {
                     subdir.resubmitKeyMessage(op, localOpMetadata);
@@ -522,7 +523,7 @@ export class SharedDirectory extends SharedObject {
                     subdir.processSetMessage(msg, op, localValue, local, localOpMetadata);
                 }
             },
-            submit: (op, localOpMetadata) => {
+            resubmit: (op, localOpMetadata) => {
                 const subdir = this.getWorkingDirectory(op.path);
                 if (subdir) {
                     subdir.resubmitKeyMessage(op, localOpMetadata);
@@ -538,7 +539,7 @@ export class SharedDirectory extends SharedObject {
                     parentSubdir.processCreateSubDirectoryMessage(msg, op, local, localOpMetadata);
                 }
             },
-            submit: (op, localOpMetadata) => {
+            resubmit: (op, localOpMetadata) => {
                 const parentSubdir = this.getWorkingDirectory(op.path);
                 if (parentSubdir) {
                     // We don't reuse the metadata but send a new one on each submit.
@@ -555,7 +556,7 @@ export class SharedDirectory extends SharedObject {
                     parentSubdir.processDeleteSubDirectoryMessage(msg, op, local, localOpMetadata);
                 }
             },
-            submit: (op, localOpMetadata) => {
+            resubmit: (op, localOpMetadata) => {
                 const parentSubdir = this.getWorkingDirectory(op.path);
                 if (parentSubdir) {
                     // We don't reuse the metadata but send a new one on each submit.
@@ -655,28 +656,6 @@ export class SharedDirectory extends SharedObject {
         return builder.getSummaryTree();
     }
 }
-/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access */
-function isKeyEditLocalOpMetadata(metadata) {
-    return (metadata !== undefined &&
-        typeof metadata.pendingMessageId === "number" &&
-        metadata.type === "edit");
-}
-function isClearLocalOpMetadata(metadata) {
-    return (metadata !== undefined &&
-        metadata.type === "clear" &&
-        typeof metadata.pendingMessageId === "number" &&
-        typeof metadata.previousStorage === "object");
-}
-function isSubDirLocalOpMetadata(metadata) {
-    return (metadata !== undefined &&
-        (metadata.type === "createSubDir" || metadata.type === "deleteSubDir"));
-}
-function isDirectoryLocalOpMetadata(metadata) {
-    return (isKeyEditLocalOpMetadata(metadata) ||
-        isClearLocalOpMetadata(metadata) ||
-        isSubDirLocalOpMetadata(metadata));
-}
-/* eslint-enable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access */
 // eslint-disable-next-line @rushstack/no-new-null
 function assertNonNullClientId(clientId) {
     assert(clientId !== null, 0x6af /* client id should never be null */);
@@ -713,21 +692,10 @@ class SubDirectory extends TypedEventEmitter {
          * String representation for the class.
          */
         this[_b] = "SubDirectory";
-        /**
-         * The in-memory data the directory is storing.
-         */
-        this._storage = new Map();
         /**
          * The subdirectories the directory is holding.
          */
         this._subdirectories = new Map();
-        /**
-         * Keys that have been modified locally but not yet ack'd from the server. This is for operations on keys like
-         * set/delete operations on keys. The value of this map is list of pendingMessageIds at which that key
-         * was modified. We don't store the type of ops, and behaviour of key ops are different from behaviour of sub
-         * directory ops, so we have separate map from subDirectories tracker.
-         */
-        this.pendingKeys = new Map();
         /**
          * Subdirectories that have been deleted locally but not yet ack'd from the server. This maintains the record
          * of delete op that are pending or yet to be acked from server. This is maintained just to track the locally
@@ -741,18 +709,111 @@ class SubDirectory extends TypedEventEmitter {
          */
         this.pendingCreateSubDirectoriesTracker = new Map();
         /**
-         * This is used to assign a unique id to every outgoing operation and helps in tracking unack'd ops.
+         * Assigns a unique ID to each subdirectory created locally but pending for acknowledgement, facilitating the tracking
+         * of the creation order.
          */
-        this.pendingMessageId = -1;
+        this.localCreationSeq = 0;
         /**
-         * The pending ids of any clears that have been performed locally but not yet ack'd from the server
+         * The data this SubDirectory instance is storing, but only including sequenced values (no local pending
+         * modifications are included).
          */
-        this.pendingClearMessageIds = [];
+        this.sequencedStorageData = new Map();
         /**
-         * Assigns a unique ID to each subdirectory created locally but pending for acknowledgement, facilitating the tracking
-         * of the creation order.
+         * A data structure containing all local pending storage modifications, which is used in combination
+         * with the sequencedStorageData to compute optimistic values.
+         *
+         * Pending sets are aggregated into "lifetimes", which permit correct relative iteration order
+         * even across remote operations and rollbacks.
          */
-        this.localCreationSeq = 0;
+        this.pendingStorageData = [];
+        /**
+         * An internal iterator that iterates over the entries in the directory.
+         */
+        this.internalIterator = () => {
+            // We perform iteration in two steps - first by iterating over members of the sequenced storage data that are not
+            // optimistically deleted or cleared, and then over the pending data lifetimes that have not subsequently
+            // been deleted or cleared.  In total, this give an ordering of members based on when they were initially
+            // added to the sub directory (even if they were later modified), similar to the native Map.
+            const sequencedStorageDataIterator = this.sequencedStorageData.keys();
+            const pendingStorageDataIterator = this.pendingStorageData.values();
+            const next = () => {
+                let nextSequencedKey = sequencedStorageDataIterator.next();
+                while (!nextSequencedKey.done) {
+                    const key = nextSequencedKey.value;
+                    // If we have any pending deletes or clears, then we won't iterate to this key yet (if at all).
+                    // Either it is optimistically deleted and will not be part of the iteration, or it was
+                    // re-added later and we'll iterate to it when we get to the pending data.
+                    if (!this.pendingStorageData.some((entry) => entry.type === "clear" || (entry.type === "delete" && entry.key === key))) {
+                        assert(this.has(key), 0xc03 /* key should exist in sequenced or pending data */);
+                        const optimisticValue = this.getOptimisticValue(key);
+                        return { value: [key, optimisticValue], done: false };
+                    }
+                    nextSequencedKey = sequencedStorageDataIterator.next();
+                }
+                let nextPending = pendingStorageDataIterator.next();
+                while (!nextPending.done) {
+                    const nextPendingEntry = nextPending.value;
+                    // A lifetime entry may need to be iterated.
+                    if (nextPendingEntry.type === "lifetime") {
+                        const nextPendingEntryIndex = this.pendingStorageData.indexOf(nextPendingEntry);
+                        const mostRecentDeleteOrClearIndex = findLastIndex(this.pendingStorageData, (entry) => entry.type === "clear" ||
+                            (entry.type === "delete" && entry.key === nextPendingEntry.key));
+                        // Only iterate the pending entry now if it hasn't been deleted or cleared.
+                        if (nextPendingEntryIndex > mostRecentDeleteOrClearIndex) {
+                            const latestPendingValue = 
+                            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                            nextPendingEntry.keySets[nextPendingEntry.keySets.length - 1];
+                            // Skip iterating if we would have would have already iterated it as part of the sequenced data.
+                            // This is not a perfect check in the case the map has changed since the iterator was created
+                            // (e.g. if a remote client added the same key in the meantime).
+                            if (!this.sequencedStorageData.has(nextPendingEntry.key) ||
+                                mostRecentDeleteOrClearIndex !== -1) {
+                                return { value: [nextPendingEntry.key, latestPendingValue.value], done: false };
+                            }
+                        }
+                    }
+                    nextPending = pendingStorageDataIterator.next();
+                }
+                return { value: undefined, done: true };
+            };
+            const iterator = {
+                next,
+                [Symbol.iterator]() {
+                    return this;
+                },
+            };
+            return iterator;
+        };
+        /**
+         * Compute the optimistic local value for a given key. This combines the sequenced data with
+         * any pending changes that have not yet been sequenced.
+         */
+        this.getOptimisticValue = (key) => {
+            const latestPendingEntry = findLast(this.pendingStorageData, (entry) => entry.type === "clear" || entry.key === key);
+            if (latestPendingEntry === undefined) {
+                return this.sequencedStorageData.get(key);
+            }
+            else if (latestPendingEntry.type === "lifetime") {
+                const latestPendingSet = 
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                latestPendingEntry.keySets[latestPendingEntry.keySets.length - 1];
+                return latestPendingSet.value;
+            }
+            else {
+                // Delete or clear
+                return undefined;
+            }
+        };
+        /**
+         * Determine if the directory optimistically has the key.
+         * This will return true even if the value is undefined.
+         */
+        this.optimisticallyHas = (key) => {
+            const latestPendingEntry = findLast(this.pendingStorageData, (entry) => entry.type === "clear" || entry.key === key);
+            return latestPendingEntry === undefined
+                ? this.sequencedStorageData.has(key)
+                : latestPendingEntry.type === "lifetime";
+        };
         this.localCreationSeqTracker = new DirectoryCreationTracker();
         this.ackedCreationSeqTracker = new DirectoryCreationTracker();
     }
@@ -782,14 +843,13 @@ class SubDirectory extends TypedEventEmitter {
      */
     has(key) {
         this.throwIfDisposed();
-        return this._storage.has(key);
+        return this.optimisticallyHas(key);
     }
     /**
      * {@inheritDoc IDirectory.get}
      */
     get(key) {
-        this.throwIfDisposed();
-        return this._storage.get(key);
+        return this.getOptimisticValue(key);
     }
     /**
      * {@inheritDoc IDirectory.set}
@@ -800,21 +860,60 @@ class SubDirectory extends TypedEventEmitter {
         if (key === undefined || key === null) {
             throw new Error("Undefined and null keys are not supported");
         }
+        const previousOptimisticLocalValue = this.getOptimisticValue(key);
         // Create a local value and serialize it.
         bindHandles(value, this.serializer, this.directory.handle);
-        // Set the value locally.
-        const previousValue = this.setCore(key, value, true);
         // If we are not attached, don't submit the op.
         if (!this.directory.isAttached()) {
+            this.sequencedStorageData.set(key, value);
+            const event = {
+                key,
+                path: this.absolutePath,
+                previousValue: previousOptimisticLocalValue,
+            };
+            this.directory.emit("valueChanged", event, true, this.directory);
+            const containedEvent = {
+                key,
+                previousValue: previousOptimisticLocalValue,
+            };
+            this.emit("containedValueChanged", containedEvent, true, this);
             return this;
         }
+        // A new pending key lifetime is created if:
+        // 1. There isn't any pending entry for the key yet
+        // 2. The most recent pending entry for the key was a deletion (as this terminates the prior lifetime)
+        // 3. A clear was sent after the last pending entry for the key (which also terminates the prior lifetime)
+        let latestPendingEntry = findLast(this.pendingStorageData, (entry) => entry.type === "clear" || entry.key === key);
+        if (latestPendingEntry === undefined ||
+            latestPendingEntry.type === "delete" ||
+            latestPendingEntry.type === "clear") {
+            latestPendingEntry = { type: "lifetime", path: this.absolutePath, key, keySets: [] };
+            this.pendingStorageData.push(latestPendingEntry);
+        }
+        const pendingKeySet = {
+            type: "set",
+            path: this.absolutePath,
+            value,
+        };
+        latestPendingEntry.keySets.push(pendingKeySet);
         const op = {
             key,
             path: this.absolutePath,
             type: "set",
             value: { type: ValueType[ValueType.Plain], value },
         };
-        this.submitKeyMessage(op, previousValue);
+        this.submitKeyMessage(op, pendingKeySet);
+        const directoryValueChanged = {
+            key,
+            path: this.absolutePath,
+            previousValue: previousOptimisticLocalValue,
+        };
+        this.directory.emit("valueChanged", directoryValueChanged, true, this.directory);
+        const valueChanged = {
+            key,
+            previousValue: previousOptimisticLocalValue,
+        };
+        this.emit("containedValueChanged", valueChanged, true, this);
         return this;
     }
     /**
@@ -976,37 +1075,76 @@ class SubDirectory extends TypedEventEmitter {
      */
     delete(key) {
         this.throwIfDisposed();
-        // Delete the key locally first.
-        const previousValue = this.deleteCore(key, true);
-        // If we are not attached, don't submit the op.
+        const previousOptimisticLocalValue = this.getOptimisticValue(key);
         if (!this.directory.isAttached()) {
-            return previousValue !== undefined;
+            const successfullyRemoved = this.sequencedStorageData.delete(key);
+            // Only emit if we actually deleted something.
+            if (previousOptimisticLocalValue !== undefined && successfullyRemoved) {
+                const event = {
+                    key,
+                    path: this.absolutePath,
+                    previousValue: previousOptimisticLocalValue,
+                };
+                this.directory.emit("valueChanged", event, true, this.directory);
+                const containedEvent = {
+                    key,
+                    previousValue: previousOptimisticLocalValue,
+                };
+                this.emit("containedValueChanged", containedEvent, true, this);
+            }
+            return successfullyRemoved;
         }
+        const pendingKeyDelete = {
+            type: "delete",
+            path: this.absolutePath,
+            key,
+        };
+        this.pendingStorageData.push(pendingKeyDelete);
         const op = {
             key,
-            path: this.absolutePath,
             type: "delete",
+            path: this.absolutePath,
         };
-        this.submitKeyMessage(op, previousValue);
-        return previousValue !== undefined;
+        this.submitKeyMessage(op, pendingKeyDelete);
+        // Only emit if we locally believe we deleted something.  Otherwise we still send the op
+        // (permitting speculative deletion even if we don't see anything locally) but don't emit
+        // a valueChanged since we in fact did not locally observe a value change.
+        if (previousOptimisticLocalValue !== undefined) {
+            const event = {
+                key,
+                path: this.absolutePath,
+                previousValue: previousOptimisticLocalValue,
+            };
+            this.directory.emit("valueChanged", event, true, this.directory);
+            const containedEvent = {
+                key,
+                previousValue: previousOptimisticLocalValue,
+            };
+            this.emit("containedValueChanged", containedEvent, true, this);
+        }
+        return true;
     }
     /**
      * Deletes all keys from within this IDirectory.
      */
     clear() {
         this.throwIfDisposed();
-        // If we are not attached, don't submit the op.
         if (!this.directory.isAttached()) {
-            this.clearCore(true);
+            this.sequencedStorageData.clear();
+            this.directory.emit("clear", true, this.directory);
             return;
         }
-        const copy = new Map(this._storage);
-        this.clearCore(true);
-        const op = {
+        const pendingClear = {
+            type: "clear",
             path: this.absolutePath,
+        };
+        this.pendingStorageData.push(pendingClear);
+        this.directory.emit("clear", true, this.directory);
+        const op = {
             type: "clear",
+            path: this.absolutePath,
         };
-        this.submitClearMessage(op, copy);
+        this.submitClearMessage(op, pendingClear);
     }
     /**
      * Issue a callback on each entry under this IDirectory.
@@ -1014,17 +1152,16 @@ class SubDirectory extends TypedEventEmitter {
      */
     forEach(callback) {
         this.throwIfDisposed();
-        // eslint-disable-next-line unicorn/no-array-for-each
-        this._storage.forEach((value, key) => {
-            callback(value, key, this);
-        });
+        for (const [key, localValue] of this.internalIterator()) {
+            callback(localValue.value, key, this);
+        }
     }
     /**
      * The number of entries under this IDirectory.
      */
     get size() {
         this.throwIfDisposed();
-        return this._storage.size;
+        return [...this.internalIterator()].length;
     }
     /**
      * Get an iterator over the entries under this IDirectory.
@@ -1032,7 +1169,23 @@ class SubDirectory extends TypedEventEmitter {
      */
     entries() {
         this.throwIfDisposed();
-        return this._storage.entries();
+        const internalIterator = this.internalIterator();
+        const next = () => {
+            const nextResult = internalIterator.next();
+            if (nextResult.done) {
+                return { value: undefined, done: true };
+            }
+            // Unpack the stored value
+            const [key, localValue] = nextResult.value;
+            return { value: [key, localValue], done: false };
+        };
+        const iterator = {
+            next,
+            [Symbol.iterator]() {
+                return this;
+            },
+        };
+        return iterator;
     }
     /**
      * Get an iterator over the keys under this IDirectory.
@@ -1040,7 +1193,22 @@ class SubDirectory extends TypedEventEmitter {
      */
     keys() {
         this.throwIfDisposed();
-        return this._storage.keys();
+        const internalIterator = this.internalIterator();
+        const next = () => {
+            const nextResult = internalIterator.next();
+            if (nextResult.done) {
+                return { value: undefined, done: true };
+            }
+            const [key] = nextResult.value;
+            return { value: key, done: false };
+        };
+        const iterator = {
+            next,
+            [Symbol.iterator]() {
+                return this;
+            },
+        };
+        return iterator;
     }
     /**
      * Get an iterator over the values under this IDirectory.
@@ -1048,7 +1216,22 @@ class SubDirectory extends TypedEventEmitter {
      */
     values() {
         this.throwIfDisposed();
-        return this._storage.values();
+        const internalIterator = this.internalIterator();
+        const next = () => {
+            const nextResult = internalIterator.next();
+            if (nextResult.done) {
+                return { value: undefined, done: true };
+            }
+            const [, localValue] = nextResult.value;
+            return { value: localValue, done: false };
+        };
+        const iterator = {
+            next,
+            [Symbol.iterator]() {
+                return this;
+            },
+        };
+        return iterator;
     }
     /**
      * Get an iterator over the entries under this IDirectory.
@@ -1056,7 +1239,7 @@ class SubDirectory extends TypedEventEmitter {
      */
     [(_b = Symbol.toStringTag, Symbol.iterator)]() {
         this.throwIfDisposed();
-        return this.entries();
+        return this.internalIterator();
     }
     /**
      * Process a clear operation.
@@ -1072,12 +1255,35 @@ class SubDirectory extends TypedEventEmitter {
             return;
         }
         if (local) {
-            assert(isClearLocalOpMetadata(localOpMetadata), 0x00f /* pendingMessageId is missing from the local client's operation */);
-            const pendingClearMessageId = this.pendingClearMessageIds.shift();
-            assert(pendingClearMessageId === localOpMetadata.pendingMessageId, 0x32a /* pendingMessageId does not match */);
-            return;
+            this.sequencedStorageData.clear();
+            const pendingClear = this.pendingStorageData.shift();
+            assert(pendingClear !== undefined &&
+                pendingClear.type === "clear" &&
+                pendingClear === localOpMetadata, 0xc04 /* Got a local clear message we weren't expecting */);
+        }
+        else {
+            // For pending set operations, collect the previous values before clearing sequenced data
+            const pendingSets = [];
+            for (const entry of this.pendingStorageData) {
+                if (entry.type === "lifetime") {
+                    const previousValue = this.sequencedStorageData.get(entry.key);
+                    pendingSets.push({ key: entry.key, previousValue });
+                }
+            }
+            this.sequencedStorageData.clear();
+            // Only emit for remote ops, we would have already emitted for local ops. Only emit if there
+            // is no optimistically-applied local pending clear that would supersede this remote clear.
+            if (!this.pendingStorageData.some((entry) => entry.type === "clear")) {
+                this.directory.emit("clear", local, this.directory);
+            }
+            // For pending set operations, emit valueChanged events
+            for (const { key, previousValue } of pendingSets) {
+                this.directory.emit("valueChanged", {
+                    key,
+                    previousValue,
+                }, local, this.directory);
+            }
         }
-        this.clearExceptPendingKeys(false);
     }
     /**
      * Process a delete operation.
@@ -1089,11 +1295,33 @@ class SubDirectory extends TypedEventEmitter {
      */
     processDeleteMessage(msg, op, local, localOpMetadata) {
         this.throwIfDisposed();
-        if (!(this.isMessageForCurrentInstanceOfSubDirectory(msg) &&
-            this.needProcessStorageOperation(op, local, localOpMetadata))) {
+        if (!this.isMessageForCurrentInstanceOfSubDirectory(msg)) {
             return;
         }
-        this.deleteCore(op.key, local);
+        if (local) {
+            const pendingEntryIndex = this.pendingStorageData.findIndex((entry) => entry.type !== "clear" && entry.key === op.key);
+            const pendingEntry = this.pendingStorageData[pendingEntryIndex];
+            assert(pendingEntry !== undefined &&
+                pendingEntry.type === "delete" &&
+                pendingEntry.key === op.key, 0xc05 /* Got a local delete message we weren't expecting */);
+            this.pendingStorageData.splice(pendingEntryIndex, 1);
+            this.sequencedStorageData.delete(op.key);
+        }
+        else {
+            const previousValue = this.sequencedStorageData.get(op.key);
+            this.sequencedStorageData.delete(op.key);
+            // Suppress the event if local changes would cause the incoming change to be invisible optimistically.
+            if (!this.pendingStorageData.some((entry) => entry.type === "clear" || entry.key === op.key)) {
+                const event = {
+                    key: op.key,
+                    path: this.absolutePath,
+                    previousValue,
+                };
+                this.directory.emit("valueChanged", event, local, this.directory);
+                const containedEvent = { key: op.key, previousValue };
+                this.emit("containedValueChanged", containedEvent, local, this);
+            }
+        }
     }
     /**
      * Process a set operation.
@@ -1105,13 +1333,33 @@ class SubDirectory extends TypedEventEmitter {
      */
     processSetMessage(msg, op, value, local, localOpMetadata) {
         this.throwIfDisposed();
-        if (!(this.isMessageForCurrentInstanceOfSubDirectory(msg) &&
-            this.needProcessStorageOperation(op, local, localOpMetadata))) {
+        if (!this.isMessageForCurrentInstanceOfSubDirectory(msg)) {
             return;
         }
-        // needProcessStorageOperation should have returned false if local is true
-        // so we can assume localValue is not undefined
-        this.setCore(op.key, value, local);
+        const { key } = op;
+        if (local) {
+            const pendingEntryIndex = this.pendingStorageData.findIndex((entry) => entry.type !== "clear" && entry.key === key);
+            const pendingEntry = this.pendingStorageData[pendingEntryIndex];
+            assert(pendingEntry !== undefined && pendingEntry.type === "lifetime", 0xc06 /* Couldn't match local set message to pending lifetime */);
+            const pendingKeySet = pendingEntry.keySets.shift();
+            assert(pendingKeySet !== undefined && pendingKeySet === localOpMetadata, 0xc07 /* Got a local set message we weren't expecting */);
+            if (pendingEntry.keySets.length === 0) {
+                this.pendingStorageData.splice(pendingEntryIndex, 1);
+            }
+            this.sequencedStorageData.set(key, pendingKeySet.value);
+        }
+        else {
+            // Get the previous value before setting the new value
+            const previousValue = this.sequencedStorageData.get(key);
+            this.sequencedStorageData.set(key, value);
+            // Suppress the event if local changes would cause the incoming change to be invisible optimistically.
+            if (!this.pendingStorageData.some((entry) => entry.type === "clear" || entry.key === key)) {
+                const event = { key, path: this.absolutePath, previousValue };
+                this.directory.emit("valueChanged", event, local, this.directory);
+                const containedEvent = { key, previousValue };
+                this.emit("containedValueChanged", containedEvent, local, this);
+            }
+        }
     }
     /**
      * Process a create subdirectory operation.
@@ -1149,57 +1397,33 @@ class SubDirectory extends TypedEventEmitter {
     /**
      * Submit a clear operation.
      * @param op - The operation
+     * @param localOpMetadata - The pending operation metadata
      */
-    submitClearMessage(op, previousValue) {
+    submitClearMessage(op, localOpMetadata) {
         this.throwIfDisposed();
-        const pendingMsgId = ++this.pendingMessageId;
-        this.pendingClearMessageIds.push(pendingMsgId);
-        const metadata = {
-            type: "clear",
-            pendingMessageId: pendingMsgId,
-            previousStorage: previousValue,
-        };
-        this.directory.submitDirectoryMessage(op, metadata);
+        this.directory.submitDirectoryMessage(op, localOpMetadata);
     }
     /**
      * Resubmit a clear operation.
      * @param op - The operation
      */
     resubmitClearMessage(op, localOpMetadata) {
-        assert(isClearLocalOpMetadata(localOpMetadata), 0x32b /* Invalid localOpMetadata for clear */);
-        // We don't reuse the metadata pendingMessageId but send a new one on each submit.
-        const pendingClearMessageId = this.pendingClearMessageIds.shift();
         // Only submit the op, if we have record for it, otherwise it is possible that the older instance
         // is already deleted, in which case we don't need to submit the op.
-        if (pendingClearMessageId === localOpMetadata.pendingMessageId) {
-            this.submitClearMessage(op, localOpMetadata.previousStorage);
+        const pendingEntryIndex = this.pendingStorageData.findIndex((entry) => entry.type === "clear");
+        const pendingEntry = this.pendingStorageData[pendingEntryIndex];
+        if (pendingEntry !== undefined) {
+            this.submitClearMessage(op, localOpMetadata);
         }
     }
-    /**
-     * Get a new pending message id for the op and cache it to track the pending op
-     */
-    getKeyMessageId(op) {
-        // We don't reuse the metadata pendingMessageId but send a new one on each submit.
-        const pendingMessageId = ++this.pendingMessageId;
-        const pendingMessageIds = this.pendingKeys.get(op.key);
-        if (pendingMessageIds === undefined) {
-            this.pendingKeys.set(op.key, [pendingMessageId]);
-        }
-        else {
-            pendingMessageIds.push(pendingMessageId);
-        }
-        return pendingMessageId;
-    }
     /**
      * Submit a key operation.
      * @param op - The operation
-     * @param previousValue - The value of the key before this op
+     * @param localOpMetadata - The pending operation metadata
      */
-    submitKeyMessage(op, previousValue) {
+    submitKeyMessage(op, localOpMetadata) {
         this.throwIfDisposed();
-        const pendingMessageId = this.getKeyMessageId(op);
-        const localMetadata = { type: "edit", pendingMessageId, previousValue };
-        this.directory.submitDirectoryMessage(op, localMetadata);
+        this.directory.submitDirectoryMessage(op, localOpMetadata);
     }
     /**
      * Submit a key message to remote clients based on a previous submit.
@@ -1207,21 +1431,12 @@ class SubDirectory extends TypedEventEmitter {
      * @param localOpMetadata - Metadata from the previous submit
      */
     resubmitKeyMessage(op, localOpMetadata) {
-        assert(isKeyEditLocalOpMetadata(localOpMetadata), 0x32d /* Invalid localOpMetadata in submit */);
-        // clear the old pending message id
-        const pendingMessageIds = this.pendingKeys.get(op.key);
         // Only submit the op, if we have record for it, otherwise it is possible that the older instance
         // is already deleted, in which case we don't need to submit the op.
-        if (pendingMessageIds !== undefined) {
-            const index = pendingMessageIds.indexOf(localOpMetadata.pendingMessageId);
-            if (index === -1) {
-                return;
-            }
-            pendingMessageIds.splice(index, 1);
-            if (pendingMessageIds.length === 0) {
-                this.pendingKeys.delete(op.key);
-            }
-            this.submitKeyMessage(op, localOpMetadata.previousValue);
+        const pendingEntryIndex = this.pendingStorageData.findIndex((entry) => entry.type !== "clear" && entry.key === op.key);
+        const pendingEntry = this.pendingStorageData[pendingEntryIndex];
+        if (pendingEntry !== undefined) {
+            this.submitKeyMessage(op, localOpMetadata);
         }
     }
     incrementPendingSubDirCount(map, subDirName) {
@@ -1279,7 +1494,6 @@ class SubDirectory extends TypedEventEmitter {
      * @param localOpMetadata - metadata submitted with the op originally
      */
     resubmitSubDirectoryMessage(op, localOpMetadata) {
-        assert(isSubDirLocalOpMetadata(localOpMetadata), 0x32f /* Invalid localOpMetadata for sub directory op */);
         // Only submit the op, if we have record for it, otherwise it is possible that the older instance
         // is already deleted, in which case we don't need to submit the op.
         if (localOpMetadata.type === "createSubDir" &&
@@ -1294,8 +1508,9 @@ class SubDirectory extends TypedEventEmitter {
             this.decrementPendingSubDirCount(this.pendingCreateSubDirectoriesTracker, op.subdirName);
             this.submitCreateSubDirectoryMessage(op);
         }
-        else {
+        else if (localOpMetadata.type === "deleteSubDir") {
             this.decrementPendingSubDirCount(this.pendingDeleteSubDirectoriesTracker, op.subdirName);
+            assert(localOpMetadata.subDirectory !== undefined, 0xc08 /* localOpMetadata.subDirectory should be defined */);
             this.submitDeleteSubDirectoryMessage(op, localOpMetadata.subDirectory);
         }
     }
@@ -1306,7 +1521,7 @@ class SubDirectory extends TypedEventEmitter {
      */
     *getSerializedStorage(serializer) {
         this.throwIfDisposed();
-        for (const [key, value] of this._storage) {
+        for (const [key, value] of this.sequencedStorageData.entries()) {
             const serializedValue = serializeValue(value, serializer, this.directory.handle);
             const res = [key, serializedValue];
             yield res;
@@ -1327,7 +1542,7 @@ class SubDirectory extends TypedEventEmitter {
      */
     populateStorage(key, value) {
         this.throwIfDisposed();
-        this._storage.set(key, value);
+        this.sequencedStorageData.set(key, value);
     }
     /**
      * Populate a subdirectory into this subdirectory, to be used when loading from snapshot.
@@ -1339,32 +1554,6 @@ class SubDirectory extends TypedEventEmitter {
         this.registerEventsOnSubDirectory(newSubDir, subdirName);
         this._subdirectories.set(subdirName, newSubDir);
     }
-    /**
-     * Retrieve the local value at the given key.  This is used to get value type information stashed on the local
-     * value so op handlers can be retrieved
-     * @param key - The key to retrieve from
-     * @returns The local value
-     */
-    getLocalValue(key) {
-        this.throwIfDisposed();
-        return this._storage.get(key);
-    }
-    /**
-     * Remove the pendingMessageId from the map tracking it on rollback
-     * @param map - map tracking the pending messages
-     * @param key - key of the edit in the op
-     */
-    rollbackPendingMessageId(map, key, pendingMessageId) {
-        const pendingMessageIds = map.get(key);
-        const lastPendingMessageId = pendingMessageIds?.pop();
-        if (!pendingMessageIds || lastPendingMessageId !== pendingMessageId) {
-            throw new Error("Rollback op does not match last pending");
-        }
-        if (pendingMessageIds.length === 0) {
-            map.delete(key);
-        }
-    }
-    /* eslint-disable @typescript-eslint/no-unsafe-member-access */
     /**
      * Rollback a local op
      * @param op - The operation to rollback
@@ -1372,41 +1561,80 @@ class SubDirectory extends TypedEventEmitter {
      */
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     rollback(op, localOpMetadata) {
-        if (!isDirectoryLocalOpMetadata(localOpMetadata)) {
-            throw new Error("Invalid localOpMetadata");
-        }
-        if (op.type === "clear" && localOpMetadata.type === "clear") {
-            for (const [key, localValue] of localOpMetadata.previousStorage.entries()) {
-                this.setCore(key, localValue, true);
-            }
-            const lastPendingClearId = this.pendingClearMessageIds.pop();
-            if (lastPendingClearId === undefined ||
-                lastPendingClearId !== localOpMetadata.pendingMessageId) {
-                throw new Error("Rollback op does match last clear");
+        const directoryOp = op;
+        if (directoryOp.type === "clear") {
+            // A pending clear will be last in the list, since it terminates all prior lifetimes.
+            const pendingClear = this.pendingStorageData.pop();
+            assert(pendingClear !== undefined &&
+                pendingClear.type === "clear" &&
+                localOpMetadata.type === "clear", 0xc09 /* Unexpected clear rollback */);
+            for (const [key] of this.internalIterator()) {
+                const event = {
+                    key,
+                    path: this.absolutePath,
+                    previousValue: undefined,
+                };
+                this.directory.emit("valueChanged", event, true, this.directory);
+                const containedEvent = { key, previousValue: undefined };
+                this.emit("containedValueChanged", containedEvent, true, this);
             }
         }
-        else if ((op.type === "delete" || op.type === "set") &&
-            localOpMetadata.type === "edit") {
-            const key = op.key;
-            assert(key !== undefined, 0x8ad /* "key" property is missing from edit operation. */);
-            assert(typeof key === "string", 0x8ae /* "key" property in edit operation is misconfigured. Expected a string. */);
-            if (localOpMetadata.previousValue === undefined) {
-                this.deleteCore(key, true);
+        else if ((directoryOp.type === "delete" || directoryOp.type === "set") &&
+            (localOpMetadata.type === "set" || localOpMetadata.type === "delete")) {
+            // A pending set/delete may not be last in the list, as the lifetimes' order is based on when
+            // they were created, not when they were last modified.
+            const pendingEntryIndex = findLastIndex(this.pendingStorageData, (entry) => entry.type !== "clear" && entry.key === directoryOp.key);
+            const pendingEntry = this.pendingStorageData[pendingEntryIndex];
+            assert(pendingEntry !== undefined &&
+                (pendingEntry.type === "delete" || pendingEntry.type === "lifetime"), 0xc0a /* Unexpected pending data for set/delete op */);
+            if (pendingEntry.type === "delete") {
+                assert(pendingEntry === localOpMetadata, 0xc0b /* Unexpected delete rollback */);
+                this.pendingStorageData.splice(pendingEntryIndex, 1);
+                // Only emit if rolling back the delete actually results in a value becoming visible.
+                if (this.getOptimisticValue(directoryOp.key) !== undefined) {
+                    const event = {
+                        key: directoryOp.key,
+                        path: this.absolutePath,
+                        previousValue: undefined,
+                    };
+                    this.directory.emit("valueChanged", event, true, this.directory);
+                    const containedEvent = {
+                        key: directoryOp.key,
+                        previousValue: undefined,
+                    };
+                    this.emit("containedValueChanged", containedEvent, true, this);
+                }
             }
-            else {
-                this.setCore(key, localOpMetadata.previousValue, true);
+            else if (pendingEntry.type === "lifetime") {
+                const pendingKeySet = pendingEntry.keySets.pop();
+                assert(pendingKeySet !== undefined && pendingKeySet === localOpMetadata, 0xc0c /* Unexpected set rollback */);
+                if (pendingEntry.keySets.length === 0) {
+                    this.pendingStorageData.splice(pendingEntryIndex, 1);
+                }
+                const event = {
+                    key: directoryOp.key,
+                    path: this.absolutePath,
+                    previousValue: pendingKeySet.value,
+                };
+                this.directory.emit("valueChanged", event, true, this.directory);
+                const containedEvent = {
+                    key: directoryOp.key,
+                    previousValue: pendingKeySet.value,
+                };
+                this.emit("containedValueChanged", containedEvent, true, this);
             }
-            this.rollbackPendingMessageId(this.pendingKeys, key, localOpMetadata.pendingMessageId);
         }
-        else if (op.type === "createSubDirectory" && localOpMetadata.type === "createSubDir") {
-            const subdirName = op.subdirName;
+        else if (directoryOp.type === "createSubDirectory" &&
+            localOpMetadata.type === "createSubDir") {
+            const subdirName = directoryOp.subdirName;
             assert(subdirName !== undefined, 0x8af /* "subdirName" property is missing from "createSubDirectory" operation. */);
             assert(typeof subdirName === "string", 0x8b0 /* "subdirName" property in "createSubDirectory" operation is misconfigured. Expected a string. */);
             this.deleteSubDirectoryCore(subdirName, true);
             this.decrementPendingSubDirCount(this.pendingCreateSubDirectoriesTracker, subdirName);
         }
-        else if (op.type === "deleteSubDirectory" && localOpMetadata.type === "deleteSubDir") {
-            const subdirName = op.subdirName;
+        else if (directoryOp.type === "deleteSubDirectory" &&
+            localOpMetadata.type === "deleteSubDir") {
+            const subdirName = directoryOp.subdirName;
             assert(subdirName !== undefined, 0x8b1 /* "subdirName" property is missing from "deleteSubDirectory" operation. */);
             assert(typeof subdirName === "string", 0x8b2 /* "subdirName" property in "deleteSubDirectory" operation is misconfigured. Expected a string. */);
             if (localOpMetadata.subDirectory !== undefined) {
@@ -1432,7 +1660,6 @@ class SubDirectory extends TypedEventEmitter {
             throw new Error("Unsupported op for rollback");
         }
     }
-    /* eslint-enable @typescript-eslint/no-unsafe-member-access */
     /**
      * Converts the given relative path into an absolute path.
      * @param path - Relative path to convert
@@ -1441,71 +1668,6 @@ class SubDirectory extends TypedEventEmitter {
     makeAbsolute(relativePath) {
         return posix.resolve(this.absolutePath, relativePath);
     }
-    /**
-     * If our local operations that have not yet been ack'd will eventually overwrite an incoming operation, we should
-     * not process the incoming operation.
-     * @param op - Operation to check
-     * @param local - Whether the operation originated from the local client
-     * @param localOpMetadata - For local client ops, this is the metadata that was submitted with the op.
-     * For ops from a remote client, this will be undefined.
-     * @returns True if the operation should be processed, false otherwise
-     */
-    needProcessStorageOperation(op, local, localOpMetadata) {
-        const firstPendingClearMessageId = this.pendingClearMessageIds[0];
-        if (firstPendingClearMessageId !== undefined) {
-            if (local) {
-                assert(localOpMetadata !== undefined &&
-                    isKeyEditLocalOpMetadata(localOpMetadata) &&
-                    localOpMetadata.pendingMessageId < firstPendingClearMessageId, 0x010 /* "Received out of order storage op when there is an unackd clear message" */);
-                // Remove all pendingMessageIds lower than first pendingClearMessageId.
-                const lowestPendingClearMessageId = firstPendingClearMessageId;
-                const pendingKeyMessageIdArray = this.pendingKeys.get(op.key);
-                if (pendingKeyMessageIdArray !== undefined) {
-                    let index = 0;
-                    let pendingKeyMessageId = pendingKeyMessageIdArray[index];
-                    while (pendingKeyMessageId !== undefined &&
-                        pendingKeyMessageId < lowestPendingClearMessageId) {
-                        index += 1;
-                        pendingKeyMessageId = pendingKeyMessageIdArray[index];
-                    }
-                    const newPendingKeyMessageId = pendingKeyMessageIdArray.splice(index);
-                    if (newPendingKeyMessageId.length === 0) {
-                        this.pendingKeys.delete(op.key);
-                    }
-                    else {
-                        this.pendingKeys.set(op.key, newPendingKeyMessageId);
-                    }
-                }
-            }
-            // If I have a NACK clear, we can ignore all ops.
-            return false;
-        }
-        const pendingKeyMessageIds = this.pendingKeys.get(op.key);
-        if (pendingKeyMessageIds !== undefined) {
-            // Found an NACK op, clear it from the directory if the latest sequence number in the directory
-            // match the message's and don't process the op.
-            if (local) {
-                assert(localOpMetadata !== undefined && isKeyEditLocalOpMetadata(localOpMetadata), 0x011 /* pendingMessageId is missing from the local client's operation */);
-                if (pendingKeyMessageIds[0] !== localOpMetadata.pendingMessageId) {
-                    // TODO: AB#7742: Hitting this block indicates that the pending message Id received
-                    // is not consistent with the "next" local op
-                    this.logger.sendTelemetryEvent({
-                        eventName: "unexpectedPendingMessage",
-                        expectedPendingMessage: pendingKeyMessageIds[0],
-                        actualPendingMessage: localOpMetadata.pendingMessageId,
-                        expectedPendingMessagesLength: pendingKeyMessageIds.length,
-                    });
-                }
-                pendingKeyMessageIds.shift();
-                if (pendingKeyMessageIds.length === 0) {
-                    this.pendingKeys.delete(op.key);
-                }
-            }
-            return false;
-        }
-        // If we don't have a NACK op on the key, we need to process the remote ops.
-        return !local;
-    }
     /**
      * This return true if the message is for the current instance of this sub directory. As the sub directory
      * can be deleted and created again, then this finds if the message is for current instance of directory or not.
@@ -1536,7 +1698,7 @@ class SubDirectory extends TypedEventEmitter {
         if ((pendingDeleteCount !== undefined && pendingDeleteCount > 0) ||
             (pendingCreateCount !== undefined && pendingCreateCount > 0)) {
             if (local) {
-                assert(isSubDirLocalOpMetadata(localOpMetadata), 0x012 /* pendingMessageId is missing from the local client's operation */);
+                assert(localOpMetadata !== undefined, 0xc0d /* localOpMetadata should be defined */);
                 if (localOpMetadata.type === "deleteSubDir") {
                     assert(pendingDeleteCount !== undefined && pendingDeleteCount > 0, 0x6c2 /* pendingDeleteCount should exist */);
                     this.decrementPendingSubDirCount(this.pendingDeleteSubDirectoriesTracker, op.subdirName);
@@ -1553,7 +1715,8 @@ class SubDirectory extends TypedEventEmitter {
                     }
                     // If this is delete op and we have keys in this subDirectory, then we need to delete these
                     // keys except the pending ones as they will be sequenced after this delete.
-                    directory.clearExceptPendingKeys(local);
+                    directory.sequencedStorageData.clear();
+                    directory.emit("clear", true, directory);
                     // In case of delete op, we need to reset the creation seqNum, clientSeqNum and client ids of
                     // creators as the previous directory is getting deleted and we will initialize again when
                     // we will receive op for the create again.
@@ -1609,68 +1772,6 @@ class SubDirectory extends TypedEventEmitter {
         }
         return !local;
     }
-    /**
-     * Clear all keys in memory in response to a remote clear, but retain keys we have modified but not yet been ack'd.
-     */
-    clearExceptPendingKeys(local) {
-        // Assuming the pendingKeys is small and the map is large
-        // we will get the value for the pendingKeys and clear the map
-        const temp = new Map();
-        for (const [key] of this.pendingKeys) {
-            const value = this._storage.get(key);
-            // If this key is already deleted, then we don't need to add it again.
-            if (value !== undefined) {
-                temp.set(key, value);
-            }
-        }
-        this.clearCore(local);
-        for (const [key, value] of temp.entries()) {
-            this.setCore(key, value, true);
-        }
-    }
-    /**
-     * Clear implementation used for both locally sourced clears as well as incoming remote clears.
-     * @param local - Whether the message originated from the local client
-     */
-    clearCore(local) {
-        this._storage.clear();
-        this.directory.emit("clear", local, this.directory);
-    }
-    /**
-     * Delete implementation used for both locally sourced deletes as well as incoming remote deletes.
-     * @param key - The key being deleted
-     * @param local - Whether the message originated from the local client
-     * @returns Previous local value of the key if it existed, undefined if it did not exist
-     */
-    deleteCore(key, local) {
-        const previousLocalValue = this._storage.get(key);
-        const previousValue = previousLocalValue;
-        const successfullyRemoved = this._storage.delete(key);
-        if (successfullyRemoved) {
-            const event = { key, path: this.absolutePath, previousValue };
-            this.directory.emit("valueChanged", event, local, this.directory);
-            const containedEvent = { key, previousValue };
-            this.emit("containedValueChanged", containedEvent, local, this);
-        }
-        return previousLocalValue;
-    }
-    /**
-     * Set implementation used for both locally sourced sets as well as incoming remote sets.
-     * @param key - The key being set
-     * @param value - The value being set
-     * @param local - Whether the message originated from the local client
-     * @returns Previous local value of the key, if any
-     */
-    setCore(key, value, local) {
-        const previousLocalValue = this._storage.get(key);
-        const previousValue = previousLocalValue;
-        this._storage.set(key, value);
-        const event = { key, path: this.absolutePath, previousValue };
-        this.directory.emit("valueChanged", event, local, this.directory);
-        const containedEvent = { key, previousValue };
-        this.emit("containedValueChanged", containedEvent, local, this);
-        return previousLocalValue;
-    }
     /**
      * Create subdirectory implementation used for both locally sourced creation as well as incoming remote creation.
      * @param subdirName - The name of the subdirectory being created