@fluidframework/map 2.0.0-dev.7.3.0.212138 → 2.0.0-dev.7.4.0.215366

package/lib/directory.mjs

@@ -11,6 +11,7 @@ import { MessageType } from "@fluidframework/protocol-definitions";
 import { SharedObject, ValueType } from "@fluidframework/shared-object-base";
 import { SummaryTreeBuilder } from "@fluidframework/runtime-utils";
 import * as path from "path-browserify";
+import { RedBlackTree } from "@fluidframework/merge-tree";
 import { LocalValueMaker, makeSerializable } from "./localValues.mjs";
 import { pkgVersion } from "./packageVersion.mjs";
 // We use path-browserify since this code can run safely on the server or the browser.
@@ -65,6 +66,100 @@ DirectoryFactory.Attributes = {
     snapshotFormatVersion: "0.1",
     packageVersion: pkgVersion,
 };
+/**
+ * The comparator essentially performs the following procedure to determine the order of subdirectory creation:
+ * 1. If subdirectory A has a non-negative 'seq' and subdirectory B has a negative 'seq', subdirectory A is always placed first due to
+ * the policy that acknowledged subdirectories precede locally created ones that have not been committed yet.
+ *
+ * 2. When both subdirectories A and B have a non-negative 'seq', they are compared as follows:
+ * - If A and B have different 'seq', they are ordered based on 'seq', and the one with the lower 'seq' will be positioned ahead. Notably this rule
+ * should not be applied in the directory ordering, since the lowest 'seq' is -1, when the directory is created locally but not acknowledged yet.
+ * - In the case where A and B have equal 'seq', the one with the lower 'clientSeq' will be positioned ahead. This scenario occurs when grouped
+ * batching is enabled, and a lower 'clientSeq' indicates that it was processed earlier after the batch was ungrouped.
+ *
+ * 3. When both subdirectories A and B have a negative 'seq', they are compared as follows:
+ * - If A and B have different 'seq', the one with lower 'seq' will be positioned ahead, which indicates the corresponding creation message was
+ * acknowledged by the server earlier.
+ * - If A and B have equal 'seq', the one with lower 'clientSeq' will be placed at the front. This scenario suggests that both subdirectories A
+ * and B were created locally and not acknowledged yet, with the one possessing the lower 'clientSeq' being created earlier.
+ *
+ * 4. A 'seq' value of zero indicates that the subdirectory was created in detached state, and it is considered acknowledged for the
+ * purpose of ordering.
+ */
+const seqDataComparator = (a, b) => {
+    if (isAcknowledgedOrDetached(a)) {
+        if (isAcknowledgedOrDetached(b)) {
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            return a.seq !== b.seq ? a.seq - b.seq : a.clientSeq - b.clientSeq;
+        }
+        else {
+            return -1;
+        }
+    }
+    else {
+        if (!isAcknowledgedOrDetached(b)) {
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            return a.seq !== b.seq ? a.seq - b.seq : a.clientSeq - b.clientSeq;
+        }
+        else {
+            return 1;
+        }
+    }
+};
+function isAcknowledgedOrDetached(seqData) {
+    return seqData.seq >= 0;
+}
+/**
+ * A utility class for tracking associations between keys and their creation indices.
+ * This is relevant to support map iteration in insertion order, see
+ * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Iterator/%40%40iterator
+ *
+ * TODO: It can be combined with the creation tracker utilized in SharedMap
+ */
+class DirectoryCreationTracker {
+    constructor() {
+        this.indexToKey = new RedBlackTree(seqDataComparator);
+        this.keyToIndex = new Map();
+    }
+    set(key, seqData) {
+        this.indexToKey.put(seqData, key);
+        this.keyToIndex.set(key, seqData);
+    }
+    has(keyOrSeqData) {
+        return typeof keyOrSeqData === "string"
+            ? this.keyToIndex.has(keyOrSeqData)
+            : this.indexToKey.get(keyOrSeqData) !== undefined;
+    }
+    delete(keyOrSeqData) {
+        if (this.has(keyOrSeqData)) {
+            if (typeof keyOrSeqData === "string") {
+                const seqData = this.keyToIndex.get(keyOrSeqData);
+                this.keyToIndex.delete(keyOrSeqData);
+                this.indexToKey.remove(seqData);
+            }
+            else {
+                const key = this.indexToKey.get(keyOrSeqData)?.data;
+                this.indexToKey.remove(keyOrSeqData);
+                this.keyToIndex.delete(key);
+            }
+        }
+    }
+    /**
+     * Retrieves all subdirectories with creation order that satisfy an optional constraint function.
+     * @param constraint - An optional constraint function that filters keys.
+     * @returns An array of keys that satisfy the constraint (or all keys if no constraint is provided).
+     */
+    keys(constraint) {
+        const keys = [];
+        this.indexToKey.mapRange((node) => {
+            if (!constraint || constraint(node.data)) {
+                keys.push(node.data);
+            }
+            return true;
+        }, keys);
+        return keys;
+    }
+}
 /**
  * {@inheritDoc ISharedDirectory}
  *
@@ -120,7 +215,7 @@ export class SharedDirectory extends SharedObject {
         /**
          * Root of the SharedDirectory, most operations on the SharedDirectory itself act on the root.
          */
-        this.root = new SubDirectory(0, new Set(), this, this.runtime, this.serializer, posix.sep);
+        this.root = new SubDirectory({ seq: 0, clientSeq: 0 }, new Set(), this, this.runtime, this.serializer, posix.sep);
         /**
          * Mapping of op types to message handlers.
          */
@@ -349,18 +444,42 @@ export class SharedDirectory extends SharedObject {
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
             const [currentSubDir, currentSubDirObject] = stack.pop();
             if (currentSubDirObject.subdirectories) {
+                // Utilize a map to store the seq -> clientSeq for the newly created subdirectory
+                const tempSeqNums = new Map();
                 for (const [subdirName, subdirObject] of Object.entries(currentSubDirObject.subdirectories)) {
                     let newSubDir = currentSubDir.getSubDirectory(subdirName);
+                    let seqData;
                     if (!newSubDir) {
                         const createInfo = subdirObject.ci;
-                        newSubDir = new SubDirectory(
-                        // If csn is -1, then initialize it with 0, otherwise we will never process ops for this
-                        // sub directory. This could be done at serialization time too, but we need to maintain
-                        // back compat too and also we will actually know the state when it was serialized.
-                        createInfo !== undefined && createInfo.csn > -1 ? createInfo.csn : 0, createInfo !== undefined
+                        // We do not store the client sequence number in the storage because the order has already been
+                        // guaranteed during the serialization process. As a result, it is only essential to utilize the
+                        // "fake" client sequence number to signify the loading order, and there is no need to retain
+                        // the actual client sequence number at this point.
+                        if (createInfo !== undefined && createInfo.csn > -1) {
+                            // If csn is -1, then initialize it with 0, otherwise we will never process ops for this
+                            // sub directory. This could be done at serialization time too, but we need to maintain
+                            // back compat too and also we will actually know the state when it was serialized.
+                            if (!tempSeqNums.has(createInfo.csn)) {
+                                tempSeqNums.set(createInfo.csn, 0);
+                            }
+                            let fakeClientSeq = tempSeqNums.get(createInfo.csn);
+                            seqData = { seq: createInfo.csn, clientSeq: fakeClientSeq };
+                            tempSeqNums.set(createInfo.csn, ++fakeClientSeq);
+                        }
+                        else {
+                            seqData = {
+                                seq: 0,
+                                clientSeq: ++currentSubDir.localCreationSeq,
+                            };
+                        }
+                        newSubDir = new SubDirectory(seqData, createInfo !== undefined
                             ? new Set(createInfo.ccIds)
                             : new Set(), this, this.runtime, this.serializer, posix.join(currentSubDir.absolutePath, subdirName));
                         currentSubDir.populateSubDirectory(subdirName, newSubDir);
+                        // Record the newly inserted subdirectory to the creation tracker
+                        currentSubDir.ackedCreationSeqTracker.set(subdirName, {
+                            ...seqData,
+                        });
                     }
                     stack.push([newSubDir, subdirObject]);
                 }
@@ -673,9 +792,9 @@ class SubDirectory extends TypedEventEmitter {
      * @param serializer - The serializer to serialize / parse handles
      * @param absolutePath - The absolute path of this IDirectory
      */
-    constructor(sequenceNumber, clientIds, directory, runtime, serializer, absolutePath) {
+    constructor(seqData, clientIds, directory, runtime, serializer, absolutePath) {
         super();
-        this.sequenceNumber = sequenceNumber;
+        this.seqData = seqData;
         this.clientIds = clientIds;
         this.directory = directory;
         this.runtime = runtime;
@@ -724,6 +843,13 @@ class SubDirectory extends TypedEventEmitter {
          * The pending ids of any clears that have been performed locally but not yet ack'd from the server
          */
         this.pendingClearMessageIds = [];
+        /**
+         * Assigns a unique ID to each subdirectory created locally but pending for acknowledgement, facilitating the tracking
+         * of the creation order.
+         */
+        this.localCreationSeq = 0;
+        this.localCreationSeqTracker = new DirectoryCreationTracker();
+        this.ackedCreationSeqTracker = new DirectoryCreationTracker();
     }
     dispose(error) {
         this._deleted = true;
@@ -825,14 +951,18 @@ class SubDirectory extends TypedEventEmitter {
         return subDir;
     }
     /**
-     * @returns A sequenceNumber which should be used for local changes.
+     * @returns The Sequence Data which should be used for local changes.
      * @remarks While detached, 0 is used rather than -1 to represent a change which should be universally known (as opposed to known
      * only by the local client). This ensures that if the directory is later attached, none of its data needs to be updated (the values
      * last set while detached will now be known to any new client, until they are changed).
+     *
+     * The client sequence number is incremented by 1 for maintaining the internal order of locally created subdirectories
      * TODO: Convert these conventions to named constants. The semantics used here match those for merge-tree.
      */
     getLocalSeq() {
-        return this.directory.isAttached() ? -1 : 0;
+        return this.directory.isAttached()
+            ? { seq: -1, clientSeq: ++this.localCreationSeq }
+            : { seq: 0, clientSeq: ++this.localCreationSeq };
     }
     /**
      * {@inheritDoc IDirectory.getSubDirectory}
@@ -875,7 +1005,26 @@ class SubDirectory extends TypedEventEmitter {
      */
     subdirectories() {
         this.throwIfDisposed();
-        return this._subdirectories.entries();
+        const ackedSubdirsInOrder = this.ackedCreationSeqTracker.keys();
+        const localSubdirsInOrder = this.localCreationSeqTracker.keys((key) => !this.ackedCreationSeqTracker.has(key));
+        const subdirNames = [...ackedSubdirsInOrder, ...localSubdirsInOrder];
+        assert(subdirNames.length === this._subdirectories.size, "The count of keys for iteration should be consistent with the size of actual data");
+        const entriesIterator = {
+            index: 0,
+            dirs: this._subdirectories,
+            next() {
+                if (this.index < subdirNames.length) {
+                    const subdirName = subdirNames[this.index++];
+                    const subdir = this.dirs.get(subdirName);
+                    return { value: [subdirName, subdir], done: false };
+                }
+                return { value: undefined, done: true };
+            },
+            [Symbol.iterator]() {
+                return this;
+            },
+        };
+        return entriesIterator;
     }
     /**
      * {@inheritDoc IDirectory.getWorkingDirectory}
@@ -1135,7 +1284,7 @@ class SubDirectory extends TypedEventEmitter {
             return;
         }
         assertNonNullClientId(msg.clientId);
-        this.createSubDirectoryCore(op.subdirName, local, msg.sequenceNumber, msg.clientId);
+        this.createSubDirectoryCore(op.subdirName, local, { seq: msg.sequenceNumber, clientSeq: msg.clientSequenceNumber }, msg.clientId);
     }
     /**
      * Apply createSubDirectory operation locally and generate metadata
@@ -1357,7 +1506,7 @@ class SubDirectory extends TypedEventEmitter {
     getSerializableCreateInfo() {
         this.throwIfDisposed();
         const createInfo = {
-            csn: this.sequenceNumber,
+            csn: this.seqData.seq,
             ccIds: Array.from(this.clientIds),
         };
         return createInfo;
@@ -1447,6 +1596,17 @@ class SubDirectory extends TypedEventEmitter {
                 this.undeleteSubDirectoryTree(localOpMetadata.subDirectory);
                 // don't need to register events because deleting never unregistered
                 this._subdirectories.set(op.subdirName, localOpMetadata.subDirectory);
+                // Restore the record in creation tracker
+                if (isAcknowledgedOrDetached(localOpMetadata.subDirectory.seqData)) {
+                    this.ackedCreationSeqTracker.set(op.subdirName, {
+                        ...localOpMetadata.subDirectory.seqData,
+                    });
+                }
+                else {
+                    this.localCreationSeqTracker.set(op.subdirName, {
+                        ...localOpMetadata.subDirectory.seqData,
+                    });
+                }
                 this.emit("subDirectoryCreated", op.subdirName, true, this);
             }
             this.decrementPendingSubDirCount(this.pendingDeleteSubDirectoriesTracker, op.subDirName);
@@ -1527,7 +1687,7 @@ class SubDirectory extends TypedEventEmitter {
         // and the op was created after the directory was created then apply this op.
         return ((msg.clientId !== null && this.clientIds.has(msg.clientId)) ||
             this.clientIds.has("detached") ||
-            (this.sequenceNumber !== -1 && this.sequenceNumber <= msg.referenceSequenceNumber));
+            (this.seqData.seq !== -1 && this.seqData.seq <= msg.referenceSequenceNumber));
     }
     /**
      * If our local operations that have not yet been ack'd will eventually overwrite an incoming operation, we should
@@ -1564,10 +1724,11 @@ class SubDirectory extends TypedEventEmitter {
                     // If this is delete op and we have keys in this subDirectory, then we need to delete these
                     // keys except the pending ones as they will be sequenced after this delete.
                     directory.clearExceptPendingKeys(local);
-                    // In case of delete op, we need to reset the creation seq number and client ids of
+                    // In case of delete op, we need to reset the creation seqNum, clientSeqNum and client ids of
                     // creators as the previous directory is getting deleted and we will initialize again when
                     // we will receive op for the create again.
-                    directory.sequenceNumber = -1;
+                    directory.seqData.seq = -1;
+                    directory.seqData.clientSeq = -1;
                     directory.clientIds.clear();
                     // Do the same thing for the subtree of the directory. If create is not pending for a child, then just
                     // delete it.
@@ -1581,21 +1742,35 @@ class SubDirectory extends TypedEventEmitter {
                     }
                 };
                 const subDirectory = this._subdirectories.get(op.subdirName);
+                // Clear the creation tracker record
+                this.ackedCreationSeqTracker.delete(op.subdirName);
                 resetSubDirectoryTree(subDirectory);
             }
             if (op.type === "createSubDirectory") {
                 const dir = this._subdirectories.get(op.subdirName);
                 // Child sub directory create seq number can't be lower than the parent subdirectory.
                 // The sequence number for multiple ops can be the same when multiple createSubDirectory occurs with grouped batching enabled, thus <= and not just <.
-                if (this.sequenceNumber !== -1 && this.sequenceNumber <= msg.sequenceNumber) {
-                    if (dir?.sequenceNumber === -1) {
-                        // Only set the seq on the first message, could be more
-                        dir.sequenceNumber = msg.sequenceNumber;
+                if (this.seqData.seq !== -1 && this.seqData.seq <= msg.sequenceNumber) {
+                    if (dir?.seqData.seq === -1) {
+                        // Only set the sequence data based on the first message
+                        dir.seqData.seq = msg.sequenceNumber;
+                        dir.seqData.clientSeq = msg.clientSequenceNumber;
+                        // set the creation seq in tracker
+                        if (!this.ackedCreationSeqTracker.has(op.subdirName) &&
+                            !this.pendingDeleteSubDirectoriesTracker.has(op.subdirName)) {
+                            this.ackedCreationSeqTracker.set(op.subdirName, {
+                                seq: msg.sequenceNumber,
+                                clientSeq: msg.clientSequenceNumber,
+                            });
+                            if (local) {
+                                this.localCreationSeqTracker.delete(op.subdirName);
+                            }
+                        }
                     }
                     // The client created the dir at or after the dirs seq, so list its client id as a creator.
                     if (dir !== undefined &&
                         !dir.clientIds.has(msg.clientId) &&
-                        dir.sequenceNumber <= msg.sequenceNumber) {
+                        dir.seqData.seq <= msg.sequenceNumber) {
                         dir.clientIds.add(msg.clientId);
                     }
                 }
@@ -1670,15 +1845,25 @@ class SubDirectory extends TypedEventEmitter {
      * Create subdirectory implementation used for both locally sourced creation as well as incoming remote creation.
      * @param subdirName - The name of the subdirectory being created
      * @param local - Whether the message originated from the local client
-     * @param seq - Sequence number at which this directory is created
+     * @param seqData - Sequence number and client sequence number at which this directory is created
      * @param clientId - Id of client which created this directory.
      * @returns True if is newly created, false if it already existed.
      */
-    createSubDirectoryCore(subdirName, local, seq, clientId) {
+    createSubDirectoryCore(subdirName, local, seqData, clientId) {
         const subdir = this._subdirectories.get(subdirName);
         if (subdir === undefined) {
             const absolutePath = posix.join(this.absolutePath, subdirName);
-            const subDir = new SubDirectory(seq, new Set([clientId]), this.directory, this.runtime, this.serializer, absolutePath);
+            const subDir = new SubDirectory({ ...seqData }, new Set([clientId]), this.directory, this.runtime, this.serializer, absolutePath);
+            /**
+             * Store the sequnce numbers of newly created subdirectory to the proper creation tracker, based
+             * on whether the creation behavior has been ack'd or not
+             */
+            if (!isAcknowledgedOrDetached(seqData)) {
+                this.localCreationSeqTracker.set(subdirName, { ...seqData });
+            }
+            else {
+                this.ackedCreationSeqTracker.set(subdirName, { ...seqData });
+            }
             this.registerEventsOnSubDirectory(subDir, subdirName);
             this._subdirectories.set(subdirName, subDir);
             this.emit("subDirectoryCreated", subdirName, local, this);
@@ -1708,6 +1893,16 @@ class SubDirectory extends TypedEventEmitter {
         // Might want to consider cleaning out the structure more exhaustively though? But not when rollback.
         if (previousValue !== undefined) {
             this._subdirectories.delete(subdirName);
+            /**
+             * Remove the corresponding record from the proper creation tracker, based on whether the subdirectory has been
+             * ack'd already or still not committed yet (could be both).
+             */
+            if (this.ackedCreationSeqTracker.has(subdirName)) {
+                this.ackedCreationSeqTracker.delete(subdirName);
+            }
+            if (this.localCreationSeqTracker.has(subdirName)) {
+                this.localCreationSeqTracker.delete(subdirName);
+            }
             this.disposeSubDirectoryTree(previousValue);
             this.emit("subDirectoryDeleted", subdirName, local, this);
         }