@fluidframework/merge-tree 2.2.0 → 2.3.0

package/lib/mergeTree.js

@@ -9,7 +9,7 @@ import { DataProcessingError, UsageError } from "@fluidframework/telemetry-utils
 import { DoublyLinkedList } from "./collections/index.js";
 import { NonCollabClient, TreeMaintenanceSequenceNumber, UnassignedSequenceNumber, UniversalSequenceNumber, } from "./constants.js";
 import { EndOfTreeSegment, StartOfTreeSegment } from "./endOfTreeSegment.js";
-import { LocalReferenceCollection, SlidingPreference, anyLocalReferencePosition, filterLocalReferencePositions, } from "./localReference.js";
+import { LocalReferenceCollection, SlidingPreference, anyLocalReferencePosition, createDetachedLocalReferencePosition, filterLocalReferencePositions, } from "./localReference.js";
 import { MergeTreeMaintenanceType, } from "./mergeTreeDeltaCallback.js";
 import { NodeAction, backwardExcursion, depthFirstNodeWalk, forwardExcursion, walkAllChildSegments, } from "./mergeTreeNodeWalk.js";
 import { 
@@ -24,11 +24,8 @@ import { DetachedReferencePosition, refGetTileLabels, refHasTileLabel, refTypeIn
 // eslint-disable-next-line import/no-deprecated
 import { PropertiesRollback } from "./segmentPropertiesManager.js";
 import { endpointPosAndSide } from "./sequencePlace.js";
+import { SortedSegmentSet } from "./sortedSegmentSet.js";
 import { zamboniSegments } from "./zamboni.js";
-function wasRemovedAfter(seg, seq) {
-    return (seg.removedSeq !== UnassignedSequenceNumber &&
-        (seg.removedSeq === undefined || seg.removedSeq > seq));
-}
 function markSegmentMoved(seg, moveInfo) {
     seg.moveDst = moveInfo.moveDst;
     seg.movedClientIds = [...moveInfo.movedClientIds];
@@ -176,6 +173,67 @@ export function getSlideToSegoff(segoff, slidingPreference = SlidingPreference.F
 }
 const forwardPred = (ref) => ref.slidingPreference !== SlidingPreference.BACKWARD;
 const backwardPred = (ref) => ref.slidingPreference === SlidingPreference.BACKWARD;
+class Obliterates {
+    constructor(mergeTree) {
+        this.mergeTree = mergeTree;
+        /**
+         * Array containing the all move operations within the
+         * collab window.
+         *
+         * The moves are stored in sequence order which accelerates clean up in setMinSeq
+         *
+         * See https://github.com/microsoft/FluidFramework/blob/main/packages/dds/merge-tree/docs/Obliterate.md#remote-perspective
+         * for additional context
+         */
+        // eslint-disable-next-line import/no-deprecated
+        this.seqOrdered = new DoublyLinkedList();
+        /**
+         * This contains a sorted lists of all obliterate starts
+         * and is used to accelerate finding overlapping obliterates
+         * as well as determining if there are any obliterates at all.
+         */
+        this.startOrdered = new SortedSegmentSet();
+    }
+    setMinSeq(minSeq) {
+        // eslint-disable-next-line @typescript-eslint/no-non-null-asserted-optional-chain
+        while (!this.seqOrdered.empty && this.seqOrdered.first?.data.seq <= minSeq) {
+            const ob = this.seqOrdered.shift();
+            this.startOrdered.remove(ob.data.start);
+            this.mergeTree.removeLocalReferencePosition(ob.data.start);
+            this.mergeTree.removeLocalReferencePosition(ob.data.end);
+        }
+    }
+    // eslint-disable-next-line import/no-deprecated
+    addOrUpdate(obliterateInfo) {
+        const { seq, start } = obliterateInfo;
+        if (seq !== UnassignedSequenceNumber) {
+            this.seqOrdered.push(obliterateInfo);
+        }
+        this.startOrdered.addOrUpdate(start);
+    }
+    empty() {
+        return this.startOrdered.size === 0;
+    }
+    // eslint-disable-next-line import/no-deprecated
+    findOverlapping(seg) {
+        // eslint-disable-next-line import/no-deprecated
+        const overlapping = [];
+        for (const start of this.startOrdered.items) {
+            if (start.getSegment().ordinal <= seg.ordinal) {
+                // eslint-disable-next-line import/no-deprecated
+                const ob = start.properties?.obliterate;
+                if (ob.end.getSegment().ordinal >= seg.ordinal) {
+                    overlapping.push(ob);
+                }
+            }
+            else {
+                // the start is past the seg, so exit
+                break;
+            }
+        }
+        return overlapping;
+    }
+}
 /**
  * @internal
  */
@@ -196,36 +254,7 @@ export class MergeTree {
         // for now assume only markers have ids and so point directly at the Segment
         // if we need to have pointers to non-markers, we can change to point at local refs
         this.idToMarker = new Map();
-        /**
-         * Array containing the sequence number of all move operations within the
-         * collab window
-         *
-         * When a segment is inserted, we must traverse to the left and right of it
-         * to determine whether the segment was inserted into an obliterated range.
-         * By keeping track of all move seqs, we can significantly reduce the search
-         * space we must traverse.
-         *
-         * Sequence numbers in `moveSeqs` are sorted to accelerate bookkeeping.
-         *
-         * See https://github.com/microsoft/FluidFramework/blob/main/packages/dds/merge-tree/docs/Obliterate.md#remote-perspective
-         * for additional context
-         */
-        this.moveSeqs = [];
-        /**
-         * Similar to moveSeqs, but tracks local moves. These are not the move
-         * operations within the collab window, but rather local moves that have
-         * not been acked.
-         */
-        this.localMoveSeqs = new Set();
-        /**
-         * Groups of segments moved by local moves/obliterates
-         *
-         * When a local obliterate is acked, we must also ack segments that were
-         * concurrently obliterated on insert. We check this segment group to find
-         * such segments
-         */
-        // eslint-disable-next-line import/no-deprecated
-        this.locallyMovedSegments = new Map();
+        this.obliterates = new Obliterates(this);
         this.splitLeafSegment = (segment, pos) => {
             if (!(pos > 0 && segment)) {
                 return {};
@@ -341,7 +370,6 @@ export class MergeTree {
                  childIndex++, nodeIndex++ // Advance to next child & node
                 ) {
                     // Insert the next node into the current block
-                    // TODO Non null asserting, why is this not null?
                     this.addNode(block, nodes[nodeIndex]);
                 }
                 // Calculate this block's info.  Previously this was inlined into the above loop as a micro-optimization,
@@ -350,8 +378,7 @@ export class MergeTree {
                 this.blockUpdate(block);
             }
             return blocks.length === 1 // If there is only one block at this layer...
-                ? // Non null asserting here because of the length check above
-                    blocks[0] // ...then we're done.  Return the root.
+                ? blocks[0] // ...then we're done.  Return the root.
                 : buildMergeBlock(blocks); // ...otherwise recursively build the next layer above blocks.
         };
         if (segments.length > 0) {
@@ -399,7 +426,6 @@ export class MergeTree {
         while (parent) {
             const children = parent.children;
             for (let childIndex = 0; childIndex < parent.childCount; childIndex++) {
-                // TODO Non null asserting, why is this not null?
                 const child = children[childIndex];
                 if ((!!prevParent && child === prevParent) || child === node) {
                     break;
@@ -614,8 +640,7 @@ export class MergeTree {
         assert(this.collabWindow.minSeq <= minSeq, 0x04f /* "minSeq of collab window > target minSeq!" */);
         if (minSeq > this.collabWindow.minSeq) {
             this.collabWindow.minSeq = minSeq;
-            const firstMoveSeqIdx = this.moveSeqs.findIndex((seq) => seq >= minSeq);
-            this.moveSeqs = firstMoveSeqIdx === -1 ? [] : this.moveSeqs.slice(firstMoveSeqIdx);
+            this.obliterates.setMinSeq(minSeq);
             if (MergeTree.options.zamboniSegments) {
                 zamboniSegments(this);
             }
@@ -723,34 +748,8 @@ export class MergeTree {
             const deltaSegments = [];
             const overlappingRemoves = [];
             pendingSegmentGroup.segments.map((pendingSegment) => {
-                const localMovedSeq = pendingSegment.localMovedSeq;
                 const overlappingRemove = !pendingSegment.ack(pendingSegmentGroup, opArgs);
-                if (opArgs.op.type === MergeTreeDeltaType.OBLITERATE && localMovedSeq !== undefined) {
-                    const locallyMovedSegments = this.locallyMovedSegments.get(localMovedSeq);
-                    if (locallyMovedSegments) {
-                        // Disabling because a for of loop causes the type of segment to be ISegment, which does not have parent information stored
-                        // eslint-disable-next-line unicorn/no-array-for-each
-                        locallyMovedSegments.segments.forEach((segment) => {
-                            segment.localMovedSeq = undefined;
-                            if (!nodesToUpdate.includes(segment.parent)) {
-                                nodesToUpdate.push(segment.parent);
-                            }
-                            if (segment.movedSeq === UnassignedSequenceNumber) {
-                                segment.movedSeq = seq;
-                            }
-                        });
-                        this.locallyMovedSegments.delete(localMovedSeq);
-                    }
-                }
                 overwrite = overlappingRemove || overwrite;
-                if (opArgs.op.type === MergeTreeDeltaType.OBLITERATE) {
-                    if (seq !== this.moveSeqs[this.moveSeqs.length - 1]) {
-                        this.moveSeqs.push(seq);
-                    }
-                    if (localMovedSeq !== undefined) {
-                        this.localMoveSeqs.delete(localMovedSeq);
-                    }
-                }
                 overlappingRemoves.push(overlappingRemove);
                 if (MergeTree.options.zamboniSegments) {
                     this.addToLRUSet(pendingSegment, seq);
@@ -762,6 +761,9 @@ export class MergeTree {
                     segment: pendingSegment,
                 });
             });
+            if (opArgs.op.type === MergeTreeDeltaType.OBLITERATE) {
+                this.obliterates.addOrUpdate(pendingSegmentGroup.obliterateInfo);
+            }
             // Perform slides after all segments have been acked, so that
             // positions after slide are final
             if (opArgs.op.type === MergeTreeDeltaType.REMOVE ||
@@ -967,90 +969,54 @@ export class MergeTree {
                 this.updateRoot(splitNode);
                 saveIfLocal(newSegment);
                 insertPos += newSegment.cachedLength;
-                if (!this.options?.mergeTreeEnableObliterate) {
-                    continue;
-                }
-                let moveUpperBound = Number.POSITIVE_INFINITY;
-                const smallestSeqMoveOp = this.getSmallestSeqMoveOp();
-                if (smallestSeqMoveOp === undefined) {
+                if (!this.options?.mergeTreeEnableObliterate || this.obliterates.empty()) {
                     continue;
                 }
-                const leftAckedSegments = {};
-                const leftLocalSegments = {};
-                let _localMovedSeq;
-                let _movedSeq;
-                let movedClientIds;
-                const findLeftMovedSegment = (seg) => {
-                    const movedSeqs = seg.movedSeqs?.filter((movedSeq) => movedSeq >= refSeq) ?? [];
-                    const localMovedSeqs = seg.localMovedSeq ? [seg.localMovedSeq] : [];
-                    for (const movedSeq of movedSeqs) {
-                        leftAckedSegments[movedSeq] = seg;
-                    }
-                    for (const localMovedSeq of localMovedSeqs) {
-                        leftLocalSegments[localMovedSeq] = seg;
-                    }
-                    if ((seg.movedSeqs?.length ?? 0) > 0 || localMovedSeqs.length > 0) {
-                        return true;
-                    }
-                    if (!isRemoved(seg) || wasRemovedAfter(seg, moveUpperBound)) {
-                        moveUpperBound = Math.min(moveUpperBound, seg.seq ?? Number.POSITIVE_INFINITY);
-                    }
-                    // If we've reached a segment that existed before any of our in-collab-window move ops
-                    // happened, no need to continue.
-                    return moveUpperBound >= smallestSeqMoveOp;
-                };
-                const findRightMovedSegment = (seg) => {
-                    const movedSeqs = seg.movedSeqs?.filter((movedSeq) => movedSeq >= refSeq) ?? [];
-                    const localMovedSeqs = seg.localMovedSeq ? [seg.localMovedSeq] : [];
-                    for (const movedSeq of movedSeqs) {
-                        const left = leftAckedSegments[movedSeq];
-                        if (left) {
-                            _movedSeq = movedSeq;
-                            const clientIdIdx = left.movedSeqs?.indexOf(movedSeq) ?? -1;
-                            const movedClientId = left.movedClientIds?.[clientIdIdx];
-                            assert(movedClientId !== undefined, 0x869 /* expected client id to exist */);
-                            movedClientIds = [movedClientId];
-                            return false;
+                // eslint-disable-next-line import/no-deprecated
+                let oldest;
+                let normalizedOldestSeq = 0;
+                // eslint-disable-next-line import/no-deprecated
+                let newest;
+                let normalizedNewestSeq = 0;
+                const movedClientIds = [];
+                const movedSeqs = [];
+                for (const ob of this.obliterates.findOverlapping(newSegment)) {
+                    // compute a normalized seq that takes into account local seqs
+                    // but is still comparable to remote seqs to keep the checks below easy
+                    // REMOTE SEQUENCE NUMBERS                                     LOCAL SEQUENCE NUMBERS
+                    // [0, 1, 2, 3, ..., 100, ..., 1000, ..., (MAX - MaxLocalSeq), L1, L2, L3, L4, ..., L100, ..., L1000, ...(MAX)]
+                    const normalizedObSeq = ob.seq === UnassignedSequenceNumber
+                        ? Number.MAX_SAFE_INTEGER - this.collabWindow.localSeq + ob.localSeq
+                        : ob.seq;
+                    if (normalizedObSeq > refSeq) {
+                        if (oldest === undefined || normalizedOldestSeq > normalizedObSeq) {
+                            normalizedOldestSeq = normalizedObSeq;
+                            oldest = ob;
+                            movedClientIds.unshift(ob.clientId);
+                            movedSeqs.unshift(ob.seq);
                         }
-                    }
-                    for (const localMovedSeq of localMovedSeqs) {
-                        const left = leftLocalSegments[localMovedSeq];
-                        if (left) {
-                            _localMovedSeq = localMovedSeq;
-                            const clientIdIdx = left.movedSeqs?.indexOf(UnassignedSequenceNumber) ?? -1;
-                            const movedClientId = left.movedClientIds?.[clientIdIdx];
-                            assert(movedClientId !== undefined, 0x86a /* expected client id to exist */);
-                            movedClientIds = [movedClientId];
-                            return false;
+                        else {
+                            if (newest === undefined || normalizedNewestSeq < normalizedObSeq) {
+                                normalizedNewestSeq = normalizedObSeq;
+                                newest = ob;
+                            }
+                            movedClientIds.push(ob.clientId);
+                            movedSeqs.push(ob.seq);
                         }
                     }
-                    if ((seg.movedSeqs?.length ?? 0) || localMovedSeqs.length > 0) {
-                        return true;
-                    }
-                    if (!isRemoved(seg) || wasRemovedAfter(seg, moveUpperBound)) {
-                        moveUpperBound = Math.min(moveUpperBound, seg.seq ?? Number.POSITIVE_INFINITY);
-                    }
-                    // If we've reached a segment that existed before any of our in-collab-window move ops
-                    // happened, no need to continue.
-                    return moveUpperBound >= smallestSeqMoveOp;
-                };
-                backwardExcursion(newSegment, findLeftMovedSegment);
-                moveUpperBound = Number.POSITIVE_INFINITY;
-                forwardExcursion(newSegment, findRightMovedSegment);
-                if (_localMovedSeq !== undefined || _movedSeq !== undefined) {
-                    assert(movedClientIds !== undefined, 0x86b /* movedClientIds should be set if local/moved seq is set */);
+                }
+                if (oldest && newest?.clientId !== clientId) {
                     const moveInfo = {
                         movedClientIds,
-                        movedSeq: _movedSeq ?? UnassignedSequenceNumber,
-                        movedSeqs: _movedSeq === undefined ? [UnassignedSequenceNumber] : [_movedSeq],
-                        localMovedSeq: _localMovedSeq,
-                        wasMovedOnInsert: (_movedSeq ?? -1) !== UnassignedSequenceNumber,
+                        movedSeq: oldest.seq,
+                        movedSeqs,
+                        localMovedSeq: oldest.localSeq,
+                        wasMovedOnInsert: oldest.seq !== UnassignedSequenceNumber,
                     };
                     markSegmentMoved(newSegment, moveInfo);
                     if (moveInfo.localMovedSeq !== undefined) {
-                        const movedSegmentGroup = this.locallyMovedSegments.get(moveInfo.localMovedSeq);
-                        assert(movedSegmentGroup !== undefined, 0x86c /* expected segment group to exist */);
-                        this.addToPendingList(newSegment, movedSegmentGroup, localSeq);
+                        assert(oldest.segmentGroup !== undefined, 0x86c /* expected segment group to exist */);
+                        this.addToPendingList(newSegment, oldest.segmentGroup);
                     }
                     if (newSegment.parent) {
                         this.blockUpdatePathLengths(newSegment.parent, seq, clientId);
@@ -1088,9 +1054,6 @@ export class MergeTree {
             return true;
         }
     }
-    getSmallestSeqMoveOp() {
-        return this.moveSeqs[0] ?? (this.localMoveSeqs.size > 0 ? -1 : undefined);
-    }
     insertingWalk(block, pos, refSeq, clientId, seq, context, isLastChildBlock = true) {
         let _pos;
         if (pos === "start") {
@@ -1108,7 +1071,6 @@ export class MergeTree {
         let newNode;
         let fromSplit;
         for (childIndex = 0; childIndex < block.childCount; childIndex++) {
-            // TODO Non null asserting, why is this not null?
             child = children[childIndex];
             // ensure we walk down the far edge of the tree, even if all sub-tree is eligible for zamboni
             const isLastNonLeafBlock = isLastChildBlock && !child.isLeaf() && childIndex === block.childCount - 1;
@@ -1173,9 +1135,7 @@ export class MergeTree {
         }
         if (newNode) {
             for (let i = block.childCount; i > childIndex; i--) {
-                // TODO Non null asserting, why is this not null?
                 block.children[i] = block.children[i - 1];
-                // TODO Non null asserting, why is this not null?
                 block.children[i].index = i;
             }
             block.assignChild(newNode, childIndex, false);
@@ -1207,7 +1167,6 @@ export class MergeTree {
         // Update ordinals to reflect lowered child count
         this.nodeUpdateOrdinals(node);
         for (let i = 0; i < halfCount; i++) {
-            // TODO Non null asserting, why is this not null?
             newNode.assignChild(node.children[halfCount + i], i, false);
             node.children[halfCount + i] = undefined;
         }
@@ -1217,7 +1176,6 @@ export class MergeTree {
     }
     nodeUpdateOrdinals(block) {
         for (let i = 0; i < block.childCount; i++) {
-            // TODO Non null asserting, why is this not null?
             const child = block.children[i];
             block.setOrdinal(child, i);
             if (!child.isLeaf()) {
@@ -1294,20 +1252,30 @@ export class MergeTree {
         const localOverlapWithRefs = [];
         const movedSegments = [];
         const localSeq = seq === UnassignedSequenceNumber ? ++this.collabWindow.localSeq : undefined;
-        if (seq !== UnassignedSequenceNumber && seq !== this.moveSeqs[this.moveSeqs.length - 1]) {
-            this.moveSeqs.push(seq);
-        }
-        else if (seq === UnassignedSequenceNumber && localSeq !== undefined) {
-            this.localMoveSeqs.add(localSeq);
-        }
         // eslint-disable-next-line import/no-deprecated
-        let segmentGroup;
+        const obliterate = {
+            clientId,
+            end: createDetachedLocalReferencePosition(undefined),
+            refSeq,
+            seq,
+            start: createDetachedLocalReferencePosition(undefined),
+            localSeq,
+            segmentGroup: undefined,
+        };
+        const normalizedStartPos = startPos === "start" || startPos === undefined ? 0 : startPos;
+        const normalizedEndPos = endPos === "end" || endPos === undefined ? this.getLength(refSeq, clientId) : endPos;
+        const { segment: startSeg } = this.getContainingSegment(normalizedStartPos, refSeq, clientId);
+        const { segment: endSeg } = this.getContainingSegment(normalizedEndPos - 1, refSeq, clientId);
+        assert(startSeg !== undefined && endSeg !== undefined, 0xa3f /* segments cannot be undefined */);
+        obliterate.start = this.createLocalReferencePosition(startSeg, 0, ReferenceType.StayOnRemove, {
+            obliterate,
+        });
+        obliterate.end = this.createLocalReferencePosition(endSeg, endSeg.cachedLength - 1, ReferenceType.StayOnRemove, {
+            obliterate,
+        });
         const markMoved = (segment, pos, _start, _end) => {
+            var _a;
             const existingMoveInfo = toMoveInfo(segment);
-            if (startSide)
-                segment.startSide = startSide;
-            if (endSide)
-                segment.endSide = endSide;
             if (clientId !== segment.clientId &&
                 segment.seq !== undefined &&
                 seq !== UnassignedSequenceNumber &&
@@ -1347,7 +1315,8 @@ export class MergeTree {
             if (this.collabWindow.collaborating) {
                 if (segment.movedSeq === UnassignedSequenceNumber &&
                     clientId === this.collabWindow.clientId) {
-                    segmentGroup = this.addToPendingList(segment, segmentGroup, localSeq);
+                    obliterate.segmentGroup = this.addToPendingList(segment, obliterate.segmentGroup, localSeq);
+                    (_a = obliterate.segmentGroup).obliterateInfo ?? (_a.obliterateInfo = obliterate);
                 }
                 else {
                     if (MergeTree.options.zamboniSegments) {
@@ -1367,6 +1336,7 @@ export class MergeTree {
             return true;
         };
         this.nodeMap(refSeq, clientId, markMoved, undefined, afterMarkMoved, start, end, undefined, seq === UnassignedSequenceNumber ? undefined : seq);
+        this.obliterates.addOrUpdate(obliterate);
         this.slideAckedRemovedSegmentReferences(localOverlapWithRefs);
         // opArgs == undefined => test code
         if (movedSegments.length > 0) {
@@ -1375,9 +1345,6 @@ export class MergeTree {
                 deltaSegments: movedSegments,
             });
         }
-        if (segmentGroup && localSeq !== undefined) {
-            this.locallyMovedSegments.set(localSeq, segmentGroup);
-        }
         // these events are newly removed
         // so we slide after eventing in case the consumer wants to make reference
         // changes at remove time, like add a ref to track undo redo.
@@ -1524,7 +1491,6 @@ export class MergeTree {
                     this.markRangeRemoved(start, start + segment.cachedLength, UniversalSequenceNumber, this.collabWindow.clientId, UniversalSequenceNumber, false, { op: removeOp });
                 } /* op.type === MergeTreeDeltaType.ANNOTATE */
                 else {
-                    // TODO Non null asserting, why is this not null?
                     const props = pendingSegmentGroup.previousProps[i];
                     const annotateOp = createAnnotateRangeOp(start, start + segment.cachedLength, props);
                     this.annotateRange(start, start + segment.cachedLength, props, UniversalSequenceNumber, this.collabWindow.clientId, UniversalSequenceNumber, { op: annotateOp }, 
@@ -1571,7 +1537,7 @@ export class MergeTree {
         if (_segment !== "start" &&
             _segment !== "end" &&
             isRemovedAndAckedOrMovedAndAcked(_segment) &&
-            !refTypeIncludesFlag(refType, ReferenceType.SlideOnRemove | ReferenceType.Transient) &&
+            !refTypeIncludesFlag(refType, ReferenceType.SlideOnRemove | ReferenceType.Transient | ReferenceType.StayOnRemove) &&
             _segment.endpointType === undefined) {
             throw new UsageError("Can only create SlideOnRemove or Transient local reference position on a removed or obliterated segment");
         }
@@ -1644,7 +1610,6 @@ export class MergeTree {
             }
         }
         for (let i = 0; i < newOrder.length; i++) {
-            // TODO Non null asserting, why is this not null?
             const seg = newOrder[i];
             const { parent, index, ordinal } = currentOrder[i];
             parent?.assignChild(seg, index, false);
@@ -1731,7 +1696,6 @@ export class MergeTree {
         const rightmostTiles = createMap();
         const leftmostTiles = createMap();
         for (let i = 0; i < block.childCount; i++) {
-            // TODO Non null asserting, why is this not null?
             const node = block.children[i];
             const nodeLength = nodeTotalLength(this, node);
             if (nodeLength !== undefined) {