@fluidframework/merge-tree 1.2.6 → 2.0.0-dev.1.3.0.96595

package/src/client.ts

@@ -16,17 +16,17 @@ import { LoggingError } from "@fluidframework/telemetry-utils";
 import { IIntegerRange } from "./base";
 import { RedBlackTree } from "./collections";
 import { UnassignedSequenceNumber, UniversalSequenceNumber } from "./constants";
-import { LocalReference, LocalReferencePosition } from "./localReference";
+import { LocalReferencePosition } from "./localReference";
 import {
     CollaborationWindow,
     compareStrings,
     IConsensusInfo,
+    IMergeNode,
     ISegment,
     ISegmentAction,
     Marker,
-    MergeTree,
     SegmentGroup,
-} from "./mergeTree";
+} from "./mergeTreeNodes";
 import { MergeTreeDeltaCallback } from "./mergeTreeDeltaCallback";
 import {
     createAnnotateMarkerOp,
@@ -51,9 +51,11 @@ import {
 import { PropertySet } from "./properties";
 import { SnapshotLegacy } from "./snapshotlegacy";
 import { SnapshotLoader } from "./snapshotLoader";
-import { MergeTreeTextHelper } from "./textSegment";
+import { IMergeTreeTextHelper } from "./textSegment";
 import { SnapshotV1 } from "./snapshotV1";
-import { ReferencePosition, RangeStackMap } from "./referencePositions";
+import { ReferencePosition, RangeStackMap, DetachedReferencePosition } from "./referencePositions";
+import { MergeTree } from "./mergeTree";
+import { MergeTreeTextHelper } from "./MergeTreeTextHelper";
 import {
     IMergeTreeClientSequenceArgs,
     IMergeTreeDeltaOpArgs,
@@ -88,6 +90,10 @@ export class Client {
         this.mergeTree.mergeTreeMaintenanceCallback = callback;
     }
 
+    /**
+     * @deprecated  for internal use only. public export will be removed.
+     * @internal
+     */
     protected readonly mergeTree: MergeTree;
 
     private readonly clientNameToIds = new RedBlackTree<string, number>(compareStrings);
@@ -167,11 +173,7 @@ export class Client {
         const annotateOp =
             createAnnotateMarkerOp(marker, props, combiningOp)!;
 
-        if (this.applyAnnotateRangeOp({ op: annotateOp })) {
-            return annotateOp;
-        } else {
-            return undefined;
-        }
+        return this.applyAnnotateRangeOp({ op: annotateOp }) ? annotateOp : undefined;
     }
     /**
      * Annotates the range with the provided properties
@@ -241,7 +243,7 @@ export class Client {
             this.getCurrentSeq(),
             this.getClientId());
 
-        if (pos === LocalReference.DetachedPosition) {
+        if (pos === DetachedReferencePosition) {
             return undefined;
         }
         const op = createInsertSegmentOp(
@@ -289,9 +291,17 @@ export class Client {
      * serializer which keeps track of all serialized handles.
      */
     public serializeGCData(handle: IFluidHandle, handleCollectingSerializer: IFluidSerializer): void {
+        let localInserts = 0;
+        let localRemoves = 0;
         this.mergeTree.walkAllSegments(
             this.mergeTree.root,
             (seg) => {
+                if (seg.seq === UnassignedSequenceNumber) {
+                    localInserts++;
+                }
+                if (seg.removedSeq === UnassignedSequenceNumber) {
+                    localRemoves++;
+                }
                 // Only serialize segments that have not been removed.
                 if (seg.removedSeq === undefined) {
                     handleCollectingSerializer.stringify(
@@ -301,6 +311,10 @@ export class Client {
                 return true;
             },
         );
+
+        if (localInserts > 0 || localRemoves > 0) {
+            this.logger.sendErrorEvent({ eventName: "LocalEditsInProcessGCData", localInserts, localRemoves });
+        }
     }
 
     public getCollabWindow(): CollaborationWindow {
@@ -312,30 +326,17 @@ export class Client {
      * does not exist in this merge tree
      * @param segment - The segment to get the position of
      */
-    public getPosition(segment: ISegment): number {
+    public getPosition(segment: ISegment, localSeq?: number): number {
         if (segment?.parent === undefined) {
             return -1;
         }
-        return this.mergeTree.getPosition(segment, this.getCurrentSeq(), this.getClientId());
-    }
-    /**
-     * @deprecated - use createReferencePosition instead
-     */
-    public addLocalReference(lref: LocalReference) {
-        return this.mergeTree.addLocalReference(lref);
-    }
-
-    /**
-     * @deprecated - use removeReferencePosition instead
-     */
-    public removeLocalReference(lref: LocalReference) {
-        return this.removeLocalReferencePosition(lref);
+        return this.mergeTree.getPosition(segment, this.getCurrentSeq(), this.getClientId(), localSeq);
     }
 
     public createLocalReferencePosition(
         segment: ISegment, offset: number | undefined, refType: ReferenceType, properties: PropertySet | undefined,
     ): LocalReferencePosition {
-        return this.mergeTree.createLocalReferencePosition(segment, offset, refType, properties, this);
+        return this.mergeTree.createLocalReferencePosition(segment, offset ?? 0, refType, properties);
     }
 
     public removeLocalReferencePosition(lref: LocalReferencePosition) {
@@ -359,6 +360,13 @@ export class Client {
         return this.mergeTree.getMarkerFromId(id);
     }
 
+    /**
+     * Revert an op
+     */
+    public rollback?(op: any, localOpMetadata: unknown) {
+        this.mergeTree.rollback(op as IMergeTreeDeltaOp, localOpMetadata as SegmentGroup);
+    }
+
     /**
      * Performs the remove based on the provided op
      * @param opArgs - The ops args for the op
@@ -655,11 +663,7 @@ export class Client {
         return this.clientNameToIds.get(longClientId)!.data;
     }
     getLongClientId(shortClientId: number) {
-        if (shortClientId >= 0) {
-            return this.shortClientIdMap[shortClientId];
-        } else {
-            return "original";
-        }
+        return shortClientId >= 0 ? this.shortClientIdMap[shortClientId] : "original";
     }
     addLongClientId(longClientId: string) {
         this.clientNameToIds.put(longClientId, this.shortClientIdMap.length);
@@ -677,36 +681,8 @@ export class Client {
      */
     protected findReconnectionPosition(segment: ISegment, localSeq: number) {
         assert(localSeq <= this.mergeTree.collabWindow.localSeq, 0x032 /* "localSeq greater than collab window" */);
-        let segmentPosition = 0;
-        /*
-            Walk the segments up to the current segment, and calculate it's
-            position taking into account local segments that were modified,
-            after the current segment.
-
-            TODO: Consider embedding this information into the tree for
-            more efficient look up of pending segment positions.
-        */
-        this.mergeTree.walkAllSegments(this.mergeTree.root, (seg) => {
-            // If we've found the desired segment, terminate the walk and return 'segmentPosition'.
-            if (seg === segment) {
-                return false;
-            }
-
-            // Otherwise, advance segmentPosition if the segment has been inserted and not removed
-            // with respect to the given 'localSeq'.
-            //
-            // Note that all ACKed / remote ops are applied and we only need concern ourself with
-            // determining if locally pending ops fall before/after the given 'localSeq'.
-            if ((seg.localSeq === undefined || seg.localSeq <= localSeq)                // Is inserted
-                && (seg.removedSeq === undefined || seg.localRemovedSeq! > localSeq)     // Not removed
-            ) {
-                segmentPosition += seg.cachedLength;
-            }
-
-            return true;
-        });
-
-        return segmentPosition;
+        const { currentSeq, clientId } = this.getCollabWindow();
+        return this.mergeTree.getPosition(segment, currentSeq, clientId, localSeq);
     }
 
     /**
@@ -716,6 +692,12 @@ export class Client {
      * If the position refers to a segment/offset that was removed by some operation between `seqNumberFrom` and
      * the current sequence number, the returned position will align with the position of a reference given
      * `SlideOnRemove` semantics.
+     *
+     * If a reference was initially given `StayOnRemove` semantics, with intent to later change to `SlideOnRemove`,
+     * that isn't equivalent, and so local bookkeeping needs to be updated.
+     *
+     * If the position has slid off and there is no nearest position (i.e. the
+     * tree is empty), returns `DetachedReferencePosition`
      */
     public rebasePosition(
         pos: number,
@@ -723,45 +705,31 @@ export class Client {
         localSeq: number,
     ): number {
         assert(localSeq <= this.mergeTree.collabWindow.localSeq, 0x300 /* localSeq greater than collab window */);
-        let segment: ISegment | undefined;
-        let posAccumulated = 0;
-        let offset = pos;
-        const isInsertedInView = (seg: ISegment) =>
-            (seg.seq !== undefined && seg.seq !== UnassignedSequenceNumber && seg.seq <= seqNumberFrom)
-            || (seg.localSeq !== undefined && seg.localSeq <= localSeq);
-
-        const isRemovedFromView = ({ removedSeq, localRemovedSeq }: ISegment) =>
-            (removedSeq !== undefined && removedSeq !== UnassignedSequenceNumber && removedSeq <= seqNumberFrom)
-            || (localRemovedSeq !== undefined && localRemovedSeq <= localSeq);
-
-        this.mergeTree.walkAllSegments(this.mergeTree.root, (seg) => {
-            assert(seg.seq !== undefined || seg.localSeq !== undefined,
-                0x301 /* Either seq or localSeq should be defined */);
-            segment = seg;
-
-            if (isInsertedInView(seg) && !isRemovedFromView(seg)) {
-                posAccumulated += seg.cachedLength;
-                if (offset >= seg.cachedLength) {
-                    offset -= seg.cachedLength;
-                }
+        const { clientId } = this.getCollabWindow();
+        let { segment, offset } = this.mergeTree.getContainingSegment(pos, seqNumberFrom, clientId, localSeq);
+        if (segment === undefined && offset === undefined) {
+            // getContainingSegment will only return non-removed segments. This means the position was past
+            // all non-removed segments in the tree, so we take the last one instead as an approximation.
+            let finalSegment: IMergeNode = this.mergeTree.root;
+            while (!finalSegment.isLeaf()) {
+                finalSegment = finalSegment.children[finalSegment.childCount - 1];
             }
+            segment = finalSegment;
+            offset = 0;
+        }
 
-            // Keep going while we've yet to reach the segment at the desired position
-            return posAccumulated <= pos;
-        });
-
+        // if segment is undefined, it slid off the string
         assert(segment !== undefined, 0x302 /* No segment found */);
-        const seqNumberTo = this.getCollabWindow().currentSeq;
-        if ((segment.removedSeq !== undefined &&
-             segment.removedSeq !== UnassignedSequenceNumber &&
-             segment.removedSeq <= seqNumberTo)
-            || (segment.localRemovedSeq !== undefined && segment.localRemovedSeq <= localSeq)) {
-            // Segment that the position was in has been removed: null out offset.
-            offset = 0;
+
+        const segoff = this.getSlideToSegment({ segment, offset }) ?? segment;
+
+        // case happens when rebasing op, but concurrently entire string has been deleted
+        if (segoff.segment === undefined || segoff.offset === undefined) {
+            return DetachedReferencePosition;
         }
 
-        assert(0 <= offset && offset < segment.cachedLength, 0x303 /* Invalid offset */);
-        return this.findReconnectionPosition(segment, localSeq) + offset;
+        assert(offset !== undefined && 0 <= offset && offset < segment.cachedLength, 0x303 /* Invalid offset */);
+        return this.findReconnectionPosition(segoff.segment, localSeq) + segoff.offset;
     }
 
     private resetPendingDeltaToOps(
@@ -776,7 +744,7 @@ export class Client {
         // We need to sort the segments by ordinal, as the segments are not sorted in the segment group.
         // The reason they need them sorted, as they have the same local sequence number and which means
         // farther segments will  take into account nearer segments when calculating their position.
-        // By sorting we ensure the nearer segment will be applied and sequenced before the father segments
+        // By sorting we ensure the nearer segment will be applied and sequenced before the farther segments
         // so their recalculated positions will be correct.
         for (const segment of segmentGroup.segments.sort((a, b) => a.ordinal < b.ordinal ? -1 : 1)) {
             const segmentSegGroup = segment.segmentGroups.dequeue();
@@ -791,7 +759,8 @@ export class Client {
                     // if the segment has been removed, there's no need to send the annotate op
                     // unless the remove was local, in which case the annotate must have come
                     // before the remove
-                    if (segment.removedSeq === undefined || segment.localRemovedSeq !== undefined) {
+                    if (segment.removedSeq === undefined ||
+                        (segment.localRemovedSeq !== undefined && segment.removedSeq === UnassignedSequenceNumber)) {
                         newOp = createAnnotateRangeOp(
                             segmentPosition,
                             segmentPosition + segment.cachedLength,
@@ -815,7 +784,7 @@ export class Client {
                     break;
 
                 case MergeTreeDeltaType.REMOVE:
-                    if (segment.localRemovedSeq !== undefined) {
+                    if (segment.localRemovedSeq !== undefined && segment.removedSeq === UnassignedSequenceNumber) {
                         newOp = createRemoveRangeOp(
                             segmentPosition,
                             segmentPosition + segment.cachedLength);
@@ -942,8 +911,8 @@ export class Client {
     }
 
     /**
-     *  Given an pending operation and segment group, regenerate the op, so it
-     *  can be resubmitted
+     * Given an pending operation and segment group, regenerate the op, so it
+     * can be resubmitted
      * @param resetOp - The op to reset
      * @param segmentGroup - The segment group associated with the op
      */
@@ -979,7 +948,7 @@ export class Client {
         return opList.length === 1 ? opList[0] : createGroupOp(...opList);
     }
 
-    public createTextHelper() {
+    public createTextHelper(): IMergeTreeTextHelper {
         return new MergeTreeTextHelper(this.mergeTree);
     }
 
@@ -1028,18 +997,17 @@ export class Client {
 
         return loader.initialize(storage);
     }
-
+    /**
+     * @deprecated  for internal use only. public export will be removed.
+     * @internal
+     */
     getStackContext(startPos: number, rangeLabels: string[]): RangeStackMap {
         return this.mergeTree.getStackContext(startPos, this.getCollabWindow().clientId, rangeLabels);
     }
 
     private getLocalSequenceNumber() {
         const segWindow = this.getCollabWindow();
-        if (segWindow.collaborating) {
-            return UnassignedSequenceNumber;
-        } else {
-            return UniversalSequenceNumber;
-        }
+        return segWindow.collaborating ? UnassignedSequenceNumber : UniversalSequenceNumber;
     }
     localTransaction(groupOp: IMergeTreeGroupMsg) {
         for (const op of groupOp.ops) {
@@ -1086,9 +1054,9 @@ export class Client {
         }
     }
 
-    getContainingSegment<T extends ISegment>(pos: number, op?: ISequencedDocumentMessage) {
+    getContainingSegment<T extends ISegment>(pos: number, op?: ISequencedDocumentMessage, localSeq?: number) {
         const args = this.getClientSequenceArgsForMessage(op);
-        return this.mergeTree.getContainingSegment<T>(pos, args.referenceSequenceNumber, args.clientId);
+        return this.mergeTree.getContainingSegment<T>(pos, args.referenceSequenceNumber, args.clientId, localSeq);
     }
 
     /**
@@ -1097,7 +1065,19 @@ export class Client {
      * @returns - segment and offset to slide the reference to
      */
     getSlideToSegment(segoff: { segment: ISegment | undefined; offset: number | undefined; }) {
-        return this.mergeTree._getSlideToSegment(segoff);
+        if (segoff.segment === undefined) {
+            return segoff;
+        }
+        const segment = this.mergeTree._getSlideToSegment(segoff.segment);
+        if (segment === segoff.segment) {
+            return segoff;
+        }
+        const offset =
+            segment && segment.ordinal < segoff.segment.ordinal ? segment.cachedLength - 1 : 0;
+        return {
+            segment,
+            offset,
+        };
     }
 
     getPropertiesAtPosition(pos: number) {

package/src/collections/heap.ts

@@ -0,0 +1,75 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/**
+ * @deprecated  for internal use only. public export will be removed.
+ * @internal
+ */
+export interface Comparer<T> {
+    compare(a: T, b: T): number;
+    min: T;
+}
+
+/**
+ * @deprecated  for internal use only. public export will be removed.
+ * @internal
+ */
+export class Heap<T> {
+    private L: T[];
+    public count() {
+        return this.L.length - 1;
+    }
+    constructor(a: T[], public comp: Comparer<T>) {
+        this.L = [comp.min];
+        for (let i = 0, len = a.length; i < len; i++) {
+            this.add(a[i]);
+        }
+    }
+    public peek() {
+        return this.L[1];
+    }
+
+    public get() {
+        const x = this.L[1];
+        this.L[1] = this.L[this.count()];
+        this.L.pop();
+        this.fixDown(1);
+        return x;
+    }
+
+    public add(x: T) {
+        this.L.push(x);
+        this.fixup(this.count());
+    }
+
+    /* eslint-disable no-bitwise */
+    private fixup(k: number) {
+        let _k = k;
+        while (_k > 1 && (this.comp.compare(this.L[_k >> 1], this.L[_k]) > 0)) {
+            const tmp = this.L[_k >> 1];
+            this.L[_k >> 1] = this.L[_k];
+            this.L[_k] = tmp;
+            _k = _k >> 1;
+        }
+    }
+
+    private fixDown(k: number) {
+        let _k = k;
+        while ((_k << 1) <= (this.count())) {
+            let j = _k << 1;
+            if ((j < this.count()) && (this.comp.compare(this.L[j], this.L[j + 1]) > 0)) {
+                j++;
+            }
+            if (this.comp.compare(this.L[_k], this.L[j]) <= 0) {
+                break;
+            }
+            const tmp = this.L[_k];
+            this.L[_k] = this.L[j];
+            this.L[j] = tmp;
+            _k = j;
+        }
+    }
+    /* eslint-enable no-bitwise */
+}

package/src/collections/index.ts

@@ -0,0 +1,11 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export * from "./heap";
+export * from "./intervalTree";
+export * from "./list";
+export * from "./rbTree";
+export * from "./stack";
+export * from "./tst";

package/src/collections/intervalTree.ts

@@ -0,0 +1,146 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import {
+    IIntegerRange,
+} from "../base";
+import { ConflictAction, IRBAugmentation, IRBMatcher, RBNode, RBNodeActions, RedBlackTree } from "./rbTree";
+
+/**
+ * @deprecated  for internal use only. public export will be removed.
+ * @internal
+ */
+export interface AugmentedIntervalNode {
+    minmax: IInterval;
+}
+
+/**
+ * @deprecated  for internal use only. public export will be removed.
+ * @internal
+ */
+export const integerRangeToString = (range: IIntegerRange) => `[${range.start},${range.end})`;
+
+/**
+ * @deprecated  for internal use only. public export will be removed.
+ * @internal
+ */
+export interface IInterval {
+    clone(): IInterval;
+    compare(b: IInterval): number;
+    compareStart(b: IInterval): number;
+    compareEnd(b: IInterval): number;
+    modify(
+        label: string,
+        start: number,
+        end: number,
+        op?: ISequencedDocumentMessage,
+        localSeq?: number
+    ): IInterval | undefined;
+    overlaps(b: IInterval): boolean;
+    union(b: IInterval): IInterval;
+}
+
+const intervalComparer = (a: IInterval, b: IInterval) => a.compare(b);
+/**
+ * @deprecated  for internal use only. public export will be removed.
+ * @internal
+ */
+export type IntervalNode<T extends IInterval> = RBNode<T, AugmentedIntervalNode>;
+/**
+ * @deprecated  for internal use only. public export will be removed.
+ * @internal
+ */
+export type IntervalConflictResolver<TInterval> = (a: TInterval, b: TInterval) => TInterval;
+
+/**
+ * @deprecated  for internal use only. public export will be removed.
+ * @internal
+ */
+export class IntervalTree<T extends IInterval> implements IRBAugmentation<T, AugmentedIntervalNode>,
+    IRBMatcher<T, AugmentedIntervalNode> {
+    public intervals = new RedBlackTree<T, AugmentedIntervalNode>(intervalComparer, this);
+
+    public remove(x: T) {
+        this.intervals.remove(x);
+    }
+
+    public removeExisting(x: T) {
+        this.intervals.removeExisting(x);
+    }
+
+    public put(x: T, conflict?: IntervalConflictResolver<T>) {
+        let rbConflict: ConflictAction<T, AugmentedIntervalNode> | undefined;
+        if (conflict) {
+            rbConflict = (key: T, currentKey: T) => {
+                const ival = conflict(key, currentKey);
+                return {
+                    key: ival,
+                };
+            };
+        }
+        this.intervals.put(x, { minmax: x.clone() }, rbConflict);
+    }
+
+    public map(fn: (x: T) => void) {
+        const actions: RBNodeActions<T, AugmentedIntervalNode> = {
+            infix: (node) => {
+                fn(node.key);
+                return true;
+            },
+            showStructure: true,
+        };
+        this.intervals.walk(actions);
+    }
+
+    public mapUntil(fn: (X: T) => boolean) {
+        const actions: RBNodeActions<T, AugmentedIntervalNode> = {
+            infix: (node) => {
+                return fn(node.key);
+            },
+            showStructure: true,
+        };
+        this.intervals.walk(actions);
+    }
+
+    public mapBackward(fn: (x: T) => void) {
+        const actions: RBNodeActions<T, AugmentedIntervalNode> = {
+            infix: (node) => {
+                fn(node.key);
+                return true;
+            },
+            showStructure: true,
+        };
+        this.intervals.walkBackward(actions);
+    }
+
+    // TODO: toString()
+    public match(x: T) {
+        return this.intervals.gather(x, this);
+    }
+
+    public matchNode(node: IntervalNode<T> | undefined, key: T) {
+        return !!node && node.key.overlaps(key);
+    }
+
+    public continueSubtree(node: IntervalNode<T> | undefined, key: T) {
+        return !!node && node.data.minmax.overlaps(key);
+    }
+
+    public update(node: IntervalNode<T>) {
+        if (node.left && node.right) {
+            node.data.minmax = node.key.union(
+                node.left.data.minmax.union(node.right.data.minmax));
+        } else {
+            if (node.left) {
+                node.data.minmax = node.key.union(node.left.data.minmax);
+            } else if (node.right) {
+                node.data.minmax = node.key.union(node.right.data.minmax);
+            } else {
+                node.data.minmax = node.key.clone();
+            }
+        }
+    }
+}