@fluidframework/merge-tree 0.59.4002 → 1.0.2

package/src/client.ts

@@ -16,7 +16,7 @@ import { LoggingError } from "@fluidframework/telemetry-utils";
 import { IIntegerRange } from "./base";
 import { RedBlackTree } from "./collections";
 import { UnassignedSequenceNumber, UniversalSequenceNumber } from "./constants";
-import { LocalReference } from "./localReference";
+import { LocalReference, _validateReferenceType } from "./localReference";
 import {
     CollaborationWindow,
     compareStrings,
@@ -575,13 +575,14 @@ export class Client {
 
     /**
      * Gets the client args from the op if remote, otherwise uses the local clients info
-     * @param opArgs - The op arg to get the client sequence args for
+     * @param sequencedMessage - The sequencedMessage to get the client sequence args for
      */
-    private getClientSequenceArgs(opArgs: IMergeTreeDeltaOpArgs): IMergeTreeClientSequenceArgs {
+     private getClientSequenceArgsForMessage(sequencedMessage: ISequencedDocumentMessage | undefined):
+        IMergeTreeClientSequenceArgs {
         // If there this no sequenced message, then the op is local
         // and unacked, so use this clients sequenced args
         //
-        if (!opArgs.sequencedMessage) {
+        if (!sequencedMessage) {
             const segWindow = this.getCollabWindow();
             return {
                 clientId: segWindow.clientId,
@@ -590,13 +591,21 @@ export class Client {
             };
         } else {
             return {
-                clientId: this.getShortClientId(opArgs.sequencedMessage.clientId),
-                referenceSequenceNumber: opArgs.sequencedMessage.referenceSequenceNumber,
-                sequenceNumber: opArgs.sequencedMessage.sequenceNumber,
+                clientId: this.getOrAddShortClientId(sequencedMessage.clientId),
+                referenceSequenceNumber: sequencedMessage.referenceSequenceNumber,
+                sequenceNumber: sequencedMessage.sequenceNumber,
             };
         }
     }
 
+    /**
+     * Gets the client args from the op if remote, otherwise uses the local clients info
+     * @param opArgs - The op arg to get the client sequence args for
+     */
+    private getClientSequenceArgs(opArgs: IMergeTreeDeltaOpArgs): IMergeTreeClientSequenceArgs {
+        return this.getClientSequenceArgsForMessage(opArgs.sequencedMessage);
+    }
+
     private ackPendingSegment(opArgs: IMergeTreeDeltaOpArgs) {
         const ackOp = (deltaOpArgs: IMergeTreeDeltaOpArgs) => {
             let trace: Trace | undefined;
@@ -1021,17 +1030,17 @@ export class Client {
     }
 
     getContainingSegment<T extends ISegment>(pos: number, op?: ISequencedDocumentMessage) {
-        let seq: number;
-        let clientId: number;
-        if (op) {
-            clientId = this.getOrAddShortClientId(op.clientId);
-            seq = op.referenceSequenceNumber;
-        } else {
-            const segWindow = this.mergeTree.getCollabWindow();
-            seq = segWindow.currentSeq;
-            clientId = segWindow.clientId;
-        }
-        return this.mergeTree.getContainingSegment<T>(pos, seq, clientId);
+        const args = this.getClientSequenceArgsForMessage(op);
+        return this.mergeTree.getContainingSegment<T>(pos, args.referenceSequenceNumber, args.clientId);
+    }
+
+    /**
+     * Returns the position to slide a reference to if a slide is required.
+     * @param segoff - The segment and offset to slide from
+     * @returns - segment and offset to slide the reference to
+     */
+    getSlideToSegment(segoff: { segment: ISegment | undefined; offset: number | undefined; }) {
+        return this.mergeTree._getSlideToSegment(segoff);
     }
 
     getPropertiesAtPosition(pos: number) {

package/src/collections.ts

@@ -245,10 +245,11 @@ export class Heap<T> {
     /* eslint-enable no-bitwise */
 }
 
-export const enum RBColor {
-    RED,
-    BLACK,
-}
+export const RBColor = {
+    RED: 0,
+    BLACK: 1,
+} as const;
+export type RBColor = typeof RBColor[keyof typeof RBColor];
 
 export interface RBNode<TKey, TData> {
     key: TKey;

package/src/index.ts

@@ -7,7 +7,7 @@ export * from "./base";
 export * from "./client";
 export * from "./collections";
 export * from "./constants";
-export * from "./localReference";
+export { LocalReference, LocalReferenceCollection } from "./localReference";
 export * from "./mergeTree";
 export * from "./mergeTreeDeltaCallback";
 export * from "./mergeTreeTracking";

package/src/localReference.ts

@@ -4,6 +4,7 @@
  */
 
 import { assert } from "@fluidframework/common-utils";
+import { UsageError } from "@fluidframework/container-utils";
 import { Client } from "./client";
 import { List, ListMakeHead, ListRemoveEntry } from "./collections";
 import {
@@ -25,6 +26,26 @@ import {
     refTypeIncludesFlag,
 } from "./referencePositions";
 
+/**
+ * @internal
+ */
+export function _validateReferenceType(refType: ReferenceType) {
+    let exclusiveCount = 0;
+    if (refTypeIncludesFlag(refType, ReferenceType.Transient)) {
+        ++exclusiveCount;
+    }
+    if (refTypeIncludesFlag(refType, ReferenceType.SlideOnRemove)) {
+        ++exclusiveCount;
+    }
+    if (refTypeIncludesFlag(refType, ReferenceType.StayOnRemove)) {
+        ++exclusiveCount;
+    }
+    if (exclusiveCount > 1) {
+        throw new UsageError(
+            "Reference types can only be one of Transient, SlideOnRemove, and StayOnRemove");
+    }
+}
+
 /**
  * @deprecated - Use ReferencePosition
  */
@@ -57,6 +78,7 @@ export class LocalReference implements ReferencePosition {
         public refType = ReferenceType.Simple,
         properties?: PropertySet,
     ) {
+        _validateReferenceType(refType);
         this.segment = initSegment;
         this.properties = properties;
     }
@@ -312,8 +334,8 @@ export class LocalReferenceCollection {
             !refTypeIncludesFlag(lref, ReferenceType.Transient),
             0x2df /* "transient references cannot be bound to segments" */);
         assertLocalReferences(lref);
-        const refsAtOffset = this.refsByOffset[lref.getOffset()] =
-            this.refsByOffset[lref.getOffset()]
+        const refsAtOffset = this.refsByOffset[lref.offset] =
+            this.refsByOffset[lref.offset]
             ?? { at: ListMakeHead() };
         const atRefs = refsAtOffset.at =
             refsAtOffset.at

package/src/mergeTree.ts

@@ -9,6 +9,7 @@
 /* eslint-disable @typescript-eslint/prefer-optional-chain, no-bitwise */
 
 import { assert } from "@fluidframework/common-utils";
+import { UsageError } from "@fluidframework/container-utils";
 import {
     Comparer,
     Heap,
@@ -109,6 +110,15 @@ export function toRemovalInfo(maybe: Partial<IRemovalInfo> | undefined): IRemova
         0x2bf /* "both removedClientIds and removedSeq should be set or not set" */);
 }
 
+function isRemoved(segment: ISegment): boolean {
+    return toRemovalInfo(segment) !== undefined;
+}
+
+function isRemovedAndAcked(segment: ISegment): boolean {
+    const removalInfo = toRemovalInfo(segment);
+    return removalInfo !== undefined && removalInfo.removedSeq !== UnassignedSequenceNumber;
+}
+
 /**
  * A segment representing a portion of the merge tree.
  */
@@ -554,7 +564,6 @@ export abstract class BaseSegment extends MergeNode implements ISegment {
                 return true;
 
             case MergeTreeDeltaType.REMOVE:
-
                 const removalInfo: IRemovalInfo | undefined = toRemovalInfo(this);
                 assert(removalInfo !== undefined, 0x046 /* "On remove ack, missing removal info!" */);
                 this.localRemovedSeq = undefined;
@@ -562,7 +571,6 @@ export abstract class BaseSegment extends MergeNode implements ISegment {
                     removalInfo.removedSeq = opArgs.sequencedMessage!.sequenceNumber;
                     return true;
                 }
-
                 return false;
 
             default:
@@ -1414,6 +1422,109 @@ export class MergeTree {
         return { segment, offset };
     }
 
+    /**
+     * @internal must only be used by client
+     * @param segoff - The segment and offset to slide from
+     * @returns The segment and offset to slide to
+     */
+    public _getSlideToSegment(segoff: { segment: ISegment | undefined; offset: number | undefined; }) {
+        if (!segoff.segment || !isRemovedAndAcked(segoff.segment)) {
+            return segoff;
+        }
+        // Slide to the next farthest valid segment in the tree. If no such segment is found
+        // slide to the last valid segment.
+        // TODO this walks the whole tree to find the segment - could write a more efficient
+        // walk that starts at the segment
+        let foundStart = false;
+        let foundSegmentPastStart = false;
+        let slideToSegment: ISegment | undefined;
+        this.walkAllSegments(this.root, (seg) => {
+            if (seg.seq !== UnassignedSequenceNumber && !isRemovedAndAcked(seg)) {
+                slideToSegment = seg;
+                if (foundStart) {
+                    foundSegmentPastStart = true;
+                    return false;
+                }
+            }
+            if (!foundStart && seg === segoff.segment) {
+                foundStart = true;
+            }
+            return true;
+        });
+        let offset = 0;
+        if (slideToSegment && !foundSegmentPastStart) {
+            // If slid nearer then offset should be at the end of the segment
+            offset = slideToSegment.cachedLength - 1;
+        }
+        return { segment: slideToSegment, offset };
+    }
+
+    /**
+     * This method should only be called when the current client sequence number is
+     * max(remove segment sequence number, add reference sequence number).
+     * Otherwise eventual consistency is not guaranteed.
+     * See `packages\dds\merge-tree\REFERENCEPOSITIONS.md`
+     */
+    private slideReferences(segment: ISegment, refsToSlide: LocalReference[]) {
+        assert(
+            isRemovedAndAcked(segment),
+            0x2f1 /* slideReferences from a segment which has not been removed and acked */);
+        assert(!!segment.localRefs, 0x2f2 /* Ref not in the segment localRefs */);
+        const newSegoff = this._getSlideToSegment({ segment, offset: 0 });
+        const newSegment = newSegoff.segment;
+        if (newSegment && !newSegment.localRefs) {
+            newSegment.localRefs = new LocalReferenceCollection(newSegment);
+        }
+        for (const ref of refsToSlide) {
+            const removedRef = segment.localRefs.removeLocalRef(ref);
+            assert(ref === removedRef, 0x2f3 /* Ref not in the segment localRefs */);
+            if (!newSegment) {
+                // No valid segments (all nodes removed or not yet created)
+                ref.segment = undefined;
+                ref.offset = 0;
+            } else {
+                ref.segment = newSegment;
+                ref.offset = newSegoff.offset ?? 0;
+                assert(!!newSegment.localRefs, 0x2f4 /* localRefs must be allocated */);
+                newSegment.localRefs.addLocalRef(ref);
+            }
+        }
+        // TODO is it required to update the path lengths?
+        if (newSegment) {
+            this.blockUpdatePathLengths(newSegment.parent, TreeMaintenanceSequenceNumber,
+                LocalClientId);
+        }
+    }
+
+    private updateSegmentRefsAfterMarkRemoved(segment: ISegment, pending: boolean) {
+        if (!segment.localRefs || segment.localRefs.empty) {
+            return;
+        }
+        const refsToSlide: LocalReference[] = [];
+        const refsToStay: LocalReference[] = [];
+        for (const lref of segment.localRefs) {
+            if (refTypeIncludesFlag(lref, ReferenceType.StayOnRemove)) {
+                refsToStay.push(lref);
+            } else if (refTypeIncludesFlag(lref, ReferenceType.SlideOnRemove)) {
+                if (pending) {
+                    refsToStay.push(lref);
+                } else {
+                    refsToSlide.push(lref);
+                }
+            }
+        }
+        // Rethink implementation of keeping and sliding refs once other reference
+        // changes are complete. This works but is fragile and possibly slow.
+        if (!pending) {
+            this.slideReferences(segment, refsToSlide);
+        }
+        segment.localRefs.clear();
+        for (const lref of refsToStay) {
+            lref.segment = segment;
+            segment.localRefs.addLocalRef(lref);
+        }
+    }
+
     private blockLength(node: IMergeBlock, refSeq: number, clientId: number) {
         if ((this.collabWindow.collaborating) && (clientId !== this.collabWindow.clientId)) {
             return node.partialLengths!.getPartialLength(refSeq, clientId);
@@ -1676,7 +1787,13 @@ export class MergeTree {
         if (pendingSegmentGroup !== undefined) {
             const deltaSegments: IMergeTreeSegmentDelta[] = [];
             pendingSegmentGroup.segments.map((pendingSegment) => {
-                overwrite = !pendingSegment.ack(pendingSegmentGroup, opArgs, this) || overwrite;
+                const modified = pendingSegment.ack(pendingSegmentGroup, opArgs, this);
+                // This computation of overwrite appears incorrect. Leaving as is to avoid breaking something.
+                overwrite = !modified || overwrite;
+
+                if (modified && opArgs.op.type === MergeTreeDeltaType.REMOVE) {
+                    this.updateSegmentRefsAfterMarkRemoved(pendingSegment, false);
+                }
                 if (MergeTree.options.zamboniSegments) {
                     this.addToLRUSet(pendingSegment, seq);
                 }
@@ -2338,7 +2455,7 @@ export class MergeTree {
         this.ensureIntervalBoundary(end, refSeq, clientId);
         let segmentGroup: SegmentGroup;
         const removedSegments: IMergeTreeSegmentDelta[] = [];
-        const savedLocalRefs: LocalReferenceCollection[] = [];
+        const segmentsWithRefs: ISegment[] = [];
         const localSeq = seq === UnassignedSequenceNumber ? ++this.collabWindow.localSeq : undefined;
         const markRemoved = (segment: ISegment, pos: number, _start: number, _end: number) => {
             const existingRemovalInfo = toRemovalInfo(segment);
@@ -2362,10 +2479,9 @@ export class MergeTree {
                 segment.localRemovedSeq = localSeq;
 
                 removedSegments.push({ segment });
-                if (segment.localRefs && !segment.localRefs.empty) {
-                    savedLocalRefs.push(segment.localRefs);
-                }
-                segment.localRefs = undefined;
+            }
+            if (segment.localRefs && !segment.localRefs.empty) {
+                segmentsWithRefs.push(segment);
             }
 
             // Save segment so can assign removed sequence number when acked by server
@@ -2389,37 +2505,9 @@ export class MergeTree {
             return true;
         };
         this.mapRange({ leaf: markRemoved, post: afterMarkRemoved }, refSeq, clientId, undefined, start, end);
-        if (savedLocalRefs.length > 0) {
-            const length = this.getLength(refSeq, clientId);
-            let refSegment: ISegment | undefined;
-            if (start < length) {
-                const afterSegOff = this.getContainingSegment(start, refSeq, clientId);
-                refSegment = afterSegOff.segment;
-                assert(!!refSegment, 0x052 /* "Missing reference segment!" */);
-                if (!refSegment.localRefs) {
-                    refSegment.localRefs = new LocalReferenceCollection(refSegment);
-                }
-                refSegment.localRefs.addBeforeTombstones(...savedLocalRefs);
-            } else if (length > 0) {
-                const beforeSegOff = this.getContainingSegment(length - 1, refSeq, clientId);
-                refSegment = beforeSegOff.segment;
-                assert(!!refSegment, 0x053 /* "Missing reference segment!" */);
-                if (!refSegment.localRefs) {
-                    refSegment.localRefs = new LocalReferenceCollection(refSegment);
-                }
-                refSegment.localRefs.addAfterTombstones(...savedLocalRefs);
-            } else {
-                // TODO: The tree is empty, so there isn't anywhere to put these
-                // they should be preserved somehow
-                for (const refsCollection of savedLocalRefs) {
-                    refsCollection.clear();
-                }
-            }
-
-            if (refSegment) {
-                this.blockUpdatePathLengths(refSegment.parent, TreeMaintenanceSequenceNumber,
-                    LocalClientId);
-            }
+        const pending = this.collabWindow.collaborating && clientId === this.collabWindow.clientId;
+        for (const segment of segmentsWithRefs) {
+            this.updateSegmentRefsAfterMarkRemoved(segment, pending);
         }
 
         // opArgs == undefined => test code
@@ -2460,6 +2548,12 @@ export class MergeTree {
         segment: ISegment, offset: number, refType: ReferenceType, properties: PropertySet | undefined,
         client: Client,
     ): ReferencePosition {
+        if (isRemoved(segment)) {
+            if (!refTypeIncludesFlag(refType, ReferenceType.SlideOnRemove | ReferenceType.Transient)) {
+                throw new UsageError(
+                    "Can only create SlideOnRemove or Transient local reference position on a removed segment");
+            }
+        }
         const localRefs = segment.localRefs ?? new LocalReferenceCollection(segment);
         segment.localRefs = localRefs;
 

package/src/mergeTreeDeltaCallback.ts

@@ -13,28 +13,26 @@ import { PropertySet } from "./properties";
 import { ISegment } from "./mergeTree";
 
 export type MergeTreeDeltaOperationType =
-    MergeTreeDeltaType.ANNOTATE | MergeTreeDeltaType.INSERT | MergeTreeDeltaType.REMOVE;
+    typeof MergeTreeDeltaType.ANNOTATE | typeof MergeTreeDeltaType.INSERT | typeof MergeTreeDeltaType.REMOVE;
 
 // Note: Assigned negative integers to avoid clashing with MergeTreeDeltaType
-export const enum MergeTreeMaintenanceType {
-    APPEND = -1,
-    SPLIT = -2,
+export const MergeTreeMaintenanceType = {
+    APPEND: -1,
+    SPLIT: -2,
     /**
      * Notification that a segment has been unlinked from the MergeTree.  This occurs during
      * Zamboni when:
      *
-     *    a) The minSeq has moved past the segment's removeSeq, in which case the segment
-     *       can no longer be referenced by incoming remote ops, and...
-     *
      *    b) The segment's tracking collection is empty (e.g., not being tracked for undo/redo).
      */
-    UNLINK = -3,
+    UNLINK: -3,
     /**
      * Notification that a local change has been acknowledged by the server.
      * This means that it has made the round trip to the server and has had a sequence number assigned.
      */
-    ACKNOWLEDGED = -4,
-}
+    ACKNOWLEDGED: -4,
+} as const;
+export type MergeTreeMaintenanceType = typeof MergeTreeMaintenanceType[keyof typeof MergeTreeMaintenanceType];
 
 export type MergeTreeDeltaOperationTypes = MergeTreeDeltaOperationType | MergeTreeMaintenanceType;
 

package/src/ops.ts

@@ -11,6 +11,7 @@ export enum ReferenceType {
     RangeBegin = 0x10,
     RangeEnd = 0x20,
     SlideOnRemove = 0x40,
+    StayOnRemove = 0x80,
     Transient = 0x100,
 }
 
@@ -19,12 +20,14 @@ export interface IMarkerDef {
 }
 
 // Note: Assigned positive integers to avoid clashing with MergeTreeMaintenanceType
-export const enum MergeTreeDeltaType {
-    INSERT = 0,
-    REMOVE = 1,
-    ANNOTATE = 2,
-    GROUP = 3,
-}
+export const MergeTreeDeltaType = {
+    INSERT: 0,
+    REMOVE: 1,
+    ANNOTATE: 2,
+    GROUP: 3,
+} as const;
+
+export type MergeTreeDeltaType = typeof MergeTreeDeltaType[keyof typeof MergeTreeDeltaType];
 
 export interface IMergeTreeDelta {
     /**
@@ -54,7 +57,7 @@ export interface IRelativePosition {
 }
 
 export interface IMergeTreeInsertMsg extends IMergeTreeDelta {
-    type: MergeTreeDeltaType.INSERT;
+    type: typeof MergeTreeDeltaType.INSERT;
     pos1?: number;
     relativePos1?: IRelativePosition;
     pos2?: number;
@@ -63,7 +66,7 @@ export interface IMergeTreeInsertMsg extends IMergeTreeDelta {
 }
 
 export interface IMergeTreeRemoveMsg extends IMergeTreeDelta {
-    type: MergeTreeDeltaType.REMOVE;
+    type: typeof MergeTreeDeltaType.REMOVE;
     pos1?: number;
     relativePos1?: IRelativePosition;
     pos2?: number;
@@ -78,7 +81,7 @@ export interface ICombiningOp {
 }
 
 export interface IMergeTreeAnnotateMsg extends IMergeTreeDelta {
-    type: MergeTreeDeltaType.ANNOTATE;
+    type: typeof MergeTreeDeltaType.ANNOTATE;
     pos1?: number;
     relativePos1?: IRelativePosition;
     pos2?: number;
@@ -88,7 +91,7 @@ export interface IMergeTreeAnnotateMsg extends IMergeTreeDelta {
 }
 
 export interface IMergeTreeGroupMsg extends IMergeTreeDelta {
-    type: MergeTreeDeltaType.GROUP;
+    type: typeof MergeTreeDeltaType.GROUP;
     ops: IMergeTreeDeltaOp[];
 }
 

package/src/referencePositions.ts

@@ -11,9 +11,10 @@ import { PropertySet, MapLike } from "./properties";
 export const reservedTileLabelsKey = "referenceTileLabels";
 export const reservedRangeLabelsKey = "referenceRangeLabels";
 
-export function refTypeIncludesFlag(refPos: ReferencePosition, flags: ReferenceType): boolean {
+export function refTypeIncludesFlag(refPosOrType: ReferencePosition | ReferenceType, flags: ReferenceType): boolean {
+    const refType = typeof refPosOrType === "number" ? refPosOrType : refPosOrType.refType;
     // eslint-disable-next-line no-bitwise
-    return (refPos.refType & flags) !== 0;
+    return (refType & flags) !== 0;
 }
 
 export const refGetTileLabels = (refPos: ReferencePosition): string[] | undefined =>