@fluidframework/merge-tree 2.0.0-rc.1.0.3 → 2.0.0-rc.2.0.0

package/src/revertibles.ts

@@ -5,17 +5,17 @@
 
 import { assert, unreachableCase } from "@fluidframework/core-utils";
 import { UsageError } from "@fluidframework/telemetry-utils";
-import { DoublyLinkedList } from "./collections";
-import { EndOfTreeSegment } from "./endOfTreeSegment";
-import { LocalReferenceCollection, LocalReferencePosition } from "./localReference";
-import { IMergeTreeDeltaCallbackArgs } from "./mergeTreeDeltaCallback";
-import { IMergeLeaf, ISegment, toRemovalInfo } from "./mergeTreeNodes";
-import { depthFirstNodeWalk } from "./mergeTreeNodeWalk";
-import { ITrackingGroup, Trackable, UnorderedTrackingGroup } from "./mergeTreeTracking";
-import { IJSONSegment, MergeTreeDeltaType, ReferenceType } from "./ops";
-import { matchProperties, PropertySet } from "./properties";
-import { DetachedReferencePosition } from "./referencePositions";
-import { MergeTree, findRootMergeBlock } from "./mergeTree";
+import { DoublyLinkedList } from "./collections/index.js";
+import { EndOfTreeSegment } from "./endOfTreeSegment.js";
+import { LocalReferenceCollection, LocalReferencePosition } from "./localReference.js";
+import { IMergeTreeDeltaCallbackArgs } from "./mergeTreeDeltaCallback.js";
+import { ISegmentLeaf, ISegment, toRemovalInfo } from "./mergeTreeNodes.js";
+import { depthFirstNodeWalk } from "./mergeTreeNodeWalk.js";
+import { ITrackingGroup, Trackable, UnorderedTrackingGroup } from "./mergeTreeTracking.js";
+import { IJSONSegment, MergeTreeDeltaType, ReferenceType } from "./ops.js";
+import { matchProperties, PropertySet } from "./properties.js";
+import { DetachedReferencePosition } from "./referencePositions.js";
+import { MergeTree, findRootMergeBlock } from "./mergeTree.js";
 
 /**
  * @alpha
@@ -94,9 +94,7 @@ function findMergeTreeWithRevert(trackable: Trackable): MergeTreeWithRevert {
 		const refCallbacks: MergeTreeWithRevert["__mergeTreeRevertible"]["refCallbacks"] = {
 			afterSlide: (r: LocalReferencePosition) => {
 				if (mergeTree.referencePositionToLocalPosition(r) === DetachedReferencePosition) {
-					const refs = (detachedReferences.localRefs ??= new LocalReferenceCollection(
-						detachedReferences,
-					));
+					const refs = LocalReferenceCollection.setOrGet(detachedReferences);
 					refs.addAfterTombstones([r]);
 				}
 			},
@@ -284,7 +282,7 @@ function revertLocalRemove(
 
 		const props = tracked.properties as RemoveSegmentRefProperties;
 		driver.insertFromSpec(realPos, props.segSpec);
-		const insertSegment: IMergeLeaf | undefined = mergeTreeWithRevert.getContainingSegment(
+		const insertSegment: ISegmentLeaf | undefined = mergeTreeWithRevert.getContainingSegment(
 			realPos,
 			mergeTreeWithRevert.collabWindow.currentSeq,
 			mergeTreeWithRevert.collabWindow.clientId,
@@ -339,9 +337,7 @@ function revertLocalRemove(
 		}
 
 		if (insertRef !== undefined) {
-			const localRefs = (insertSegment.localRefs ??= new LocalReferenceCollection(
-				insertSegment,
-			));
+			const localRefs = LocalReferenceCollection.setOrGet(insertSegment);
 			if (insertRef.before?.empty === false) {
 				localRefs.addBeforeTombstones(insertRef.before.map((n) => n.data));
 			}

package/src/segmentGroupCollection.ts

@@ -3,9 +3,9 @@
  * Licensed under the MIT License.
  */
 
-import { DoublyLinkedList, walkList } from "./collections";
+import { DoublyLinkedList, walkList } from "./collections/index.js";
 // eslint-disable-next-line import/no-deprecated
-import { ISegment, SegmentGroup } from "./mergeTreeNodes";
+import { ISegment, SegmentGroup } from "./mergeTreeNodes.js";
 
 /**
  * @alpha

package/src/segmentPropertiesManager.ts

@@ -6,10 +6,10 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 
 import { assert } from "@fluidframework/core-utils";
-import { UnassignedSequenceNumber, UniversalSequenceNumber } from "./constants";
-import { IMergeTreeAnnotateMsg } from "./ops";
+import { UnassignedSequenceNumber, UniversalSequenceNumber } from "./constants.js";
+import { IMergeTreeAnnotateMsg } from "./ops.js";
 // eslint-disable-next-line import/no-deprecated
-import { createMap, MapLike, PropertySet } from "./properties";
+import { createMap, MapLike, PropertySet } from "./properties.js";
 
 /**
  * @alpha

package/src/snapshotChunks.ts

@@ -8,10 +8,10 @@
 import { IFluidHandle } from "@fluidframework/core-interfaces";
 import { IFluidSerializer } from "@fluidframework/shared-object-base";
 import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
-import { PropertySet } from "./properties";
-import { SnapshotLegacy } from "./snapshotlegacy";
-import { IJSONSegment } from "./ops";
-import { SerializedAttributionCollection } from "./attributionCollection";
+import { PropertySet } from "./properties.js";
+import { SnapshotLegacy } from "./snapshotlegacy.js";
+import { IJSONSegment } from "./ops.js";
+import { SerializedAttributionCollection } from "./attributionCollection.js";
 
 export interface VersionedMergeTreeChunk {
 	version: undefined | "1";

package/src/snapshotLoader.ts

@@ -20,14 +20,14 @@ import {
 } from "@fluidframework/datastore-definitions";
 import { AttachState } from "@fluidframework/container-definitions";
 // eslint-disable-next-line import/no-deprecated
-import { Client } from "./client";
-import { NonCollabClient, UniversalSequenceNumber } from "./constants";
-import { ISegment } from "./mergeTreeNodes";
-import { IJSONSegment } from "./ops";
-import { IJSONSegmentWithMergeInfo, hasMergeInfo, MergeTreeChunkV1 } from "./snapshotChunks";
-import { SnapshotV1 } from "./snapshotV1";
-import { SnapshotLegacy } from "./snapshotlegacy";
-import { MergeTree } from "./mergeTree";
+import { Client } from "./client.js";
+import { NonCollabClient, UniversalSequenceNumber } from "./constants.js";
+import { ISegment } from "./mergeTreeNodes.js";
+import { IJSONSegment } from "./ops.js";
+import { IJSONSegmentWithMergeInfo, hasMergeInfo, MergeTreeChunkV1 } from "./snapshotChunks.js";
+import { SnapshotV1 } from "./snapshotV1.js";
+import { SnapshotLegacy } from "./snapshotlegacy.js";
+import { MergeTree } from "./mergeTree.js";
 
 export class SnapshotLoader {
 	private readonly logger: ITelemetryLoggerExt;
@@ -83,7 +83,7 @@ export class SnapshotLoader {
 			// TODO: The 'Snapshot.catchupOps' tree entry is purely for backwards compatibility.
 			//       (See https://github.com/microsoft/FluidFramework/issues/84)
 
-			return this.loadCatchupOps(services.readBlob(blobs[0]));
+			return this.loadCatchupOps(services.readBlob(blobs[0]), this.serializer);
 		} else if (blobs.length !== headerChunk.headerMetadata!.orderedChunkMetadata.length) {
 			throw new Error("Unexpected blobs in snapshot");
 		}
@@ -301,12 +301,15 @@ export class SnapshotLoader {
 	/**
 	 * If loading from a snapshot, get the catchup messages.
 	 * @param rawMessages - The messages in original encoding
-	 * @returns The decoded messages, but handles aren't parsed.  Matches the format that will be passed in
+	 * @returns The decoded messages with parsed+hydrated handles.  Matches the format that will be passed in
 	 * SharedObject.processCore.
 	 */
 	private async loadCatchupOps(
 		rawMessages: Promise<ArrayBufferLike>,
+		serializer: IFluidSerializer,
 	): Promise<ISequencedDocumentMessage[]> {
-		return JSON.parse(bufferToString(await rawMessages, "utf8")) as ISequencedDocumentMessage[];
+		return serializer.parse(
+			bufferToString(await rawMessages, "utf8"),
+		) as ISequencedDocumentMessage[];
 	}
 }

package/src/snapshotV1.ts

@@ -11,9 +11,9 @@ import { bufferToString } from "@fluid-internal/client-utils";
 import { IChannelStorageService } from "@fluidframework/datastore-definitions";
 import { AttributionKey, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
 import { SummaryTreeBuilder } from "@fluidframework/runtime-utils";
-import { UnassignedSequenceNumber } from "./constants";
-import { ISegment } from "./mergeTreeNodes";
-import { matchProperties, PropertySet } from "./properties";
+import { UnassignedSequenceNumber } from "./constants.js";
+import { ISegment } from "./mergeTreeNodes.js";
+import { matchProperties, PropertySet } from "./properties.js";
 import {
 	IJSONSegmentWithMergeInfo,
 	JsonSegmentSpecs,
@@ -21,11 +21,11 @@ import {
 	MergeTreeChunkV1,
 	toLatestVersion,
 	serializeAsMaxSupportedVersion,
-} from "./snapshotChunks";
-import { SnapshotLegacy } from "./snapshotlegacy";
-import { MergeTree } from "./mergeTree";
-import { walkAllChildSegments } from "./mergeTreeNodeWalk";
-import { IAttributionCollection } from "./attributionCollection";
+} from "./snapshotChunks.js";
+import { SnapshotLegacy } from "./snapshotlegacy.js";
+import { MergeTree } from "./mergeTree.js";
+import { walkAllChildSegments } from "./mergeTreeNodeWalk.js";
+import { IAttributionCollection } from "./attributionCollection.js";
 
 export class SnapshotV1 {
 	// Split snapshot into two entries - headers (small) and body (overflow) for faster loading initial content
@@ -181,12 +181,16 @@ export class SnapshotV1 {
 		const mergeTree = this.mergeTree;
 		const minSeq = this.header.minSequenceNumber;
 
+		let originalSegments = 0;
+		let segmentsAfterCombine = 0;
+
 		// Helper to add the given `MergeTreeChunkV0SegmentSpec` to the snapshot.
 		const pushSegRaw = (
 			json: JsonSegmentSpecs,
 			length: number,
 			attribution: IAttributionCollection<AttributionKey> | undefined,
 		) => {
+			segmentsAfterCombine += 1;
 			this.segments.push(json);
 			this.segmentLengths.push(length);
 			if (attribution) {
@@ -223,9 +227,14 @@ export class SnapshotV1 {
 				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 				segment.movedSeq! <= minSeq
 			) {
+				if (segment.seq !== UnassignedSequenceNumber) {
+					originalSegments += 1;
+				}
 				return true;
 			}
 
+			originalSegments += 1;
+
 			// Next determine if the snapshot needs to preserve information required for merging the segment
 			// (seq, client, etc.)  This information is only needed if the segment is above the MSN (and doesn't
 			// have a pending remove.)
@@ -322,7 +331,7 @@ export class SnapshotV1 {
 					0x066 /* "Corrupted preservation of segment metadata!" */,
 				);
 
-				// Record the segment with it's required metadata.
+				// Record the segment with its required metadata.
 				pushSegRaw(raw, segment.cachedLength, segment.attribution);
 			}
 			return true;
@@ -333,6 +342,17 @@ export class SnapshotV1 {
 		// If the last segment in the walk was coalescable, push it now.
 		pushSeg(prev);
 
+		// To reduce potential spam from this telemetry, we sample only a small
+		// percentage of summaries
+		if (Math.abs(originalSegments - segmentsAfterCombine) > 500 && Math.random() < 0.005) {
+			this.logger.sendTelemetryEvent({
+				eventName: "MergeTreeV1SummarizeSegmentCount",
+				originalSegments,
+				segmentsAfterCombine,
+				segmentsLen: this.segments.length,
+			});
+		}
+
 		return this.segments;
 	}
 

package/src/snapshotlegacy.ts

@@ -12,15 +12,15 @@ import { IFluidSerializer } from "@fluidframework/shared-object-base";
 import { ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { SummaryTreeBuilder } from "@fluidframework/runtime-utils";
-import { NonCollabClient, UnassignedSequenceNumber } from "./constants";
-import { ISegment } from "./mergeTreeNodes";
-import { matchProperties } from "./properties";
+import { NonCollabClient, UnassignedSequenceNumber } from "./constants.js";
+import { ISegment } from "./mergeTreeNodes.js";
+import { matchProperties } from "./properties.js";
 import {
 	JsonSegmentSpecs,
 	MergeTreeChunkLegacy,
 	serializeAsMinSupportedVersion,
-} from "./snapshotChunks";
-import { MergeTree } from "./mergeTree";
+} from "./snapshotChunks.js";
+import { MergeTree } from "./mergeTree.js";
 
 interface SnapshotHeader {
 	chunkCount?: number;
@@ -197,6 +197,8 @@ export class SnapshotLegacy {
 			seq: this.mergeTree.collabWindow.minSeq,
 		};
 
+		let originalSegments = 0;
+
 		const segs: ISegment[] = [];
 		let prev: ISegment | undefined;
 		const extractSegment = (
@@ -214,6 +216,7 @@ export class SnapshotLegacy {
 					segment.removedSeq === UnassignedSequenceNumber ||
 					segment.removedSeq > this.seq!)
 			) {
+				originalSegments += 1;
 				if (
 					prev?.canAppend(segment) &&
 					matchProperties(prev.properties, segment.properties)
@@ -246,6 +249,17 @@ export class SnapshotLegacy {
 			this.segments!.push(segment);
 		});
 
+		// To reduce potential spam from this telemetry, we sample only a small
+		// percentage of summaries
+		if (Math.abs(originalSegments - segs.length) > 500 && Math.random() < 0.005) {
+			this.logger.sendTelemetryEvent({
+				eventName: "MergeTreeLegacySummarizeSegmentCount",
+				originalSegments,
+				segmentsAfterCombine: segs.length,
+				segmentsLen: this.segments.length,
+			});
+		}
+
 		// We observed this.header.segmentsTotalLength < totalLength to happen in some cases
 		// When this condition happens, we might not write out all segments in getSeqLengthSegs()
 		// when writing out "body". Issue #1995 tracks following up on the core of the problem.

package/src/sortedSegmentSet.ts

@@ -3,10 +3,10 @@
  * Licensed under the MIT License.
  */
 
-import { LocalReferencePosition } from "./localReference";
-import { ISegment } from "./mergeTreeNodes";
+import { LocalReferencePosition } from "./localReference.js";
+import { ISegment } from "./mergeTreeNodes.js";
 // eslint-disable-next-line import/no-deprecated
-import { SortedSet } from "./sortedSet";
+import { SortedSet } from "./sortedSet.js";
 
 /**
  * @deprecated This functionality was not meant to be exported and will be removed in a future release
@@ -39,9 +39,11 @@ export class SortedSegmentSet<T extends SortedSegmentSetItem = ISegment> extends
 		const maybeRef = item as Partial<LocalReferencePosition>;
 		if (maybeRef.getSegment !== undefined && maybeRef.isLeaf?.() === false) {
 			const lref = maybeRef as LocalReferencePosition;
-			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-			const segment = lref.getSegment()!;
-			return segment.ordinal;
+			// If the reference position has no associated segment, assign it a sentinel value.
+			// The particular value for comparison doesn't matter because `findItemPosition` tolerates
+			// elements with duplicate keys (as it must, since local references use the same key as their segment).
+			// All that matters is that it's consistent.
+			return lref.getSegment()?.ordinal ?? "";
 		}
 		const maybeObject = item as { readonly segment: ISegment };
 		if (maybeObject?.segment) {

package/src/textSegment.ts

@@ -4,9 +4,9 @@
  */
 
 import { assert } from "@fluidframework/core-utils";
-import { BaseSegment, ISegment } from "./mergeTreeNodes";
-import { IJSONSegment } from "./ops";
-import { PropertySet } from "./properties";
+import { BaseSegment, ISegment } from "./mergeTreeNodes.js";
+import { IJSONSegment } from "./ops.js";
+import { PropertySet } from "./properties.js";
 
 // Maximum length of text segment to be considered to be merged with other segment.
 // Maximum segment length is at least 2x of it (not taking into account initial segment creation).

package/src/zamboni.ts

@@ -5,9 +5,9 @@
 
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 
-import { UnassignedSequenceNumber } from "./constants";
-import { MergeTree } from "./mergeTree";
-import { MergeTreeMaintenanceType } from "./mergeTreeDeltaCallback";
+import { UnassignedSequenceNumber } from "./constants.js";
+import { MergeTree } from "./mergeTree.js";
+import { MergeTreeMaintenanceType } from "./mergeTreeDeltaCallback.js";
 import {
 	IMergeBlock,
 	IMergeNode,
@@ -17,8 +17,8 @@ import {
 	seqLTE,
 	toMoveInfo,
 	toRemovalInfo,
-} from "./mergeTreeNodes";
-import { matchProperties } from "./properties";
+} from "./mergeTreeNodes.js";
+import { matchProperties } from "./properties.js";
 
 export const zamboniSegmentsMax = 2;
 function underflow(node: IMergeBlock) {
@@ -40,8 +40,8 @@ export function zamboniSegments(
 		}
 		segmentToScour = mergeTree.segmentsToScour.get()!;
 		// Only skip scouring if needs scour is explicitly false, not true or undefined
-		if (segmentToScour.segment!.parent && segmentToScour.segment!.parent.needsScour !== false) {
-			const block = segmentToScour.segment!.parent;
+		if (segmentToScour?.segment?.parent && segmentToScour.segment.parent.needsScour !== false) {
+			const block = segmentToScour.segment.parent;
 			const childrenCopy: IMergeNode[] = [];
 			scourNode(block, childrenCopy, mergeTree);
 			// This will avoid the cost of re-scouring nodes

package/tsconfig.cjs.json

@@ -0,0 +1,7 @@
+{
+	// This config must be used in a "type": "commonjs" environment. (Use fluid-tsc commonjs.)
+	"extends": "./tsconfig.json",
+	"compilerOptions": {
+		"outDir": "./dist",
+	},
+}

package/tsconfig.json

@@ -1,13 +1,10 @@
 {
-	"extends": [
-		"../../../common/build/build-common/tsconfig.base.json",
-		"../../../common/build/build-common/tsconfig.cjs.json",
-	],
+	"extends": "../../../common/build/build-common/tsconfig.node16.json",
 	"include": ["src/**/*"],
 	"exclude": ["src/test/**/*"],
 	"compilerOptions": {
 		"rootDir": "./src",
-		"outDir": "./dist",
+		"outDir": "./lib",
 		"preserveConstEnums": true,
 		"noImplicitAny": true,
 	},

package/api-extractor-esm.json

@@ -1,17 +0,0 @@
-{
-	"$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
-	"extends": "../../../common/build/build-common/api-extractor-base-esm.json",
-	"messages": {
-		"extractorMessageReporting": {
-			// The ESM types complain about missing exports; I don't understand why CJS doesn't. The errors are:
-			//
-			// Error: src/localReference.ts:241:2 - (ae-forgotten-export) The symbol "IRefsAtOffset" needs to be exported by the entry point index.d.mts
-			// Error: src/mergeTreeNodes.ts:402:2 - (ae-forgotten-export) The symbol "ISegmentLeaf" needs to be exported by
-			// the entry point index.d.mts
-			// TODO: AB#6647
-			"ae-forgotten-export": {
-				"logLevel": "none"
-			}
-		}
-	}
-}

package/dist/test/beastTest.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"beastTest.d.ts","sourceRoot":"","sources":["../../src/test/beastTest.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAUH,OAAO,EAAE,OAAO,EAAc,MAAM,sCAAsC,CAAC;AAa3E,OAAO,EAEN,UAAU,EAKV,MAAM,mBAAmB,CAAC;AAU3B,OAAO,EAA2B,UAAU,EAAE,MAAM,cAAc,CAAC;AAmHnE,wBAAgB,UAAU,SAYzB;AAcD,wBAAgB,YAAY,WAsC3B;AAED,wBAAgB,SAAS,SAoDxB;AAQD,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CAExD;AA6ED,wBAAgB,cAAc,SAwB7B;AAED,wBAAgB,kBAAkB,SAoFjC;AAED,wBAAgB,oBAAoB,WAwMnC;AAID,wBAAgB,QAAQ,CAAC,OAAO,UAAO;;;+BA4DJ,MAAM;;EAimBxC;AAYD,qBAAa,UAAU;IACtB,MAAM,EAAE,OAAO,CAAC;;IAKhB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;IAIpC,UAAU,CAAC,SAAS,EAAE,MAAM;CA+B5B;AAED,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,YAAY,CAAC;AACjD;;;;;;;GAOG;AACH,qBAAa,YAAY;IAOhB,IAAI,EAAE,MAAM;IACZ,QAAQ,EAAE,YAAY,EAAE;IAPhC,GAAG,SAAK;IACR,GAAG;;;MAAsB;IACzB,EAAE,EAAE,MAAM,GAAG,SAAS,CAAC;IACvB,MAAM,CAAC,QAAQ,aAAoB;gBAG3B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,YAAY,EAAE;IAGhC,cAAc,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY;IA2CxD,MAAM,CAAC,gBAAgB;IAKvB,MAAM,CAAC,eAAe,CAAC,cAAc,EAAE,MAAM;IAuB7C,MAAM,CAAC,iBAAiB;IAOxB,MAAM,CAAC,WAAW,CAAC,cAAc,EAAE,MAAM;IAUzC,MAAM,CAAC,WAAW,CAAC,cAAc,EAAE,MAAM;CAGzC"}