@fluidframework/tree 2.2.0 → 2.3.0

package/src/feature-libraries/chunked-forest/chunkTree.ts

@@ -20,6 +20,7 @@ import {
 	type Value,
 	mapCursorFields,
 	Multiplicity,
+	ValueSchema,
 } from "../../core/index.js";
 import { fail, getOrCreate } from "../../util/index.js";
 import type { FullSchemaPolicy } from "../modular-schema/index.js";
@@ -28,6 +29,8 @@ import { BasicChunk } from "./basicChunk.js";
 import { type TreeChunk, tryGetChunk } from "./chunk.js";
 import { SequenceChunk } from "./sequenceChunk.js";
 import { type FieldShape, TreeShape, UniformChunk } from "./uniformChunk.js";
+import { isStableNodeKey } from "../node-key/index.js";
+import type { IIdCompressor } from "@fluidframework/id-compressor";
 
 export interface Disposable {
 	/**
@@ -157,7 +160,7 @@ export class Chunker implements IChunker {
  *
  * @param cursor - cursor in nodes mode
  */
-export function chunkTree(cursor: ITreeCursorSynchronous, policy: ChunkPolicy): TreeChunk {
+export function chunkTree(cursor: ITreeCursorSynchronous, policy: ChunkCompressor): TreeChunk {
 	return chunkRange(cursor, policy, 1, true)[0] ?? oob();
 }
 
@@ -165,7 +168,10 @@ export function chunkTree(cursor: ITreeCursorSynchronous, policy: ChunkPolicy):
  * Get a TreeChunk[] for the current field (and its children) of cursor.
  * This will copy if needed, but add refs to existing chunks which hold the data.
  */
-export function chunkField(cursor: ITreeCursorSynchronous, policy: ChunkPolicy): TreeChunk[] {
+export function chunkField(
+	cursor: ITreeCursorSynchronous,
+	policy: ChunkCompressor,
+): TreeChunk[] {
 	const length = cursor.getFieldLength();
 	const started = cursor.firstNode();
 	assert(started, 0x57c /* field to chunk should have at least one node */);
@@ -178,7 +184,7 @@ export function chunkField(cursor: ITreeCursorSynchronous, policy: ChunkPolicy):
  */
 export function chunkFieldSingle(
 	cursor: ITreeCursorSynchronous,
-	policy: ChunkPolicy,
+	policy: ChunkCompressor,
 ): TreeChunk {
 	const chunks = chunkField(cursor, policy);
 	if (chunks.length === 1) {
@@ -193,7 +199,7 @@ export function chunkFieldSingle(
  */
 export function basicChunkTree(
 	cursor: ITreeCursorSynchronous,
-	policy: ChunkPolicy,
+	policy: ChunkCompressor,
 ): BasicChunk {
 	// symbol based fast path to check for BasicChunk:
 	// return existing chunk with a increased ref count if possible.
@@ -242,7 +248,13 @@ export function tryShapeFromSchema(
 	return getOrCreate(shapes, type, () => {
 		const treeSchema = schema.nodeSchema.get(type) ?? fail("missing schema");
 		if (treeSchema instanceof LeafNodeStoredSchema) {
-			return new TreeShape(type, true, []);
+			// Allow all string values (but only string values) to be compressed by the id compressor.
+			// This allows compressing all compressible identifiers without requiring additional context to know which values could be identifiers.
+			// Attempting to compress other string shouldn't have significant overhead,
+			// and if any of them do end up compressing, that's a benefit not a bug.
+			return treeSchema.leafValue === ValueSchema.String
+				? new TreeShape(type, true, [], true)
+				: new TreeShape(type, true, [], false);
 		}
 		if (treeSchema instanceof ObjectNodeStoredSchema) {
 			const fieldsArray: FieldShape[] = [];
@@ -337,7 +349,21 @@ export interface ChunkPolicy {
 	shapeFromSchema(schema: TreeNodeSchemaIdentifier): ShapeInfo;
 }
 
-function newBasicChunkTree(cursor: ITreeCursorSynchronous, policy: ChunkPolicy): BasicChunk {
+export interface ChunkCompressor {
+	readonly policy: ChunkPolicy;
+	/**
+	 * If the idCompressor is provided, {@link UniformChunk}s with identifiers will be encoded for its in-memory representation.
+	 * @remarks
+	 * This compression applies to {@link UniformChunk}s when {@link TreeShape.maybeDecompressedStringAsNumber} is set.
+	 * If the `policy` does not use UniformChunks or does not set `maybeDecompressedStringAsNumber`, then no compression will be applied even when providing `idCompressor`.
+	 */
+	readonly idCompressor: IIdCompressor | undefined;
+}
+
+function newBasicChunkTree(
+	cursor: ITreeCursorSynchronous,
+	policy: ChunkCompressor,
+): BasicChunk {
 	return new BasicChunk(
 		cursor.type,
 		new Map(mapCursorFields(cursor, () => [cursor.getFieldKey(), chunkField(cursor, policy)])),
@@ -353,7 +379,7 @@ function newBasicChunkTree(cursor: ITreeCursorSynchronous, policy: ChunkPolicy):
  */
 export function chunkRange(
 	cursor: ITreeCursorSynchronous,
-	policy: ChunkPolicy,
+	chunkCompressor: ChunkCompressor,
 	length: number,
 	skipLastNavigation: boolean,
 ): TreeChunk[] {
@@ -373,7 +399,7 @@ export function chunkRange(
 				if (chunk !== undefined) {
 					if (
 						chunk instanceof SequenceChunk &&
-						chunk.subChunks.length <= policy.sequenceChunkInlineThreshold
+						chunk.subChunks.length <= chunkCompressor.policy.sequenceChunkInlineThreshold
 					) {
 						// If sequence chunk, and its very short, inline it.
 						// Note that this is not recursive: there may be short sequences nested below this which are not inlined.
@@ -399,11 +425,11 @@ export function chunkRange(
 			assert(cursor.mode === CursorLocationType.Nodes, 0x580 /* should be in nodes */);
 			// TODO: if provided, use schema to consider using UniformChunks
 			const type = cursor.type;
-			const shape = policy.shapeFromSchema(type);
+			const shape = chunkCompressor.policy.shapeFromSchema(type);
 			if (shape instanceof TreeShape) {
 				const nodesPerTopLevelNode = shape.positions.length;
 				const maxTopLevelLength = Math.ceil(
-					nodesPerTopLevelNode / policy.uniformChunkNodeCount,
+					nodesPerTopLevelNode / chunkCompressor.policy.uniformChunkNodeCount,
 				);
 				const maxLength = Math.min(maxTopLevelLength, remaining);
 				const newChunk = uniformChunkFromCursor(
@@ -411,12 +437,13 @@ export function chunkRange(
 					shape,
 					maxLength,
 					maxLength === remaining && skipLastNavigation,
+					chunkCompressor.idCompressor,
 				);
 				remaining -= newChunk.topLevelLength;
 				output.push(newChunk);
 			} else {
 				// Slow path: copy tree into new basic chunk
-				output.push(newBasicChunkTree(cursor, policy));
+				output.push(newBasicChunkTree(cursor, chunkCompressor));
 				remaining -= 1;
 				if (!skipLastNavigation || remaining !== 0) {
 					cursor.nextNode();
@@ -428,8 +455,10 @@ export function chunkRange(
 	// TODO: maybe make a pass over output to coalesce UniformChunks and/or convert other formats to UniformChunks where possible.
 
 	// If output is large, group it into a tree of sequence chunks.
-	while (output.length > policy.sequenceChunkSplitThreshold) {
-		const chunkCount = Math.ceil(output.length / policy.sequenceChunkSplitThreshold);
+	while (output.length > chunkCompressor.policy.sequenceChunkSplitThreshold) {
+		const chunkCount = Math.ceil(
+			output.length / chunkCompressor.policy.sequenceChunkSplitThreshold,
+		);
 		const newOutput: TreeChunk[] = [];
 		// Rounding down, and add an extra item to some of the chunks.
 		const chunkSize = Math.floor(output.length / chunkCount);
@@ -448,11 +477,15 @@ export function chunkRange(
 
 	return output;
 }
-
+/**
+ * @param idCompressor - compressor used to encoded string values that are compressible by the idCompressor for in-memory representation.
+ * If the idCompressor is not provided, the values will be the original uncompressed values.
+ */
 export function insertValues(
 	cursor: ITreeCursorSynchronous,
 	shape: TreeShape,
 	values: Value[],
+	idCompressor?: IIdCompressor,
 ): void {
 	assert(shape.type === cursor.type, 0x582 /* shape and type must match */);
 
@@ -461,13 +494,21 @@ export function insertValues(
 
 	// Slow path: walk shape and cursor together, inserting values.
 	if (shape.hasValue) {
-		values.push(cursor.value);
+		if (
+			typeof cursor.value === "string" &&
+			idCompressor !== undefined &&
+			isStableNodeKey(cursor.value)
+		) {
+			values.push(idCompressor.tryRecompress(cursor.value) ?? cursor.value);
+		} else {
+			values.push(cursor.value);
+		}
 	}
 	for (const [key, childShape, length] of shape.fieldsArray) {
 		cursor.enterField(key);
 		let count = 0;
 		for (let inNodes = cursor.firstNode(); inNodes; inNodes = cursor.nextNode()) {
-			insertValues(cursor, childShape, values);
+			insertValues(cursor, childShape, values, idCompressor);
 			count++;
 		}
 		cursor.exitField();
@@ -484,12 +525,16 @@ export function insertValues(
  * If this stops early due to the type changing, `skipLastNavigation` is not involved:
  * `skipLastNavigation` only determines if the cursor will be left on the node after the last one (possibly exiting the field)
  * if the full length is used.
+ *
+ * @param idCompressor - compressor used to encoded string values that are compressible by the idCompressor for in-memory representation.
+ * If the idCompressor is not provided, the values will be the original uncompressed values.
  */
 export function uniformChunkFromCursor(
 	cursor: ITreeCursorSynchronous,
 	shape: TreeShape,
 	maxTopLevelLength: number,
 	skipLastNavigation: boolean,
+	idCompressor?: IIdCompressor,
 ): UniformChunk {
 	// TODO:
 	// This could have a fast path for consuming already uniformly chunked data with matching shape.
@@ -497,7 +542,7 @@ export function uniformChunkFromCursor(
 	const values: TreeValue[] = [];
 	let topLevelLength = 1;
 	while (topLevelLength <= maxTopLevelLength) {
-		insertValues(cursor, shape, values);
+		insertValues(cursor, shape, values, idCompressor);
 		if (topLevelLength === maxTopLevelLength) {
 			if (!skipLastNavigation) {
 				cursor.nextNode();
@@ -510,5 +555,5 @@ export function uniformChunkFromCursor(
 		}
 		topLevelLength += 1;
 	}
-	return new UniformChunk(shape.withTopLevelLength(topLevelLength), values);
+	return new UniformChunk(shape.withTopLevelLength(topLevelLength), values, idCompressor);
 }

package/src/feature-libraries/chunked-forest/chunkedForest.ts

@@ -34,6 +34,7 @@ import { assertValidRange, brand, fail, getOrAddEmptyToMap } from "../../util/in
 import { BasicChunk, BasicChunkCursor, type SiblingsOrKey } from "./basicChunk.js";
 import type { ChunkedCursor, TreeChunk } from "./chunk.js";
 import { type IChunker, basicChunkTree, chunkTree } from "./chunkTree.js";
+import type { IIdCompressor } from "@fluidframework/id-compressor";
 
 function makeRoot(): BasicChunk {
 	return new BasicChunk(aboveRootPlaceholder, new Map());
@@ -65,6 +66,7 @@ export class ChunkedForest implements IEditableForest {
 		public readonly schema: TreeStoredSchemaSubscription,
 		public readonly chunker: IChunker,
 		public readonly anchors: AnchorSet = new AnchorSet(),
+		public readonly idCompressor?: IIdCompressor,
 	) {}
 
 	public get isEmpty(): boolean {
@@ -122,7 +124,12 @@ export class ChunkedForest implements IEditableForest {
 			},
 			create(content: ProtoNodes, destination: FieldKey): void {
 				this.forest.events.emit("beforeChange");
-				const chunks: TreeChunk[] = content.map((c) => chunkTree(c, this.forest.chunker));
+				const chunks: TreeChunk[] = content.map((c) =>
+					chunkTree(c, {
+						policy: this.forest.chunker,
+						idCompressor: this.forest.idCompressor,
+					}),
+				);
 				this.forest.roots.fields.set(destination, chunks);
 				this.forest.events.emit("afterRootFieldCreated", destination);
 			},
@@ -221,7 +228,10 @@ export class ChunkedForest implements IEditableForest {
 					//
 					// Maybe build path when visitor navigates then lazily sync to chunk tree when editing?
 					const newChunks = mapCursorField(found.cursor(), (cursor) =>
-						basicChunkTree(cursor, this.forest.chunker),
+						basicChunkTree(cursor, {
+							policy: this.forest.chunker,
+							idCompressor: this.forest.idCompressor,
+						}),
 					);
 					// TODO: this could fail for really long chunks being split (due to argument count limits).
 					// Current implementations of chunks shouldn't ever be that long, but it could be an issue if they get bigger.
@@ -445,6 +455,10 @@ class Cursor extends BasicChunkCursor implements ITreeSubscriptionCursor {
 /**
  * @returns an implementation of {@link IEditableForest} with no data or schema.
  */
-export function buildChunkedForest(chunker: IChunker, anchors?: AnchorSet): ChunkedForest {
-	return new ChunkedForest(makeRoot(), chunker.schema, chunker, anchors);
+export function buildChunkedForest(
+	chunker: IChunker,
+	anchors?: AnchorSet,
+	idCompressor?: IIdCompressor,
+): ChunkedForest {
+	return new ChunkedForest(makeRoot(), chunker.schema, chunker, anchors, idCompressor);
 }

package/src/feature-libraries/chunked-forest/uniformChunk.ts

@@ -19,6 +19,8 @@ import { ReferenceCountedBase, fail } from "../../util/index.js";
 import { SynchronousCursor, prefixFieldPath, prefixPath } from "../treeCursorUtils.js";
 
 import { type ChunkedCursor, type TreeChunk, cursorChunk, dummyRoot } from "./chunk.js";
+import type { SessionSpaceCompressedId, IIdCompressor } from "@fluidframework/id-compressor";
+import { UsageError } from "@fluidframework/telemetry-utils/internal";
 
 /**
  * Create a tree chunk with ref count 1.
@@ -37,6 +39,7 @@ export function uniformChunk(shape: ChunkShape, values: TreeValue[]): TreeChunk
  * allowing deduplication of shape information and storing of content as a flat sequence of values.
  */
 export class UniformChunk extends ReferenceCountedBase implements TreeChunk {
+	public idCompressor: undefined | IIdCompressor;
 	/**
 	 * Create a tree chunk with ref count 1.
 	 *
@@ -46,8 +49,10 @@ export class UniformChunk extends ReferenceCountedBase implements TreeChunk {
 	public constructor(
 		public shape: ChunkShape,
 		public values: TreeValue[],
+		idCompressor?: IIdCompressor,
 	) {
 		super();
+		this.idCompressor = idCompressor;
 		assert(
 			shape.treeShape.valuesPerTopLevelNode * shape.topLevelLength === values.length,
 			0x4c3 /* invalid number of values for shape */,
@@ -93,11 +98,30 @@ export class TreeShape {
 	// TODO: this is only needed at chunk roots. Optimize it base on that.
 	public readonly positions: readonly NodePositionInfo[];
 
+	/**
+	 *
+	 * @param type - {@link TreeNodeSchemaIdentifier} used to compare shapes.
+	 * @param hasValue - whether or not the TreeShape has a value.
+	 * @param fieldsArray - an array of {@link FieldShape} values, which contains a TreeShape for each FieldKey.
+	 *
+	 * @param maybeDecompressedStringAsNumber - used to check whether or not the value could have been compressed by the idCompressor.
+	 * This flag can only be set on string leaf nodes, and will throw a usage error otherwise.
+	 * If set to true, an additional check can be made (example: getting the value of {@link Cursor}) to return the original uncompressed value.
+	 */
 	public constructor(
 		public readonly type: TreeNodeSchemaIdentifier,
 		public readonly hasValue: boolean,
 		public readonly fieldsArray: readonly FieldShape[],
+		public readonly maybeDecompressedStringAsNumber: boolean = false,
 	) {
+		if (
+			maybeDecompressedStringAsNumber &&
+			!(hasValue && type === "com.fluidframework.leaf.string")
+		) {
+			throw new UsageError(
+				"maybeDecompressedStringAsNumber flag can only be set to true for string leaf node.",
+			);
+		}
 		const fields: Map<FieldKey, OffsetShape> = new Map();
 		let numberOfValues = hasValue ? 1 : 0;
 		const infos: NodePositionInfo[] = [
@@ -519,7 +543,16 @@ class Cursor extends SynchronousCursor implements ChunkedCursor {
 	}
 
 	public get value(): Value {
+		const idCompressor = this.chunk.idCompressor;
 		const info = this.nodeInfo(CursorLocationType.Nodes);
+		// If the maybeDecompressedStringAsNumber flag is set to true, we check if the value is a number.
+		// This flag can only ever be set on string leaf nodes, so if the value is a number, we can assume it is a compressible, known stable id.
+		if (info.shape.hasValue && info.shape.maybeDecompressedStringAsNumber) {
+			const value = this.chunk.values[info.valueOffset];
+			if (typeof value === "number" && idCompressor !== undefined) {
+				return idCompressor.decompress(value as SessionSpaceCompressedId);
+			}
+		}
 		return info.shape.hasValue ? this.chunk.values[info.valueOffset] : undefined;
 	}
 }

package/src/feature-libraries/default-schema/defaultEditBuilder.ts

@@ -42,6 +42,7 @@ import {
 	sequence,
 	required as valueFieldKind,
 } from "./defaultFieldKinds.js";
+import type { IIdCompressor } from "@fluidframework/id-compressor";
 
 export type DefaultChangeset = ModularChangeset;
 
@@ -168,6 +169,7 @@ export class DefaultEditBuilder implements ChangeFamilyEditor, IDefaultEditBuild
 	public constructor(
 		family: ChangeFamily<ChangeFamilyEditor, DefaultChangeset>,
 		changeReceiver: (change: DefaultChangeset) => void,
+		private readonly idCompressor?: IIdCompressor,
 	) {
 		this.modularBuilder = new ModularEditBuilder(family, fieldKinds, changeReceiver);
 	}
@@ -188,7 +190,7 @@ export class DefaultEditBuilder implements ChangeFamilyEditor, IDefaultEditBuild
 			set: (newContent: ITreeCursorSynchronous): void => {
 				const fillId = this.modularBuilder.generateId();
 
-				const build = this.modularBuilder.buildTrees(fillId, newContent);
+				const build = this.modularBuilder.buildTrees(fillId, newContent, this.idCompressor);
 				const change: FieldChangeset = brand(
 					valueFieldKind.changeHandler.editor.set({
 						fill: fillId,
@@ -216,7 +218,7 @@ export class DefaultEditBuilder implements ChangeFamilyEditor, IDefaultEditBuild
 				let optionalChange: OptionalChangeset;
 				if (newContent !== undefined) {
 					fillId = this.modularBuilder.generateId();
-					const build = this.modularBuilder.buildTrees(fillId, newContent);
+					const build = this.modularBuilder.buildTrees(fillId, newContent, this.idCompressor);
 					edits.push(build);
 
 					optionalChange = optional.changeHandler.editor.set(wasEmpty, {
@@ -339,7 +341,7 @@ export class DefaultEditBuilder implements ChangeFamilyEditor, IDefaultEditBuild
 				}
 
 				const firstId = this.modularBuilder.generateId(length);
-				const build = this.modularBuilder.buildTrees(firstId, content);
+				const build = this.modularBuilder.buildTrees(firstId, content, this.idCompressor);
 				const change: FieldChangeset = brand(
 					sequence.changeHandler.editor.insert(index, length, firstId),
 				);

package/src/feature-libraries/flex-map-tree/index.ts

@@ -10,4 +10,5 @@ export {
 	isMapTreeSequenceField,
 	getOrCreateMapTreeNode,
 	tryGetMapTreeNode,
+	UnhydratedContext,
 } from "./mapTreeNode.js";