@fluidframework/tree 2.93.0 → 2.100.1

package/src/feature-libraries/chunked-forest/basicChunk.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { assert, oob, fail } from "@fluidframework/core-utils/internal";
+import { assert, oob, fail, debugAssert } from "@fluidframework/core-utils/internal";
 
 import {
 	CursorLocationType,
@@ -166,10 +166,19 @@ export class BasicChunkCursor extends SynchronousCursor implements ChunkedCursor
 		if (this.nestedCursor !== undefined) {
 			return this.nestedCursor.mode;
 		}
-		// Compute the number of nodes deep the current depth is.
-		// We want the floor of the result, which can computed using a bitwise shift assuming the depth is less than 2^31, which seems safe.
-		// eslint-disable-next-line no-bitwise
-		const halfHeight = (this.siblingStack.length + 1) >> 1;
+		this.assertChunkStacksMatchNodeDepth();
+		return this.siblingStack.length % 2 === 0
+			? CursorLocationType.Fields
+			: CursorLocationType.Nodes;
+	}
+
+	/**
+	 * Asserts that the node-only stacks (`indexOfChunkStack` and `indexWithinChunkStack`) are in sync with `siblingStack`.
+	 * Since `siblingStack` interleaves field and node levels while the node-only stacks are pushed/popped only on node-level transitions,
+	 * their length should always equal the number of node levels traversed.
+	 */
+	private assertChunkStacksMatchNodeDepth(): void {
+		const halfHeight = this.getNodeOnlyHeightFromHeight();
 		assert(
 			this.indexOfChunkStack.length === halfHeight,
 			0x51c /* unexpected indexOfChunkStack */,
@@ -178,9 +187,6 @@ export class BasicChunkCursor extends SynchronousCursor implements ChunkedCursor
 			this.indexWithinChunkStack.length === halfHeight,
 			0x51d /* unexpected indexWithinChunkStack */,
 		);
-		return this.siblingStack.length % 2 === 0
-			? CursorLocationType.Fields
-			: CursorLocationType.Nodes;
 	}
 
 	public getFieldKey(): FieldKey {
@@ -203,9 +209,32 @@ export class BasicChunkCursor extends SynchronousCursor implements ChunkedCursor
 		return this.indexStack[height] ?? oob();
 	}
 
-	private getStackedNode(height: number): BasicChunk {
-		const index = this.getStackedNodeIndex(height);
-		return (this.siblingStack[height] as readonly TreeChunk[])[index] as BasicChunk;
+	private getStackedChunkIndex(height: number): number {
+		assert(height % 2 === 1, 0xcf3 /* must be node height */);
+		assert(height >= 0, 0xcf4 /* must not be above root */);
+		return this.indexOfChunkStack[this.getNodeOnlyHeightFromHeight(height)] ?? oob();
+	}
+
+	private getStackedChunk(height: number): BasicChunk {
+		const index = this.getStackedChunkIndex(height);
+		const chunk = (this.siblingStack[height] as readonly TreeChunk[])[index];
+		debugAssert(() => chunk instanceof BasicChunk || "only basic chunks are expected");
+		return chunk as BasicChunk;
+	}
+
+	/**
+	 * Converts a {@link height}, which contains field and node levels, into the corresponding depth/index
+	 * for the node-only stacks ({@link indexOfChunkStack} and {@link indexWithinChunkStack}), which are
+	 * only pushed on node-level transitions.
+	 *
+	 * @param height - A depth in {@link siblingStack} to convert. Defaults to {@link siblingStack}'s
+	 * current length, which gives the current depth of the node-only stacks.
+	 * @returns `floor(height / 2)` — the number of node levels at or below the given stack height.
+	 */
+	private getNodeOnlyHeightFromHeight(height: number = this.siblingStack.length): number {
+		// The bitwise shift computes the floor, which is valid assuming the depth is less than 2^31, which seems safe.
+		// eslint-disable-next-line no-bitwise
+		return height >> 1;
 	}
 
 	public getFieldLength(): number {
@@ -322,6 +351,11 @@ export class BasicChunkCursor extends SynchronousCursor implements ChunkedCursor
 		assert(this.mode === CursorLocationType.Nodes, 0x528 /* must be in nodes mode */);
 		this.siblingStack.push(this.siblings);
 		this.indexStack.push(this.index);
+		// Save the chunk array position of the current node. When siblings contain
+		// multi node chunks, the flat node index diverges from the array position,
+		// so getField needs this to locate the parent in the sibling array.
+		this.indexOfChunkStack.push(this.indexOfChunk);
+		this.indexWithinChunkStack.push(this.indexWithinChunk);
 
 		// For fields, siblings are only used for key lookup and
 		// nextField and which has arbitrary iteration order,
@@ -330,6 +364,7 @@ export class BasicChunkCursor extends SynchronousCursor implements ChunkedCursor
 		// at the cost of an allocation here.
 		this.index = 0;
 		this.siblings = [key];
+		this.assertChunkStacksMatchNodeDepth();
 	}
 
 	public nextField(): boolean {
@@ -355,8 +390,11 @@ export class BasicChunkCursor extends SynchronousCursor implements ChunkedCursor
 
 		this.siblingStack.push(this.siblings);
 		this.indexStack.push(this.index);
+		this.indexOfChunkStack.push(this.indexOfChunk);
+		this.indexWithinChunkStack.push(this.indexWithinChunk);
 		this.index = 0;
 		this.siblings = [...fields.keys()]; // TODO: avoid this copy
+		this.assertChunkStacksMatchNodeDepth();
 		return true;
 	}
 
@@ -422,12 +460,11 @@ export class BasicChunkCursor extends SynchronousCursor implements ChunkedCursor
 		}
 		this.siblingStack.push(this.siblings);
 		this.indexStack.push(this.index);
-		this.indexOfChunkStack.push(this.indexOfChunk);
-		this.indexWithinChunkStack.push(this.indexWithinChunk);
 		this.index = 0;
 		this.siblings = siblings;
 		this.indexOfChunk = 0;
 		this.indexWithinChunk = 0;
+		this.assertChunkStacksMatchNodeDepth();
 		this.initNestedCursor();
 		return true;
 	}
@@ -486,6 +523,12 @@ export class BasicChunkCursor extends SynchronousCursor implements ChunkedCursor
 		this.siblings =
 			this.siblingStack.pop() ?? fail(0xaf0 /* Unexpected siblingStack.length */);
 		this.index = this.indexStack.pop() ?? fail(0xaf1 /* Unexpected indexStack.length */);
+		this.indexOfChunk =
+			this.indexOfChunkStack.pop() ?? fail(0xcf5 /* Unexpected indexOfChunkStack.length */);
+		this.indexWithinChunk =
+			this.indexWithinChunkStack.pop() ??
+			fail(0xcf6 /* Unexpected indexWithinChunkStack.length */);
+		this.assertChunkStacksMatchNodeDepth();
 	}
 
 	public exitNode(): void {
@@ -502,18 +545,27 @@ export class BasicChunkCursor extends SynchronousCursor implements ChunkedCursor
 		this.siblings =
 			this.siblingStack.pop() ?? fail(0xaf2 /* Unexpected siblingStack.length */);
 		this.index = this.indexStack.pop() ?? fail(0xaf3 /* Unexpected indexStack.length */);
-		this.indexOfChunk =
-			this.indexOfChunkStack.pop() ?? fail(0xaf4 /* Unexpected indexOfChunkStack.length */);
-		this.indexWithinChunk =
-			this.indexWithinChunkStack.pop() ??
-			fail(0xaf5 /* Unexpected indexWithinChunkStack.length */);
+		// At the Fields level these aren't semantically used, but reset for consistent state
+		// (so a fully-iterated cursor matches a fresh cursor at the same logical position).
+		this.indexOfChunk = 0;
+		this.indexWithinChunk = 0;
+		this.assertChunkStacksMatchNodeDepth();
 	}
 
 	private getNode(): BasicChunk {
 		assert(this.mode === CursorLocationType.Nodes, 0x52f /* can only get node when in node */);
-		return (this.siblings as TreeChunk[])[this.index] as BasicChunk;
+		const chunk = (this.siblings as TreeChunk[])[this.indexOfChunk];
+		debugAssert(() => chunk instanceof BasicChunk || "only basic chunks are expected");
+		return chunk as BasicChunk;
 	}
 
+	/**
+	 * Resolves the chunks that make up the field the cursor is currently in. At the root, this is
+	 * {@link root} directly. Otherwise, the cursor must be in {@link CursorLocationType.Fields} mode,
+	 * and the result is looked up on the parent node using the current field key.
+	 *
+	 * @returns The chunks that make up the field the cursor is currently in.
+	 */
 	private getField(): readonly TreeChunk[] {
 		if (this.siblingStack.length === 0) {
 			return this.root;
@@ -522,7 +574,11 @@ export class BasicChunkCursor extends SynchronousCursor implements ChunkedCursor
 			this.mode === CursorLocationType.Fields,
 			0x530 /* can only get field when in fields */,
 		);
-		const parent = this.getStackedNode(this.indexStack.length - 1);
+		// The parent node is the `BasicChunk` in the node array at the top of
+		// `siblingStack` while we are in `CursorLocationType.Fields` mode. We need the parent
+		// since a field's chunks are stored on the parent node's `BasicChunk.fields` map, not on
+		// the cursor itself.
+		const parent = this.getStackedChunk(this.siblingStack.length - 1);
 		const key: FieldKey = this.getFieldKey();
 		const field = parent.fields.get(key) ?? [];
 		return field;

package/src/feature-libraries/chunked-forest/codec/chunkDecoding.ts

@@ -9,6 +9,7 @@ import type {
 	OpSpaceCompressedId,
 	SessionId,
 } from "@fluidframework/id-compressor";
+import { v5 as uuidV5 } from "uuid";
 
 import { DiscriminatedUnionDispatcher } from "../../../codec/index.js";
 import type {
@@ -37,7 +38,8 @@ import {
 	decode as genericDecode,
 	readStreamIdentifier,
 } from "./chunkDecodingGeneric.js";
-import type { IncrementalDecoder } from "./codecs.js";
+// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Referenced by doc comments
+import type { FieldBatchEncodingContext, IncrementalDecoder } from "./codecs.js";
 import {
 	type EncodedAnyShape,
 	type EncodedChunkShapeV1OrV2,
@@ -49,8 +51,8 @@ import {
 	type EncodedNestedArrayShape,
 	type EncodedNodeShape,
 	type EncodedValueShape,
-	FieldBatchFormatVersion,
 	SpecialField,
+	supportsIncrementalEncoding,
 } from "./format/index.js";
 
 export interface IdDecodingContext {
@@ -59,13 +61,32 @@ export interface IdDecodingContext {
 	 * The creator of any local Ids to be decoded.
 	 */
 	originatorId: SessionId;
+	/**
+	 * {@inheritdoc FieldBatchEncodingContext.isSummary}
+	 */
+	isSummary: boolean;
+	/**
+	 * See {@link FieldBatchEncodingContext.healUnresolvableIdentifiersOnDecode}.
+	 */
+	healUnresolvableIdentifiersOnDecode?: boolean;
+	/**
+	 * See {@link FieldBatchEncodingContext.sharedObjectId}.
+	 */
+	sharedObjectId?: string;
 }
+
+/**
+ * Random v4 UUID generated as a namespace for the "heal an unresolvable identifier into a stable UUID"
+ * path in {@link readValue}. This scheme requires consensus across all clients to function.
+ */
+const healingNamespace = "f8a89df3-6882-400f-b913-4c1f6f0157bd";
+
 /**
  * Decode `chunk` into a TreeChunk.
  */
 export function decode(
 	chunk: EncodedFieldBatchV1OrV2,
-	idDecodingContext: { idCompressor: IIdCompressor; originatorId: SessionId },
+	idDecodingContext: IdDecodingContext,
 	incrementalDecoder?: IncrementalDecoder,
 ): TreeChunk[] {
 	return genericDecode(
@@ -126,15 +147,43 @@ export function readValue(
 				typeof streamValue === "number" || typeof streamValue === "string",
 				0x997 /* identifier must be string or number. */,
 			);
+			if (typeof streamValue === "string") {
+				return streamValue;
+			}
 			const idCompressor = idDecodingContext.idCompressor;
-			return typeof streamValue === "number"
-				? idCompressor.decompress(
-						idCompressor.normalizeToSessionSpace(
-							streamValue as OpSpaceCompressedId,
-							idDecodingContext.originatorId,
-						),
-					)
-				: streamValue;
+			// OpSpaceCompressedIds are negative, and require a session-id to compute their value.
+			// Due to a bug, we have some special casing for them (see below).
+			// TODO: isFinalId should probably be exported from id-compressor and that could be used to do the narrowing here.
+			if (idDecodingContext.isSummary === true && streamValue < 0) {
+				if (
+					idDecodingContext.healUnresolvableIdentifiersOnDecode === true &&
+					idDecodingContext.sharedObjectId !== undefined
+				) {
+					// Documents written before the encode-side fix for non-finalized identifier
+					// values can persist negative op-space IDs that are no
+					// longer resolvable once the originating session's local state has been stripped.
+					// When loading such a summary with the heal-on-decode option on, synthesize a deterministic
+					// stable UUID so all readers of the same blob agree on the resulting value.
+					//
+					// The heal path is intentionally restricted to summary loads — an
+					// unresolvable ID encountered while applying an op should still surface as
+					// an error, since it indicates a real bug rather than a recoverable state.
+					return uuidV5(
+						`${idDecodingContext.sharedObjectId}|${streamValue}`,
+						healingNamespace,
+					);
+				}
+				// See `SharedTreeOptionsBeta.healUnresolvableIdentifiersOnDecode` for details on this error.
+				throw new Error(
+					"Summary could not be loaded due incorrectly encoded identifier. See SharedTreeOptionsBeta.healUnresolvableIdentifiersOnDecode for mitigation.",
+				);
+			}
+			return idCompressor.decompress(
+				idCompressor.normalizeToSessionSpace(
+					streamValue as OpSpaceCompressedId,
+					idDecodingContext.originatorId,
+				),
+			);
 		} else {
 			// EncodedCounter case:
 			unreachableCase(shape, "decoding values as deltas is not yet supported");
@@ -256,7 +305,7 @@ export class IncrementalChunkDecoder implements ChunkDecoder {
 
 		const chunkDecoder = (batch: EncodedFieldBatchV2): TreeChunk => {
 			assert(
-				batch.version >= FieldBatchFormatVersion.v2,
+				supportsIncrementalEncoding(batch.version),
 				0xc9f /* Unsupported FieldBatchFormatVersion for incremental chunks; must be v2 or higher */,
 			);
 			const context = new DecoderContext(

package/src/feature-libraries/chunked-forest/codec/codecs.ts

@@ -30,6 +30,7 @@ import {
 	EncodedFieldBatchV1,
 	EncodedFieldBatchV2,
 	FieldBatchFormatVersion,
+	supportsIncrementalEncoding,
 	type EncodedFieldBatchV1OrV2,
 } from "./format/index.js";
 import type { IncrementalEncodingPolicy } from "./incrementalEncodingPolicy.js";
@@ -105,6 +106,33 @@ export interface FieldBatchEncodingContext {
 	 * This will be defined if incremental encoding is supported and enabled.
 	 */
 	readonly incrementalEncoderDecoder?: IncrementalEncoderDecoder;
+	/**
+	 * `true` when encoding to or decoding from a summary blob. `false` for
+	 * op-stream encode/decode paths and for utility encoders that are not
+	 * tied to a persisted document. Healing behavior is gated on this flag.
+	 */
+	readonly isSummary: boolean;
+	/**
+	 * If `true`, when an op-space compressed ID encountered while decoding
+	 * cannot be resolved by the local id-compressor (e.g. the attach-summary
+	 * blob's originator session state was stripped), a deterministic stable
+	 * UUID derived from `sharedObjectId` is returned instead of throwing.
+	 * @remarks
+	 * Off by default. Used only to recover documents whose attach summary was
+	 * written with non-finalized op-space IDs before the encode-side fix
+	 * shipped. Only takes effect when `isSummary` is also `true`.
+	 * See {@link SharedTreeOptionsBeta.healUnresolvableIdentifiersOnDecode}.
+	 */
+	readonly healUnresolvableIdentifiersOnDecode?: boolean;
+	/**
+	 * The SharedTree's shared-object id, used as input to the deterministic
+	 * UUID derivation when `healUnresolvableIdentifiersOnDecode` triggers. Required
+	 * for that path; ignored otherwise.
+	 * @remarks
+	 * This allows us to ensure that multiple attaches,
+	 * in the same or different documents, with the same session offsets, get different UUIDs.
+	 */
+	readonly sharedObjectId?: string;
 }
 /**
  * @remarks
@@ -147,7 +175,7 @@ function makeFieldBatchCodecForVersion(
 				}
 				case TreeCompressionStrategy.CompressedIncremental: {
 					assert(
-						version >= FieldBatchFormatVersion.v2,
+						supportsIncrementalEncoding(version),
 						0xca0 /* Unsupported FieldBatchFormatVersion for incremental encoding; must be v2 or higher */,
 					);
 					// Incremental encoding is only supported for CompressedIncremental.
@@ -189,6 +217,9 @@ function makeFieldBatchCodecForVersion(
 				{
 					idCompressor: context.idCompressor,
 					originatorId: context.originatorId,
+					isSummary: context.isSummary,
+					healUnresolvableIdentifiersOnDecode: context.healUnresolvableIdentifiersOnDecode,
+					sharedObjectId: context.sharedObjectId,
 				},
 				context.incrementalEncoderDecoder,
 			).map((chunk) => chunk.cursor());

package/src/feature-libraries/chunked-forest/codec/compressedEncode.ts

@@ -36,8 +36,9 @@ import {
 	type EncodedFieldBatchV1OrV2,
 	type EncodedNestedArrayShape,
 	type EncodedValueShape,
-	FieldBatchFormatVersion,
+	type FieldBatchFormatVersion,
 	SpecialField,
+	supportsIncrementalEncoding,
 } from "./format/index.js";
 
 /**
@@ -461,7 +462,7 @@ export const incrementalFieldEncoder: FieldEncoder = {
 			0xc88 /* incremental encoder must be defined to use incrementalFieldEncoder */,
 		);
 		assert(
-			context.version >= FieldBatchFormatVersion.v2,
+			supportsIncrementalEncoding(context.version),
 			0xca1 /* Unsupported FieldBatchFormatVersion for incremental encoding; must be v2 or higher */,
 		);
 

package/src/feature-libraries/chunked-forest/codec/format/formatGeneric.ts

@@ -28,7 +28,6 @@ export const Count = Type.Number({ multipleOf: 1, minimum: 0 });
 
 const EncodedFieldBatchBase = Type.Object(
 	{
-		version: Type.Number(),
 		identifiers: Type.Array(Type.String()),
 		/**
 		 * Top level array is list of field from batch.

package/src/feature-libraries/chunked-forest/codec/format/index.ts

@@ -20,6 +20,7 @@ export {
 	FieldBatchFormatVersion,
 	EncodedFieldBatchV1,
 	EncodedFieldBatchV2,
+	supportsIncrementalEncoding,
 	type EncodedFieldBatchV1OrV2,
 	type EncodedFieldBatchV1AndV2,
 	type EncodedChunkShapeV1OrV2,

package/src/feature-libraries/chunked-forest/codec/format/versions.ts

@@ -31,6 +31,21 @@ export const FieldBatchFormatVersion = strictEnum("FieldBatchFormatVersion", {
 	v2: 2,
 });
 
+/**
+ * Whether the given format version supports incremental chunk encoding.
+ *
+ * @remarks
+ * This helper should be used for comparison since experimental versions
+ * can be a string.
+ */
+export function supportsIncrementalEncoding(version: FieldBatchFormatVersion): boolean {
+	if (version === FieldBatchFormatVersion.v1) {
+		return false;
+	}
+
+	return true;
+}
+
 /**
  * Encoded {@link FieldBatch} using V1 format.
  * @remarks

package/src/feature-libraries/chunked-forest/uniformChunk.ts

@@ -20,7 +20,7 @@ import {
 	cursorChunk,
 	dummyRoot,
 } from "../../core/index.js";
-import { ReferenceCountedBase, hasSome } from "../../util/index.js";
+import { ReferenceCountedBase, getOrCreate, hasSome } from "../../util/index.js";
 import { SynchronousCursor, prefixFieldPath, prefixPath } from "../treeCursorUtils.js";
 
 /**
@@ -83,6 +83,23 @@ export class UniformChunk extends ReferenceCountedBase implements TreeChunk {
  */
 export type FieldShape = readonly [FieldKey, TreeShape, number];
 
+/**
+ * Maximum topLevelLength value (exclusive) for which {@link TreeShape.withTopLevelLength}
+ * caches the resulting {@link ChunkShape}. Values at or above this threshold always
+ * create a new instance to prevent unbounded cache growth.
+ *
+ * @remarks
+ * This value is an estimation of the general size needed to cover current workflows,
+ * not a researched constant, and is safe to tune as workloads change.
+ *
+ * Raising this value captures more chunk sizes in the cache, at the cost of
+ * each `TreeShape` retaining up to `chunkShapeCacheLimit - 1` cached entries for the
+ * lifetime of the shape. Lowering it reduces memory held per `TreeShape` but forces
+ * small chunks, where the relative cost of rebuilding `positions` is highest, to pay
+ * the construction cost on every call.
+ */
+const chunkShapeCacheLimit = 8;
+
 /**
  * The "shape" of a tree.
  * Does not contain the actual values from  the tree, but describes everything else,
@@ -110,7 +127,13 @@ export class TreeShape {
 	public readonly mayContainCompressedIds: boolean;
 
 	/**
-	 *
+	 * Cache for ChunkShape instances created by {@link withTopLevelLength}.
+	 * `topLevelLength` is always a positive integer (enforced by the {@link ChunkShape} constructor),
+	 * so the cache only ever holds entries for values in `1..chunkShapeCacheLimit - 1` to prevent unbounded growth.
+	 */
+	private readonly chunkShapeCache: Map<number, ChunkShape> = new Map();
+
+	/**
 	 * @param type - {@link TreeNodeSchemaIdentifier} used to compare shapes.
 	 * @param hasValue - whether or not the TreeShape has a value.
 	 * @param fieldsArray - an array of {@link FieldShape} values, which contains a TreeShape for each FieldKey.
@@ -179,6 +202,13 @@ export class TreeShape {
 	}
 
 	public withTopLevelLength(topLevelLength: number): ChunkShape {
+		if (topLevelLength < chunkShapeCacheLimit) {
+			return getOrCreate(
+				this.chunkShapeCache,
+				topLevelLength,
+				() => new ChunkShape(this, topLevelLength),
+			);
+		}
 		return new ChunkShape(this, topLevelLength);
 	}
 }