@fluidframework/tree 2.101.0 → 2.102.0

package/src/feature-libraries/chunked-forest/chunkTree.ts

@@ -25,7 +25,7 @@ import {
 	type SchemaAndPolicy,
 	type SchemaPolicy,
 } from "../../core/index.js";
-import { getOrCreate } from "../../util/index.js";
+import { assertNonNegativeSafeInteger, getOrCreate } from "../../util/index.js";
 import { isStableNodeIdentifier } from "../node-identifier/index.js";
 
 import { BasicChunk } from "./basicChunk.js";
@@ -53,6 +53,7 @@ export function makeTreeChunker(
 		defaultChunkPolicy.sequenceChunkInlineThreshold,
 		defaultChunkPolicy.sequenceChunkInlineThreshold,
 		defaultChunkPolicy.uniformChunkNodeCount,
+		defaultChunkPolicy.uniformChunkNodeCountDynamicTargetMax,
 		(type: TreeNodeSchemaIdentifier, shapes: Map<TreeNodeSchemaIdentifier, ShapeInfo>) =>
 			tryShapeFromNodeSchema(
 				{
@@ -117,6 +118,7 @@ export class Chunker implements IChunker {
 		public readonly sequenceChunkSplitThreshold: number,
 		public readonly sequenceChunkInlineThreshold: number,
 		public readonly uniformChunkNodeCount: number,
+		public readonly uniformChunkNodeCountDynamicTargetMax: number,
 		// eslint-disable-next-line @typescript-eslint/no-shadow
 		private readonly tryShapeFromNodeSchema: (
 			type: TreeNodeSchemaIdentifier,
@@ -133,6 +135,7 @@ export class Chunker implements IChunker {
 			this.sequenceChunkSplitThreshold,
 			this.sequenceChunkInlineThreshold,
 			this.uniformChunkNodeCount,
+			this.uniformChunkNodeCountDynamicTargetMax,
 			this.tryShapeFromNodeSchema,
 		);
 	}
@@ -379,6 +382,7 @@ export const defaultChunkPolicy: ChunkPolicy = {
 	sequenceChunkInlineThreshold: Number.POSITIVE_INFINITY,
 	// Current UniformChunk handling doesn't scale well to large chunks, so set a modest size limit:
 	uniformChunkNodeCount: 400,
+	uniformChunkNodeCountDynamicTargetMax: 25,
 	// Without knowing what the schema is, all shapes are possible.
 	// Use `makeTreeChunker` to do better.
 	shapeFromSchema: () => polymorphic,
@@ -388,6 +392,7 @@ export const basicOnlyChunkPolicy: ChunkPolicy = {
 	sequenceChunkSplitThreshold: Number.POSITIVE_INFINITY,
 	sequenceChunkInlineThreshold: Number.POSITIVE_INFINITY,
 	uniformChunkNodeCount: 0,
+	uniformChunkNodeCountDynamicTargetMax: 0,
 	shapeFromSchema: () => polymorphic,
 };
 
@@ -413,6 +418,24 @@ export interface ChunkPolicy {
 	 */
 	readonly uniformChunkNodeCount: number;
 
+	/**
+	 * Target maximum top level length for a UniformChunk while a field is being edited.
+	 *
+	 * @remarks
+	 * When {@link splitFieldAtIndex} has to split a chunk to land an attach/detach on a chunk
+	 * boundary, chunks whose {@link TreeChunk.topLevelLength} exceeds this value are bisected
+	 * recursively until each piece is at or below it, and only the piece holding the target index
+	 * is split exactly. This bounds N splits inside an M-sized chunk at the cost of producing a
+	 * few extra intermediate chunks.
+	 *
+	 * Future merge/extend logic for adjacent small chunks could use the same value as the
+	 * upper bound it tries to stay under, so dynamic chunk sizes settle around this target.
+	 *
+	 * Independent of {@link ChunkPolicy.uniformChunkNodeCount}, which only bounds the size of
+	 * chunks produced by the initial chunking pass.
+	 */
+	readonly uniformChunkNodeCountDynamicTargetMax: number;
+
 	/**
 	 * Returns information about the shapes trees of type `schema` can take.
 	 */
@@ -556,6 +579,67 @@ export function chunkRange(
 
 	return output;
 }
+
+/**
+ * Walks the `chunks` array of a field and splits a chunk if needed so that `nodeIndex` sits on
+ * a chunk boundary. After the call, inserting a chunk at the returned index would place its
+ * first top-level node at index `nodeIndex` when treating `chunks` as a field.
+ *
+ * @remarks
+ * When splitting chunks, large chunks are split evenly so that repeated calls to this method (or similar operations)
+ * avoid poor worst-case behavior. See {@link ChunkPolicy.uniformChunkNodeCountDynamicTargetMax} for details.
+ *
+ * @param chunks - The array of {@link TreeChunk}s for the field to split. Mutated in place.
+ * @param nodeIndex - The index to split at, measured in top-level nodes within the field.
+ * Must be in `[0, totalNodes]`, where `totalNodes` is the sum of {@link TreeChunk.topLevelLength}
+ * across all chunks.
+ * @param policy - The {@link ChunkCompressor} to use when splitting chunks and re-chunking each side
+ * of the split via {@link chunkRange}.
+ *
+ * @returns The index in `chunks` (after modifications made by this function) where if a chunk were inserted at that index its first top level node would have index `nodeIndex` when treating `chunks` as a field.
+ */
+export function splitFieldAtIndex(
+	chunks: TreeChunk[],
+	nodeIndex: number,
+	policy: ChunkCompressor,
+): number {
+	assertNonNegativeSafeInteger(nodeIndex);
+	const bisectThreshold = policy.policy.uniformChunkNodeCountDynamicTargetMax;
+	let remaining = nodeIndex;
+	let chunkIndex = 0;
+	while (chunkIndex < chunks.length) {
+		if (remaining === 0) {
+			return chunkIndex;
+		}
+		const chunk = chunks[chunkIndex] ?? oob();
+		const total = chunk.topLevelLength;
+		if (remaining >= total) {
+			// nodeIndex is not in this chunk, so move forward one chunk and continue.
+			remaining -= total;
+			chunkIndex++;
+			continue;
+		}
+
+		// nodeIndex falls within this chunk, so split the chunk.
+		// This does not move the chunkIndex forward: the next iteration might need to split again at the same index.
+		//
+		// For chunks above the bisect threshold, cut at the midpoint and let the loop descend
+		// into whichever half holds nodeIndex. The other half is left untouched.
+		const splitPoint = total > bisectThreshold ? Math.floor(total / 2) : remaining;
+		const cursor = chunk.cursor();
+		cursor.firstNode();
+		const before = chunkRange(cursor, policy, splitPoint, false);
+		const after = chunkRange(cursor, policy, total - splitPoint, true);
+		// TODO: this could fail for really long chunks being split (due to argument count limits).
+		chunks.splice(chunkIndex, 1, ...before, ...after);
+		// The spliced-out slot held a ref to the original chunk. The two new chunks come with
+		// their own refs from chunkRange, so the slot's ref to the original needs to be released.
+		chunk.referenceRemoved();
+	}
+	assert(remaining === 0, 0xcf9 /* nodeIndex exceeds total node count in field */);
+	return chunks.length;
+}
+
 /**
  * Extracts values from the current cursor position according to the provided tree shape.
  *

package/src/feature-libraries/chunked-forest/chunkedForest.ts

@@ -36,7 +36,6 @@ import {
 	type DeltaDetachedNodeId,
 } from "../../core/index.js";
 import {
-	assertValidRange,
 	brand,
 	getLast,
 	getOrAddEmptyToMap,
@@ -45,7 +44,14 @@ import {
 } from "../../util/index.js";
 
 import { BasicChunk, BasicChunkCursor, type SiblingsOrKey } from "./basicChunk.js";
-import { type IChunker, basicChunkTree, chunkField, chunkTree } from "./chunkTree.js";
+import {
+	type ChunkCompressor,
+	type IChunker,
+	basicChunkTree,
+	chunkField,
+	chunkTree,
+	splitFieldAtIndex,
+} from "./chunkTree.js";
 
 function makeRoot(): BasicChunk {
 	return new BasicChunk(aboveRootPlaceholder, new Map());
@@ -173,8 +179,12 @@ export class ChunkedForest implements IEditableForest {
 
 				const parent = this.getParent();
 				const destinationField = getOrAddEmptyToMap(parent.mutableChunk.fields, parent.key);
+				const destinationChunkIndex = splitFieldAtIndex(destinationField, destination, {
+					policy: this.forest.chunker,
+					idCompressor: this.forest.idCompressor,
+				});
 				// TODO: this will fail for very large moves due to argument limits.
-				destinationField.splice(destination, 0, ...sourceField);
+				destinationField.splice(destinationChunkIndex, 0, ...sourceField);
 			},
 			/**
 			 * Detaches the range from the current field and transfers it to the given destination if any.
@@ -193,9 +203,19 @@ export class ChunkedForest implements IEditableForest {
 				this.forest.#events.emit("beforeChange");
 				const parent = this.getParent();
 				const sourceField = parent.mutableChunk.fields.get(parent.key) ?? [];
-
-				assertValidRange(source, sourceField);
-				const newField = sourceField.splice(source.start, source.end - source.start);
+				assert(source.start <= source.end, 0xcf8 /* detach range start must not exceed end */);
+
+				const policy: ChunkCompressor = {
+					policy: this.forest.chunker,
+					idCompressor: this.forest.idCompressor,
+				};
+				// Split start first: splitting end later only expands chunks at positions >= startChunkIndex,
+				// which leaves startChunkIndex valid. The reverse order would shift endChunkIndex when
+				// source.start and source.end land in different chunks.
+				// Performance: It's practical to have a variant of splitFieldAtIndex which can split at multiple locations in a single pass if the performance of this becomes worth optimizing.
+				const startChunkIndex = splitFieldAtIndex(sourceField, source.start, policy);
+				const endChunkIndex = splitFieldAtIndex(sourceField, source.end, policy);
+				const newField = sourceField.splice(startChunkIndex, endChunkIndex - startChunkIndex);
 
 				if (destination === undefined) {
 					for (const child of newField) {
@@ -225,12 +245,12 @@ export class ChunkedForest implements IEditableForest {
 				let indexOfChunk = 0;
 				let chunk = chunks[indexOfChunk] ?? oob();
 				while (indexWithinChunk >= chunk.topLevelLength) {
-					chunk = chunks[indexOfChunk] ?? oob();
 					indexWithinChunk -= chunk.topLevelLength;
 					indexOfChunk++;
 					if (indexOfChunk === chunks.length) {
 						fail(0xaf7 /* missing edited node */);
 					}
+					chunk = chunks[indexOfChunk] ?? oob();
 				}
 				let found = chunks[indexOfChunk] ?? oob();
 				if (!(found instanceof BasicChunk)) {

package/src/feature-libraries/chunked-forest/codec/chunkDecoding.ts

@@ -9,6 +9,7 @@ import type {
 	OpSpaceCompressedId,
 	SessionId,
 } from "@fluidframework/id-compressor";
+import { isFinalId } from "@fluidframework/id-compressor/internal";
 import { v5 as uuidV5 } from "uuid";
 
 import { DiscriminatedUnionDispatcher } from "../../../codec/index.js";
@@ -42,7 +43,7 @@ import {
 import type { FieldBatchEncodingContext, IncrementalDecoder } from "./codecs.js";
 import {
 	type EncodedAnyShape,
-	type EncodedChunkShapeV1OrV2,
+	type EncodedChunkShape,
 	type EncodedChunkShapeV2,
 	type EncodedFieldBatchV1OrV2,
 	type EncodedFieldBatchV2,
@@ -50,7 +51,9 @@ import {
 	type EncodedInlineArrayShape,
 	type EncodedNestedArrayShape,
 	type EncodedNodeShape,
+	type EncodedSpecializedNodeShape,
 	type EncodedValueShape,
+	type ShapeIndex,
 	SpecialField,
 	supportsIncrementalEncoding,
 } from "./format/index.js";
@@ -97,9 +100,137 @@ export function decode(
 	);
 }
 
+/**
+ * Resolves `shapeIndex` to a fully-resolved {@link EncodedNodeShape}, normalizing away any
+ * specialized node shapes (`f`) along the way by applying their overlays via
+ * {@link applySpecialization} until a concrete node shape is reached.
+ *
+ * @param input - The index of the shape to resolve, which must be a concrete or specialized node shape.
+ * @param context - The decoding context containing the shape definitions.
+ * @param pendingResolution - (Internal) A set of shape indices visited so far in the current resolution chain, used to detect cycles in the specialization chain. Most callers should not provide this argument.
+ *
+ * @remarks
+ * Exported for testing.
+ */
+export function normalizeToNodeShape(
+	input: EncodedNodeShape | EncodedSpecializedNodeShape,
+	context: DecoderContext<EncodedChunkShape>,
+	pendingResolution: Set<ShapeIndex> = new Set(),
+): EncodedNodeShape {
+	if (!("base" in input)) {
+		return input;
+	}
+
+	const baseIndex = input.base;
+	assert(!pendingResolution.has(baseIndex), 0xcfb /* cyclic specialized node shape chain */);
+	pendingResolution.add(baseIndex);
+	const encoded = context.shapes[baseIndex];
+	assert(encoded !== undefined, 0xcfc /* shape index out of bounds */);
+
+	const baseShape = encoded.c ?? ("f" in encoded ? encoded.f : undefined);
+	assert(
+		baseShape !== undefined,
+		0xcfd /* shape at index must be a concrete (c) or specialized (f) node shape */,
+	);
+
+	return applySpecialization(
+		normalizeToNodeShape(baseShape, context, pendingResolution),
+		input,
+		context,
+	);
+}
+
+/**
+ * Produces a specialized {@link EncodedNodeShape} by overlaying `overrides` onto `base`.
+ *
+ * See {@link EncodedSpecializedNodeShape} for the override/inherit/clear semantics.
+ *
+ * @remarks
+ * Exported for testing.
+ */
+export function applySpecialization(
+	base: EncodedNodeShape,
+	overrides: EncodedSpecializedNodeShape,
+	context: DecoderContext<EncodedChunkShape>,
+): EncodedNodeShape {
+	const fields = [...(base.fields ?? [])];
+	const indexFromKey = new Map<FieldKey, number>();
+	for (const [i, [keyEncoded]] of fields.entries()) {
+		const key = context.identifier<FieldKey>(keyEncoded);
+		assert(!indexFromKey.has(key), 0xcfe /* duplicate field key in base node shape */);
+		indexFromKey.set(key, i);
+	}
+
+	// Replace fields in base with overrides, append new keys in overrides in the order they are specified.
+	const seenOverrideKeys = new Set<FieldKey>();
+	for (const [keyEncoded, shapeIndex] of overrides.fields ?? []) {
+		const key = context.identifier<FieldKey>(keyEncoded);
+		assert(
+			!seenOverrideKeys.has(key),
+			0xcff /* duplicate field key in specialized node shape */,
+		);
+		seenOverrideKeys.add(key);
+		const existingIndex = indexFromKey.get(key);
+		if (existingIndex === undefined) {
+			fields.push([keyEncoded, shapeIndex]);
+		} else {
+			const index = fields[existingIndex];
+			assert(index !== undefined, 0xd00 /* expected existing field index */);
+			fields[existingIndex] = [index[0], shapeIndex];
+		}
+	}
+
+	return {
+		type: base.type,
+		value: resolveOverride(overrides.value, base.value),
+		fields: fields.length > 0 ? fields : undefined,
+		extraFields: resolveOverride(overrides.extraFields, base.extraFields),
+	};
+}
+
+/**
+ * Resolves an override against a base value.
+ *
+ * @param override - `undefined` means the override is absent (inherit from base); `null` is the
+ * explicit-clear sentinel needed because JSON.stringify drops `undefined`-valued properties, making
+ * property-presence indistinguishable from absent on the wire.
+ * @param baseValue - The value to inherit when the override is absent.
+ */
+function resolveOverride<T>(
+	// eslint-disable-next-line @rushstack/no-new-null
+	override: T | null | undefined,
+	baseValue: T | undefined,
+): T | undefined {
+	if (override === undefined) {
+		return baseValue;
+	}
+	if (override === null) {
+		return undefined;
+	}
+	return override;
+}
+
+/**
+ * Decoder for {@link EncodedSpecializedNodeShape}s.
+ * Applies the specialization's field overrides to the resolved base node shape, then delegates
+ * to a {@link NodeDecoder} built from the resulting shape.
+ */
+export class SpecializedNodeDecoder implements ChunkDecoder {
+	private readonly inner: NodeDecoder;
+	public constructor(
+		shape: EncodedSpecializedNodeShape,
+		context: DecoderContext<EncodedChunkShape>,
+	) {
+		this.inner = new NodeDecoder(normalizeToNodeShape(shape, context), context);
+	}
+	public decode(decoders: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk {
+		return this.inner.decode(decoders, stream);
+	}
+}
+
 const decoderLibrary = new DiscriminatedUnionDispatcher<
-	EncodedChunkShapeV1OrV2,
-	[context: DecoderContext<EncodedChunkShapeV1OrV2>],
+	EncodedChunkShape,
+	[context: DecoderContext<EncodedChunkShape>],
 	ChunkDecoder
 >({
 	a(shape: EncodedNestedArrayShape, context): ChunkDecoder {
@@ -120,6 +251,9 @@ const decoderLibrary = new DiscriminatedUnionDispatcher<
 	): ChunkDecoder {
 		return new IncrementalChunkDecoder(context);
 	},
+	f(shape: EncodedSpecializedNodeShape, context): ChunkDecoder {
+		return new SpecializedNodeDecoder(shape, context);
+	},
 });
 
 /**
@@ -153,8 +287,10 @@ export function readValue(
 			const idCompressor = idDecodingContext.idCompressor;
 			// OpSpaceCompressedIds are negative, and require a session-id to compute their value.
 			// Due to a bug, we have some special casing for them (see below).
-			// TODO: isFinalId should probably be exported from id-compressor and that could be used to do the narrowing here.
-			if (idDecodingContext.isSummary === true && streamValue < 0) {
+			if (
+				idDecodingContext.isSummary === true &&
+				!isFinalId(streamValue as OpSpaceCompressedId)
+			) {
 				if (
 					idDecodingContext.healUnresolvableIdentifiersOnDecode === true &&
 					idDecodingContext.sharedObjectId !== undefined
@@ -349,7 +485,7 @@ type BasicFieldDecoder = (
  * Get a decoder for fields of a provided (via `shape` and `context`).
  */
 function fieldDecoder(
-	context: DecoderContext<EncodedChunkShapeV1OrV2>,
+	context: DecoderContext<EncodedChunkShape>,
 	key: FieldKey,
 	shape: number,
 ): BasicFieldDecoder {
@@ -368,7 +504,7 @@ export class NodeDecoder implements ChunkDecoder {
 	private readonly fieldDecoders: readonly BasicFieldDecoder[];
 	public constructor(
 		private readonly shape: EncodedNodeShape,
-		private readonly context: DecoderContext<EncodedChunkShapeV1OrV2>,
+		private readonly context: DecoderContext<EncodedChunkShape>,
 	) {
 		this.type = shape.type === undefined ? undefined : context.identifier(shape.type);
 

package/src/feature-libraries/chunked-forest/codec/codecs.ts

@@ -9,8 +9,9 @@ import { lowestMinVersionForCollab } from "@fluidframework/runtime-utils/interna
 import type { TSchema } from "@sinclair/typebox";
 
 import {
-	ClientVersionDispatchingCodecBuilder,
+	VersionDispatchingCodecBuilder,
 	type CodecAndSchema,
+	type VersionDispatchingCodec,
 	FluidClientVersion,
 } from "../../../codec/index.js";
 import {
@@ -138,7 +139,11 @@ export interface FieldBatchEncodingContext {
  * @remarks
  * Fields in this batch currently don't have field schema for the root, which limits optimizations.
  */
-export type FieldBatchCodec = ReturnType<typeof fieldBatchCodecBuilder.build>;
+export type FieldBatchCodec = VersionDispatchingCodec<
+	FieldBatch,
+	FieldBatchEncodingContext,
+	FieldBatchFormatVersion
+>;
 
 /**
  * Creates the encode/decode functions for a specific FieldBatch format version.
@@ -231,30 +236,27 @@ function makeFieldBatchCodecForVersion(
 }
 
 /**
- * {@link ClientVersionDispatchingCodecBuilder} for field batch codecs.
+ * {@link VersionDispatchingCodecBuilder} for field batch codecs.
  */
-export const fieldBatchCodecBuilder = ClientVersionDispatchingCodecBuilder.build(
-	"FieldBatch",
-	[
-		{
-			minVersionForCollab: lowestMinVersionForCollab,
-			formatVersion: FieldBatchFormatVersion.v1,
-			codec: makeFieldBatchCodecForVersion(
-				FieldBatchFormatVersion.v1,
-				uncompressedEncodeV1,
-				schemaCompressedEncodeV1,
-				EncodedFieldBatchV1,
-			),
-		},
-		{
-			minVersionForCollab: FluidClientVersion.v2_73,
-			formatVersion: FieldBatchFormatVersion.v2,
-			codec: makeFieldBatchCodecForVersion(
-				FieldBatchFormatVersion.v2,
-				uncompressedEncodeV2,
-				schemaCompressedEncodeV2,
-				EncodedFieldBatchV2,
-			),
-		},
-	],
-);
+export const fieldBatchCodecBuilder = VersionDispatchingCodecBuilder.build("FieldBatch", [
+	{
+		minVersionForCollab: lowestMinVersionForCollab,
+		formatVersion: FieldBatchFormatVersion.v1,
+		codec: makeFieldBatchCodecForVersion(
+			FieldBatchFormatVersion.v1,
+			uncompressedEncodeV1,
+			schemaCompressedEncodeV1,
+			EncodedFieldBatchV1,
+		),
+	},
+	{
+		minVersionForCollab: FluidClientVersion.v2_73,
+		formatVersion: FieldBatchFormatVersion.v2,
+		codec: makeFieldBatchCodecForVersion(
+			FieldBatchFormatVersion.v2,
+			uncompressedEncodeV2,
+			schemaCompressedEncodeV2,
+			EncodedFieldBatchV2,
+		),
+	},
+]);

package/src/feature-libraries/chunked-forest/codec/compressedEncode.ts

@@ -32,7 +32,7 @@ import type { FieldBatch } from "./fieldBatch.js";
 import {
 	type EncodedAnyShape,
 	type EncodedChunkShapeV1,
-	type EncodedChunkShapeV1OrV2,
+	type EncodedChunkShape,
 	type EncodedChunkShapeV2,
 	type EncodedFieldBatchV1OrV2,
 	type EncodedNestedArrayShape,
@@ -64,8 +64,8 @@ export function compressedEncode(
 	return updateShapesAndIdentifiersEncoding(context.version, batchBuffer);
 }
 
-export type BufferFormat = BufferFormatGeneric<EncodedChunkShapeV1OrV2>;
-export type Shape = ShapeGeneric<EncodedChunkShapeV1OrV2>;
+export type BufferFormat = BufferFormatGeneric<EncodedChunkShape>;
+export type Shape = ShapeGeneric<EncodedChunkShape>;
 
 /**
  * Like {@link FieldEncoder}, except data will be prefixed with the key.
@@ -167,7 +167,7 @@ export function asNodesEncoder(encoder: NodeEncoder): NodesEncoder {
 /**
  * Encodes a chunk with {@link EncodedAnyShape} by prefixing the data with its shape.
  */
-export class AnyShape extends ShapeGeneric<EncodedChunkShapeV1OrV2> {
+export class AnyShape extends ShapeGeneric<EncodedChunkShape> {
 	private constructor() {
 		super();
 	}
@@ -272,7 +272,7 @@ export const anyFieldEncoder: FieldEncoder = {
  * which is an easy way to keep all the related code together without extra objects.
  */
 export class InlineArrayEncoder
-	extends ShapeGeneric<EncodedChunkShapeV1OrV2>
+	extends ShapeGeneric<EncodedChunkShape>
 	implements NodesEncoder, FieldEncoder
 {
 	public static readonly empty: InlineArrayEncoder = new InlineArrayEncoder(0, {
@@ -356,7 +356,7 @@ export class InlineArrayEncoder
 /**
  * Encodes the shape for a nested array as {@link EncodedNestedArrayShape} shape.
  */
-export class NestedArrayShape extends ShapeGeneric<EncodedChunkShapeV1OrV2> {
+export class NestedArrayShape extends ShapeGeneric<EncodedChunkShape> {
 	/**
 	 * @param innerShape - The shape of each item in this nested array.
 	 */
@@ -367,7 +367,7 @@ export class NestedArrayShape extends ShapeGeneric<EncodedChunkShapeV1OrV2> {
 	public encodeShape(
 		identifiers: DeduplicationTable<string>,
 		shapes: DeduplicationTable<Shape>,
-	): EncodedChunkShapeV1OrV2 {
+	): EncodedChunkShape {
 		const shape: EncodedNestedArrayShape =
 			shapes.valueToIndex.get(this.innerShape) ??
 			fail(0xb4f /* index for shape not found in table */);

package/src/feature-libraries/chunked-forest/codec/format/formatV2.ts

@@ -16,16 +16,20 @@ import { shapesV1 } from "./formatV1.js";
 export type EncodedIncrementalChunkShape = Static<typeof EncodedIncrementalChunkShape>;
 export const EncodedIncrementalChunkShape = Type.Literal(0);
 
+/**
+ * The chunk shapes supported by the V2 format.
+ * @remarks
+ * See {@link EncodedChunkShapeV2}.
+ */
+export const shapesV2 = {
+	...shapesV1,
+	e: Type.Optional(EncodedIncrementalChunkShape),
+} as const;
+
 /**
  * V2 extension of {@link EncodedChunkShapeV1}.
  * @remarks
  * See {@link DiscriminatedUnionDispatcher} for more information on this pattern.
  */
 export type EncodedChunkShapeV2 = Static<typeof EncodedChunkShapeV2>;
-export const EncodedChunkShapeV2 = Type.Object(
-	{
-		...shapesV1,
-		e: Type.Optional(EncodedIncrementalChunkShape),
-	},
-	unionOptions,
-);
+export const EncodedChunkShapeV2 = Type.Object(shapesV2, unionOptions);

package/src/feature-libraries/chunked-forest/codec/format/formatVText.ts

@@ -0,0 +1,83 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { type Static, Type } from "@sinclair/typebox";
+
+import { unionOptions } from "../../../../codec/index.js";
+
+import { ShapeIndex } from "./formatGeneric.js";
+import { EncodedFieldShape, EncodedValueShape } from "./formatV1.js";
+import { shapesV2 } from "./formatV2.js";
+
+/**
+ * A node shape that derives from another node shape by overlaying property-level overrides.
+ *
+ * @remarks
+ * Compresses runs of node shapes that differ only in a few properties: a base node shape
+ * defines the structural skeleton, and the specialization narrows specific properties.
+ *
+ * For example, a base `FormatNode` with a variable-boolean `bold` field can be specialized
+ * to a shape that pins `bold` to a constant `true` — every node decoded with the
+ * specialization contributes zero stream tokens for `bold`.
+ *
+ * Specialization rules: `type` is always inherited from the resolved base. `fields` overrides
+ * apply per-key: entries whose key matches a base field replace that entry's shape index in
+ * place; entries with new keys are appended after all base fields. For `value` and
+ * `extraFields`: if the property is absent on the wire, the base's value is inherited; if
+ * `null`, the resulting shape has no value / no extraFields (explicit clear); any other value
+ * replaces the base's.
+ *
+ * The `null` sentinel exists because JSON does not preserve `undefined`-valued properties,
+ * so override-vs-inherit cannot be discriminated by property presence after persistence.
+ *
+ * Decoded by {@link applySpecialization}.
+ */
+export type EncodedSpecializedNodeShape = Static<typeof EncodedSpecializedNodeShape>;
+export const EncodedSpecializedNodeShape = Type.Object(
+	{
+		/**
+		 * Index into the enclosing batch's shapes array of the shape this specializes.
+		 * Must resolve to either an {@link EncodedNodeShape} or another
+		 * `EncodedSpecializedNodeShape`; chains are followed transitively until a node shape
+		 * is reached. This restriction is enforced at runtime, not by the schema.
+		 */
+		base: ShapeIndex,
+		/**
+		 * Field-level overrides applied to the resolved base's `fields`. Entries whose key
+		 * matches a base field replace that field's shape index in place; entries with new
+		 * keys are appended after all base fields, in the order given here. Base field order
+		 * is preserved — this is the stream consumption order at decode time, so encoders
+		 * must serialize per-field tokens in the resulting field order, not in this list's order.
+		 */
+		fields: Type.Optional(Type.Array(EncodedFieldShape)),
+		/**
+		 * If absent, inherits the resolved base's value shape. If `null`, the resulting shape
+		 * has no value shape (explicit clear). Any other value replaces the base's.
+		 */
+		value: Type.Optional(Type.Union([EncodedValueShape, Type.Null()])),
+		/**
+		 * If absent, inherits the resolved base's extraFields shape. If `null`, the resulting
+		 * shape has no extraFields (explicit clear). Any other value replaces the base's.
+		 */
+		extraFields: Type.Optional(Type.Union([ShapeIndex, Type.Null()])),
+	},
+	{ additionalProperties: false },
+);
+
+/**
+ * Experimental extension of {@link EncodedChunkShapeV2}.
+ * @remarks
+ * See {@link DiscriminatedUnionDispatcher} for more information on this pattern.
+ */
+export type EncodedChunkShapeVTextExperimental = Static<
+	typeof EncodedChunkShapeVTextExperimental
+>;
+export const EncodedChunkShapeVTextExperimental = Type.Object(
+	{
+		...shapesV2,
+		f: Type.Optional(EncodedSpecializedNodeShape),
+	},
+	unionOptions,
+);

package/src/feature-libraries/chunked-forest/codec/format/index.ts

@@ -16,14 +16,19 @@ export {
 	SpecialField,
 } from "./formatV1.js";
 export { EncodedIncrementalChunkShape, EncodedChunkShapeV2 } from "./formatV2.js";
+export {
+	EncodedChunkShapeVTextExperimental,
+	EncodedSpecializedNodeShape,
+} from "./formatVText.js";
 export {
 	FieldBatchFormatVersion,
 	EncodedFieldBatchV1,
 	EncodedFieldBatchV2,
+	EncodedFieldBatchVTextExperimental,
 	supportsIncrementalEncoding,
 	type EncodedFieldBatchV1OrV2,
 	type EncodedFieldBatchV1AndV2,
-	type EncodedChunkShapeV1OrV2,
+	type EncodedChunkShape,
 } from "./versions.js";
 export type {
 	ShapeIndex,