@fluidframework/tree 2.51.0 → 2.53.0-350190

package/src/feature-libraries/chunked-forest/codec/schemaBasedEncode.ts

@@ -20,18 +20,18 @@ import {
 import type { FullSchemaPolicy } from "../../modular-schema/index.js";
 
 import {
-	EncoderCache,
+	EncoderContext,
 	type FieldEncoder,
-	type FieldShaper,
+	type FieldEncodeBuilder,
 	type KeyedFieldEncoder,
-	type TreeShaper,
+	type NodeEncodeBuilder,
 	anyNodeEncoder,
 	asFieldEncoder,
 	compressedEncode,
 } from "./compressedEncode.js";
 import type { FieldBatch } from "./fieldBatch.js";
 import { type EncodedFieldBatch, type EncodedValueShape, SpecialField } from "./format.js";
-import { NodeShape } from "./nodeShape.js";
+import { NodeShapeBasedEncoder } from "./nodeEncoder.js";
 
 /**
  * Encode data from `fieldBatch` in into an `EncodedChunk`.
@@ -45,37 +45,38 @@ export function schemaCompressedEncode(
 	fieldBatch: FieldBatch,
 	idCompressor: IIdCompressor,
 ): EncodedFieldBatch {
-	return compressedEncode(fieldBatch, buildCache(schema, policy, idCompressor));
+	return compressedEncode(fieldBatch, buildContext(schema, policy, idCompressor));
 }
 
-export function buildCache(
-	schema: StoredSchemaCollection,
+export function buildContext(
+	storedSchema: StoredSchemaCollection,
 	policy: FullSchemaPolicy,
 	idCompressor: IIdCompressor,
-): EncoderCache {
-	const cache: EncoderCache = new EncoderCache(
-		(fieldHandler: FieldShaper, schemaName: TreeNodeSchemaIdentifier) =>
-			treeShaper(schema, policy, fieldHandler, schemaName),
-		(treeHandler: TreeShaper, field: TreeFieldStoredSchema) =>
-			fieldShaper(treeHandler, field, cache, schema),
+): EncoderContext {
+	const context: EncoderContext = new EncoderContext(
+		(fieldBuilder: FieldEncodeBuilder, schemaName: TreeNodeSchemaIdentifier) =>
+			getNodeEncoder(fieldBuilder, storedSchema, schemaName),
+		(nodeBuilder: NodeEncodeBuilder, fieldSchema: TreeFieldStoredSchema) =>
+			getFieldEncoder(nodeBuilder, fieldSchema, context, storedSchema),
 		policy.fieldKinds,
 		idCompressor,
 	);
-	return cache;
+	return context;
 }
 
 /**
- * Selects shapes to use to encode fields.
+ * Selects an encoder to use to encode fields.
  */
-export function fieldShaper(
-	treeHandler: TreeShaper,
+export function getFieldEncoder(
+	nodeBuilder: NodeEncodeBuilder,
 	field: TreeFieldStoredSchema,
-	cache: EncoderCache,
+	context: EncoderContext,
 	storedSchema: StoredSchemaCollection,
 ): FieldEncoder {
-	const kind = cache.fieldShapes.get(field.kind) ?? fail(0xb52 /* missing FieldKind */);
+	const kind = context.fieldShapes.get(field.kind) ?? fail(0xb52 /* missing FieldKind */);
 	const type = oneFromSet(field.types);
-	const nodeEncoder = type !== undefined ? treeHandler.shapeFromTree(type) : anyNodeEncoder;
+	const nodeEncoder =
+		type !== undefined ? nodeBuilder.nodeEncoderFromSchema(type) : anyNodeEncoder;
 	if (kind.multiplicity === Multiplicity.Single) {
 		if (field.kind === identifierFieldKindIdentifier) {
 			assert(type !== undefined, 0x999 /* field type must be defined in identifier field */);
@@ -89,7 +90,7 @@ export function fieldShaper(
 				nodeSchema.leafValue === ValueSchema.String,
 				0x99c /* identifier field can only be type string */,
 			);
-			const identifierNodeEncoder = new NodeShape(
+			const identifierNodeEncoder = new NodeShapeBasedEncoder(
 				type,
 				SpecialField.Identifier,
 				[],
@@ -99,21 +100,20 @@ export function fieldShaper(
 		}
 		return asFieldEncoder(nodeEncoder);
 	} else {
-		return cache.nestedArray(nodeEncoder);
+		return context.nestedArrayEncoder(nodeEncoder);
 	}
 }
 
 /**
- * Selects shapes to use to encode trees.
+ * Selects an encoder to use to encode nodes.
  */
-export function treeShaper(
-	fullSchema: StoredSchemaCollection,
-	policy: FullSchemaPolicy,
-	fieldHandler: FieldShaper,
+export function getNodeEncoder(
+	fieldBuilder: FieldEncodeBuilder,
+	storedSchema: StoredSchemaCollection,
 	schemaName: TreeNodeSchemaIdentifier,
-): NodeShape {
+): NodeShapeBasedEncoder {
 	const schema =
-		fullSchema.nodeSchema.get(schemaName) ?? fail(0xb53 /* missing node schema */);
+		storedSchema.nodeSchema.get(schemaName) ?? fail(0xb53 /* missing node schema */);
 
 	if (schema instanceof ObjectNodeStoredSchema) {
 		// TODO:Performance:
@@ -122,14 +122,14 @@ export function treeShaper(
 
 		const objectNodeFields: KeyedFieldEncoder[] = [];
 		for (const [key, field] of schema.objectNodeFields ?? []) {
-			objectNodeFields.push({ key, encoder: fieldHandler.shapeFromField(field) });
+			objectNodeFields.push({ key, encoder: fieldBuilder.fieldEncoderFromSchema(field) });
 		}
 
-		const shape = new NodeShape(schemaName, false, objectNodeFields, undefined);
+		const shape = new NodeShapeBasedEncoder(schemaName, false, objectNodeFields, undefined);
 		return shape;
 	}
 	if (schema instanceof LeafNodeStoredSchema) {
-		const shape = new NodeShape(
+		const shape = new NodeShapeBasedEncoder(
 			schemaName,
 			valueShapeFromSchema(schema.leafValue),
 			[],
@@ -138,11 +138,11 @@ export function treeShaper(
 		return shape;
 	}
 	if (schema instanceof MapNodeStoredSchema) {
-		const shape = new NodeShape(
+		const shape = new NodeShapeBasedEncoder(
 			schemaName,
 			false,
 			[],
-			fieldHandler.shapeFromField(schema.mapFields),
+			fieldBuilder.fieldEncoderFromSchema(schema.mapFields),
 		);
 		return shape;
 	}

package/src/feature-libraries/chunked-forest/codec/uncompressedEncode.ts

@@ -13,8 +13,8 @@ import type { FluidSerializableReadOnly } from "../../valueUtilities.js";
 import type { FieldBatch } from "./fieldBatch.js";
 import {
 	type EncodedFieldBatch,
-	type EncodedNestedArray,
-	type EncodedTreeShape,
+	type EncodedNestedArrayShape,
+	type EncodedNodeShape,
 	version,
 } from "./format.js";
 import type { ShapeIndex } from "./formatGeneric.js";
@@ -33,7 +33,7 @@ export function uncompressedEncode(batch: FieldBatch): EncodedFieldBatch {
 		version,
 		identifiers: [],
 		// A single shape used to encode all fields.
-		shapes: [{ c: anyTreeShape }, { a: anyArray }],
+		shapes: [{ c: anyNodeShape }, { a: anyArray }],
 		// Wrap up each field as an indicator to use the above shape, and its encoded data.
 		data: rootFields.map((data) => [arrayIndex, data]),
 	};
@@ -42,11 +42,11 @@ export function uncompressedEncode(batch: FieldBatch): EncodedFieldBatch {
 const treeIndex: ShapeIndex = 0;
 const arrayIndex: ShapeIndex = 1;
 
-const anyTreeShape: EncodedTreeShape = {
+const anyNodeShape: EncodedNodeShape = {
 	extraFields: arrayIndex,
 };
 
-const anyArray: EncodedNestedArray = treeIndex;
+const anyArray: EncodedNestedArrayShape = treeIndex;
 
 /**
  * Encode a field using the hard coded shape above.

package/src/feature-libraries/default-schema/defaultFieldKinds.ts

@@ -41,6 +41,7 @@ export const noChangeHandler: FieldChangeHandler<0> = {
 		compose: (change1: 0, change2: 0) => 0,
 		invert: (changes: 0) => 0,
 		rebase: (change: 0, over: 0) => 0,
+		mute: (changes: 0) => 0,
 	}),
 	codecsFactory: () => noChangeCodecFamily,
 	editor: { buildChildChanges: () => fail(0xb0d /* Child changes not supported */) },
@@ -97,6 +98,11 @@ export const required = new FieldKindWithEditor(
 	Multiplicity.Single,
 	valueChangeHandler,
 	(types, other) =>
+		// By omitting Identifier here,
+		// this is making a policy choice that a schema upgrade cannot be done from required to identifier.
+		// Since an identifier can be upgraded into a required field,
+		// preventing the inverse helps ensure that schema upgrades are monotonic.
+		// Which direction is allowed is a subjective policy choice.
 		(other.kind === sequence.identifier ||
 			other.kind === requiredIdentifier ||
 			other.kind === optional.identifier ||
@@ -125,6 +131,10 @@ const nodeKeyIdentifier = "NodeKey";
 
 /**
  * Exactly one identifier.
+ *
+ * TODO: this is almost the same as identifier, but apparently unused.
+ * Confirm if this is truly unused since before the document format was stabilized, and remove if possible.
+ * @deprecated Superseded by {@link identifier}.
  */
 export const nodeKey = new FieldKindWithEditor(
 	nodeKeyIdentifier,
@@ -149,6 +159,7 @@ export const identifier = new FieldKindWithEditor(
 	Multiplicity.Single,
 	noChangeHandler,
 	(types, other) =>
+		// Allows upgrading from identifier to required: which way this upgrade is allowed to go is a subjective policy choice.
 		(other.kind === sequence.identifier ||
 			other.kind === requiredIdentifier ||
 			other.kind === optional.identifier ||

package/src/feature-libraries/default-schema/defaultSchema.ts

@@ -12,6 +12,4 @@ import { fieldKinds } from "./defaultFieldKinds.js";
  */
 export const defaultSchemaPolicy: FullSchemaPolicy = {
 	fieldKinds,
-	validateSchema: false,
-	allowUnknownOptionalFields: () => false,
 };

package/src/feature-libraries/default-schema/index.ts

@@ -30,7 +30,7 @@ export {
 	SchemaValidationError,
 	isNodeInSchema,
 	isFieldInSchema,
-	inSchemaOrThrow,
+	throwOutOfSchema,
 } from "./schemaChecker.js";
 
 export { defaultSchemaPolicy } from "./defaultSchema.js";

package/src/feature-libraries/default-schema/schemaChecker.ts

@@ -32,66 +32,78 @@ export enum SchemaValidationError {
 }
 
 /**
- * Throws a UsageError if maybeError indicates a tree is out of schema.
+ * Throws a UsageError indicating a tree is out of schema.
  */
-export function inSchemaOrThrow(maybeError: SchemaValidationError | undefined): void {
-	if (maybeError !== undefined) {
-		throw new UsageError(
-			`Tree does not conform to schema: ${SchemaValidationError[maybeError]}`,
-		);
-	}
+export function throwOutOfSchema(maybeError: SchemaValidationError): never {
+	throw new UsageError(
+		`Tree does not conform to schema: ${SchemaValidationError[maybeError]}`,
+	);
 }
 
+type NotUndefined = number | string | boolean | bigint | symbol | object;
+
 /**
  * Deeply checks that the provided node complies with the schema based on its identifier.
+ *
+ * @param onError - Called with the first error (if any).
+ *
+ * @returns the return value from `onError` if the node or anything inside of it is out of schema, otherwise `undefined`.
  */
-export function isNodeInSchema(
+export function isNodeInSchema<T extends NotUndefined>(
 	node: MinimalMapTreeNodeView,
 	schemaAndPolicy: SchemaAndPolicy,
-): SchemaValidationError | undefined {
+	onError: (error: SchemaValidationError) => T,
+): T | undefined {
 	// Validate the schema declared by the node exists
 	const schema = schemaAndPolicy.schema.nodeSchema.get(node.type);
 	if (schema === undefined) {
-		return SchemaValidationError.Node_MissingSchema;
+		return onError(SchemaValidationError.Node_MissingSchema);
 	}
 
 	// Validate the node is well formed according to its schema
 
 	if (schema instanceof LeafNodeStoredSchema) {
 		if (iterableHasSome(node.fields)) {
-			return SchemaValidationError.LeafNode_FieldsNotAllowed;
+			return onError(SchemaValidationError.LeafNode_FieldsNotAllowed);
 		}
 		if (!allowsValue(schema.leafValue, node.value)) {
-			return SchemaValidationError.LeafNode_InvalidValue;
+			return onError(SchemaValidationError.LeafNode_InvalidValue);
 		}
 	} else {
 		if (node.value !== undefined) {
-			return SchemaValidationError.NonLeafNode_ValueNotAllowed;
+			return onError(SchemaValidationError.NonLeafNode_ValueNotAllowed);
 		}
 
 		if (schema instanceof ObjectNodeStoredSchema) {
 			const uncheckedFieldsFromNode = new Set(mapIterable(node.fields, ([key, field]) => key));
 			for (const [fieldKey, fieldSchema] of schema.objectNodeFields) {
 				const nodeField = node.fields.get(fieldKey) ?? [];
-				const fieldInSchemaResult = isFieldInSchema(nodeField, fieldSchema, schemaAndPolicy);
+				const fieldInSchemaResult = isFieldInSchema(
+					nodeField,
+					fieldSchema,
+					schemaAndPolicy,
+					onError,
+				);
 				if (fieldInSchemaResult !== undefined) {
 					return fieldInSchemaResult;
 				}
 				uncheckedFieldsFromNode.delete(fieldKey);
 			}
-			// The node has fields that we did not check as part of looking at every field defined in the node's schema
-			if (
-				uncheckedFieldsFromNode.size !== 0 &&
-				// TODO: AB#43547: This check is wrong. If a given view schema allows an unknown optional field, that does NOT mean the stored schema should allow unknown:
-				// In-fact, any data the view schema does not know about must still comply with the stored schema:
-				// if this were not the case schema evolution could not add any fields since they might already have out of schema data.
-				!schemaAndPolicy.policy.allowUnknownOptionalFields(node.type)
-			) {
-				return SchemaValidationError.ObjectNode_FieldNotInSchema;
+			// The node has fields that we did not check as part of looking at every field defined in the node's schema.
+			// Since this is testing compatibility with a stored schema (not view schema), "allowUnknownOptionalFields" does not exist at this layer.
+			// Code using this with a stored schema derived from a view schema rather than the document can be problematic because it may be missing unknown fields that the actual document has.
+			// Other schema evolution features like "staged" allowed types will likely cause similar issues elsewhere in this checker.
+			if (uncheckedFieldsFromNode.size !== 0) {
+				return onError(SchemaValidationError.ObjectNode_FieldNotInSchema);
 			}
 		} else if (schema instanceof MapNodeStoredSchema) {
 			for (const [_key, field] of node.fields) {
-				const fieldInSchemaResult = isFieldInSchema(field, schema.mapFields, schemaAndPolicy);
+				const fieldInSchemaResult = isFieldInSchema(
+					field,
+					schema.mapFields,
+					schemaAndPolicy,
+					onError,
+				);
 				if (fieldInSchemaResult !== undefined) {
 					return fieldInSchemaResult;
 				}
@@ -106,34 +118,39 @@ export function isNodeInSchema(
 
 /**
  * Deeply checks that the nodes comply with the field schema and included schema.
+ *
+ * @param onError - Called with the first error (if any).
+ *
+ * @returns the return value from `onError` if the field or anything inside of it is out of schema, otherwise `undefined`.
  */
-export function isFieldInSchema(
+export function isFieldInSchema<T extends NotUndefined>(
 	childNodes: MapTreeFieldViewGeneric<MinimalMapTreeNodeView>,
 	schema: TreeFieldStoredSchema,
 	schemaAndPolicy: SchemaAndPolicy,
-): SchemaValidationError | undefined {
+	onError: (error: SchemaValidationError) => T,
+): T | undefined {
 	// Validate that the field kind is handled by the schema policy
 	const kind = schemaAndPolicy.policy.fieldKinds.get(schema.kind);
 	if (kind === undefined) {
-		return SchemaValidationError.Field_KindNotInSchemaPolicy;
+		return onError(SchemaValidationError.Field_KindNotInSchemaPolicy);
 	}
 
 	// Validate that the field doesn't contain more nodes than its type supports
 	{
 		const multiplicityCheck = compliesWithMultiplicity(childNodes.length, kind.multiplicity);
 		if (multiplicityCheck !== undefined) {
-			return multiplicityCheck;
+			return onError(multiplicityCheck);
 		}
 	}
 
 	for (const node of childNodes) {
 		// Validate the type declared by the node is allowed in this field
 		if (schema.types !== undefined && !schema.types.has(node.type)) {
-			return SchemaValidationError.Field_NodeTypeNotAllowed;
+			return onError(SchemaValidationError.Field_NodeTypeNotAllowed);
 		}
 
 		// Validate the node complies with the type it declares to be.
-		const nodeInSchemaResult = isNodeInSchema(node, schemaAndPolicy);
+		const nodeInSchemaResult = isNodeInSchema(node, schemaAndPolicy, onError);
 		if (nodeInSchemaResult !== undefined) {
 			return nodeInSchemaResult;
 		}

package/src/feature-libraries/index.ts

@@ -136,7 +136,7 @@ export {
 	SchemaValidationError,
 	isNodeInSchema,
 	isFieldInSchema,
-	inSchemaOrThrow,
+	throwOutOfSchema,
 } from "./default-schema/index.js";
 
 export {
@@ -193,5 +193,3 @@ export {
 	type TreeIndexKey,
 	type TreeIndexNodes,
 } from "./indexing/index.js";
-
-export { initializeForest } from "./initializeForest.js";

package/src/feature-libraries/modular-schema/comparison.ts

@@ -23,7 +23,7 @@ import { withEditor } from "./fieldKindWithEditor.js";
 import { isNeverTree } from "./isNeverTree.js";
 
 /**
- * @returns true iff `superset` is a superset of `original`.
+ * Returns true iff `superset` is a superset of `original`.
  *
  * This does not require a strict (aka proper) superset: equivalent schema will return true.
  *
@@ -65,12 +65,7 @@ export function allowsTreeSuperset(
 
 	if (original instanceof MapNodeStoredSchema) {
 		if (superset instanceof MapNodeStoredSchema) {
-			return allowsFieldSuperset(
-				policy,
-				originalData,
-				normalizeField(original.mapFields),
-				normalizeField(superset.mapFields),
-			);
+			return allowsFieldSuperset(policy, originalData, original.mapFields, superset.mapFields);
 		}
 		return false;
 	}
@@ -78,14 +73,7 @@ export function allowsTreeSuperset(
 	assert(original instanceof ObjectNodeStoredSchema, 0x895 /* unsupported node kind */);
 	if (superset instanceof MapNodeStoredSchema) {
 		for (const [_key, field] of original.objectNodeFields) {
-			if (
-				!allowsFieldSuperset(
-					policy,
-					originalData,
-					normalizeField(field),
-					normalizeField(superset.mapFields),
-				)
-			) {
+			if (!allowsFieldSuperset(policy, originalData, field, superset.mapFields)) {
 				return false;
 			}
 		}
@@ -102,13 +90,13 @@ export function allowsTreeSuperset(
 				originalData,
 				original.objectNodeFields.get(originalField) ??
 					fail(0xb17 /* missing expected field */),
-				normalizeField(undefined),
+				storedEmptyFieldSchema,
 			),
 		bExtra: (supersetField) =>
 			allowsFieldSuperset(
 				policy,
 				originalData,
-				normalizeField(undefined),
+				storedEmptyFieldSchema,
 				superset.objectNodeFields.get(supersetField) ??
 					fail(0xb18 /* missing expected field */),
 			),
@@ -123,7 +111,7 @@ export function allowsTreeSuperset(
 }
 
 /**
- * @returns true iff `superset` is a superset of `original`.
+ * Returns true iff `superset` is a superset of `original`.
  *
  * This does not require a strict (aka proper) superset: equivalent schema will return true.
  */
@@ -135,7 +123,7 @@ export function allowsValueSuperset(
 }
 
 /**
- * @returns true iff `superset` is a superset of `original`.
+ * Returns true iff `superset` is a superset of `original`.
  *
  * This does not require a strict (aka proper) superset: equivalent schema will return true.
  */
@@ -151,7 +139,7 @@ export function allowsFieldSuperset(
 }
 
 /**
- * @returns true iff `superset` is a superset of `original`.
+ * Returns true iff `superset` is a superset of `original`.
  *
  * This does not require a strict (aka proper) superset: equivalent schema will return true.
  */
@@ -168,7 +156,7 @@ export function allowsTreeSchemaIdentifierSuperset(
 }
 
 /**
- * @returns true iff `superset` is a superset of `original`.
+ * Returns true iff `superset` is a superset of `original`.
  *
  * This does not require a strict (aka proper) superset: equivalent schema will return true.
  *
@@ -182,7 +170,6 @@ export function allowsRepoSuperset(
 	superset: TreeStoredSchema,
 ): boolean {
 	{
-		// TODO: I think its ok to use the field from superset here, but I should confirm it is, and document why.
 		if (
 			!allowsFieldSuperset(
 				policy,
@@ -194,17 +181,15 @@ export function allowsRepoSuperset(
 			return false;
 		}
 	}
+	// Check if all schema in original are included in superset, and permit a superset of the node content.
+	// Note that any schema from `original.nodeSchema` can be used as the schema for a node at the root of a detached field,
+	// so we must check all of them, even if they are not reachable from the root field schema.
 	for (const [key, schema] of original.nodeSchema) {
-		// TODO: I think its ok to use the tree from superset here, but I should confirm it is, and document why.
 		if (!allowsTreeSuperset(policy, original, schema, superset.nodeSchema.get(key))) {
 			return false;
 		}
 	}
+	// Any schema in superset not in original are already known to be superset of original since they are "never" due to being missing.
+	// Therefore, we do not need to check them.
 	return true;
 }
-
-export function normalizeField(
-	schema: TreeFieldStoredSchema | undefined,
-): TreeFieldStoredSchema {
-	return schema ?? storedEmptyFieldSchema;
-}

package/src/feature-libraries/modular-schema/fieldChangeHandler.ts

@@ -169,6 +169,12 @@ export interface FieldChangeRebaser<TChangeset> {
 		oldRevisions: Set<RevisionTag | undefined>,
 		newRevisions: RevisionTag | undefined,
 	): TChangeset;
+
+	/**
+	 * Returns a copy of the given changeset with the same declarations (e.g., new cells) but no actual changes.
+	 * This is a kludge. TODO: remove once AB#46104 is completed.
+	 */
+	mute(change: TChangeset): TChangeset;
 }
 
 /**
@@ -179,11 +185,13 @@ export function referenceFreeFieldChangeRebaser<TChangeset>(data: {
 	compose: (change1: TChangeset, change2: TChangeset) => TChangeset;
 	invert: (change: TChangeset) => TChangeset;
 	rebase: (change: TChangeset, over: TChangeset) => TChangeset;
+	mute: (change: TChangeset) => TChangeset;
 }): FieldChangeRebaser<TChangeset> {
 	return isolatedFieldChangeRebaser({
 		compose: (change1, change2, _composeChild, _genId) => data.compose(change1, change2),
 		invert: (change, _invertChild, _genId) => data.invert(change),
 		rebase: (change, over, _rebaseChild, _genId) => data.rebase(change, over),
+		mute: (change) => data.mute(change),
 	});
 }
 
@@ -191,6 +199,7 @@ export function isolatedFieldChangeRebaser<TChangeset>(data: {
 	compose: FieldChangeRebaser<TChangeset>["compose"];
 	invert: FieldChangeRebaser<TChangeset>["invert"];
 	rebase: FieldChangeRebaser<TChangeset>["rebase"];
+	mute: FieldChangeRebaser<TChangeset>["mute"];
 }): FieldChangeRebaser<TChangeset> {
 	return {
 		...data,

package/src/feature-libraries/modular-schema/genericFieldKind.ts

@@ -39,6 +39,7 @@ export const genericChangeHandler: FieldChangeHandler<GenericChangeset> = {
 		rebase: rebaseGenericChange,
 		prune: pruneGenericChange,
 		replaceRevisions,
+		mute: (change: GenericChangeset): GenericChangeset => change,
 	},
 	codecsFactory: makeGenericChangeCodec,
 	editor: {