@fluidframework/tree 2.52.0 → 2.53.0-350190

package/src/feature-libraries/modular-schema/modularChangeFamily.ts

@@ -39,6 +39,7 @@ import {
 	newChangeAtomIdRangeMap,
 	type DeltaDetachedNodeChanges,
 	type DeltaDetachedNodeRename,
+	mapTaggedChange,
 } from "../../core/index.js";
 import {
 	type IdAllocationState,
@@ -213,24 +214,16 @@ export class ModularChangeFamily
 	}
 
 	private composeAllFields(
-		change1: ModularChangeset,
-		change2: ModularChangeset,
+		potentiallyConflictedChange1: ModularChangeset,
+		potentiallyConflictedChange2: ModularChangeset,
 		revInfos: RevisionInfo[],
 		idState: IdAllocationState,
 	): ModularChangesetContent {
-		if (hasConflicts(change1) && hasConflicts(change2)) {
-			return {
-				fieldChanges: new Map(),
-				nodeChanges: newTupleBTree(),
-				nodeToParent: newTupleBTree(),
-				nodeAliases: newTupleBTree(),
-				crossFieldKeys: newCrossFieldKeyTable(),
-			};
-		} else if (hasConflicts(change1)) {
-			return change2;
-		} else if (hasConflicts(change2)) {
-			return change1;
-		}
+		// Our current cell ordering scheme in sequences depends on being able to rebase over a change with conflicts.
+		// This means that compose must preserve declarations (e.g., new cells) made by conflicted changes (so that we can rebase over the composition).
+		// TODO: remove once AB#46104 is completed
+		const change1 = this.getEffectiveChange(potentiallyConflictedChange1);
+		const change2 = this.getEffectiveChange(potentiallyConflictedChange2);
 
 		const genId: IdAllocator = idAllocatorFromState(idState);
 		const revisionMetadata: RevisionMetadataSource = revisionMetadataSourceFromInfo(revInfos);
@@ -857,12 +850,17 @@ export class ModularChangeFamily
 
 	public rebase(
 		taggedChange: TaggedChange<ModularChangeset>,
-		over: TaggedChange<ModularChangeset>,
+		potentiallyConflictedOver: TaggedChange<ModularChangeset>,
 		revisionMetadata: RevisionMetadataSource,
 	): ModularChangeset {
-		if (hasConflicts(over.change)) {
-			return taggedChange.change;
-		}
+		// Our current cell ordering scheme in sequences depends on being able to rebase over a change with conflicts.
+		// This means that we must rebase over a muted version of the conflicted changeset.
+		// That is, a version that includes its declarations (e.g., new cells) but not its changes.
+		// TODO: remove once AB#46104 is completed
+		const over = mapTaggedChange(
+			potentiallyConflictedOver,
+			this.getEffectiveChange(potentiallyConflictedOver.change),
+		);
 
 		const change = taggedChange.change;
 		const maxId = Math.max(change.maxId ?? -1, over.change.maxId ?? -1);
@@ -1711,6 +1709,50 @@ export class ModularChangeFamily
 
 		return numChildren;
 	}
+
+	private getEffectiveChange(change: ModularChangeset): ModularChangeset {
+		if (hasConflicts(change)) {
+			return this.muteChange(change);
+		}
+		return change;
+	}
+
+	/**
+	 * Returns a copy of the given changeset with the same declarations (e.g., new cells) but no actual changes.
+	 */
+	private muteChange(change: ModularChangeset): ModularChangeset {
+		const muted: Mutable<ModularChangeset> = {
+			...change,
+			crossFieldKeys: newCrossFieldKeyTable(),
+			fieldChanges: this.muteFieldChanges(change.fieldChanges),
+			nodeChanges: brand(change.nodeChanges.mapValues((v) => this.muteNodeChange(v))),
+		};
+		return muted;
+	}
+
+	private muteNodeChange(change: NodeChangeset): NodeChangeset {
+		if (change.fieldChanges === undefined) {
+			return change;
+		}
+		return {
+			...change,
+			fieldChanges: this.muteFieldChanges(change.fieldChanges),
+		};
+	}
+
+	private muteFieldChanges(change: FieldChangeMap): FieldChangeMap {
+		return new Map(
+			Array.from(change.entries(), ([key, value]) => [key, this.muteFieldChange(value)]),
+		);
+	}
+
+	private muteFieldChange(change: FieldChange): FieldChange {
+		const handler = getChangeHandler(this.fieldKinds, change.fieldKind);
+		return {
+			fieldKind: change.fieldKind,
+			change: brand(handler.rebaser.mute(change.change)),
+		};
+	}
 }
 
 function replaceCrossFieldKeyTableRevisions(

package/src/feature-libraries/optional-field/optionalField.ts

@@ -465,6 +465,10 @@ export const optionalChangeRebaser: FieldChangeRebaser<OptionalChangeset> = {
 
 		return updated;
 	},
+
+	mute: (change: OptionalChangeset): OptionalChangeset => {
+		return { childChanges: change.childChanges, moves: [] };
+	},
 };
 
 function replaceReplaceRevisions(

package/src/feature-libraries/sequence-field/sequenceFieldChangeRebaser.ts

@@ -11,6 +11,7 @@ import { prune } from "./prune.js";
 import { rebase } from "./rebase.js";
 import { replaceRevisions } from "./replaceRevisions.js";
 import type { Changeset } from "./types.js";
+import { omitMarkEffect } from "./utils.js";
 
 export type SequenceChangeRebaser = FieldChangeRebaser<Changeset>;
 
@@ -20,4 +21,7 @@ export const sequenceFieldChangeRebaser = {
 	rebase,
 	prune,
 	replaceRevisions,
+	mute: (change: Changeset): Changeset => {
+		return change.map((mark) => omitMarkEffect(mark));
+	},
 };

package/src/index.ts

@@ -118,6 +118,7 @@ export {
 	type InternalTreeNode,
 	type WithType,
 	type NodeChangedData,
+	type SchemaUpgrade,
 	// Types not really intended for public use, but used in links.
 	// Can not be moved to internalTypes since doing so causes app code to throw errors like:
 	// Error: src/simple-tree/objectNode.ts:72:1 - (ae-unresolved-link) The @link reference could not be resolved: The package "@fluidframework/tree" does not have an export "TreeNodeApi"
@@ -168,9 +169,9 @@ export {
 	type UnannotateAllowedTypes,
 	type UnannotateAllowedType,
 	type UnannotateAllowedTypesList,
-	type UnannotateAllowedTypeOrLazyItem,
 	type UnannotateImplicitFieldSchema,
 	type UnannotateSchemaRecord,
+	type SchemaStaticsAlpha,
 	// Beta APIs
 	TreeBeta,
 	type TreeChangeEventsBeta,

package/src/jsonDomainSchema.ts

@@ -11,9 +11,11 @@ import type {
 	ValidateRecursiveSchema,
 	// #region Unused imports to make d.ts cleaner
 	/* eslint-disable unused-imports/no-unused-imports, @typescript-eslint/no-unused-vars */
+	ArrayNodeCustomizableSchemaUnsafe,
 	System_Unsafe,
 	TreeNodeSchemaNonClass,
 	TreeNodeSchemaClass,
+	TreeRecordNodeUnsafe,
 	NodeKind,
 	TreeNodeSchemaCore,
 	WithType,
@@ -52,6 +54,7 @@ export namespace JsonAsTree {
 	] as const satisfies AllowedTypes;
 
 	/**
+	 * {@inheritDoc JsonAsTree.(Primitive:variable)}
 	 * @alpha
 	 */
 	export type Primitive = TreeNodeFromImplicitAllowedTypes<typeof Primitive>;
@@ -72,6 +75,7 @@ export namespace JsonAsTree {
 	export const Tree = [...Primitive, () => JsonObject, () => Array] as const;
 
 	/**
+	 * {@inheritDoc JsonAsTree.(Tree:variable)}
 	 * @alpha
 	 */
 	export type Tree = TreeNodeFromImplicitAllowedTypes<typeof Tree>;

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/tree";
-export const pkgVersion = "2.52.0";
+export const pkgVersion = "2.53.0-350190";

package/src/shared-tree/schematizeTree.ts

@@ -19,7 +19,7 @@ import {
 	defaultSchemaPolicy,
 	mapTreeFromCursor,
 } from "../feature-libraries/index.js";
-import { toStoredSchema, type SchemaCompatibilityTester } from "../simple-tree/index.js";
+import { toUpgradeSchema, type SchemaCompatibilityTester } from "../simple-tree/index.js";
 import { isReadonlyArray } from "../util/index.js";
 
 import type { ITreeCheckout } from "./treeCheckout.js";
@@ -219,7 +219,7 @@ export function ensureSchema(
 			return false;
 		}
 		case UpdateType.SchemaCompatible: {
-			checkout.updateSchema(toStoredSchema(viewSchema.viewSchema.root));
+			checkout.updateSchema(toUpgradeSchema(viewSchema.viewSchema.root));
 			return true;
 		}
 		default: {

package/src/shared-tree/schematizingTreeView.ts

@@ -51,10 +51,10 @@ import {
 	areImplicitFieldSchemaEqual,
 	prepareForInsertionContextless,
 	type FieldSchema,
-	toStoredSchema,
 	tryDisposeTreeNode,
 	FieldSchemaAlpha,
 	TreeViewConfigurationAlpha,
+	toInitialSchema,
 } from "../simple-tree/index.js";
 import {
 	type Breakable,
@@ -172,7 +172,7 @@ export class SchematizingSimpleTreeView<
 		}
 
 		this.runSchemaEdit(() => {
-			const schema = toStoredSchema(this.config.schema);
+			const schema = toInitialSchema(this.config.schema);
 			const mapTree = prepareForInsertionContextless(
 				content as InsertableContent | undefined,
 				this.rootFieldSchema,
@@ -181,6 +181,7 @@ export class SchematizingSimpleTreeView<
 					policy: defaultSchemaPolicy,
 				},
 				this,
+				schema.rootFieldSchema,
 			);
 
 			initialize(this.checkout, {
@@ -436,7 +437,12 @@ export class SchematizingSimpleTreeView<
 			);
 		}
 		const view = this.getFlexTreeContext();
-		setField(view.root, this.rootFieldSchema, newRoot as InsertableContent | undefined);
+		setField(
+			view.root,
+			this.rootFieldSchema,
+			newRoot as InsertableContent | undefined,
+			this.checkout.storedSchema.rootFieldSchema,
+		);
 	}
 
 	// #region Branching

package/src/shared-tree/treeAlpha.ts

@@ -36,7 +36,6 @@ import {
 	verboseFromCursor,
 	type TreeEncodingOptions,
 	type VerboseTree,
-	toStoredSchema,
 	extractPersistedSchema,
 	type TreeBranch,
 	TreeViewConfigurationAlpha,
@@ -52,6 +51,9 @@ import {
 	tryGetTreeNodeForField,
 	isObjectNodeSchema,
 	isTreeNode,
+	toInitialSchema,
+	convertField,
+	toUnhydratedSchema,
 } from "../simple-tree/index.js";
 import { brand, extractFromOpaque, type JsonCompatible } from "../util/index.js";
 import {
@@ -493,7 +495,11 @@ export const TreeAlpha: TreeAlpha = {
 			return undefined as Unhydrated<TreeFieldFromImplicitField<TSchema>>;
 		}
 		const cursor = cursorFromVerbose(data, schemalessConfig);
-		return createFromCursor(schema, cursor);
+		return createFromCursor(
+			schema,
+			cursor,
+			convertField(normalizeFieldSchema(schema), toUnhydratedSchema),
+		);
 	},
 
 	exportConcise,
@@ -523,11 +529,18 @@ export const TreeAlpha: TreeAlpha = {
 		const batch: FieldBatch = [cursor];
 		// If none provided, create a compressor which will not compress anything.
 		const idCompressor = options.idCompressor ?? createIdCompressor();
+
+		// Grabbing an existing stored schema from the node is important to ensure that unknown optional fields can be preserved.
+		// Note that if the node is unhydrated, this can result in all staged allowed types being included in the schema, which might be undesired.
+		const storedSchema = isTreeNode(node)
+			? getKernel(node).context.flexContext.schema
+			: toInitialSchema(schema);
+
 		const context: FieldBatchEncodingContext = {
 			encodeType: TreeCompressionStrategy.Compressed,
 			idCompressor,
 			originatorId: idCompressor.localSessionId, // TODO: Why is this needed?
-			schema: { schema: toStoredSchema(schema), policy: defaultSchemaPolicy },
+			schema: { schema: storedSchema, policy: defaultSchemaPolicy },
 		};
 		const result = codec.encode(batch, context);
 		return result;
@@ -543,7 +556,8 @@ export const TreeAlpha: TreeAlpha = {
 		const config = new TreeViewConfigurationAlpha({ schema });
 		const content: ViewContent = {
 			// Always use a v1 schema codec for consistency.
-			schema: extractPersistedSchema(config, FluidClientVersion.v2_0),
+			// TODO: reevaluate how staged schema should behave in schema import/export APIs before stabilizing this.
+			schema: extractPersistedSchema(config.schema, FluidClientVersion.v2_0, () => true),
 			tree: compressedData,
 			idCompressor: options.idCompressor ?? createIdCompressor(),
 		};

package/src/simple-tree/api/configuration.ts

@@ -25,7 +25,11 @@ import {
 	evaluateLazySchema,
 	markSchemaMostDerived,
 } from "../core/index.js";
-import { toStoredSchema } from "../toStoredSchema.js";
+import {
+	permissiveStoredSchemaGenerationOptions,
+	restrictiveStoredSchemaGenerationOptions,
+	toStoredSchema,
+} from "../toStoredSchema.js";
 import {
 	isArrayNodeSchema,
 	isMapNodeSchema,
@@ -161,6 +165,9 @@ export interface ITreeViewConfiguration<
 
 /**
  * Configuration for {@link ViewableTree.viewWith}.
+ * @privateRemarks
+ * When `ImplicitAnnotatedFieldSchema` is stabilized, TSchema should be updated to use it.
+ * When doing this, the example for `staged` will need to be updated/simplified.
  * @sealed @public
  */
 export class TreeViewConfiguration<
@@ -213,7 +220,8 @@ export class TreeViewConfiguration<
 
 		// Eagerly perform this conversion to surface errors sooner.
 		// Includes detection of duplicate schema identifiers.
-		toStoredSchema(config.schema);
+		toStoredSchema(config.schema, restrictiveStoredSchemaGenerationOptions);
+		toStoredSchema(config.schema, permissiveStoredSchemaGenerationOptions);
 
 		const definitions = new Map<string, SimpleNodeSchema & TreeNodeSchema>();
 

package/src/simple-tree/api/create.ts

@@ -13,12 +13,9 @@ import {
 	mapCursorFields,
 	type ITreeCursorSynchronous,
 	type SchemaAndPolicy,
+	type TreeFieldStoredSchema,
 } from "../../core/index.js";
-import {
-	normalizeFieldSchema,
-	type ImplicitFieldSchema,
-	type TreeFieldFromImplicitField,
-} from "../fieldSchema.js";
+import type { ImplicitFieldSchema, TreeFieldFromImplicitField } from "../fieldSchema.js";
 import {
 	type Context,
 	getOrCreateNodeFromInnerNode,
@@ -32,33 +29,31 @@ import {
 	throwOutOfSchema,
 } from "../../feature-libraries/index.js";
 import { getUnhydratedContext } from "../createContext.js";
-import { convertField } from "../toStoredSchema.js";
 import { unknownTypeError } from "./customTree.js";
 
 /**
  * Creates an unhydrated simple-tree field from a cursor in nodes mode.
  * @remarks
  * Does not support providing missing defaults values.
- * Validates the field is in schema using the provided `contextForNewNodes` of the default unhydrated context if not provided.
+ * Validates the field is in schema using `destinationSchema` the provided `contextForNewNodes` or the default unhydrated context if not provided.
  */
 export function createFromCursor<const TSchema extends ImplicitFieldSchema>(
 	schema: TSchema,
 	cursor: ITreeCursorSynchronous | undefined,
+	destinationSchema: TreeFieldStoredSchema,
 	contextForNewNodes?: Context,
 ): Unhydrated<TreeFieldFromImplicitField<TSchema>> {
 	const context = contextForNewNodes ?? getUnhydratedContext(schema);
 	assert(context.flexContext.isHydrated() === false, 0xbfe /* Expected unhydrated context */);
 	const mapTrees = cursor === undefined ? [] : [unhydratedFlexTreeFromCursor(context, cursor)];
 
-	const rootFieldSchema = convertField(normalizeFieldSchema(schema));
-
 	const schemaAndPolicy: SchemaAndPolicy = {
 		policy: defaultSchemaPolicy,
 		schema: context.flexContext.schema,
 	};
 
 	// Assuming the caller provides the correct `contextForNewNodes`, this should handle unknown optional fields.
-	isFieldInSchema(mapTrees, rootFieldSchema, schemaAndPolicy, throwOutOfSchema);
+	isFieldInSchema(mapTrees, destinationSchema, schemaAndPolicy, throwOutOfSchema);
 
 	if (mapTrees.length === 0) {
 		return undefined as Unhydrated<TreeFieldFromImplicitField<TSchema>>;

package/src/simple-tree/api/discrepancies.ts

@@ -89,11 +89,13 @@ export interface FieldDiscrepancyLocation {
 export interface AllowedTypeDiscrepancy extends FieldDiscrepancyLocation {
 	readonly mismatch: "allowedTypes";
 	/**
-	 * List of annotated allowed types in viewed schema which are not allowed in stored schema
+	 * Annotated allowed types in viewed schema
+	 * (excluding {@link SchemaStaticsAlpha.staged | staged} schema) which are not allowed in stored schema.
 	 */
 	readonly view: readonly AnnotatedAllowedType<TreeNodeSchema>[];
 	/**
-	 * List of allowed type identifiers in stored schema which are not allowed in view schema
+	 * Allowed type identifiers in stored schema which are not allowed in view schema
+	 * (including the view schema's {@link SchemaStaticsAlpha.staged | staged} schema).
 	 */
 	readonly stored: readonly TreeNodeSchemaIdentifier[];
 }
@@ -296,28 +298,28 @@ function* getNodeDiscrepancies(
 
 /**
  * Returns the allowed types that are not present in both the given view and stored schemas.
- * It returns a tuple containing two arrays:
- * 1. The first array contains the allowed types that are present in the view schema but not in the stored schema.
- * 2. The second array contains the allowed types that are present in the stored schema but not in the view schema.
  */
 export function findExtraAllowedTypes(
 	viewAllowedTypes: readonly AnnotatedAllowedType<TreeNodeSchema>[],
 	storedAllowedTypes: TreeTypeSet,
-): {
-	viewExtra: readonly AnnotatedAllowedType<TreeNodeSchema>[];
-	storedExtra: TreeNodeSchemaIdentifier[];
-} {
+): Pick<AllowedTypeDiscrepancy, "view" | "stored"> {
 	const viewNodeSchemaIdentifiers = new Set(
 		viewAllowedTypes.map((value) => value.type.identifier),
 	);
 
-	const viewExtra = viewAllowedTypes.filter(
-		(value) => !storedAllowedTypes.has(brand(value.type.identifier)),
+	// TODO:
+	// Add detecting which SchemaUpgrades have been made.
+	// When doing so, each will fall into one of three states: staged, partially applied, or fully applied.
+	// This would allow for nicer compatibility errors and more detailed compatibility status in the future.
+	const view = viewAllowedTypes.filter(
+		(value) =>
+			!storedAllowedTypes.has(brand(value.type.identifier)) &&
+			value.metadata.stagedSchemaUpgrade === undefined,
 	);
-	const storedExtra = [...storedAllowedTypes].filter(
+	const stored = [...storedAllowedTypes].filter(
 		(value) => !viewNodeSchemaIdentifiers.has(value),
 	);
-	return { viewExtra, storedExtra };
+	return { view, stored };
 }
 
 /**
@@ -384,14 +386,13 @@ function* getAllowedTypeDiscrepancies(
 	identifier: TreeNodeSchemaIdentifier | undefined,
 	fieldKey: FieldKey | undefined,
 ): Iterable<FieldDiscrepancy> {
-	const { viewExtra, storedExtra } = findExtraAllowedTypes(view, stored);
-	if (viewExtra.length > 0 || storedExtra.length > 0) {
+	const discrepancies = findExtraAllowedTypes(view, stored);
+	if (discrepancies.view.length > 0 || discrepancies.stored.length > 0) {
 		yield {
+			...discrepancies,
 			identifier,
 			fieldKey,
 			mismatch: "allowedTypes",
-			view: viewExtra,
-			stored: storedExtra,
 		} satisfies AllowedTypeDiscrepancy;
 	}
 }

package/src/simple-tree/api/index.ts

@@ -24,16 +24,15 @@ export type {
 	ITreeAlpha,
 } from "./tree.js";
 export { asTreeViewAlpha } from "./tree.js";
+export { type SchemaStatics, schemaStatics } from "./schemaStatics.js";
 export {
 	SchemaFactory,
 	type ScopedSchemaName,
 	type SchemaFactoryObjectOptions,
-	type SchemaStatics,
-	schemaStatics,
 	type NodeSchemaOptions,
 	type NodeSchemaOptionsAlpha,
 } from "./schemaFactory.js";
-export { SchemaFactoryAlpha } from "./schemaFactoryAlpha.js";
+export { SchemaFactoryAlpha, type SchemaStaticsAlpha } from "./schemaFactoryAlpha.js";
 export type {
 	ValidateRecursiveSchema,
 	FixRecursiveArraySchema,

package/src/simple-tree/api/schemaCompatibilityTester.ts

@@ -8,7 +8,7 @@ import { allowsRepoSuperset, defaultSchemaPolicy } from "../../feature-libraries
 
 import type { SchemaCompatibilityStatus } from "./tree.js";
 import { getDiscrepanciesInAllowedContent } from "./discrepancies.js";
-import { toStoredSchema } from "../toStoredSchema.js";
+import { toUpgradeSchema } from "../toStoredSchema.js";
 import type { TreeSchema } from "./configuration.js";
 
 /**
@@ -51,18 +51,14 @@ export class SchemaCompatibilityTester {
 			break;
 		}
 
-		const canUpgrade = allowsRepoSuperset(
-			policy,
-			stored,
-			toStoredSchema(this.viewSchema.root),
-		);
+		const wouldUpgradeTo = toUpgradeSchema(this.viewSchema.root);
+
+		const canUpgrade = allowsRepoSuperset(policy, stored, wouldUpgradeTo);
 
 		// If true, then upgrading has no effect on what can be stored in the document.
 		// TODO: This should likely be changed to indicate up a schema upgrade would be a no-op, including stored schema metadata.
 		const isEquivalent =
-			canView &&
-			canUpgrade &&
-			allowsRepoSuperset(policy, toStoredSchema(this.viewSchema.root), stored);
+			canView && canUpgrade && allowsRepoSuperset(policy, wouldUpgradeTo, stored);
 
 		return {
 			canView,