@fluidframework/tree 2.0.0-dev-rc.1.0.0.224419

package/src/core/schema-stored/schema.ts

@@ -0,0 +1,313 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { DiscriminatedUnionDispatcher } from "../../codec";
+import {
+	Brand,
+	Erased,
+	MakeNominal,
+	Named,
+	brand,
+	brandErased,
+	compareNamed,
+	fail,
+	invertMap,
+} from "../../util";
+import {
+	FieldKey,
+	FieldKindIdentifier,
+	FieldSchemaFormat,
+	NamedFieldSchemaFormat,
+	PersistedValueSchema,
+	TreeNodeSchemaDataFormat,
+	TreeNodeSchemaIdentifier,
+} from "./format";
+
+/**
+ * Schema for what {@link TreeValue} is allowed on a Leaf node.
+ * @internal
+ */
+export enum ValueSchema {
+	Number,
+	String,
+	Boolean,
+	FluidHandle,
+	Null,
+}
+
+/**
+ * Set of allowed tree types.
+ * Providing multiple values here allows polymorphism, tagged union style.
+ *
+ * If not specified, types are unconstrained
+ * (equivalent to the set containing every TreeNodeSchemaIdentifier defined in the document).
+ *
+ * Note that even when unconstrained, children must still be in-schema for their own type.
+ *
+ * In the future, this could be extended to allow inlining a TreeNodeStoredSchema here
+ * (or some similar structural schema system).
+ * For structural types which could go here, there are a few interesting options:
+ *
+ * - Allow replacing the whole set with a structural type for terminal / non-tree data,
+ * and use this as a replacement for values on the tree nodes.
+ *
+ * - Allow expression structural constraints for child trees, for example requiring specific traits
+ * (ex: via TreeNodeStoredSchema), instead of by type.
+ *
+ * There are two ways this could work:
+ *
+ * - Constrain the child nodes based on their shape:
+ * this makes schema safe editing difficult because nodes would incur extra editing constraints to prevent them
+ * from going out of schema based on their location in such a field.
+ *
+ * - Constrain the types allowed based on which types guarantee their data will always meet the constraints.
+ *
+ * Care would need to be taken to make sure this is sound for the schema updating mechanisms.
+ * @internal
+ */
+export type TreeTypeSet = ReadonlySet<TreeNodeSchemaIdentifier> | undefined;
+
+/**
+ * Specifies which field kind to use.
+ *
+ * @remarks
+ * This is used instead of just the FieldKindIdentifier so that it can be subtyped into a more expressive type with additional information.
+ *
+ * @internal
+ */
+export interface FieldKindSpecifier<T = FieldKindIdentifier> {
+	identifier: T;
+}
+
+/**
+ * Schema for a field.
+ * Object implementing this interface should never be modified.
+ * @internal
+ */
+export interface TreeFieldStoredSchema {
+	readonly kind: FieldKindSpecifier;
+	/**
+	 * The set of allowed child types.
+	 * If not specified, types are unconstrained.
+	 */
+	readonly types?: TreeTypeSet;
+}
+
+/**
+ * Identifier used for the FieldKind for fields which must be empty.
+ *
+ * @remarks
+ * This mainly show up in:
+ * 1. The root default field for documents.
+ * 2. The schema used for out of schema fields (which thus must be empty/not exist) on object and leaf nodes.
+ *
+ * @internal
+ */
+export const forbiddenFieldKindIdentifier = "Forbidden";
+
+/**
+ * A schema for empty fields (fields which must always be empty).
+ * There are multiple ways this could be encoded, but this is the most explicit.
+ */
+export const storedEmptyFieldSchema: TreeFieldStoredSchema = {
+	// This kind requires the field to be empty.
+	kind: { identifier: brand(forbiddenFieldKindIdentifier) },
+	// This type set also forces the field to be empty not not allowing any types as all.
+	types: new Set(),
+};
+
+/**
+ * Opaque type erased handle to the encoded representation of the contents of a stored schema.
+ * @internal
+ */
+export interface ErasedTreeNodeSchemaDataFormat extends Erased<"TreeNodeSchemaDataFormat"> {}
+export interface BrandedTreeNodeSchemaDataFormat
+	extends Brand<TreeNodeSchemaDataFormat, ErasedTreeNodeSchemaDataFormat> {}
+
+/**
+ * @internal
+ */
+export abstract class TreeNodeStoredSchema {
+	protected _typeCheck!: MakeNominal;
+
+	/**
+	 * @privateRemarks
+	 * Returns TreeNodeSchemaDataFormat.
+	 * This is uses an opaque type to avoid leaking these types out of the package,
+	 * and is runtime validated by the codec.
+	 */
+	public abstract encode(): ErasedTreeNodeSchemaDataFormat;
+}
+
+/**
+ * @internal
+ */
+export class ObjectNodeStoredSchema extends TreeNodeStoredSchema {
+	/**
+	 * @param objectNodeFields -
+	 * Schema for fields with keys scoped to this TreeNodeStoredSchema.
+	 * This refers to the TreeFieldStoredSchema directly
+	 * (as opposed to just supporting FieldSchemaIdentifier and having a central FieldKey -\> TreeFieldStoredSchema map).
+	 * This allows os short friendly field keys which can ergonomically used as field names in code.
+	 * It also interoperates well with mapFields being used as a map with arbitrary data as keys.
+	 */
+	public constructor(
+		public readonly objectNodeFields: ReadonlyMap<FieldKey, TreeFieldStoredSchema>,
+	) {
+		super();
+	}
+
+	public override encode(): ErasedTreeNodeSchemaDataFormat {
+		// Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant).
+		// This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken.
+		return brandErased<BrandedTreeNodeSchemaDataFormat>({
+			object: [...this.objectNodeFields]
+				.map(([k, v]) => encodeNamedField(k, v))
+				.sort(compareNamed),
+		});
+	}
+}
+
+/**
+ * @internal
+ */
+export class MapNodeStoredSchema extends TreeNodeStoredSchema {
+	/**
+	 * @param mapFields -
+	 * Allows using using the fields as a map, with the keys being
+	 * FieldKeys and the values being constrained by this TreeFieldStoredSchema.
+	 * Usually `FieldKind.Value` should NOT be used here
+	 * since no nodes can ever be in schema are in schema if you use `FieldKind.Value` here
+	 * (that would require infinite children).
+	 */
+	public constructor(public readonly mapFields: TreeFieldStoredSchema) {
+		super();
+	}
+
+	public override encode(): ErasedTreeNodeSchemaDataFormat {
+		return brandErased<BrandedTreeNodeSchemaDataFormat>({
+			map: encodeFieldSchema(this.mapFields),
+		});
+	}
+}
+
+/**
+ * @internal
+ */
+export class LeafNodeStoredSchema extends TreeNodeStoredSchema {
+	/**
+	 * @param leafValue -
+	 * There are several approaches for how to store actual data in the tree
+	 * (special node types, special field contents, data on nodes etc.)
+	 * as well as several options about how the data should be modeled at this level
+	 * (byte sequence? javascript type? json?),
+	 * as well as options for how much of this would be exposed in the schema language
+	 * (ex: would all nodes with values be special built-ins, or could any schema add them?)
+	 * A simple easy to do in javascript approach is taken here:
+	 * this is not intended to be a suggestion of what approach to take, or what to expose in the schema language.
+	 * This is simply one approach that can work for modeling them in the internal schema representation.
+	 */
+	public constructor(public readonly leafValue: ValueSchema) {
+		super();
+	}
+
+	public override encode(): ErasedTreeNodeSchemaDataFormat {
+		return brandErased<BrandedTreeNodeSchemaDataFormat>({
+			leaf: encodeValueSchema(this.leafValue),
+		});
+	}
+}
+
+export const storedSchemaDecodeDispatcher: DiscriminatedUnionDispatcher<
+	TreeNodeSchemaDataFormat,
+	[],
+	TreeNodeStoredSchema
+> = new DiscriminatedUnionDispatcher({
+	leaf: (data: PersistedValueSchema) => new LeafNodeStoredSchema(decodeValueSchema(data)),
+	object: (data: NamedFieldSchemaFormat[]) =>
+		new ObjectNodeStoredSchema(
+			new Map(
+				data.map((field): [FieldKey, TreeFieldStoredSchema] => [
+					brand(field.name),
+					decodeFieldSchema(field),
+				]),
+			),
+		),
+	map: (data: FieldSchemaFormat) => new MapNodeStoredSchema(decodeFieldSchema(data)),
+});
+
+const valueSchemaEncode = new Map([
+	[ValueSchema.Number, PersistedValueSchema.Number],
+	[ValueSchema.String, PersistedValueSchema.String],
+	[ValueSchema.Boolean, PersistedValueSchema.Boolean],
+	[ValueSchema.FluidHandle, PersistedValueSchema.FluidHandle],
+	[ValueSchema.Null, PersistedValueSchema.Null],
+]);
+
+const valueSchemaDecode = invertMap(valueSchemaEncode);
+
+function encodeValueSchema(inMemory: ValueSchema): PersistedValueSchema {
+	return valueSchemaEncode.get(inMemory) ?? fail("missing PersistedValueSchema");
+}
+
+function decodeValueSchema(inMemory: PersistedValueSchema): ValueSchema {
+	return valueSchemaDecode.get(inMemory) ?? fail("missing ValueSchema");
+}
+
+export function encodeFieldSchema(schema: TreeFieldStoredSchema): FieldSchemaFormat {
+	const out: FieldSchemaFormat = {
+		kind: schema.kind.identifier,
+	};
+	if (schema.types !== undefined) {
+		out.types = [...schema.types];
+	}
+	return out;
+}
+
+function encodeNamedField<T>(name: T, schema: TreeFieldStoredSchema): FieldSchemaFormat & Named<T> {
+	return {
+		...encodeFieldSchema(schema),
+		name,
+	};
+}
+
+export function decodeFieldSchema(schema: FieldSchemaFormat): TreeFieldStoredSchema {
+	const out: TreeFieldStoredSchema = {
+		// TODO: maybe provide actual FieldKind objects here, error on unrecognized kinds.
+		kind: { identifier: schema.kind },
+		types: schema.types === undefined ? undefined : new Set(schema.types),
+	};
+	return out;
+}
+
+/**
+ * Document schema data that can be stored in a document.
+ *
+ * @remarks
+ * Note: the owner of this may modify it over time:
+ * thus if needing to hand onto a specific version, make a copy.
+ * @internal
+ */
+export interface TreeStoredSchema extends StoredSchemaCollection {
+	/**
+	 * Schema for the root field which contains the whole tree.
+	 */
+	readonly rootFieldSchema: TreeFieldStoredSchema;
+}
+
+/**
+ * Collection of TreeNodeSchema data that can be stored in a document.
+ *
+ * @remarks
+ * Note: the owner of this may modify it over time:
+ * thus if needing to hang onto a specific version, make a copy.
+ * @internal
+ */
+export interface StoredSchemaCollection {
+	/**
+	 * {@inheritdoc StoredSchemaCollection}
+	 */
+	readonly nodeSchema: ReadonlyMap<TreeNodeSchemaIdentifier, TreeNodeStoredSchema>;
+}

package/src/core/schema-stored/storedSchemaRepository.ts

@@ -0,0 +1,136 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import BTree from "sorted-btree";
+import { createEmitter, ISubscribable } from "../../events";
+import { compareStrings } from "../../util";
+import {
+	StoredSchemaCollection,
+	TreeFieldStoredSchema,
+	TreeNodeStoredSchema,
+	TreeStoredSchema,
+	storedEmptyFieldSchema,
+} from "./schema";
+import { TreeNodeSchemaIdentifier } from "./format";
+
+/**
+ * Events for {@link TreeStoredSchemaSubscription}.
+ *
+ * TODO: consider having before and after events per subtree instead while applying anchor (and this just shows what happens at the root).
+ * @internal
+ */
+export interface SchemaEvents {
+	/**
+	 * Schema change is about to be applied.
+	 */
+	beforeSchemaChange(newSchema: TreeStoredSchema): void;
+
+	/**
+	 * Schema change was just applied.
+	 */
+	afterSchemaChange(newSchema: TreeStoredSchema): void;
+}
+
+/**
+ * A collection of stored schema that fires events in response to changes.
+ * @internal
+ */
+export interface TreeStoredSchemaSubscription
+	extends ISubscribable<SchemaEvents>,
+		TreeStoredSchema {}
+
+/**
+ * Mutable collection of stored schema.
+ * @internal
+ */
+export interface MutableTreeStoredSchema extends TreeStoredSchemaSubscription {
+	/**
+	 * Mutates the stored schema.
+	 * Replaces all schema with the provided schema.
+	 * Can over-write preexisting schema, and removes unmentioned schema.
+	 */
+	apply(newSchema: TreeStoredSchema): void;
+}
+
+/**
+ * Mutable TreeStoredSchema repository.
+ */
+export class TreeStoredSchemaRepository implements MutableTreeStoredSchema {
+	protected nodeSchemaData: BTree<TreeNodeSchemaIdentifier, TreeNodeStoredSchema>;
+	protected rootFieldSchemaData: TreeFieldStoredSchema;
+	protected readonly events = createEmitter<SchemaEvents>();
+
+	/**
+	 * Copies in the provided schema. If `data` is an TreeStoredSchemaRepository, it will be cheap-cloned.
+	 * Otherwise, it will be deep-cloned.
+	 *
+	 * We might not want to store schema in maps long term, as we might want a way to reserve a
+	 * large space of schema IDs within a schema.
+	 * The way mapFields has been structured mitigates the need for this, but it still might be useful.
+	 *
+	 * (ex: someone using data as field identifiers might want to
+	 * reserve all fields identifiers starting with "foo." to have a specific schema).
+	 * Combined with support for such namespaces in the allowed sets in the schema objects,
+	 * that might provide a decent alternative to mapFields (which is a bit odd).
+	 */
+	public constructor(data?: TreeStoredSchema) {
+		if (data === undefined) {
+			this.rootFieldSchemaData = storedEmptyFieldSchema;
+			this.nodeSchemaData = new BTree<TreeNodeSchemaIdentifier, TreeNodeStoredSchema>(
+				[],
+				compareStrings,
+			);
+		} else {
+			if (data instanceof TreeStoredSchemaRepository) {
+				this.rootFieldSchemaData = data.rootFieldSchema;
+				this.nodeSchemaData = data.nodeSchemaData.clone();
+			} else {
+				this.rootFieldSchemaData = data.rootFieldSchema;
+				this.nodeSchemaData = cloneNodeSchemaData(data.nodeSchema);
+			}
+		}
+	}
+
+	public on<K extends keyof SchemaEvents>(eventName: K, listener: SchemaEvents[K]): () => void {
+		return this.events.on(eventName, listener);
+	}
+
+	public get nodeSchema(): ReadonlyMap<TreeNodeSchemaIdentifier, TreeNodeStoredSchema> {
+		// Btree implements iterator, but not in a type-safe way
+		return this.nodeSchemaData as unknown as ReadonlyMap<
+			TreeNodeSchemaIdentifier,
+			TreeNodeStoredSchema
+		>;
+	}
+
+	public get rootFieldSchema(): TreeFieldStoredSchema {
+		return this.rootFieldSchemaData;
+	}
+
+	public apply(newSchema: TreeStoredSchema): void {
+		this.events.emit("beforeSchemaChange", newSchema);
+		const clone = new TreeStoredSchemaRepository(newSchema);
+		// In the future, we could use btree's delta functionality to do a more efficient update
+		this.rootFieldSchemaData = clone.rootFieldSchemaData;
+		this.nodeSchemaData = clone.nodeSchemaData;
+		this.events.emit("afterSchemaChange", newSchema);
+	}
+
+	public clone(): TreeStoredSchemaRepository {
+		return new TreeStoredSchemaRepository(this);
+	}
+}
+
+export function schemaDataIsEmpty(data: TreeStoredSchema): boolean {
+	return data.nodeSchema.size === 0;
+}
+
+function cloneNodeSchemaData(
+	nodeSchema: StoredSchemaCollection["nodeSchema"],
+): BTree<TreeNodeSchemaIdentifier, TreeNodeStoredSchema> {
+	// Schema objects are immutable (unlike stored schema repositories), so this shallow copy is fine.
+	const entries: [TreeNodeSchemaIdentifier, TreeNodeStoredSchema][] = [...nodeSchema.entries()];
+	return new BTree<TreeNodeSchemaIdentifier, TreeNodeStoredSchema>(entries, compareStrings);
+}

package/src/core/schema-view/README.md

@@ -0,0 +1,9 @@
+# schema-view
+
+This folder extends "Schema-Stored", adding APIs for applications to view and interact with documents which use stored-schema.
+Included are utilities for declaring schema with strong typescript types,
+ways to detect and handle when a document's stored schema does not match the application's view schema,
+libraries to help with schema evolution (changes to schema over time), and other related topics.
+
+See [Stored and View Schema](<../schema-stored/Stored and View Schema.md>)
+for more information about the current focus of this Schema library, and its overall design.

package/src/core/schema-view/index.ts

@@ -0,0 +1,6 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export { Adapters, Compatibility, TreeAdapter, AdaptedViewSchema, AllowedUpdateType } from "./view";

package/src/core/schema-view/view.ts

@@ -0,0 +1,76 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { TreeNodeSchemaIdentifier, TreeStoredSchema } from "../schema-stored";
+
+/**
+ * APIs for applying `view schema` to documents.
+ */
+
+/**
+ * How compatible a particular view schema is for some operation on some specific document.
+ */
+export enum Compatibility {
+	Incompatible,
+	// For write compatibility this can include compatible schema updates to stored schema.
+	// TODO: separate schema updates from adapters.
+	RequiresAdapters,
+	Compatible,
+}
+
+/**
+ * What kinds of updates to stored schema to permit.
+ *
+ * TODO:
+ * Currently this does not account for lazy schema updates, and/or use of adapters.
+ * @internal
+ */
+export enum AllowedUpdateType {
+	/**
+	 * Do not update the stored schema to match view schema.
+	 */
+	None,
+	/**
+	 * Update the stored schema to match the view schema if the current document contents are compatible with the view schema.
+	 * TODO: support this option.
+	 */
+	// DataCompatible,
+	/**
+	 * Update the stored schema to match view schema if all possible documents based on the current stored schema would be compatible with the view schema.
+	 */
+	SchemaCompatible,
+}
+
+/**
+ * @internal
+ */
+export interface TreeAdapter {
+	readonly output: TreeNodeSchemaIdentifier;
+	readonly input: TreeNodeSchemaIdentifier;
+
+	// TODO: include actual adapter functionality, not just what types it converts
+}
+
+/**
+ * Minimal selection of adapters (nothing for general out of schema, field level adjustments etc.).
+ * Would be used with schematize and have actual conversion/update functionality.
+ *
+ * TODO: Support more kinds of adapters
+ * TODO: support efficient lookup of adapters
+ * @internal
+ */
+export interface Adapters {
+	readonly tree?: readonly TreeAdapter[];
+}
+
+/**
+ * A collection of View information for schema, including policy.
+ */
+export class AdaptedViewSchema {
+	public constructor(
+		public readonly adapters: Adapters,
+		public readonly adaptedForViewSchema: TreeStoredSchema,
+	) {}
+}

package/src/core/tree/README.md

@@ -0,0 +1,15 @@
+# tree
+
+Types related to this package's flavor of tree.
+This means a tree with typed nodes that have named fields and contain values.
+
+TODO:
+The scope of this directory needs some refinement.
+Its relation to schema-stored should be clarified (and maybe inverted),
+and AnchorSet (at least the implementation logic) should probably move elsewhere.
+
+TODO:
+This module having the same name as the package (other than scope/qualification) but meaning something rather different is confusing.
+Consider refactoring this module out of existence or renaming it (maybe something like `tree-types`, `tree-nodes` or `typed-tree`.)
+Another option is to give a name to this specific tree abstraction (named fields with sequences + values), and use that name for the module:
+`flex-tree` has been proposed for a name for this in the past.