@fluidframework/tree 2.0.0-dev-rc.1.0.0.224419

package/src/feature-libraries/treeTextCursor.ts

@@ -0,0 +1,125 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert } from "@fluidframework/core-utils";
+import {
+	genericTreeKeys,
+	getGenericTreeField,
+	JsonableTree,
+	ITreeCursor,
+	CursorLocationType,
+	mapCursorField,
+	ITreeCursorSynchronous,
+	setGenericTreeField,
+	IForestSubscription,
+	moveToDetachedField,
+	aboveRootPlaceholder,
+	DetachedField,
+	rootField,
+	detachedFieldAsKey,
+} from "../core";
+import { CursorAdapter, stackTreeFieldCursor, stackTreeNodeCursor } from "./treeCursorUtils";
+
+/**
+ * This module provides support for reading and writing a human readable (and
+ * editable) tree format.
+ *
+ * This implementation can handle all trees (so it does not need a fallback for any special cases),
+ * and is not optimized.
+ *
+ * It's suitable for testing and debugging,
+ * though it could also reasonably be used as a fallback for edge cases or for small trees.
+ *
+ * TODO: Use placeholders.
+ * build / add operations should be able to include detached ranges instead of children directly.
+ * summaries should be able to reference unloaded chunks instead of having children directly.
+ * Leverage placeholders in the types below to accomplish this.
+ * Determine how this relates to Cursor: should cursor be generic over placeholder values?
+ * (Could use them for errors to allow non erroring cursors?)
+ *
+ * Note:
+ * Currently a lot of Tree's codebase is using json for serialization.
+ * Because putting json strings inside json works poorly (adds lots of escaping),
+ * for now this library actually outputs and inputs the Json compatible type JsonableTree
+ * rather than actual strings.
+ */
+
+/**
+ * Create a cursor, in `nodes` mode at the root of the provided tree.
+ *
+ * @returns an {@link ITreeCursorSynchronous} in nodes mode for a single {@link JsonableTree}.
+ * @remarks
+ * Do not confuse this with {@link JsonableTree} with the JSON domain:
+ * this takes in data in a specific format that is json compatible (except for FluidHandle values).
+ * That is distinct from treating arbitrary JSON data as a tree in the JSON domain.
+ * @internal
+ */
+export function cursorForJsonableTreeNode(root: JsonableTree): ITreeCursorSynchronous {
+	return stackTreeNodeCursor(adapter, root);
+}
+
+/**
+ * @returns an {@link ITreeCursorSynchronous} in fields mode for a JsonableTree field.
+ */
+export function cursorForJsonableTreeField(
+	trees: JsonableTree[],
+	detachedField: DetachedField = rootField,
+): ITreeCursorSynchronous {
+	const key = detachedFieldAsKey(detachedField);
+	return stackTreeFieldCursor(
+		adapter,
+		{ type: aboveRootPlaceholder, fields: { [key]: trees } },
+		detachedField,
+	);
+}
+
+export const adapter: CursorAdapter<JsonableTree> = {
+	value: (node) => node.value,
+	type: (node) => node.type,
+	keysFromNode: genericTreeKeys,
+	getFieldFromNode: (node, key): readonly JsonableTree[] => getGenericTreeField(node, key, false),
+};
+
+/**
+ * Extract a JsonableTree from the contents of the given ITreeCursor's current node.
+ * @internal
+ */
+export function jsonableTreeFromCursor(cursor: ITreeCursor): JsonableTree {
+	assert(cursor.mode === CursorLocationType.Nodes, 0x3ba /* must start at node */);
+	const node: JsonableTree = {
+		type: cursor.type,
+	};
+	// Normalize object by only including fields that are required.
+	if (cursor.value !== undefined) {
+		node.value = cursor.value;
+	}
+	for (let inFields = cursor.firstField(); inFields; inFields = cursor.nextField()) {
+		const field: JsonableTree[] = mapCursorField(cursor, jsonableTreeFromCursor);
+		setGenericTreeField(node, cursor.getFieldKey(), field);
+	}
+	return node;
+}
+
+/**
+ * Extract a JsonableTree from the contents of the given ITreeCursor's current node.
+ */
+export function jsonableTreeFromFieldCursor(cursor: ITreeCursor): JsonableTree[] {
+	assert(cursor.mode === CursorLocationType.Fields, 0x7ca /* must start at field */);
+	return mapCursorField(cursor, jsonableTreeFromCursor);
+}
+
+/**
+ * Copy forest content into a JsonableTree.
+ * @remarks
+ * This is not a time or memory efficient way to pass around forest content:
+ * its intended for debugging and testing purposes when forest content is needed in a human readable serializable format.
+ */
+export function jsonableTreeFromForest(forest: IForestSubscription): JsonableTree[] {
+	const readCursor = forest.allocateCursor();
+	moveToDetachedField(forest, readCursor);
+	const jsonable = jsonableTreeFromFieldCursor(readCursor);
+	readCursor.free();
+	return jsonable;
+}

package/src/feature-libraries/typed-schema/README.md

@@ -0,0 +1,6 @@
+# Typed Schema
+
+Schema building API which captures type information in both runtime data and TypeScript types.
+
+TODO:
+The portion of this that is not dependent on specific field kinds should be separated from the part that does depend on specific field kinds.

package/src/feature-libraries/typed-schema/flexList.ts

@@ -0,0 +1,167 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ArrayToUnion } from "./typeUtils";
+
+/** A symbol used to identify a `MarkedEager`. */
+const flexListEager = Symbol("FlexList Eager");
+
+/**
+ * An object that has been marked as eager (as opposed to lazy) when used as an item in a `FlexList`.
+ * It will be considered to be an eager item in a `FlexList` even if it is a function.
+ */
+interface MarkedEager {
+	[flexListEager]: true;
+}
+
+/** Returns true iff the given item is a function and is not a `MarkedEager`. */
+export function isLazy<Item>(item: LazyItem<Item>): item is () => Item {
+	return typeof item === "function" && (item as Partial<MarkedEager>)[flexListEager] !== true;
+}
+
+/**
+ * Mark the given object as an eager item in a `FlexList`.
+ * @remarks
+ * This only has an effect on function objects that would otherwise be considered to be lazy items in a `FlexList`.
+ * @param t - The object to mark as eager.
+ * @returns `t`, marked as eager if applicable.
+ */
+export function markEager<T>(t: T): T {
+	return isLazy(t)
+		? Object.defineProperty(t, flexListEager, {
+				value: true,
+				configurable: true,
+				enumerable: false,
+				writable: false,
+		  })
+		: t;
+}
+
+/**
+ * A flexible way to list values.
+ * Each item in the list can either be an "eager" **value** or a "lazy" **function that returns a value** (the latter allows cyclic references to work).
+ * @privateRemarks
+ * By default, items that are of type `"function"` will be considered lazy and all other items will be considered eager.
+ * To force a `"function"` item to be treated as an eager item, call `markEager` before putting it in the list.
+ * This is necessary e.g. when the eager list items are function types and the lazy items are functions that _return_ function types.
+ * `FlexList`s are processed by `normalizeFlexList` and `normalizeFlexListEager`.
+ * @beta
+ */
+export type FlexList<Item = unknown> = readonly LazyItem<Item>[];
+
+/**
+ * Given a `FlexList` of eager and lazy items, return an equivalent list where all items are lazy.
+ */
+export function normalizeFlexListLazy<List extends FlexList>(t: List): FlexListToLazyArray<List> {
+	return t.map((value: LazyItem) => {
+		if (isLazy(value)) {
+			return value;
+		}
+		return () => value;
+	}) as FlexListToLazyArray<List>;
+}
+
+/**
+ * Given a `FlexList` of eager and lazy items, return an equivalent list where all items are eager.
+ */
+export function normalizeFlexListEager<List extends FlexList>(
+	t: List,
+): FlexListToNonLazyArray<List> {
+	const data: readonly unknown[] = t.map((value: LazyItem) => {
+		if (isLazy(value)) {
+			return value();
+		}
+		return value;
+	});
+	return data as FlexListToNonLazyArray<List>;
+}
+
+/**
+ * An "eager" or "lazy" Item in a `FlexList`.
+ * Lazy items are wrapped in a function to allow referring to themselves before they are declared.
+ * This makes recursive and co-recursive items possible.
+ * @beta
+ */
+export type LazyItem<Item = unknown> = Item | (() => Item);
+
+/**
+ * @internal
+ */
+export type NormalizedFlexList<Item> = readonly Item[];
+
+export type NormalizedLazyFlexList<Item> = (() => Item)[];
+
+/**
+ * Get the `Item` type from a `LazyItem<Item>`.
+ * @beta
+ */
+export type ExtractItemType<Item extends LazyItem> = Item extends () => infer Result
+	? Result
+	: Item;
+
+/**
+ * @internal
+ */
+export type ExtractListItemType<List extends FlexList> = List extends FlexList<infer Item>
+	? Item
+	: unknown;
+
+type NormalizeLazyItem<List extends LazyItem> = List extends () => unknown ? List : () => List;
+
+/**
+ * Normalize FlexList type to a non-lazy array.
+ */
+export type FlexListToNonLazyArray<List extends FlexList> = ArrayHasFixedLength<List> extends true
+	? ConstantFlexListToNonLazyArray<List>
+	: NormalizedFlexList<ExtractListItemType<List>>;
+
+/**
+ * Normalize FlexList type to a union.
+ * @beta
+ */
+export type FlexListToUnion<TList extends FlexList> = ExtractItemType<ArrayToUnion<TList>>;
+
+/**
+ * Normalize FlexList type to a non-lazy array.
+ * @internal
+ */
+export type ConstantFlexListToNonLazyArray<List extends FlexList> = List extends readonly [
+	infer Head,
+	...infer Tail,
+]
+	? [ExtractItemType<Head>, ...ConstantFlexListToNonLazyArray<Tail>]
+	: [];
+
+/**
+ * Detect if an array is a Tuple (fixed length) or unknown length.
+ *
+ * Types which many have one of multiple fixed lengths (like `[] | [0]`) count as having a fixed length.
+ *
+ * @remarks
+ * Type operations designed to work on tuples can often behave very badly on regular arrays.
+ * For example recursive patterns for processing them often just return the base case,
+ * losing all the type information.
+ */
+// This works by determining if the length is `number` (and not a specific number).
+export type ArrayHasFixedLength<List extends readonly unknown[]> = number extends List["length"]
+	? false
+	: true;
+
+/**
+ * Normalize FlexList type to a lazy array.
+ */
+export type FlexListToLazyArray<List extends FlexList> = ArrayHasFixedLength<List> extends true
+	? ConstantFlexListToLazyArray<List>
+	: NormalizedLazyFlexList<ExtractListItemType<List>>;
+
+/**
+ * Normalize FlexList type to a lazy array.
+ */
+export type ConstantFlexListToLazyArray<List extends FlexList> = List extends readonly [
+	infer Head,
+	...infer Tail,
+]
+	? [NormalizeLazyItem<Head>, ...ConstantFlexListToLazyArray<Tail>]
+	: [];

package/src/feature-libraries/typed-schema/index.ts

@@ -0,0 +1,51 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export {
+	TreeNodeSchema,
+	TreeFieldSchema,
+	Any,
+	allowedTypesToTypeSet,
+	AllowedTypes,
+	LazyTreeNodeSchema,
+	LeafNodeSchema,
+	MapNodeSchema,
+	ObjectNodeSchema,
+	FieldNodeSchema,
+	FlexTreeSchema,
+	Unenforced,
+	AllowedTypeSet,
+	MapFieldSchema,
+	SchemaCollection,
+	TreeNodeSchemaBase,
+	Fields,
+	schemaIsFieldNode,
+	schemaIsLeaf,
+	schemaIsMap,
+	schemaIsObjectNode,
+	NormalizeObjectNodeFields,
+	NormalizeField,
+	intoStoredSchema,
+	allowedTypesSchemaSet,
+	intoStoredSchemaCollection,
+} from "./typedTreeSchema";
+
+export { ViewSchema } from "./view";
+
+export {
+	bannedFieldNames,
+	fieldApiPrefixes,
+	validateObjectNodeFieldName,
+	SchemaLibraryData,
+	SchemaLintConfiguration,
+	aggregateSchemaLibraries,
+	schemaLintDefault,
+} from "./schemaCollection";
+
+export { FlexList, markEager, FlexListToUnion, LazyItem, isLazy } from "./flexList";
+
+// Below here are things that are used by the above, but not part of the desired API surface.
+import * as InternalTypedSchemaTypes from "./internal";
+export { InternalTypedSchemaTypes };

package/src/feature-libraries/typed-schema/internal.ts

@@ -0,0 +1,10 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+// Used by public types, but not part of the desired API surface
+
+export { ArrayToUnion } from "./typeUtils";
+
+export { FlexList, LazyItem, ExtractItemType, FlexListToUnion } from "./flexList";

package/src/feature-libraries/typed-schema/schemaCollection.ts

@@ -0,0 +1,309 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert } from "@fluidframework/core-utils";
+import { Adapters, TreeAdapter, TreeNodeSchemaIdentifier } from "../../core";
+import { capitalize, fail, requireAssignableTo } from "../../util";
+import { defaultSchemaPolicy, FieldKinds } from "../default-schema";
+import { Multiplicity } from "../multiplicity";
+import {
+	TreeFieldSchema,
+	TreeNodeSchema,
+	allowedTypesIsAny,
+	SchemaCollection,
+	MapNodeSchema,
+	LeafNodeSchema,
+	FieldNodeSchema,
+	ObjectNodeSchema,
+} from "./typedTreeSchema";
+import { normalizeFlexListEager } from "./flexList";
+import { Sourced } from "./view";
+
+// TODO: tests for this file
+
+/**
+ * Schema data collected by a single SchemaBuilder (does not include referenced libraries).
+ * @internal
+ */
+export interface SchemaLibraryData extends SchemaCollection {
+	readonly name: string;
+	readonly adapters: Adapters;
+}
+
+/**
+ * Mutable adapter collection which records the associated factory.
+ * See {@link Adapters}.
+ */
+export interface SourcedAdapters {
+	readonly tree: (Sourced & TreeAdapter)[];
+}
+
+{
+	type _check = requireAssignableTo<SourcedAdapters, Adapters>;
+}
+
+/**
+ * Allows opting into and out of errors for some unusual schema patterns which are usually bugs.
+ * @internal
+ */
+export interface SchemaLintConfiguration {
+	readonly rejectForbidden: boolean;
+	readonly rejectEmpty: boolean;
+}
+
+export const schemaLintDefault: SchemaLintConfiguration = {
+	rejectForbidden: true,
+	rejectEmpty: true,
+};
+
+/**
+ * Build and validate a SchemaCollection.
+ *
+ * As much as possible tries to detect anything that might be a mistake made by the schema author.
+ * This will error on some valid but probably never intended to be used patterns
+ * (like libraries with the same name, nodes which are impossible to create etc).
+ *
+ * @param name - Name of the resulting library.
+ * @param lintConfiguration - configuration for what errors to generate.
+ * @param libraries - Data to aggregate into the SchemaCollection.
+ * @param rootFieldSchema - Only validated: not included in the result.
+ *
+ * @privateRemarks
+ * This checks that input works with defaultSchemaPolicy.
+ * If support fo other policies is added, this will need to take in the policy.
+ */
+export function aggregateSchemaLibraries(
+	name: string,
+	lintConfiguration: SchemaLintConfiguration,
+	libraries: Iterable<SchemaLibraryData>,
+	rootFieldSchema?: TreeFieldSchema,
+): SchemaLibraryData {
+	const nodeSchema: Map<TreeNodeSchemaIdentifier, TreeNodeSchema> = new Map();
+	const adapters: SourcedAdapters = { tree: [] };
+
+	const errors: string[] = [];
+	const librarySet: Set<SchemaLibraryData> = new Set();
+	const libraryNames: Set<string> = new Set();
+
+	for (const library of libraries) {
+		if (librarySet.has(library)) {
+			// SchemaBuilder should ensure this doesn't happen, but include it here for completeness.
+			errors.push(`Duplicate library named "${library.name}"`);
+			continue;
+		}
+		librarySet.add(library);
+		if (libraryNames.has(library.name)) {
+			// This wouldn't break anything, but could make error messages confusing, so its better to avoid duplicates.
+			errors.push(`Found another library with name "${library.name}"`);
+		}
+
+		for (const [key, tree] of library.nodeSchema) {
+			// This check is an assert since if it fails, the other error messages would be incorrect.
+			assert(
+				tree.builder.name === library.name,
+				0x6a9 /* tree must be part by the library its in */,
+			);
+			const existing = nodeSchema.get(key);
+			if (existing !== undefined) {
+				errors.push(
+					`Multiple tree schema for identifier "${key}". One from library "${existing.builder.name}" and one from "${tree.builder.name}"`,
+				);
+			} else {
+				nodeSchema.set(key, tree);
+			}
+		}
+		for (const _adapter of library.adapters.tree ?? []) {
+			fail("Adapters not yet supported");
+		}
+	}
+
+	if (errors.length !== 0) {
+		fail(errors.join("\n"));
+	}
+
+	const result = { rootFieldSchema, nodeSchema, adapters, policy: defaultSchemaPolicy };
+	const errors2 = validateSchemaCollection(lintConfiguration, result, rootFieldSchema);
+	if (errors2.length !== 0) {
+		fail(errors2.join("\n"));
+	}
+
+	return {
+		name,
+		nodeSchema,
+		adapters,
+	};
+}
+
+/**
+ * Returns an array of descriptions of errors in the collection.
+ *
+ * As much as possible tries to detect anything that might be a mistake made by the schema author.
+ * This will error on some valid but probably never intended to be used patterns (like never nodes).
+ */
+export function validateSchemaCollection(
+	lintConfiguration: SchemaLintConfiguration,
+	collection: SchemaCollection,
+	rootFieldSchema?: TreeFieldSchema,
+): string[] {
+	const errors: string[] = [];
+
+	// TODO: make this check specific to document schema. Replace check here for no tre or field schema (empty library).
+	if (collection.nodeSchema.size === 0 && lintConfiguration.rejectEmpty) {
+		errors.push("No tree schema are included, meaning no data can possibly be stored.");
+	}
+
+	// Validate that all schema referenced are included, and none are "never".
+	if (rootFieldSchema !== undefined) {
+		validateRootField(lintConfiguration, collection, rootFieldSchema, errors);
+	}
+	for (const [identifier, tree] of collection.nodeSchema) {
+		if (tree instanceof MapNodeSchema) {
+			validateField(
+				lintConfiguration,
+				collection,
+				tree.info,
+				() => `Map fields of "${identifier}" schema from library "${tree.builder.name}"`,
+				errors,
+			);
+			if ((tree.info.kind.multiplicity as Multiplicity) === Multiplicity.Single) {
+				errors.push(
+					`Map fields of "${identifier}" schema from library "${tree.builder.name}" has kind with multiplicity "Single". This is invalid since it requires all possible field keys to have a value under them.`,
+				);
+			}
+		} else if (tree instanceof LeafNodeSchema) {
+			// No validation for now.
+		} else if (tree instanceof FieldNodeSchema) {
+			const description = () =>
+				`Field node field of "${identifier}" schema from library "${tree.builder.name}"`;
+			validateField(lintConfiguration, collection, tree.info, description, errors);
+		} else if (tree instanceof ObjectNodeSchema) {
+			for (const [key, field] of tree.objectNodeFields) {
+				const description = () =>
+					`Object node field "${key}" of "${identifier}" schema from library "${tree.builder.name}"`;
+				validateField(lintConfiguration, collection, field, description, errors);
+				validateObjectNodeFieldName(key, description, errors);
+			}
+		} else {
+			// TODO: there should be a common fallback that works for cases without a specialized implementation.
+			fail("unrecognized node kind");
+		}
+	}
+
+	// TODO: validate adapters
+	return errors;
+}
+
+export function validateRootField(
+	lintConfiguration: SchemaLintConfiguration,
+	collection: SchemaCollection,
+	field: TreeFieldSchema,
+	errors: string[],
+): void {
+	const describeField = () => `Root field schema`;
+	validateField(lintConfiguration, collection, field, describeField, errors);
+}
+
+export function validateField(
+	lintConfiguration: SchemaLintConfiguration,
+	collection: SchemaCollection,
+	field: TreeFieldSchema,
+	describeField: () => string,
+	errors: string[],
+): void {
+	const types = field.allowedTypes;
+	if (!allowedTypesIsAny(types)) {
+		const normalizedTypes = normalizeFlexListEager(types);
+		for (const type of normalizedTypes) {
+			const referenced = collection.nodeSchema.get(type.name);
+			if (referenced === undefined) {
+				errors.push(
+					`${describeField()} references type "${type.name}" from library "${
+						type.builder.name
+					}" which is not defined. Perhaps another type was intended, or that library needs to be added.`,
+				);
+			}
+		}
+		if (types.length === 0 && lintConfiguration.rejectEmpty) {
+			errors.push(
+				`${describeField()} requires children to have a type from a set of zero types. This means the field must always be empty.`,
+			);
+		}
+	}
+
+	const kind = field.kind;
+	const kindFromPolicy = defaultSchemaPolicy.fieldKinds.get(kind.identifier);
+	if (kindFromPolicy === undefined) {
+		errors.push(`"${describeField()}" has unknown field kind "${kind.identifier}".`);
+	} else if (kindFromPolicy !== kind) {
+		errors.push(
+			`${describeField()} has field kind "${
+				kind.identifier
+			}" which isn't a reference to the default kind with that identifier.`,
+		);
+	} else if (kind === FieldKinds.forbidden) {
+		if (lintConfiguration.rejectForbidden) {
+			errors.push(
+				`${describeField()} explicitly uses "forbidden" kind, which is not recommended.`,
+			);
+		}
+	} // else if (kind !== counter) {
+	// 	errors.push(
+	// 		`${describeField()} explicitly uses "counter" kind, which is finished.`,
+	// 	);
+	// }
+}
+
+/**
+ * Reserved field names to avoid collisions with the API.
+ */
+export const bannedFieldNames = new Set([
+	"constructor",
+	"context",
+	"is",
+	"on",
+	"parentField",
+	"schema",
+	"treeStatus",
+	"tryGetField",
+	"type",
+	"value",
+	"localNodeKey",
+]);
+
+/**
+ * Field names starting with these must not be followed by an upper case letter
+ */
+export const fieldApiPrefixes = new Set(["set", "boxed"]);
+
+export function validateObjectNodeFieldName(
+	name: string,
+	describeField: () => string,
+	errors: string[],
+): void {
+	// TODO: Remove conflicts between possible field keys and editable-tree API members.
+	const suggestion =
+		"Pick a different field name to avoid property name collisions in the implementation. In the future this list of reserved names will be removed.";
+
+	if (bannedFieldNames.has(name)) {
+		errors.push(
+			`${describeField()} uses "${name}" one of the banned field names (${[
+				...bannedFieldNames,
+			]}). ${suggestion}`,
+		);
+	}
+
+	for (const prefix of fieldApiPrefixes) {
+		if (name.startsWith(prefix)) {
+			const afterPrefix = name.slice(prefix.length);
+			if (afterPrefix === capitalize(afterPrefix)) {
+				errors.push(
+					`${describeField()} has name that starts with one of the banned prefixes (${[
+						...fieldApiPrefixes,
+					]}) followed by something other than a lowercase letter. ${suggestion}`,
+				);
+			}
+		}
+	}
+}