@fluidframework/tree 2.0.0-dev-rc.1.0.0.224419

package/src/feature-libraries/modular-schema/modularChangeFormat.ts

@@ -0,0 +1,147 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ObjectOptions, Static, Type } from "@sinclair/typebox";
+import { schemaFormat, RevisionTagSchema, ChangesetLocalId } from "../../core";
+import {
+	brandedNumberType,
+	JsonCompatibleReadOnly,
+	JsonCompatibleReadOnlySchema,
+} from "../../util";
+import { EncodedFieldBatch } from "../chunked-forest";
+
+const noAdditionalProps: ObjectOptions = { additionalProperties: false };
+
+export const ChangesetLocalIdSchema = brandedNumberType<ChangesetLocalId>();
+
+export const EncodedChangeAtomId = Type.Object(
+	{
+		/**
+		 * Uniquely identifies the changeset within which the change was made.
+		 */
+		revision: Type.Optional(RevisionTagSchema),
+		/**
+		 * Uniquely identifies, in the scope of the changeset, the change made to the field.
+		 */
+		localId: ChangesetLocalIdSchema,
+	},
+	noAdditionalProps,
+);
+
+const EncodedValueChange = Type.Object(
+	{
+		revision: Type.Optional(RevisionTagSchema),
+		value: Type.Optional(JsonCompatibleReadOnlySchema),
+	},
+	noAdditionalProps,
+);
+type EncodedValueChange = Static<typeof EncodedValueChange>;
+
+const EncodedValueConstraint = Type.Object(
+	{
+		value: Type.Optional(JsonCompatibleReadOnlySchema),
+		violated: Type.Boolean(),
+	},
+	noAdditionalProps,
+);
+type EncodedValueConstraint = Static<typeof EncodedValueConstraint>;
+
+const EncodedFieldChange = Type.Object(
+	{
+		fieldKey: schemaFormat.FieldKeySchema,
+		fieldKind: schemaFormat.FieldKindIdentifierSchema,
+		// Implementation note: node and field change encoding is mutually recursive.
+		// This field marks a boundary in that recursion to avoid constructing excessively complex
+		// recursive types. Encoded changes are validated at this boundary at runtime--see modularChangeCodecs.ts.
+		change: JsonCompatibleReadOnlySchema,
+	},
+	noAdditionalProps,
+);
+
+export interface EncodedFieldChange extends Static<typeof EncodedFieldChange> {
+	/**
+	 * Encoded in format selected by `fieldKind`
+	 */
+	change: JsonCompatibleReadOnly;
+}
+
+const EncodedFieldChangeMap = Type.Array(EncodedFieldChange);
+
+/**
+ * Format for encoding as json.
+ *
+ * This chooses to use lists of named objects instead of maps:
+ * this choice is somewhat arbitrary, but avoids user data being used as object keys,
+ * which can sometimes be an issue (for example handling that for "__proto__" can require care).
+ */
+export type EncodedFieldChangeMap = Static<typeof EncodedFieldChangeMap>;
+
+const EncodedNodeExistsConstraint = Type.Object({
+	violated: Type.Boolean(),
+});
+type EncodedNodeExistsConstraint = Static<typeof EncodedNodeExistsConstraint>;
+
+export const EncodedNodeChangeset = Type.Object(
+	{
+		valueChange: Type.Optional(EncodedValueChange),
+		fieldChanges: Type.Optional(EncodedFieldChangeMap),
+		valueConstraint: Type.Optional(EncodedValueConstraint),
+		nodeExistsConstraint: Type.Optional(EncodedNodeExistsConstraint),
+	},
+	noAdditionalProps,
+);
+
+/**
+ * Format for encoding as json.
+ */
+export type EncodedNodeChangeset = Static<typeof EncodedNodeChangeset>;
+
+export const EncodedRevisionInfo = Type.Object(
+	{
+		revision: Type.Readonly(RevisionTagSchema),
+		rollbackOf: Type.ReadonlyOptional(RevisionTagSchema),
+	},
+	noAdditionalProps,
+);
+
+export type EncodedRevisionInfo = Static<typeof EncodedRevisionInfo>;
+
+/**
+ * Index of field in an EncodedFieldBatch.
+ */
+export const EncodedTreeIndex = Type.Number({ multipleOf: 1, minimum: 0 });
+
+// TODO:YA6307 adopt more efficient encoding, likely based on contiguous runs of IDs
+export const EncodedBuildsArray = Type.Array(
+	Type.Union([
+		Type.Tuple([ChangesetLocalIdSchema, EncodedTreeIndex]),
+		Type.Tuple([RevisionTagSchema, ChangesetLocalIdSchema, EncodedTreeIndex]),
+	]),
+);
+
+export type EncodedBuildsArray = Static<typeof EncodedBuildsArray>;
+
+export const EncodedBuilds = Type.Object({
+	builds: EncodedBuildsArray,
+	/**
+	 * Fields indexed by the EncodedTreeIndexes above.
+	 * TODO: Strongly typing this here may result in redundant schema validation of this data.
+	 */
+	trees: EncodedFieldBatch,
+});
+
+export type EncodedBuilds = Static<typeof EncodedBuilds>;
+
+export const EncodedModularChangeset = Type.Object(
+	{
+		maxId: Type.Optional(ChangesetLocalIdSchema),
+		changes: EncodedFieldChangeMap,
+		revisions: Type.ReadonlyOptional(Type.Array(EncodedRevisionInfo)),
+		builds: Type.Optional(EncodedBuilds),
+	},
+	noAdditionalProps,
+);
+
+export type EncodedModularChangeset = Static<typeof EncodedModularChangeset>;

package/src/feature-libraries/modular-schema/modularChangeTypes.ts

@@ -0,0 +1,85 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import {
+	ChangeAtomIdMap,
+	ChangesetLocalId,
+	FieldKey,
+	FieldKindIdentifier,
+	RevisionInfo,
+	RevisionTag,
+} from "../../core";
+import { Brand } from "../../util";
+import { TreeChunk } from "../chunked-forest";
+
+/**
+ * @internal
+ */
+export interface ModularChangeset extends HasFieldChanges {
+	/**
+	 * The numerically highest `ChangesetLocalId` used in this changeset.
+	 * If undefined then this changeset contains no IDs.
+	 */
+	maxId?: ChangesetLocalId;
+	/**
+	 * The revisions included in this changeset, ordered temporally (oldest to newest).
+	 * Undefined for anonymous changesets.
+	 * Should never be empty.
+	 */
+	readonly revisions?: readonly RevisionInfo[];
+	fieldChanges: FieldChangeMap;
+	constraintViolationCount?: number;
+	// TODO:YA6307 adopt more efficient representation, likely based on contiguous runs of IDs
+	readonly builds?: ChangeAtomIdMap<TreeChunk>;
+}
+
+/**
+ * @internal
+ */
+export interface NodeExistsConstraint {
+	violated: boolean;
+}
+
+/**
+ * Changeset for a subtree rooted at a specific node.
+ * @internal
+ */
+export interface NodeChangeset extends HasFieldChanges {
+	nodeExistsConstraint?: NodeExistsConstraint;
+}
+
+/**
+ * @internal
+ */
+export interface HasFieldChanges {
+	fieldChanges?: FieldChangeMap;
+}
+
+/**
+ * @internal
+ */
+export type FieldChangeMap = Map<FieldKey, FieldChange>;
+
+/**
+ * @internal
+ */
+export interface FieldChange {
+	fieldKind: FieldKindIdentifier;
+
+	/**
+	 * If defined, `change` is part of the specified revision.
+	 * Undefined in the following cases:
+	 * A) A revision is specified on an ancestor of this `FieldChange`, in which case `change` is part of that revision.
+	 * B) `change` is composed of multiple revisions.
+	 * C) `change` is part of an anonymous revision.
+	 */
+	revision?: RevisionTag;
+	change: FieldChangeset;
+}
+
+/**
+ * @internal
+ */
+export type FieldChangeset = Brand<unknown, "FieldChangeset">;

package/src/feature-libraries/multiplicity.ts

@@ -0,0 +1,54 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+/**
+ * Describes how a particular field functions.
+ *
+ * This determine its reading and editing APIs, multiplicity, and what merge resolution policies it will use.
+ * @internal
+ */
+export enum Multiplicity {
+	/**
+	 * Exactly one item.
+	 */
+	Single,
+	/**
+	 * 0 or 1 items.
+	 */
+	Optional,
+	/**
+	 * 0 or more items.
+	 */
+	Sequence,
+	/**
+	 * Exactly 0 items.
+	 *
+	 * Using Forbidden makes what types are listed for allowed in a field irrelevant
+	 * since the field will never have values in it.
+	 *
+	 * Using Forbidden is equivalent to picking a kind that permits empty (like sequence or optional)
+	 * and having no allowed types (or only never types).
+	 * Because of this, its possible to express everything constraint wise without Forbidden,
+	 * but using Forbidden can be more semantically clear than optional with no allowed types.
+	 *
+	 * For view schema, this can be useful if you need to:
+	 * - run a specific out of schema handler when a field is present,
+	 * but otherwise are ignoring or tolerating (ex: via extra fields) unmentioned fields.
+	 * - prevent a specific field from being used as an extra field
+	 * (perhaps for some past of future compatibility reason)
+	 * - keep a field in a schema for metadata purposes
+	 * (ex: for improved error messaging, error handling or documentation)
+	 * that is not used in this specific version of the schema (ex: to document what it was or will be used for).
+	 *
+	 * For stored schema, this can be useful if you need to:
+	 * - have a field which can have its schema updated to Optional or Sequence of any type.
+	 * - to exclude a field from extra fields
+	 * - for the schema system to use as a default for fields which aren't declared
+	 * (ex: when updating a field that did not exist into one that does)
+	 *
+	 * @privateRemarks
+	 * See storedEmptyFieldSchema for a constant, reusable field using Forbidden.
+	 */
+	Forbidden,
+}

package/src/feature-libraries/node-key/README.md

@@ -0,0 +1,11 @@
+# Node Key
+
+Nodes keys provide references to nodes in a shared tree.
+
+## Local Node Keys
+
+Local node keys are an efficient representation of a node key and are unique within the local session (machine memory).
+
+## Stable Node Keys
+
+Stable node keys are UUIDs that are globally unique and can be persisted outside of the document.

package/src/feature-libraries/node-key/index.ts

@@ -0,0 +1,16 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export {
+	compareLocalNodeKeys,
+	LocalNodeKey,
+	StableNodeKey,
+	nodeKeyFieldKey,
+	nodeKeyTreeIdentifier,
+} from "./nodeKey";
+
+export { NodeKeyIndex } from "./nodeKeyIndex";
+
+export { createNodeKeyManager, createMockNodeKeyManager, NodeKeyManager } from "./nodeKeyManager";

package/src/feature-libraries/node-key/nodeKey.ts

@@ -0,0 +1,58 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { SessionSpaceCompressedId, StableId } from "@fluidframework/runtime-definitions";
+import { Brand, Opaque, brand } from "../../util";
+import { TreeNodeSchemaIdentifier } from "../../core";
+
+/**
+ * A key which uniquely identifies a node in the tree within this session.
+ * @remarks {@link LocalNodeKey}s must not be serialized and stored as data without first being converted into a {@link StableNodeKey}.
+ * They are local to the current session and equivalent nodes in another session will not necessarily have the same {@link LocalNodeKey}.
+ * However, they are otherwise preferential to use over {@link StableNodeKey}s as they are much smaller.
+ * For example, they are more efficient than {@link StableNodeKey}s when used as keys in a map.
+ * {@link LocalNodeKey}s may be compared or equated via {@link compareLocalNodeKeys}.
+ * @internal
+ */
+export interface LocalNodeKey extends Opaque<Brand<SessionSpaceCompressedId, "Local Node Key">> {}
+
+/**
+ * A UUID which identifies a node in the tree.
+ * This key is universally unique and stable forever; therefore it is safe to persist as data in a SharedTree or other DDS/database.
+ * When not persisted or serialized, it is preferable to use a {@link LocalNodeKey} instead for better performance.
+ * @internal
+ */
+export type StableNodeKey = Brand<StableId, "Stable Node Key">;
+
+/**
+ * Compares two {@link LocalNodeKey}s.
+ * All {@link LocalNodeKey}s retrieved from a single SharedTree client can be totally ordered using this comparator.
+ * @param a - the first key to compare
+ * @param b - the second key to compare
+ * @returns `0` if `a` and `b` are the same key, otherwise `-1` if `a` is ordered before `b` or `1` if `a` is ordered after `b`.
+ * @internal
+ */
+export function compareLocalNodeKeys(a: LocalNodeKey, b: LocalNodeKey): -1 | 0 | 1 {
+	return a === b ? 0 : a > b ? 1 : -1;
+}
+
+/**
+ * The key for the special field for {@link LocalNodeKey}s,
+ * which allows nodes to be given keys that can be used to find the nodes via the node key index.
+ * @internal
+ * @privateRemarks TODO: Come up with a unified and collision-resistant naming schema for fields defined by the system.
+ * For now, we'll use `__` to reduce the change of collision, since this is what other internal properties use in Fluid.
+ */
+export const nodeKeyFieldKey = "__n_id__";
+
+/**
+ * The TreeNodeSchemaIdentifier for node keys.
+ * @internal
+ * @privateRemarks TODO: Come up with a unified and collision-resistant naming schema for types defined by the system.
+ * For now, we'll use `__` to reduce the change of collision, since this is what other internal properties use in Fluid.
+ */
+export const nodeKeyTreeIdentifier: TreeNodeSchemaIdentifier = brand(
+	"com.fluidframework.nodeKey.NodeKey",
+);

package/src/feature-libraries/node-key/nodeKeyIndex.ts

@@ -0,0 +1,125 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert } from "@fluidframework/core-utils";
+import { FieldKey, ValueSchema } from "../../core";
+import {
+	FlexTreeObjectNode,
+	FlexTreeContext,
+	FlexTreeField,
+	FlexTreeNode,
+	boxedIterator,
+} from "../flex-tree";
+import { FlexTreeSchema, LeafNodeSchema, schemaIsObjectNode } from "../typed-schema";
+import { LocalNodeKey, nodeKeyTreeIdentifier } from "./nodeKey";
+
+/**
+ * The node key index records nodes with {@link LocalNodeKey}s and allows them to be looked up by key.
+ */
+export class NodeKeyIndex implements ReadonlyMap<LocalNodeKey, FlexTreeObjectNode> {
+	// TODO: The data structure that holds the nodes can likely be optimized to better support cloning
+	private readonly nodes: Map<LocalNodeKey, FlexTreeObjectNode>;
+
+	public constructor(
+		public readonly fieldKey: FieldKey,
+		keys: Iterable<[LocalNodeKey, FlexTreeObjectNode]> = [],
+	) {
+		this.nodes = new Map(keys);
+	}
+
+	/**
+	 * Returns true if the given schema contains the node key type, otherwise false
+	 */
+	public static hasNodeKeyTreeSchema(schema: FlexTreeSchema): boolean {
+		// TODO: make TreeStoredSchema contain ViewSchema and compare by reference to nodeKeyTreeSchema.
+		const treeSchema = schema.nodeSchema.get(nodeKeyTreeIdentifier);
+		if (!(treeSchema instanceof LeafNodeSchema)) {
+			return false;
+		}
+		return treeSchema.leafValue === ValueSchema.String;
+	}
+
+	/**
+	 * Search the tree for all nodes with keys, and record them in this index for lookup.
+	 * This should be called each time the tree changes; each call to scan forgets all existing keys.
+	 * @param context - the editable tree context in which to search for node keys
+	 */
+	// TODO: This can be optimized by responding to deltas/changes to the tree, rather than rescanning the whole tree every time
+	public scanKeys(context: FlexTreeContext): void {
+		this.nodes.clear();
+		if (NodeKeyIndex.hasNodeKeyTreeSchema(context.schema)) {
+			for (const [id, node] of this.findKeysInField(context.root)) {
+				// TODO:
+				// This invariant (that there is only one node with a given key) is not enforced by tree, so it should not assert.
+				// Multiple nodes (including deleted ones), might occur with the same key.
+				assert(!this.nodes.has(id), 0x6e1 /* Encountered duplicate node key */);
+				this.nodes.set(id, node);
+			}
+		}
+	}
+
+	/**
+	 * Create a copy of this index which can be mutated without affecting this one.
+	 */
+	public clone(context: FlexTreeContext): NodeKeyIndex {
+		const indexClone = new NodeKeyIndex(this.fieldKey);
+		indexClone.scanKeys(context);
+		return indexClone;
+	}
+
+	// #region ReadonlyMap interface
+	public forEach(
+		callbackfn: (
+			value: FlexTreeObjectNode,
+			key: LocalNodeKey,
+			map: ReadonlyMap<LocalNodeKey, FlexTreeObjectNode>,
+		) => void,
+		thisArg?: any,
+	): void {
+		return this.nodes.forEach(callbackfn, thisArg);
+	}
+	public get(key: LocalNodeKey): FlexTreeObjectNode | undefined {
+		return this.nodes.get(key);
+	}
+	public has(key: LocalNodeKey): boolean {
+		return this.nodes.has(key);
+	}
+	public get size(): number {
+		return this.nodes.size;
+	}
+	public entries(): IterableIterator<[LocalNodeKey, FlexTreeObjectNode]> {
+		return this.nodes.entries();
+	}
+	public keys(): IterableIterator<LocalNodeKey> {
+		return this.nodes.keys();
+	}
+	public values(): IterableIterator<FlexTreeObjectNode> {
+		return this.nodes.values();
+	}
+	public [Symbol.iterator](): IterableIterator<[LocalNodeKey, FlexTreeObjectNode]> {
+		return this.nodes[Symbol.iterator]();
+	}
+	// #endregion ReadonlyMap interface
+
+	private *findKeys(node: FlexTreeNode): Iterable<[key: LocalNodeKey, node: FlexTreeObjectNode]> {
+		if (schemaIsObjectNode(node.schema)) {
+			const key = (node as FlexTreeObjectNode).localNodeKey;
+			if (key !== undefined) {
+				yield [key, node as FlexTreeObjectNode];
+			}
+		}
+		for (const f of node[boxedIterator]()) {
+			yield* this.findKeysInField(f);
+		}
+	}
+
+	private *findKeysInField(
+		f: FlexTreeField,
+	): Iterable<[key: LocalNodeKey, node: FlexTreeObjectNode]> {
+		for (const child of f[boxedIterator]()) {
+			yield* this.findKeys(child);
+		}
+	}
+}

package/src/feature-libraries/node-key/nodeKeyManager.ts

@@ -0,0 +1,107 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert } from "@fluidframework/core-utils";
+import {
+	assertIsStableId,
+	IIdCompressor,
+	SessionSpaceCompressedId,
+	StableId,
+} from "@fluidframework/id-compressor";
+import { brand, extractFromOpaque } from "../../util";
+import { StableNodeKey, LocalNodeKey } from "./nodeKey";
+
+/**
+ * An object which handles the generation of node keys as well as conversion between their two types ({@link StableNodeKey} and {@link LocalNodeKey}).
+ */
+export interface NodeKeyManager {
+	/**
+	 * Generate a {@link StableNodeKey}.
+	 */
+	generateLocalNodeKey(): LocalNodeKey;
+
+	/**
+	 * Convert the given {@link StableNodeKey} into its {@link LocalNodeKey} form.
+	 */
+	localizeNodeKey(key: StableNodeKey): LocalNodeKey;
+
+	/**
+	 * Convert the given {@link LocalNodeKey} into its {@link StableNodeKey} form.
+	 */
+	stabilizeNodeKey(key: LocalNodeKey): StableNodeKey;
+}
+
+/**
+ * Creates a {@link NodeKeyManager} from the given {@link IIdCompressor}.
+ * @param idCompressor - the compressor to use for key generation, compression, and decompression.
+ * If undefined, then attempts to generate or convert keys will throw an error.
+ */
+export function createNodeKeyManager(idCompressor?: IIdCompressor | undefined): NodeKeyManager {
+	return {
+		generateLocalNodeKey: () => {
+			assert(
+				idCompressor !== undefined,
+				0x6e4 /* Runtime IdCompressor must be available to generate local node keys */,
+			);
+			return brand(idCompressor.generateCompressedId());
+		},
+
+		localizeNodeKey: (key: StableNodeKey) => {
+			assert(
+				idCompressor !== undefined,
+				0x6e5 /* Runtime IdCompressor must be available to convert node keys */,
+			);
+			return brand(idCompressor.recompress(key));
+		},
+
+		stabilizeNodeKey: (key: LocalNodeKey) => {
+			assert(
+				idCompressor !== undefined,
+				0x6e6 /* Runtime IdCompressor must be available to convert node keys */,
+			);
+			return brand(
+				// TODO: The assert below is required for type safety but is maybe slow
+				assertIsStableId(idCompressor.decompress(extractFromOpaque(key))),
+			);
+		},
+	};
+}
+
+/**
+ * Create a {@link NodeKeyManager} that generates deterministic {@link StableNodeKey}s and {@link LocalNodeKey}s.
+ * @remarks This is useful for test environments because it will always yield the same keys in the same order.
+ * It should not be used for production environments for the same reason; the {@link StableNodeKey}s are not universally unique.
+ */
+export function createMockNodeKeyManager(): NodeKeyManager {
+	return new MockNodeKeyManager();
+}
+
+class MockNodeKeyManager implements NodeKeyManager {
+	private count = 0;
+
+	public generateStableNodeKey(): StableNodeKey {
+		return brand(this.createMockStableId(this.count++));
+	}
+
+	public generateLocalNodeKey(): LocalNodeKey {
+		return this.localizeNodeKey(this.generateStableNodeKey());
+	}
+
+	public localizeNodeKey(key: StableNodeKey): LocalNodeKey {
+		return brand(Number.parseInt(key.substring(30), 16) as SessionSpaceCompressedId);
+	}
+
+	public stabilizeNodeKey(key: LocalNodeKey): StableNodeKey {
+		return brand(this.createMockStableId(extractFromOpaque(key)));
+	}
+
+	private createMockStableId(offset: number): StableId {
+		assert(offset >= 0, 0x6e7 /* UUID offset may not be negative */);
+		assert(offset < 281_474_976_710_656, 0x6e8 /* UUID offset must be at most 16^12 */);
+		return assertIsStableId(
+			`a110ca7e-add1-4000-8000-${Math.round(offset).toString(16).padStart(12, "0")}`,
+		);
+	}
+}

package/src/feature-libraries/object-forest/README.md

@@ -0,0 +1,7 @@
+# Object Forest
+
+Implementation of Forest using in place mutated javascript objects for tree nodes.
+
+This is intended as a reference implementation, and thus is designed for minimal complexity, understandable design and clear correctness.
+
+The performance should be reasonable when avoiding large numbers of nodes and long sequences, but is not the focus on this library.

package/src/feature-libraries/object-forest/index.ts

@@ -0,0 +1,6 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export { buildForest } from "./objectForest";