@fluidframework/tree 2.0.0-dev-rc.1.0.0.224419

package/src/core/tree/delta.ts

@@ -0,0 +1,255 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { FieldKey } from "../schema-stored";
+import { ITreeCursorSynchronous } from "./cursor";
+
+/**
+ * This format describes changes that must be applied to a forest in order to update it.
+ * Instances of this format are generated based on incoming changesets and consumed by a view layer (e.g., Forest) to
+ * update itself.
+ *
+ * Because this format is only meant for updating document state, it does not fully represent user intentions.
+ * For example, if some concurrent edits A and B insert content at the same location, then a Delta that represents
+ * represents the state update for edit A would not include information that allows B's insertion to be ordered
+ * relative to A's insertion. This format is therefore not fit to be rebased in the face of concurrent changes.
+ * Instead this format is used to describe the end product of rebasing user intentions over concurrent edits.
+ *
+ * This format is self-contained in the following ways:
+ *
+ * 1. It uses integer indices (offsets, technically) to describe the locations of necessary changes.
+ * As such, it does not rely on document nodes being accessible/locatable by ID.
+ *
+ * 2. This format does not require historical information in order to apply the changes it describes.
+ * For example, if a user undoes the deletion of a subtree, then the Delta generated for the undo edit will contain all
+ * information necessary to restore that subtree.
+ *
+ * This format can be generated from any Changeset without having access to the current document state.
+ *
+ * This format is meant to serve as the lowest common denominator to represent state changes resulting from any kind
+ * of operation on any kind of field.
+ * This means all such operations must be expressible in terms of this format.
+ *
+ * Within the above design constrains, this format is designed with the following goals in mind:
+ *
+ * 1. Make it easy to walk both a document tree and the delta tree to apply the changes described in the delta
+ * with a minimum amount of backtracking over the contents of the tree.
+ * This a boon for both code simplicity and performance.
+ *
+ * 2. Make the format terse.
+ *
+ * 3. Make the format uniform.
+ *
+ * These goals are reflected in the following design choices (this is very much optional reading for users of this
+ * format):
+ *
+ * 1. All marks that apply to field elements are represented in a single linear structure where marks that affect later
+ * elements of the document field appear after marks that affect earlier elements of the document field.
+ *
+ * If the marks were not ordered in this fashion then a consumer would need to backtrack within the document field.
+ *
+ * If the marks were represented in multiple such linear structures then it would be necessary to either:
+ * - backtrack when iterating over one structure fully, then the next
+ * - maintain a pointer within each such linear structure and advance them in lock-step (like in a k-way merge-sort but
+ * more fiddly because of the offsets).
+ *
+ * 2. Nested changes are not inlined within `ProtoNode`s.
+ *
+ * Inlining them would force the consuming code to detect such changes within the `ProtoNode` and handle them
+ * within the context of the content creation.
+ * This would be cumbersome because either the code that is responsible for consuming the `ProtoNode` would need to
+ * be aware of and have the context to handle such changes, or some caller of that code would need to find and extract such
+ * change information ahead to calling that code.
+ */
+
+/**
+ * Represents the change made to a document.
+ * Immutable, therefore safe to retain for async processing.
+ * @internal
+ */
+export interface Root<TTree = ProtoNode> {
+	/**
+	 * Changes to apply to the root fields.
+	 */
+	readonly fields?: FieldMap<TTree>;
+	/**
+	 * New detached nodes to be constructed.
+	 * The ordering has no significance.
+	 *
+	 * Build instructions for a root that is undergoing a rename should be listed under the starting name.
+	 * For example, if one wishes to build a tree which is being renamed from ID A to ID B,
+	 * then the build should be listed under ID A.
+	 */
+	readonly build?: readonly DetachedNodeBuild<TTree>[];
+	/**
+	 * New detached nodes to be destroyed.
+	 * The ordering has no significance.
+	 *
+	 * Destruction instructions for a root that is undergoing a rename should be listed under the final name.
+	 * For example, if one wishes to destroy a tree which is being renamed from ID A to ID B,
+	 * then the destruction should be listed under ID B.
+	 */
+	readonly destroy?: readonly DetachedNodeDestruction[];
+}
+/**
+ * The default representation for inserted content.
+ *
+ * TODO:
+ * Ownership and lifetime of data referenced by this cursor is unclear,
+ * so it is a poor abstraction for this use-case which needs to hold onto the data in a non-exclusive (readonly) way.
+ * Cursors can be one supported way to input data, but aren't a good storage format.
+ * @internal
+ */
+export type ProtoNode = ITreeCursorSynchronous;
+
+/**
+ * The default representation a chunk (sub-sequence) of inserted content.
+ *
+ * TODO:
+ * See issue TODO with ProtoNode.
+ * Additionally, Cursors support sequences, so if using cursors, there are better ways to handle this than an array of cursors,
+ * like using a cursor over all the content (starting in fields mode).
+ * Long term something like TreeChunk should probably be used here.
+ * @internal
+ */
+export type ProtoNodes = readonly ProtoNode[];
+
+/**
+ * Represents a change being made to a part of the document tree.
+ * @internal
+ */
+export interface Mark<TTree = ProtoNode> {
+	/**
+	 * The number of nodes affected.
+	 * When `isAttachMark(mark)` is true, this is the number of new nodes being attached.
+	 * When `isAttachMark(mark)` is false, this the number of existing nodes affected.
+	 * Must be 1 when `fields` is populated.
+	 */
+	readonly count: number;
+
+	/**
+	 * Modifications to the pre-existing content.
+	 * Must be undefined when `attach` is set but `detach` is not.
+	 */
+	readonly fields?: FieldMap<TTree>;
+
+	/**
+	 * When set, indicates that some pre-existing content is being detached and sent to the given detached field.
+	 */
+	readonly detach?: DetachedNodeId;
+
+	/**
+	 * When set, indicates that some content is being attached from the given detached field.
+	 */
+	readonly attach?: DetachedNodeId;
+}
+
+/**
+ * A globally unique ID for a node in a detached field.
+ * @internal
+ */
+export interface DetachedNodeId {
+	readonly major?: string | number;
+	readonly minor: number;
+}
+
+/**
+ * @internal
+ */
+export type FieldMap<TTree = ProtoNode> = ReadonlyMap<FieldKey, FieldChanges<TTree>>;
+
+/**
+ * Represents changes made to a detached node
+ * @internal
+ */
+export interface DetachedNodeChanges<TTree = ProtoNode> {
+	readonly id: DetachedNodeId;
+	readonly fields: FieldMap<TTree>;
+}
+
+/**
+ * Represents the creation of detached nodes.
+ *
+ * Tree creation is idempotent: if a tree with the same ID already exists,
+ * then this build is ignored in favor of the existing tree.
+ * @internal
+ */
+export interface DetachedNodeBuild<TTree = ProtoNode> {
+	readonly id: DetachedNodeId;
+	readonly trees: readonly TTree[];
+}
+
+/**
+ * Represents the destruction of detached nodes
+ * @internal
+ */
+export interface DetachedNodeDestruction {
+	readonly id: DetachedNodeId;
+	readonly count: number;
+}
+
+/**
+ * Represents a detached node being assigned a new `DetachedNodeId`.
+ * @internal
+ */
+export interface DetachedNodeRename {
+	readonly count: number;
+	readonly oldId: DetachedNodeId;
+	readonly newId: DetachedNodeId;
+}
+
+/**
+ * Represents the changes to perform on a given field.
+ * @internal
+ */
+export interface FieldChanges<TTree = ProtoNode> {
+	/**
+	 * Represents a list of changes to the nodes in the field.
+	 * The index of each mark within the range of nodes, before
+	 * applying any of the changes, is not represented explicitly.
+	 * It corresponds to the sum of `mark.count` values for all previous marks for which `isAttachMark(mark)` is false.
+	 */
+	readonly local?: readonly Mark<TTree>[];
+	/**
+	 * Changes to apply to detached nodes.
+	 * The ordering has no significance.
+	 *
+	 * Nested changes for a root that is undergoing a rename should be listed under the starting name.
+	 * For example, if one wishes to change a tree which is being renamed from ID A to ID B,
+	 * then the changes should be listed under ID A.
+	 */
+	readonly global?: readonly DetachedNodeChanges<TTree>[];
+	/**
+	 * New detached nodes to be constructed.
+	 * The ordering has no significance.
+	 *
+	 * @deprecated - Builds should be set at the root.
+	 * TODO:6308 migrate all reader/writers away from this and remove it.
+	 *
+	 * Build instructions for a root that is undergoing a rename should be listed under the starting name.
+	 * For example, if one wishes to build a tree which is being renamed from ID A to ID B,
+	 * then the build should be listed under ID A.
+	 */
+	readonly build?: readonly DetachedNodeBuild<TTree>[];
+	/**
+	 * New detached nodes to be destroyed.
+	 * The ordering has no significance.
+	 *
+	 * @deprecated - Destroys should be set at the root.
+	 * TODO:6308 migrate all reader/writers away from this and remove it.
+	 *
+	 * Destruction instructions for a root that is undergoing a rename should be listed under the final name.
+	 * For example, if one wishes to destroy a tree which is being renamed from ID A to ID B,
+	 * then the destruction should be listed under ID B.
+	 */
+	readonly destroy?: readonly DetachedNodeDestruction[];
+	/**
+	 * Detached whose associated ID needs to be updated.
+	 * The ordering has no significance.
+	 * Note that the renames may need to be performed in a specific order to avoid collisions.
+	 * This ordering problem is left to the consumer of this format.
+	 */
+	readonly rename?: readonly DetachedNodeRename[];
+}

package/src/core/tree/deltaUtil.ts

@@ -0,0 +1,102 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { Mutable } from "../../util";
+import { FieldKey } from "../schema-stored";
+import { ITreeCursorSynchronous } from "./cursor";
+import { Root, DetachedNodeId, FieldChanges, Mark } from "./delta";
+import { rootFieldKey } from "./types";
+
+export const emptyDelta: Root<never> = {};
+
+export const emptyFieldChanges: FieldChanges<never> = {};
+
+export function isAttachMark(mark: Mark): boolean {
+	return mark.attach !== undefined && mark.detach === undefined;
+}
+
+export function isDetachMark(mark: Mark): boolean {
+	return mark.detach !== undefined && mark.attach === undefined;
+}
+
+export function isReplaceMark(mark: Mark): boolean {
+	return mark.detach !== undefined && mark.attach !== undefined;
+}
+
+export function isEmptyFieldChanges(fieldChanges: FieldChanges): boolean {
+	return (
+		fieldChanges.local === undefined &&
+		fieldChanges.global === undefined &&
+		fieldChanges.build === undefined &&
+		fieldChanges.rename === undefined
+	);
+}
+
+export function deltaForRootInitialization(content: readonly ITreeCursorSynchronous[]): Root {
+	if (content.length === 0) {
+		return emptyDelta;
+	}
+	const buildId = { minor: 0 };
+	const delta: Root = {
+		build: [{ id: buildId, trees: content }],
+		fields: new Map<FieldKey, FieldChanges>([
+			[
+				rootFieldKey,
+				{
+					local: [{ count: content.length, attach: buildId }],
+				},
+			],
+		]),
+	};
+	return delta;
+}
+
+export function deltaForSet(
+	newNode: ITreeCursorSynchronous,
+	buildId: DetachedNodeId,
+	detachId?: DetachedNodeId,
+): FieldChanges {
+	const mark: Mutable<Mark> = { count: 1, attach: buildId };
+	if (detachId !== undefined) {
+		mark.detach = detachId;
+	}
+	return {
+		build: [{ id: buildId, trees: [newNode] }],
+		local: [mark],
+	};
+}
+
+export function makeDetachedNodeId(
+	major: DetachedNodeId["major"],
+	minor: DetachedNodeId["minor"],
+): DetachedNodeId {
+	const out: Mutable<DetachedNodeId> = { minor };
+	if (major !== undefined) {
+		out.major = major;
+	}
+	return out;
+}
+
+export function offsetDetachId(id: DetachedNodeId, offset: number): DetachedNodeId;
+export function offsetDetachId(
+	id: DetachedNodeId | undefined,
+	offset: number,
+): DetachedNodeId | undefined;
+export function offsetDetachId(
+	id: DetachedNodeId | undefined,
+	offset: number,
+): DetachedNodeId | undefined {
+	if (id === undefined) {
+		return undefined;
+	}
+	return {
+		...id,
+		minor: id.minor + offset,
+	};
+}
+
+export function areDetachedNodeIdsEqual(a: DetachedNodeId, b: DetachedNodeId): boolean {
+	return a.major === b.major && a.minor === b.minor;
+}

package/src/core/tree/detachedFieldIndex.ts

@@ -0,0 +1,170 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert } from "@fluidframework/core-utils";
+import {
+	Brand,
+	IdAllocator,
+	NestedMap,
+	brand,
+	deleteFromNestedMap,
+	idAllocatorFromMaxId,
+	populateNestedMap,
+	setInNestedMap,
+	tryGetFromNestedMap,
+} from "../../util";
+import { FieldKey } from "../schema-stored";
+import { ICodecOptions, IJsonCodec, noopValidator } from "../../codec";
+import * as Delta from "./delta";
+import { DetachedFieldSummaryData, Major, Minor } from "./detachedFieldIndexTypes";
+import { makeDetachedNodeToFieldCodec } from "./detachedFieldIndexCodec";
+
+/**
+ * ID used to create a detached field key for a removed subtree.
+ *
+ * TODO: Move to Forest once forests can support multiple roots.
+ * @internal
+ */
+export type ForestRootId = Brand<number, "tree.ForestRootId">;
+
+/**
+ * The tree index records detached field IDs and associates them with a change atom ID.
+ */
+export class DetachedFieldIndex {
+	// TODO: don't store the field key in the index, it can be derived from the root ID
+	private detachedNodeToField: NestedMap<Major, Minor, ForestRootId> = new Map();
+	private readonly codec: IJsonCodec<DetachedFieldSummaryData, string>;
+	private readonly options: ICodecOptions;
+
+	/**
+	 * @param name - A name for the index, used as a prefix for the generated field keys.
+	 * @param rootIdAllocator - An ID allocator used to generate unique field keys.
+	 */
+	public constructor(
+		private readonly name: string,
+		private rootIdAllocator: IdAllocator<ForestRootId>,
+		options?: ICodecOptions,
+	) {
+		this.options = options ?? { jsonValidator: noopValidator };
+		this.codec = makeDetachedNodeToFieldCodec(this.options);
+	}
+
+	public clone(): DetachedFieldIndex {
+		const clone = new DetachedFieldIndex(
+			this.name,
+			idAllocatorFromMaxId(this.rootIdAllocator.getNextId()) as IdAllocator<ForestRootId>,
+			this.options,
+		);
+		populateNestedMap(this.detachedNodeToField, clone.detachedNodeToField);
+		return clone;
+	}
+
+	public *entries(): Generator<{ root: ForestRootId } & { id: Delta.DetachedNodeId }> {
+		for (const [major, innerMap] of this.detachedNodeToField) {
+			if (major !== undefined) {
+				for (const [minor, entry] of innerMap) {
+					yield { id: { major, minor }, root: entry };
+				}
+			} else {
+				for (const [minor, entry] of innerMap) {
+					yield { id: { minor }, root: entry };
+				}
+			}
+		}
+	}
+
+	public updateMajor(current: Major, updated: Major) {
+		const innerCurrent = this.detachedNodeToField.get(current);
+		if (innerCurrent !== undefined) {
+			this.detachedNodeToField.delete(current);
+			const innerUpdated = this.detachedNodeToField.get(updated);
+			if (innerUpdated === undefined) {
+				this.detachedNodeToField.set(updated, innerCurrent);
+			} else {
+				for (const [minor, entry] of innerCurrent) {
+					assert(
+						innerUpdated.get(minor) === undefined,
+						0x7a9 /* Collision during index update */,
+					);
+					innerUpdated.set(minor, entry);
+				}
+			}
+		}
+	}
+
+	/**
+	 * Returns a field key for the given ID.
+	 * This does not save the field key on the index. To do so, call {@link createEntry}.
+	 */
+	public toFieldKey(id: ForestRootId): FieldKey {
+		return brand(`${this.name}-${id}`);
+	}
+
+	/**
+	 * Returns the FieldKey associated with the given id.
+	 * Returns undefined if no such id is known to the index.
+	 */
+	public tryGetEntry(id: Delta.DetachedNodeId): ForestRootId | undefined {
+		return tryGetFromNestedMap(this.detachedNodeToField, id.major, id.minor);
+	}
+
+	/**
+	 * Returns the FieldKey associated with the given id.
+	 * Fails if no such id is known to the index.
+	 */
+	public getEntry(id: Delta.DetachedNodeId): ForestRootId {
+		const key = this.tryGetEntry(id);
+		assert(key !== undefined, 0x7aa /* Unknown removed node ID */);
+		return key;
+	}
+
+	public deleteEntry(nodeId: Delta.DetachedNodeId): void {
+		const found = deleteFromNestedMap(this.detachedNodeToField, nodeId.major, nodeId.minor);
+		assert(found, 0x7ab /* Unable to delete unknown entry */);
+	}
+
+	/**
+	 * Associates the DetachedNodeId with a field key and creates an entry for it in the index.
+	 */
+	public createEntry(nodeId?: Delta.DetachedNodeId, count: number = 1): ForestRootId {
+		const root = this.rootIdAllocator.allocate(count);
+		if (nodeId !== undefined) {
+			for (let i = 0; i < count; i++) {
+				assert(
+					tryGetFromNestedMap(
+						this.detachedNodeToField,
+						nodeId.major,
+						nodeId.minor + i,
+					) === undefined,
+					0x7ce /* Detached node ID already exists in index */,
+				);
+				setInNestedMap(this.detachedNodeToField, nodeId.major, nodeId.minor + i, root + i);
+			}
+		}
+		return root;
+	}
+
+	public encode(): string {
+		return this.codec.encode({
+			data: this.detachedNodeToField,
+			maxId: this.rootIdAllocator.getNextId(),
+		});
+	}
+
+	/**
+	 * Loads the tree index from the given string, this overrides any existing data.
+	 */
+	public loadData(data: string): void {
+		const detachedFieldIndex: {
+			data: NestedMap<Major, Minor, ForestRootId>;
+			maxId: number;
+		} = this.codec.decode(data);
+
+		this.rootIdAllocator = idAllocatorFromMaxId(
+			detachedFieldIndex.maxId,
+		) as IdAllocator<ForestRootId>;
+		this.detachedNodeToField = detachedFieldIndex.data;
+	}
+}

package/src/core/tree/detachedFieldIndexCodec.ts

@@ -0,0 +1,59 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert } from "@fluidframework/core-utils";
+import { ICodecOptions, IJsonCodec } from "../../codec";
+import { fail, forEachInNestedMap, setInNestedMap } from "../../util";
+import { ForestRootId } from "./detachedFieldIndex";
+import { Format, Versioned, version } from "./detachedFieldIndexFormat";
+import { DetachedFieldSummaryData, Major, Minor } from "./detachedFieldIndexTypes";
+
+export function makeDetachedNodeToFieldCodec({
+	jsonValidator: validator,
+}: ICodecOptions): IJsonCodec<DetachedFieldSummaryData, string> {
+	const versionedValidator = validator.compile(Versioned);
+	const formatValidator = validator.compile(Format);
+	return {
+		encode: (data: DetachedFieldSummaryData): string => {
+			const detachedNodeToFieldData: [Major, Minor, ForestRootId][] = [];
+			forEachInNestedMap(data.data, (root, key1, key2) => {
+				detachedNodeToFieldData.push([key1, key2, root]);
+			});
+			const encoded = {
+				version,
+				data: detachedNodeToFieldData,
+				maxId: data.maxId,
+			};
+			assert(
+				versionedValidator.check(encoded),
+				0x7ff /* Encoded detachedNodeToField data should be versioned */,
+			);
+			assert(formatValidator.check(encoded), 0x800 /* Encoded schema should validate */);
+			return JSON.stringify(encoded);
+		},
+		decode: (data: string): DetachedFieldSummaryData => {
+			const parsed = JSON.parse(data);
+
+			if (!versionedValidator.check(parsed)) {
+				fail("invalid serialized data: did not have a version");
+			}
+			// When more versions exist, we can switch on the version here.
+			if (parsed.version !== version) {
+				fail("Unexpected version for serialized data");
+			}
+			if (!formatValidator.check(parsed)) {
+				fail("Serialized data failed validation");
+			}
+			const map = new Map();
+			for (const [major, minor, root] of parsed.data) {
+				setInNestedMap(map, major, minor, root);
+			}
+			return {
+				data: map,
+				maxId: parsed.maxId,
+			};
+		},
+	};
+}

package/src/core/tree/detachedFieldIndexFormat.ts

@@ -0,0 +1,35 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { Static, Type } from "@sinclair/typebox";
+import { brandedNumberType } from "../../util";
+import { ForestRootId } from "./detachedFieldIndex";
+
+export const version = 1.0;
+
+// Define the Major and Minor types:
+const MajorSchema = Type.Union([Type.String(), Type.Number(), Type.Undefined()]);
+const MinorSchema = Type.Number();
+
+const ForestRootIdSchema = brandedNumberType<ForestRootId>();
+
+// Define the tuple:
+const TupleSchema = Type.Tuple([MajorSchema, MinorSchema, ForestRootIdSchema]);
+
+export const Format = Type.Object(
+	{
+		version: Type.Literal(version),
+		data: Type.Array(TupleSchema),
+		maxId: ForestRootIdSchema,
+	},
+	{ additionalProperties: false },
+);
+
+export type Format = Static<typeof Format>;
+
+export const Versioned = Type.Object({
+	version: Type.Number(),
+});
+export type Versioned = Static<typeof Versioned>;

package/src/core/tree/detachedFieldIndexTypes.ts

@@ -0,0 +1,15 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { NestedMap } from "../../util";
+import { ForestRootId } from "./detachedFieldIndex";
+
+export type Major = string | number | undefined;
+export type Minor = number;
+
+export interface DetachedFieldSummaryData {
+	data: NestedMap<Major, Minor, ForestRootId>;
+	maxId: ForestRootId;
+}