@fluid-experimental/tree 0.58.2002 → 0.58.3000-61081

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluid-experimental/tree",
-  "version": "0.58.2002",
+  "version": "0.58.3000-61081",
   "description": "Distributed tree",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -38,26 +38,30 @@
     "@fluidframework/common-utils": "^0.32.1",
     "@fluidframework/container-definitions": "^0.47.1000",
     "@fluidframework/core-interfaces": "^0.42.0",
-    "@fluidframework/datastore-definitions": "^0.58.2002",
+    "@fluidframework/datastore-definitions": "0.58.3000-61081",
     "@fluidframework/protocol-definitions": "^0.1027.1000",
-    "@fluidframework/runtime-definitions": "^0.58.2002",
-    "@fluidframework/shared-object-base": "^0.58.2002",
-    "@fluidframework/telemetry-utils": "^0.58.2002",
+    "@fluidframework/runtime-definitions": "0.58.3000-61081",
+    "@fluidframework/shared-object-base": "0.58.3000-61081",
+    "@fluidframework/telemetry-utils": "0.58.3000-61081",
     "buffer": "^6.0.3",
-    "denque": "^1.5.0",
+    "denque": "^1.5.1",
     "lru-cache": "^6.0.0",
-    "sorted-btree": "^1.5.0",
+    "sorted-btree": "^1.8.0",
     "uuid": "^8.3.1"
   },
   "devDependencies": {
+    "@fluid-tools/benchmark": "^0.40.0",
     "@fluidframework/build-common": "^0.23.0",
-    "@fluidframework/container-loader": "^0.58.2002",
-    "@fluidframework/eslint-config-fluid": "^0.27.0",
-    "@fluidframework/mocha-test-setup": "^0.58.2002",
-    "@fluidframework/runtime-utils": "^0.58.2002",
-    "@fluidframework/test-drivers": "^0.58.2002",
-    "@fluidframework/test-runtime-utils": "^0.58.2002",
-    "@fluidframework/test-utils": "^0.58.2002",
+    "@fluidframework/container-loader": "0.58.3000-61081",
+    "@fluidframework/container-runtime": "0.58.3000-61081",
+    "@fluidframework/eslint-config-fluid": "^0.27.2000-59622",
+    "@fluidframework/mocha-test-setup": "0.58.3000-61081",
+    "@fluidframework/runtime-utils": "0.58.3000-61081",
+    "@fluidframework/test-driver-definitions": "0.58.3000-61081",
+    "@fluidframework/test-drivers": "0.58.3000-61081",
+    "@fluidframework/test-runtime-utils": "0.58.3000-61081",
+    "@fluidframework/test-utils": "0.58.3000-61081",
+    "@fluidframework/undo-redo": "0.58.3000-61081",
     "@microsoft/api-extractor": "^7.16.1",
     "@rushstack/eslint-config": "^2.5.1",
     "@types/lru-cache": "^5.1.0",
@@ -77,6 +81,7 @@
     "eslint-plugin-unicorn": "~40.0.0",
     "mocha": "^8.4.0",
     "nyc": "^15.0.0",
+    "prando": "^6.0.1",
     "prettier": "^2.3.1",
     "rimraf": "^2.6.2",
     "typescript": "~4.1.3"

package/src/ChangeCompression.ts

@@ -0,0 +1,159 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { DetachedSequenceId, NodeId, OpSpaceNodeId } from './Identifiers';
+import {
+	BuildInternal,
+	BuildNodeInternal,
+	ChangeInternal,
+	CompressedChangeInternal,
+	ChangeTypeInternal,
+	CompressedBuildInternal,
+	CompressedPlaceholderTree,
+	InsertInternal,
+	DetachInternal,
+	SetValueInternal,
+	ConstraintInternal,
+} from './persisted-types';
+import { ContextualizedNodeIdNormalizer } from './NodeIdUtilities';
+import { copyPropertyIfDefined, fail, Mutable, ReplaceRecursive } from './Common';
+import { TreeCompressor } from './TreeCompressor';
+import { StringInterner } from './StringInterner';
+import { convertStablePlaceIds, convertStableRangeIds } from './IdConversion';
+
+/**
+ * Encapsulates knowledge of how to compress/decompress a change into a compressed change
+ * Compression and decompression each take in a {@link StringInterner} for deduplicating shared strings.
+ * @internal
+ */
+export class ChangeCompressor {
+	public constructor(private readonly treeCompressor: TreeCompressor<DetachedSequenceId>) {}
+
+	public compress<TId extends OpSpaceNodeId>(
+		change: ChangeInternal,
+		interner: StringInterner,
+		idNormalizer: ContextualizedNodeIdNormalizer<TId>
+	): CompressedChangeInternal<TId> {
+		if (change.type === ChangeTypeInternal.Build) {
+			const source: CompressedPlaceholderTree<TId, DetachedSequenceId>[] = [];
+			for (const node of change.source) {
+				source.push(this.treeCompressor.compress(node, interner, idNormalizer));
+			}
+			const newChange: CompressedBuildInternal<TId> = {
+				destination: change.destination,
+				source,
+				type: ChangeTypeInternal.CompressedBuild,
+			};
+			return newChange;
+		} else {
+			return normalizeChange(change, (id) => idNormalizer.normalizeToOpSpace(id));
+		}
+	}
+
+	public decompress<TId extends OpSpaceNodeId>(
+		change: CompressedChangeInternal<TId>,
+		interner: StringInterner,
+		idNormalizer: ContextualizedNodeIdNormalizer<TId>
+	): ChangeInternal {
+		if (change.type === ChangeTypeInternal.CompressedBuild) {
+			const source: BuildNodeInternal[] = [];
+			for (const node of change.source) {
+				source.push(this.treeCompressor.decompress(node, interner, idNormalizer));
+			}
+			const newChange: BuildInternal = {
+				destination: change.destination,
+				source,
+				type: ChangeTypeInternal.Build,
+			};
+			return newChange;
+		} else {
+			return normalizeChange(change, (id) => idNormalizer.normalizeToSessionSpace(id));
+		}
+	}
+}
+
+function normalizeChange<From extends NodeId | OpSpaceNodeId, To extends NodeId | OpSpaceNodeId>(
+	change: ReplaceRecursive<Exclude<ChangeInternal, BuildInternal>, NodeId, From>,
+	normalizeId: (id: From) => To
+): ReplaceRecursive<Exclude<ChangeInternal, BuildInternal>, NodeId, To> {
+	switch (change.type) {
+		case ChangeTypeInternal.Insert: {
+			const insert: ReplaceRecursive<InsertInternal, NodeId, To> = {
+				source: change.source,
+				destination: convertStablePlaceIds(change.destination, normalizeId),
+				type: ChangeTypeInternal.Insert,
+			};
+			return insert;
+		}
+		case ChangeTypeInternal.Detach: {
+			const detach: ReplaceRecursive<DetachInternal, NodeId, To> = {
+				source: convertStableRangeIds(change.source, normalizeId),
+				type: ChangeTypeInternal.Detach,
+			};
+			copyPropertyIfDefined(change, detach, 'destination');
+			return detach;
+		}
+		case ChangeTypeInternal.SetValue: {
+			const setValue: ReplaceRecursive<SetValueInternal, NodeId, To> = {
+				nodeToModify: normalizeId(change.nodeToModify),
+				payload: change.payload,
+				type: ChangeTypeInternal.SetValue,
+			};
+			return setValue;
+		}
+		case ChangeTypeInternal.Constraint: {
+			const constraint: Mutable<ReplaceRecursive<ConstraintInternal, NodeId, To>> = {
+				effect: change.effect,
+				toConstrain: convertStableRangeIds(change.toConstrain, normalizeId),
+				type: ChangeTypeInternal.Constraint,
+			};
+			copyPropertyIfDefined(change, constraint, 'contentHash');
+			copyPropertyIfDefined(change, constraint, 'identityHash');
+			copyPropertyIfDefined(change, constraint, 'label');
+			copyPropertyIfDefined(change, constraint, 'length');
+			if (change.parentNode !== undefined) {
+				constraint.parentNode = normalizeId(change.parentNode);
+			}
+			return constraint;
+		}
+		default:
+			fail('unexpected change type');
+	}
+}
+
+/**
+ * Compresses the provided edit by applying `compressor` to each change and leaving other fields
+ * untouched.
+ */
+export function compressEdit<TId extends OpSpaceNodeId, TEdit extends { changes: readonly ChangeInternal[] }>(
+	compressor: ChangeCompressor,
+	interner: StringInterner,
+	idNormalizer: ContextualizedNodeIdNormalizer<TId>,
+	edit: TEdit
+): Omit<TEdit, 'changes'> & { changes: readonly CompressedChangeInternal<TId>[] } {
+	return {
+		...edit,
+		changes: edit.changes.map((change) => compressor.compress(change, interner, idNormalizer)),
+	};
+}
+
+/**
+ * Decompresses the provided edit by applying `compressor` to each change and leaving other fields
+ * untouched.
+ */
+export function decompressEdit<
+	TId extends OpSpaceNodeId,
+	TEdit extends { changes: readonly CompressedChangeInternal<TId>[] }
+>(
+	compressor: ChangeCompressor,
+	interner: StringInterner,
+	idNormalizer: ContextualizedNodeIdNormalizer<TId>,
+	edit: TEdit
+): Omit<TEdit, 'changes'> & { changes: readonly ChangeInternal[] } {
+	return {
+		...edit,
+		changes: edit.changes.map((change) => compressor.decompress(change, interner, idNormalizer)),
+	};
+}

package/src/{default-edits/PersistedTypes.ts → ChangeTypes.ts}

@@ -3,48 +3,21 @@
  * Licensed under the MIT License.
  */
 
-// All types imported into this file inherit the requirements documented below.
-// These imports are ok because they consist only of type aliases for primitive types,
-// and thus have no impact on serialization as long as the primitive type they are an alias for does not change.
-// This does mean that the various UuidString types must remain strings, and must never change the format unless the process for changing
-// persisted types (as documented below) is followed.
-import { DetachedSequenceId, NodeId, TraitLabel, UuidString } from '../Identifiers';
-import { assertNotUndefined, assert } from '../Common';
-import { Side } from '../Snapshot';
-import { EditBase, BuildNode, NodeData, Payload, TraitLocation, TreeNodeSequence } from '../generic';
+import { NodeId, TraitLabel, UuidString } from './Identifiers';
+import { assert, assertNotUndefined } from './Common';
+import { ConstraintEffect, NodeData, Payload, Side, TreeNodeSequence } from './persisted-types';
+import { TraitLocation } from './TreeView';
+import { getNodeId } from './NodeIdUtilities';
 
 /**
- * Types for Edits in Fluid Ops and Fluid summaries.
- *
- * Types describing locations in the tree are stable in the presence of other concurrent edits.
- *
- * All types are compatible with Fluid Serializable.
- *
- * These types can only be modified in ways that are both backwards and forwards compatible since they
- * are used in edits, and thus are persisted (using Fluid serialization).
- *
- * This means these types cannot be changed in any way that impacts their Fluid serialization
- * except through a very careful process:
- *
- * 1. The planned change must support all old data, and maintain the exact semantics of it.
- * This means that the change is pretty much limited to adding optional fields,
- * or making required fields optional.
- * 2. Support for the new format must be deployed to all users (This means all applications using SharedTree must do this),
- * and this deployment must be confirmed to be stable and will not be rolled back.
- * 3. Usage of the new format may start.
- *
- * Support for the old format can NEVER be removed: it must be maintained indefinably or old documents will break.
- * Because this process puts requirements on applications using shared tree,
- * step 3 should only ever be done in a Major version update,
- * and must be explicitly called out in the release notes
- * stating which versions of SharedTree are supported for documents modified by the new version.
- */
-
-/**
- * The information included in an edit.
+ * An object which may have traits with children of the given type underneath it
  * @public
  */
-export type DefaultEditBase = EditBase<Change>;
+export interface HasVariadicTraits<TChild> {
+	readonly traits?: {
+		readonly [key: string]: TChild | TreeNodeSequence<TChild> | undefined;
+	};
+}
 
 /**
  * The type of a Change
@@ -63,11 +36,31 @@ export enum ChangeType {
  *
  * `Change` objects can be conveniently constructed with the helper methods exported on a constant of the same name.
  * @example
- * Change.insert(sourceId, destination)
+ * TreeChange.insert(sourceId, destination)
  * @public
  */
 export type Change = Insert | Detach | Build | SetValue | Constraint;
 
+/**
+ * Node or sequence of Nodes for use in a Build change.
+ *
+ * Other formats for sub-sequences of Nodes can be added here, and those formats should be supported in blobs as well.
+ * Future formats will include referenced blobs containing sequences of Nodes,
+ * template based metadata and identity deduplication, and possibly compressed and binary formats.
+ * These optimized formats should also be used within tree views.
+ * @public
+ */
+export type BuildNode = BuildTreeNode | number;
+
+/**
+ * Node for use in a Build change.
+ */
+export interface BuildTreeNode extends HasVariadicTraits<BuildNode> {
+	definition: string;
+	identifier?: NodeId;
+	payload?: Payload;
+}
+
 /**
  * Constructs a sequence of nodes, associates it with the supplied ID, and stores it for use in later changes.
  * Does not modify the document.
@@ -84,8 +77,8 @@ export type Change = Insert | Detach | Build | SetValue | Constraint;
  * @public
  */
 export interface Build {
-	readonly destination: DetachedSequenceId;
-	readonly source: TreeNodeSequence<BuildNode>;
+	readonly destination: number;
+	readonly source: BuildNode | TreeNodeSequence<BuildNode>;
 	readonly type: typeof ChangeType.Build;
 }
 
@@ -96,7 +89,7 @@ export interface Build {
  */
 export interface Insert {
 	readonly destination: StablePlace;
-	readonly source: DetachedSequenceId;
+	readonly source: number;
 	readonly type: typeof ChangeType.Insert;
 }
 
@@ -108,7 +101,7 @@ export interface Insert {
  * @public
  */
 export interface Detach {
-	readonly destination?: DetachedSequenceId;
+	readonly destination?: number;
 	readonly source: StableRange;
 	readonly type: typeof ChangeType.Detach;
 }
@@ -125,6 +118,7 @@ export interface SetValue {
 	 * SetValue may use `undefined` in future API versions to mean "don't change the payload" (which is useful if e.g. other
 	 * fields are added to SetValue that can be changed without altering the payload)
 	 */
+	// eslint-disable-next-line @rushstack/no-new-null
 	readonly payload: Payload | null;
 	readonly type: typeof ChangeType.SetValue;
 }
@@ -191,47 +185,24 @@ export interface Constraint {
 	readonly type: typeof ChangeType.Constraint;
 }
 
-/**
- * What to do when a Constraint is violated.
- * @public
- */
-export enum ConstraintEffect {
-	/**
-	 * Discard Edit.
-	 */
-	InvalidAndDiscard,
-
-	/**
-	 * Discard Edit, but record metadata that application may want to try and recover this change by recreating it.
-	 * Should this be the default policy for when another (non Constraint) change is invalid?
-	 */
-	InvalidRetry,
-
-	/**
-	 * Apply the change, but flag it for possible reconsideration by the app
-	 * (applying it is better than not, but perhaps the high level logic could produce something better).
-	 */
-	ValidRetry,
-}
-
 // Note: Documentation of this constant is merged with documentation of the `Change` interface.
 /**
  * @public
  */
 export const Change = {
-	build: (source: TreeNodeSequence<BuildNode>, destination: DetachedSequenceId): Build => ({
+	build: (source: BuildNode | TreeNodeSequence<BuildNode>, destination: number): Build => ({
 		destination,
 		source,
 		type: ChangeType.Build,
 	}),
 
-	insert: (source: DetachedSequenceId, destination: StablePlace): Insert => ({
+	insert: (source: number, destination: StablePlace): Insert => ({
 		destination,
 		source,
 		type: ChangeType.Insert,
 	}),
 
-	detach: (source: StableRange, destination?: DetachedSequenceId): Detach => ({
+	detach: (source: StableRange, destination?: number): Detach => ({
 		destination,
 		source,
 		type: ChangeType.Detach,
@@ -268,43 +239,27 @@ export const Change = {
 		label,
 		type: ChangeType.Constraint,
 	}),
-};
 
-/**
- * Helper for creating a `Delete` edit.
- * @public
- */
-export const Delete = {
+	/** Helpers for making high-level composite operations */
+
 	/**
-	 * @returns a Change that deletes the supplied part of the tree.
+	 * @returns a change that deletes the supplied part of the tree.
 	 */
-	create: (stableRange: StableRange): Change => Change.detach(stableRange),
-};
+	delete: (stableRange: StableRange): Change => Change.detach(stableRange),
 
-/**
- * Helper for creating an `Insert` edit.
- * @public
- */
-export const Insert = {
 	/**
-	 * @returns a Change that inserts 'nodes' into the specified location in the tree.
+	 * @returns changes that insert 'nodes' into the specified location in the tree.
 	 */
-	create: (nodes: TreeNodeSequence<BuildNode>, destination: StablePlace): Change[] => {
-		const build = Change.build(nodes, 0 as DetachedSequenceId);
+	insertTree: (nodes: BuildNode | TreeNodeSequence<BuildNode>, destination: StablePlace): Change[] => {
+		const build = Change.build(nodes, 0);
 		return [build, Change.insert(build.destination, destination)];
 	},
-};
 
-/**
- * Helper for creating a `Move` edit.
- * @public
- */
-export const Move = {
 	/**
-	 * @returns a Change that moves the specified content to a new location in the tree.
+	 * @returns changes that moves the specified content to a new location in the tree.
 	 */
-	create: (source: StableRange, destination: StablePlace): Change[] => {
-		const detach = Change.detach(source, 0 as DetachedSequenceId);
+	move: (source: StableRange, destination: StablePlace): Change[] => {
+		const detach = Change.detach(source, 0);
 		return [detach, Change.insert(assertNotUndefined(detach.destination), destination)];
 	},
 };
@@ -317,8 +272,10 @@ export const Move = {
  * To be well formed, either `sibling` or `trait` must be defined, but not both.
  *
  * Any given insertion location can be described by two `StablePlace` objects, one with `Side.After` and one with `Side.Before`.
- * For example, in a trait containing two strings "foo" and "bar", there are 6 places corresponding to 3 places into the trait a new node
- * could be inserted: at the start, before "foo", after "foo", before "bar", after "bar", and at the end.
+ * For example, in a trait containing two strings "foo" and "bar", there are 6 different `StablePlace`s corresponding to 3 locations in the
+ * trait a new node could be inserted: at the start, before "foo", after "foo", before "bar", after "bar", and at the end.
+ * Neither of the two ways to specify the same location are considered to be after each other.
+ *
  * The anchor (`referenceSibling` or `referenceTrait`) used for a particular `StablePlace` can have an impact in collaborative scenarios.
  *
  * `StablePlace` objects can be conveniently constructed with the helper methods exported on a constant of the same name.
@@ -352,6 +309,8 @@ export interface StablePlace {
  * Specifies the range of nodes from `start` to `end` within a trait.
  * Valid iff start and end are valid and are within the same trait and the start does not occur after the end in the trait.
  *
+ * See {@link (StablePlace:interface)} for what it means for a place to be "after" another place.
+ *
  * `StableRange` objects can be conveniently constructed with the helper methods exported on a constant of the same name.
  * @example
  * StableRange.from(StablePlace.before(startNode)).to(StablePlace.after(endNode))
@@ -378,14 +337,14 @@ export const StablePlace = {
 	/**
 	 * @returns The location directly before `node`.
 	 */
-	before: (node: NodeData | NodeId): StablePlace => ({
+	before: (node: NodeData<NodeId> | NodeId): StablePlace => ({
 		side: Side.Before,
 		referenceSibling: getNodeId(node),
 	}),
 	/**
 	 * @returns The location directly after `node`.
 	 */
-	after: (node: NodeData | NodeId): StablePlace => ({ side: Side.After, referenceSibling: getNodeId(node) }),
+	after: (node: NodeData<NodeId> | NodeId): StablePlace => ({ side: Side.After, referenceSibling: getNodeId(node) }),
 	/**
 	 * @returns The location at the start of `trait`.
 	 */
@@ -420,7 +379,10 @@ export const StableRange = {
 	 * @returns a `StableRange` which contains only the provided `node`.
 	 * Both the start and end `StablePlace` objects used to anchor this `StableRange` are in terms of the passed in node.
 	 */
-	only: (node: NodeData | NodeId): StableRange => ({ start: StablePlace.before(node), end: StablePlace.after(node) }),
+	only: (node: NodeData<NodeId> | NodeId): StableRange => ({
+		start: StablePlace.before(node),
+		end: StablePlace.after(node),
+	}),
 	/**
 	 * @returns a `StableRange` which contains everything in the trait.
 	 * This is anchored using the provided `trait`, and is independent of the actual contents of the trait:
@@ -431,23 +393,3 @@ export const StableRange = {
 		end: StablePlace.atEndOf(trait),
 	}),
 };
-
-/**
- * @returns True iff the given `node` is of type NodeData.
- * @internal
- */
-export function isNodeData(node: NodeData | NodeId): node is NodeData {
-	return (node as NodeData).definition !== undefined && (node as NodeData).identifier !== undefined;
-}
-
-/**
- * @returns The NodeId for a given node or its id.
- * @internal
- */
-export function getNodeId(node: NodeData | NodeId): NodeId {
-	if (isNodeData(node)) {
-		return node.identifier;
-	} else {
-		return node;
-	}
-}