@fluidframework/tree 2.0.0-dev-rc.1.0.0.224419

package/src/events/events.ts

@@ -0,0 +1,282 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import type { IEvent } from "@fluidframework/core-interfaces";
+import { assert } from "@fluidframework/core-utils";
+import { fail, getOrCreate } from "../util";
+
+/**
+ * Convert a union of types to an intersection of those types. Useful for `TransformEvents`.
+ */
+export type UnionToIntersection<T> = (T extends any ? (k: T) => unknown : never) extends (
+	k: infer U,
+) => unknown
+	? U
+	: never;
+
+/**
+ * `true` iff the given type is an acceptable shape for an event
+ * @beta
+ */
+export type IsEvent<Event> = Event extends (...args: any[]) => any ? true : false;
+
+/**
+ * Used to specify the kinds of events emitted by an {@link ISubscribable}.
+ *
+ * @remarks
+ *
+ * Any object type is a valid {@link Events}, but only the event-like properties of that
+ * type will be included.
+ *
+ * @example
+ *
+ * ```typescript
+ * interface MyEvents {
+ *   load: (user: string, data: IUserData) => void;
+ *   error: (errorCode: number) => void;
+ * }
+ * ```
+ *
+ * @beta
+ */
+export type Events<E> = {
+	[P in (string | symbol) & keyof E as IsEvent<E[P]> extends true ? P : never]: E[P];
+};
+
+/**
+ * Converts an `Events` type (i.e. the event registry for an {@link ISubscribable}) into a type consumable
+ * by an IEventProvider from `@fluidframework/core-interfaces`.
+ * @param E - the `Events` type to transform
+ * @param Target - an optional `IEvent` type that will be merged into the result along with the transformed `E`
+ *
+ * @example
+ *
+ * ```typescript
+ * interface MyEvents {
+ *   load: (user: string, data: IUserData) => void;
+ *   error: (errorCode: number) => void;
+ * }
+ *
+ * class MySharedObject extends SharedObject<TransformEvents<MyEvents, ISharedObjectEvents>> {
+ *    // ...
+ * }
+ * ```
+ */
+export type TransformEvents<E extends Events<E>, Target extends IEvent = IEvent> = {
+	[P in keyof Events<E>]: (event: P, listener: E[P]) => void;
+} extends Record<any, infer Z>
+	? UnionToIntersection<Z> & Target
+	: never;
+
+/**
+ * An object which allows the registration of listeners so that subscribers can be notified when an event happens.
+ *
+ * {@link createEmitter} can help implement this interface via delegation.
+ * `EventEmitter` can be used as a base class to implement this via extension.
+ * @param E - All the events that this emitter supports
+ * @example
+ * ```ts
+ * type MyEventEmitter = IEventEmitter<{
+ *   load: (user: string, data: IUserData) => void;
+ *   error: (errorCode: number) => void;
+ * }>
+ * ```
+ * @beta
+ */
+export interface ISubscribable<E extends Events<E>> {
+	/**
+	 * Register an event listener.
+	 * @param eventName - the name of the event
+	 * @param listener - the handler to run when the event is fired by the emitter
+	 * @returns a function which will deregister the listener when run. This function has undefined behavior
+	 * if called more than once.
+	 */
+	on<K extends keyof Events<E>>(eventName: K, listener: E[K]): () => void;
+}
+
+/**
+ * Interface for an event emitter that can emit typed events to subscribed listeners.
+ * @internal
+ */
+export interface IEmitter<E extends Events<E>> {
+	/**
+	 * Emits an event with the specified name and arguments, notifying all subscribers by calling their registered listener functions.
+	 * @param eventName - the name of the event to fire
+	 * @param args - the arguments passed to the event listener functions
+	 */
+	emit<K extends keyof Events<E>>(eventName: K, ...args: Parameters<E[K]>): void;
+
+	/**
+	 * Emits an event with the specified name and arguments, notifying all subscribers by calling their registered listener functions.
+	 * It also collects the return values of all listeners into an array.
+	 *
+	 * Warning: This method should be used with caution. It deviates from the standard event-based integration pattern as creates substantial coupling between the emitter and its listeners.
+	 * For the majority of use-cases it is recommended to use the standard {@link IEmitter.emit} functionality.
+	 * @param eventName - the name of the event to fire
+	 * @param args - the arguments passed to the event listener functions
+	 * @returns An array of the return values of each listener, preserving the order listeners were called.
+	 */
+	emitAndCollect<K extends keyof Events<E>>(
+		eventName: K,
+		...args: Parameters<E[K]>
+	): ReturnType<E[K]>[];
+}
+
+/**
+ * Create an {@link ISubscribable} that can be instructed to emit events via the {@link IEmitter} interface.
+ *
+ * A class can delegate handling {@link ISubscribable} to the returned value while using it to emit the events.
+ * See also `EventEmitter` which be used as a base class to implement {@link ISubscribable} via extension.
+ * @internal
+ */
+export function createEmitter<E extends Events<E>>(
+	noListeners?: NoListenersCallback<E>,
+): ISubscribable<E> & IEmitter<E> & HasListeners<E> {
+	return new ComposableEventEmitter<E>(noListeners);
+}
+
+/**
+ * Called when the last listener for `eventName` is removed.
+ * Useful for determining when to clean up resources related to detecting when the event might occurs.
+ * @internal
+ */
+export type NoListenersCallback<E extends Events<E>> = (eventName: keyof Events<E>) => void;
+
+/**
+ * @internal
+ */
+export interface HasListeners<E extends Events<E>> {
+	/**
+	 * When no `eventName` is provided, returns true iff there are any listeners.
+	 *
+	 * When `eventName` is provided, returns true iff there are listeners for that event.
+	 *
+	 * @remarks
+	 * This can be used to know when its safe to cleanup data-structures which only exist to fire events for their listeners.
+	 */
+	hasListeners(eventName?: keyof Events<E>): boolean;
+}
+
+/**
+ * Provides an API for subscribing to and listening to events.
+ *
+ * @remarks Classes wishing to emit events may either extend this class or compose over it.
+ *
+ * @example Extending this class
+ *
+ * ```typescript
+ * interface MyEvents {
+ *   "loaded": () => void;
+ * }
+ *
+ * class MyClass extends EventEmitter<MyEvents> {
+ *   private load() {
+ *     this.emit("loaded");
+ *   }
+ * }
+ * ```
+ *
+ * @example Composing over this class
+ *
+ * ```typescript
+ * class MyClass implements ISubscribable<MyEvents> {
+ *   private readonly events = EventEmitter.create<MyEvents>();
+ *
+ *   private load() {
+ *     this.events.emit("loaded");
+ *   }
+ *
+ *   public on<K extends keyof MyEvents>(eventName: K, listener: MyEvents[K]): () => void {
+ *     return this.events.on(eventName, listener);
+ *   }
+ * }
+ * ```
+ */
+export class EventEmitter<E extends Events<E>> implements ISubscribable<E>, HasListeners<E> {
+	private readonly listeners = new Map<keyof E, Set<(...args: unknown[]) => any>>();
+
+	// Because this is protected and not public, calling this externally (not from a subclass) makes sending events to the constructed instance impossible.
+	// Instead, use the static `create` function to get an instance which allows emitting events.
+	protected constructor(private readonly noListeners?: NoListenersCallback<E>) {}
+
+	protected emit<K extends keyof Events<E>>(eventName: K, ...args: Parameters<E[K]>): void {
+		const listeners = this.listeners.get(eventName);
+		if (listeners !== undefined) {
+			const argArray: unknown[] = args; // TODO: Current TS (4.5.5) cannot spread `args` into `listener()`, but future versions (e.g. 4.8.4) can.
+			for (const listener of listeners.values()) {
+				listener(...argArray);
+			}
+		}
+	}
+
+	protected emitAndCollect<K extends keyof Events<E>>(
+		eventName: K,
+		...args: Parameters<E[K]>
+	): ReturnType<E[K]>[] {
+		const listeners = this.listeners.get(eventName);
+		if (listeners !== undefined) {
+			const argArray: unknown[] = args;
+			const resultArray: ReturnType<E[K]>[] = [];
+			for (const listener of listeners.values()) {
+				resultArray.push(listener(...argArray));
+			}
+			return resultArray;
+		}
+		return [];
+	}
+
+	/**
+	 * Register an event listener.
+	 * @param eventName - the name of the event
+	 * @param listener - the handler to run when the event is fired by the emitter
+	 * @returns a function which will deregister the listener when run.
+	 * This function will error if called more than once.
+	 */
+	public on<K extends keyof Events<E>>(eventName: K, listener: E[K]): () => void {
+		getOrCreate(this.listeners, eventName, () => new Set()).add(listener);
+		return () => this.off(eventName, listener);
+	}
+
+	private off<K extends keyof Events<E>>(eventName: K, listener: E[K]): void {
+		const listeners =
+			this.listeners.get(eventName) ??
+			fail(
+				"Event has no listeners. Event deregistration functions may only be invoked once.",
+			);
+		assert(
+			listeners.delete(listener),
+			0x4c1 /* Listener does not exist. Event deregistration functions may only be invoked once. */,
+		);
+		if (listeners.size === 0) {
+			this.listeners.delete(eventName);
+			this.noListeners?.(eventName);
+		}
+	}
+
+	public hasListeners(eventName?: keyof Events<E>): boolean {
+		if (eventName === undefined) {
+			return this.listeners.size !== 0;
+		}
+		return this.listeners.has(eventName);
+	}
+}
+
+// This class exposes the constructor and the `emit` method of `EventEmitter`, elevating them from protected to public
+class ComposableEventEmitter<E extends Events<E>> extends EventEmitter<E> implements IEmitter<E> {
+	public constructor(noListeners?: NoListenersCallback<E>) {
+		super(noListeners);
+	}
+
+	public override emit<K extends keyof Events<E>>(eventName: K, ...args: Parameters<E[K]>): void {
+		return super.emit(eventName, ...args);
+	}
+
+	public override emitAndCollect<K extends keyof Events<E>>(
+		eventName: K,
+		...args: Parameters<E[K]>
+	): ReturnType<E[K]>[] {
+		return super.emitAndCollect(eventName, ...args);
+	}
+}

package/src/events/fence.json

@@ -0,0 +1,5 @@
+{
+	"tags": ["events"],
+	"exports": ["index"],
+	"imports": ["util"]
+}

package/src/events/index.ts

@@ -0,0 +1,17 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export {
+	createEmitter,
+	EventEmitter,
+	Events,
+	ISubscribable,
+	IsEvent,
+	TransformEvents,
+	UnionToIntersection,
+	IEmitter,
+	NoListenersCallback,
+	HasListeners,
+} from "./events";

package/src/external-utilities/README.md

@@ -0,0 +1,5 @@
+# External Utilities
+
+This folder contains customer-facing entrypoints for pay-to-play (in the bundle-size sense) `SharedTree` features.
+
+To keep features pay-to-play, exports should not be used by `SharedTree` production code.

package/src/external-utilities/fence.json

@@ -0,0 +1,5 @@
+{
+	"tags": ["external-utilities"],
+	"exports": ["index"],
+	"imports": ["codec"]
+}

package/src/external-utilities/index.ts

@@ -0,0 +1,6 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export { typeboxValidator } from "./typeboxValidator";

package/src/external-utilities/typeboxValidator.ts

@@ -0,0 +1,30 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+// This export is documented as supported in typebox's documentation.
+// eslint-disable-next-line import/no-internal-modules
+import { TypeCompiler } from "@sinclair/typebox/compiler";
+import type { Static, TSchema } from "@sinclair/typebox";
+import type { JsonValidator } from "../codec";
+
+/**
+ * A {@link JsonValidator} implementation which uses TypeBox's JSON schema validator.
+ *
+ * @privateRemarks Take care to not reference this validator directly in SharedTree code:
+ * the intent of factoring JSON validation into an interface is to make validation more pay-to-play
+ * (i.e. a JSON validator is only included in an application's bundle if that application references it).
+ *
+ * Defining this validator in its own file also helps to ensure it is tree-shakeable.
+ *
+ * @internal
+ */
+export const typeboxValidator: JsonValidator = {
+	compile: <Schema extends TSchema>(schema: Schema) => {
+		const compiledFormat = TypeCompiler.Compile(schema);
+		return {
+			check: (data): data is Static<Schema> => compiledFormat.Check(data),
+		};
+	},
+};

package/src/feature-libraries/README.md

@@ -0,0 +1,140 @@
+# Feature Libraries
+
+Libraries which plug into the [Core Libraries](../core/README.md) to provide specializations, either for performance, or compatibility with specific tools or use-cases.
+
+This includes concrete implementations of abstractions used to parameterize `SharedTreeCore` (Like `Index` and `ChangeFamily`), or any other entry points to the `core` libraries] package (like `Forest`).
+
+`SharedTreeCore` should be usable with alternative versions of anything within this library.
+
+Some important libraries in here:
+
+-   [defaultFieldKinds](./defaultFieldKinds.ts): Definitions of FieldsKinds for schema.
+-   [modular-schema](./modular-schema/README.md): Tools for working with documents using multiple FieldKinds, including a `change-family` and `ChangeRebaser` implementations.
+-   [defaultChangeFamily](./defaultChangeFamily.ts): `modular-schema` powered `change-family` for `defaultFieldKinds`.
+-   Families of changes/edits which can be applies for various field kinds
+-   `ChangeRebaser` implementations for these change families
+-   Implementations of Forest (Currently just [object-forest](./object-forest/README.md)).
+-   `Index` summarizers, including [schemaSummarizer](./schemaSummarizer.ts), and [forestSummarizer](./forestSummarizer.ts).
+
+## Future Plans
+
+In the future, many more libraries may be added here.
+
+Currently planned libraries for the near future include: (TODO: [roadmap](../../docs/roadmap.md) should have milestones which align with this.)
+
+-   An optimized forest implementation which supports compressed chunks.
+    -   Some compressed chunk formats to use with this forest.
+-   A node identifier index for looking up nodes based on identifiers on the nodes.
+
+Longer term there are many options (these are not all planned, and is not an exhaustive list):
+
+-   More chunk formats: Optimize more common tree shapes (including identifiers, common types with dynamically sized children, sequences with variable width members like utf-8).
+-   Schema importers: We will likely want at least one tool that takes input schema files and generates the types used by the schema libraries.
+    Some options are listed here, but little thought has gone into this list:
+
+    -   typescript DSL (like [TypeBox](https://www.npmjs.com/package/@sinclair/typebox)) that supports runtime and compile time typing without code gen.
+    -   [json schema](https://json-schema.org/)
+    -   graphQL
+
+-   Schematize configurations:
+    Likely the only schematize configuration we will want is one for use with `Schema` and `EditableTree`, but other options are possible.
+    Likely will want optional fast path for when the `EditableTree` is backed by a `Forest` using shape based compression: may be able to skip some or all validation for whole subtrees this way.
+
+-   Schematize Types: A typescript type-meta-function that subsets the `EditableTree` API to be schema aware (for both reading and editing)
+
+-   Normalizers: Libraries to convert EditableTrees into representations ideal for use in specific frameworks/platforms/tool-kits. Like schematize, may have a fast-path for specific compressed data representations.
+
+    -   react
+    -   custom?
+
+## Possible System / Service Configurations
+
+Several different DDS setups can be made with these libraries.
+
+For example, alfred (or some other service) could run a `Rebaser` sufficient to detect all conflicts, marking (or discarding) conflicted ops.
+Such a service could also optionally enforce some validation or editing permissions.
+Alternatively, conflict resolution can run fully client side.
+A service could run a `Rebaser` that outputs a stream of tree diffs and saved tree snapshot summaries allowing lighter weight clients, and optionally granular read permissions.
+Services could provide additional indexes (such as history or search).
+
+Clients may or may not (depending on their `Rebaser` setup) be able to run with no actual copy of the tree in the DDS at all.
+
+TODO: Enumerate some possible configurations and their motivations. Unify this with the list of use-cases below somehow.
+
+## Use Cases
+
+TODO: this section is just a collection of unorganized notes, and needs some work to be properly integrated into this document.
+
+### Systems with existing data
+
+There are a few different usage patterns for systems with existing data:
+
+-   Import / Copy / Convert data into Fluid system. Maybe leave old one readonly.
+-   Re-implement existing APIs / services / apps using Fluid for storage (can add Fluid as option and/or migrate data some at a time (ex: document level like whiteboard is doing), or migrate whole service).
+-   Leave existing system as source of truth, and add optional Fluid front end with bidirectional sync.
+-   Use Fluid as new source of truth, but support bidirectional sync to the other system.
+-   Have an application that works with both existing data and Fluid data (may want to have compatible type/schema systems)
+
+### Misc use-case
+
+Large documents, partial view, permissions
+
+indexing / external readonly copies: update them from deltas (or maybe from change sets?)
+
+untrusted clients
+
+trusted clients, small document minimal cost (currently Fluid)
+
+offline editing -> merge
+
+readers / platforms expecting specific formats (normalize)
+
+demand for REST APIs, GraphQL. Read, maybe even modify. App specific?
+
+encode / decode libraries or mapping to json (ex: namespace prefix / escaping)
+
+pure json vs annotated json.
+
+creating extra array nodes for sequences makes it hard to use?
+
+autocad: xml
+
+readers with schema: cover most use-cases
+
+reserve some trait prefix
+
+can generate json schema for reading data that was encoded to json without schema (can support either sequence everywhere or when only not 1 item)
+
+pure json domain: contextual schema on read
+
+pure json domain to schema on write.
+
+json path
+
+graphQl.
+
+collab over existing datasets approaches:
+
+-   import: access with new api
+-   collab in-place
+-   replace system, collab with same API
+-   bidirection sync
+-   ephemeral Fluid session then write back new data.
+
+REST: https://www.redhat.com/en/topics/api/what-is-a-rest-api
+
+rest: pure model: post data update: identity challenges
+
+rest has collab issues
+
+rest service: may rebase, may enforce schema
+
+rest: document specific schema perf
+
+cross document schema caching: hash -> rest service
+
+rest: need partial view
+
+rest: post vs put. Constraint to prevent concurrent edits. can express cancel of modified. Json patch. Reference identifier to diff against.
+
+SOAP: https://www.redhat.com/en/topics/integration/whats-the-difference-between-soap-rest

package/src/feature-libraries/chunked-forest/README.md

@@ -0,0 +1,13 @@
+# Chunked Forest
+
+Implementation of Forest using chunks.
+Optimized for small memory footprint and fast loading of chunked node data.
+
+In this context, chunks are contiguous parts of the tree which get stored together in some data format.
+Multiple chunk formats will be used to optimize for different cases.
+Chunks are copy-on-write, but optimized to be mutated in place when a chunk only has a single user (detected using reference counting).
+This allows for efficient cloning of forests, without major performance overheads for non-cloning scenarios.
+
+## Status
+
+In progress: not ready for use.