@fluidframework/tree 2.0.0-dev-rc.1.0.0.224419

package/src/feature-libraries/chunked-forest/chunkedForest.ts

@@ -0,0 +1,443 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert } from "@fluidframework/core-utils";
+import {
+	ITreeSubscriptionCursor,
+	IEditableForest,
+	TreeNavigationResult,
+	TreeStoredSchemaSubscription,
+	FieldKey,
+	DetachedField,
+	AnchorSet,
+	detachedFieldAsKey,
+	UpPath,
+	Anchor,
+	FieldAnchor,
+	ForestEvents,
+	ITreeSubscriptionCursorState,
+	rootFieldKey,
+	mapCursorField,
+	DeltaVisitor,
+	PlaceIndex,
+	Range,
+	ITreeCursorSynchronous,
+	aboveRootPlaceholder,
+	ProtoNodes,
+} from "../../core";
+import { assertValidRange, brand, fail, getOrAddEmptyToMap } from "../../util";
+import { createEmitter } from "../../events";
+import { BasicChunk, BasicChunkCursor, SiblingsOrKey } from "./basicChunk";
+import { basicChunkTree, chunkTree, IChunker } from "./chunkTree";
+import { ChunkedCursor, TreeChunk } from "./chunk";
+
+function makeRoot(): BasicChunk {
+	return new BasicChunk(aboveRootPlaceholder, new Map());
+}
+
+interface StackNode {
+	mutableChunk: BasicChunk;
+	key: FieldKey;
+}
+
+/**
+ * Implementation of IEditableForest based on copy on write chunks.
+ *
+ * This implementation focuses on performance.
+ */
+export class ChunkedForest implements IEditableForest {
+	private activeVisitor?: DeltaVisitor;
+
+	private readonly events = createEmitter<ForestEvents>();
+
+	/**
+	 * @param roots - dummy node above the root under which detached fields are stored. All content of the forest is reachable from this.
+	 * @param schema - schema which all content in this forest is assumed to comply with.
+	 * @param chunker - Chunking policy. TODO: dispose of this when forest is disposed.
+	 * @param anchors - anchorSet used to track location in this forest across changes. Callers of applyDelta must ensure this is updated accordingly.
+	 */
+	public constructor(
+		public roots: BasicChunk,
+		public readonly schema: TreeStoredSchemaSubscription,
+		public readonly chunker: IChunker,
+		public readonly anchors: AnchorSet = new AnchorSet(),
+	) {}
+
+	public get isEmpty(): boolean {
+		return this.roots.fields.size === 0;
+	}
+
+	public on<K extends keyof ForestEvents>(eventName: K, listener: ForestEvents[K]): () => void {
+		return this.events.on(eventName, listener);
+	}
+
+	public clone(schema: TreeStoredSchemaSubscription, anchors: AnchorSet): ChunkedForest {
+		this.roots.referenceAdded();
+		return new ChunkedForest(this.roots, schema, this.chunker.clone(schema), anchors);
+	}
+
+	public forgetAnchor(anchor: Anchor): void {
+		this.anchors.forget(anchor);
+	}
+
+	public acquireVisitor(): DeltaVisitor {
+		assert(
+			this.activeVisitor === undefined,
+			0x76a /* Must release existing visitor before acquiring another */,
+		);
+		this.events.emit("beforeChange");
+
+		if (this.roots.isShared()) {
+			this.roots = this.roots.clone();
+		}
+
+		const visitor = {
+			forest: this,
+			// Current location in the tree, as a non-shared BasicChunk (TODO: support in-place modification of other chunk formats when possible).
+			// Start above root detached sequences.
+			mutableChunkStack: [] as StackNode[],
+			mutableChunk: this.roots as BasicChunk | undefined,
+			getParent() {
+				assert(
+					this.mutableChunkStack.length > 0,
+					0x532 /* invalid access to root's parent */,
+				);
+				return this.mutableChunkStack[this.mutableChunkStack.length - 1];
+			},
+			free(): void {
+				this.mutableChunk = undefined;
+				this.mutableChunkStack.length = 0;
+				assert(
+					this.forest.activeVisitor !== undefined,
+					0x76b /* Multiple free calls for same visitor */,
+				);
+				this.forest.activeVisitor = undefined;
+				this.forest.events.emit("afterChange");
+			},
+			destroy(detachedField: FieldKey, count: number): void {
+				this.forest.roots.fields.delete(detachedField);
+			},
+			create(content: ProtoNodes, destination: FieldKey): void {
+				const chunks: TreeChunk[] = content.map((c) => chunkTree(c, this.forest.chunker));
+				this.forest.roots.fields.set(destination, chunks);
+			},
+			attach(source: FieldKey, count: number, destination: PlaceIndex): void {
+				this.attachEdit(source, count, destination);
+			},
+			detach(source: Range, destination: FieldKey): void {
+				this.detachEdit(source, destination);
+			},
+			/**
+			 * Attaches the range into the current field by transferring it from the given source path.
+			 * Does not invalidate dependents.
+			 * @param source - The the range to be attached.
+			 * @param destination - The index in the current field at which to attach the content.
+			 */
+			attachEdit(source: FieldKey, count: number, destination: PlaceIndex): void {
+				const sourceField = this.forest.roots.fields.get(source) ?? [];
+				this.forest.roots.fields.delete(source);
+				if (sourceField.length === 0) {
+					return; // Prevent creating 0 sized fields when inserting empty into empty.
+				}
+
+				const parent = this.getParent();
+				const destinationField = getOrAddEmptyToMap(parent.mutableChunk.fields, parent.key);
+				// TODO: this will fail for very large moves due to argument limits.
+				destinationField.splice(destination, 0, ...sourceField);
+			},
+			/**
+			 * Detaches the range from the current field and transfers it to the given destination if any.
+			 * Does not invalidate dependents.
+			 * @param source - The bounds of the range to be detached from the current field.
+			 * @param destination - If specified, the destination to transfer the detached range to.
+			 * If not specified, the detached range is destroyed.
+			 */
+			detachEdit(source: Range, destination: FieldKey | undefined): void {
+				const parent = this.getParent();
+				const sourceField = parent.mutableChunk.fields.get(parent.key) ?? [];
+
+				assertValidRange(source, sourceField);
+				const newField = sourceField.splice(source.start, source.end - source.start);
+
+				if (destination !== undefined) {
+					assert(
+						!this.forest.roots.fields.has(destination),
+						0x7af /* Destination must be a new empty detached field */,
+					);
+					if (newField.length > 0) {
+						this.forest.roots.fields.set(destination, newField);
+					}
+				} else {
+					for (const child of newField) {
+						child.referenceRemoved();
+					}
+				}
+				// This check is performed after the transfer to ensure that the field is not removed in scenarios
+				// where the source and destination are the same.
+				if (sourceField.length === 0) {
+					parent.mutableChunk.fields.delete(parent.key);
+				}
+			},
+			replace(
+				newContentSource: FieldKey,
+				range: Range,
+				oldContentDestination: FieldKey,
+			): void {
+				assert(
+					newContentSource !== oldContentDestination,
+					0x7b0 /* Replace detached source field and detached destination field must be different */,
+				);
+				this.detachEdit(range, oldContentDestination);
+				this.attachEdit(newContentSource, range.end - range.start, range.start);
+			},
+			enterNode(index: number): void {
+				assert(this.mutableChunk === undefined, 0x535 /* should be in field */);
+				const parent = this.getParent();
+				const chunks =
+					parent.mutableChunk.fields.get(parent.key) ?? fail("missing edited field");
+				let indexWithinChunk = index;
+				let indexOfChunk = 0;
+				while (indexWithinChunk >= chunks[indexOfChunk].topLevelLength) {
+					indexWithinChunk -= chunks[indexOfChunk].topLevelLength;
+					indexOfChunk++;
+					if (indexOfChunk === chunks.length) {
+						fail("missing edited node");
+					}
+				}
+				let found = chunks[indexOfChunk];
+				if (!(found instanceof BasicChunk)) {
+					// TODO:Perf: support in place editing of other chunk formats when possible:
+					// 1. Support updating values in uniform chunks.
+					// 2. Support traversing sequence chunks.
+					//
+					// Maybe build path when visitor navigates then lazily sync to chunk tree when editing?
+					const newChunks = mapCursorField(found.cursor(), (cursor) =>
+						basicChunkTree(cursor, this.forest.chunker),
+					);
+					// TODO: this could fail for really long chunks being split (due to argument count limits).
+					// Current implementations of chunks shouldn't ever be that long, but it could be an issue if they get bigger.
+					chunks.splice(indexOfChunk, 1, ...newChunks);
+					found.referenceRemoved();
+
+					found = newChunks[indexWithinChunk];
+				}
+				assert(found instanceof BasicChunk, 0x536 /* chunk should have been normalized */);
+				if (found.isShared()) {
+					this.mutableChunk = chunks[indexOfChunk] = found.clone();
+					found.referenceRemoved();
+				} else {
+					this.mutableChunk = found;
+				}
+			},
+			exitNode(index: number): void {
+				assert(this.mutableChunk !== undefined, 0x537 /* should be in node */);
+				this.mutableChunk = undefined;
+			},
+			enterField(key: FieldKey): void {
+				assert(this.mutableChunk !== undefined, 0x538 /* should be in node */);
+				this.mutableChunkStack.push({ key, mutableChunk: this.mutableChunk });
+				this.mutableChunk = undefined;
+			},
+			exitField(key: FieldKey): void {
+				const top = this.mutableChunkStack.pop() ?? fail("should not be at root");
+				assert(this.mutableChunk === undefined, 0x539 /* should be in field */);
+				this.mutableChunk = top.mutableChunk;
+			},
+		};
+		this.activeVisitor = visitor;
+		return visitor;
+	}
+
+	private nextDetachedFieldIdentifier = 0;
+	public newDetachedField(): DetachedField {
+		const field: DetachedField = brand(String(this.nextDetachedFieldIdentifier));
+		assert(
+			!this.roots.fields.has(detachedFieldAsKey(field)),
+			0x53a /* new field must not already exist */,
+		);
+		this.nextDetachedFieldIdentifier += 1;
+		return field;
+	}
+
+	public allocateCursor(): Cursor {
+		return new Cursor(
+			this,
+			ITreeSubscriptionCursorState.Cleared,
+			[],
+			[],
+			[],
+			[],
+			[],
+			[],
+			0,
+			0,
+			0,
+			undefined,
+		);
+	}
+
+	public tryMoveCursorToNode(
+		destination: Anchor,
+		cursorToMove: ITreeSubscriptionCursor,
+	): TreeNavigationResult {
+		const path = this.anchors.locate(destination);
+		if (path === undefined) {
+			return TreeNavigationResult.NotFound;
+		}
+		this.moveCursorToPath(path, cursorToMove);
+		return TreeNavigationResult.Ok;
+	}
+
+	public tryMoveCursorToField(
+		destination: FieldAnchor,
+		cursorToMove: ITreeSubscriptionCursor,
+	): TreeNavigationResult {
+		assert(
+			cursorToMove instanceof Cursor,
+			0x53b /* ChunkedForest must only be given its own Cursor type */,
+		);
+		if (destination.parent === undefined) {
+			cursorToMove.setToDetachedSequence(destination.fieldKey);
+			return TreeNavigationResult.Ok;
+		}
+		const result = this.tryMoveCursorToNode(destination.parent, cursorToMove);
+		if (result !== TreeNavigationResult.Ok) {
+			return result;
+		}
+
+		cursorToMove.enterField(destination.fieldKey);
+		return TreeNavigationResult.Ok;
+	}
+
+	public moveCursorToPath(destination: UpPath, cursorToMove: ITreeSubscriptionCursor): void {
+		assert(
+			cursorToMove instanceof Cursor,
+			0x53c /* ChunkedForest must only be given its own Cursor type */,
+		);
+		assert(
+			cursorToMove.forest === this,
+			0x53d /* ChunkedForest must only be given its own Cursor */,
+		);
+
+		const indexStack: number[] = [];
+		const keyStack: FieldKey[] = [];
+
+		let path: UpPath | undefined = destination;
+		while (path !== undefined) {
+			indexStack.push(path.parentIndex);
+			keyStack.push(path.parentField);
+			path = path.parent;
+		}
+		cursorToMove.clear();
+		while (keyStack.length > 0) {
+			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+			const key = keyStack.pop()!;
+			if (cursorToMove.state === ITreeSubscriptionCursorState.Cleared) {
+				cursorToMove.setToDetachedSequence(key);
+				cursorToMove.state = ITreeSubscriptionCursorState.Current;
+			} else {
+				cursorToMove.enterField(key);
+			}
+
+			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+			cursorToMove.enterNode(indexStack.pop()!);
+		}
+	}
+
+	public getCursorAboveDetachedFields(): ITreeCursorSynchronous {
+		const rootCursor = this.roots.cursor();
+		rootCursor.enterNode(0);
+		return rootCursor;
+	}
+}
+
+class Cursor extends BasicChunkCursor implements ITreeSubscriptionCursor {
+	public constructor(
+		public readonly forest: ChunkedForest,
+		public state: ITreeSubscriptionCursorState,
+		root: readonly TreeChunk[],
+		siblingStack: SiblingsOrKey[],
+		indexStack: number[],
+		indexOfChunkStack: number[],
+		indexWithinChunkStack: number[],
+		siblings: SiblingsOrKey,
+		index: number,
+		indexOfChunk: number,
+		indexWithinChunk: number,
+		nestedCursor: ChunkedCursor | undefined,
+	) {
+		super(
+			root,
+			siblingStack,
+			indexStack,
+			indexOfChunkStack,
+			indexWithinChunkStack,
+			siblings,
+			index,
+			indexOfChunk,
+			indexWithinChunk,
+			nestedCursor,
+		);
+	}
+
+	public setToDetachedSequence(key: FieldKey): void {
+		this.root = (this.forest.roots.fields.get(key) ?? []) as BasicChunk[];
+		this.siblingStack.length = 0;
+		this.indexStack.length = 0;
+		this.indexOfChunkStack.length = 0;
+		this.indexWithinChunkStack.length = 0;
+		this.siblings = [key];
+		this.index = 0;
+		this.indexOfChunk = 0;
+		this.indexWithinChunk = 0;
+		this.nestedCursor = undefined;
+	}
+
+	public override fork(): Cursor {
+		// Siblings arrays are not modified during navigation and do not need be be copied.
+		// This allows this copy to be shallow, and `this.siblings` below to not be copied as all.
+		return new Cursor(
+			this.forest,
+			this.state,
+			this.root,
+			[...this.siblingStack],
+			[...this.indexStack],
+			[...this.indexOfChunkStack],
+			[...this.indexWithinChunkStack],
+			this.siblings,
+			this.index,
+			this.indexOfChunk,
+			this.indexWithinChunk,
+			this.nestedCursor?.fork(),
+		);
+	}
+
+	public buildFieldAnchor(): FieldAnchor {
+		const path = this.getFieldPath();
+		const anchor =
+			path.parent === undefined ? undefined : this.forest.anchors.track(path.parent);
+		return { parent: anchor, fieldKey: path.field };
+	}
+
+	public free(): void {
+		this.state = ITreeSubscriptionCursorState.Freed;
+	}
+
+	public buildAnchor(): Anchor {
+		return this.forest.anchors.track(this.getPath());
+	}
+
+	public clear(): void {
+		this.state = ITreeSubscriptionCursorState.Cleared;
+		this.setToDetachedSequence(rootFieldKey);
+	}
+}
+
+/**
+ * @returns an implementation of {@link IEditableForest} with no data or schema.
+ */
+export function buildChunkedForest(chunker: IChunker, anchors?: AnchorSet): ChunkedForest {
+	return new ChunkedForest(makeRoot(), chunker.schema, chunker, anchors);
+}

package/src/feature-libraries/chunked-forest/codec/README.md

@@ -0,0 +1,55 @@
+# Chunked Forest Codec
+
+Provides a serializable encoding of trees which is optimized for size as well as fast encoding and decoding.
+
+## Encoding Optimizations
+
+Two main algorithms are used:
+
+-   Shape based encoding: Trees are split into "Shape" information and "Data".
+    "Shape" can (but does not have to) contain information like type, which fields exist, and how many nodes are in them and the shapes of those nodes.
+-   [Dictionary Coding](https://go-compression.github.io/algorithms/dictionary/) using `Counter` from [chunkCodexUtilities.ts](./chunkCodecUtilities.ts)
+    is applied to Identifiers (such a type names and field keys) and Shapes.
+
+The main challenge is deciding what shapes to use: if too much information is captured in the shape, there will be more shapes which take up more space,
+but if too little information is captured in the shapes, the size of the encoded tree will be large.
+
+Balancing this involves making strategic choices about what to include in the shape, and what not to.
+A few implementations are included which make different choices here:
+
+-   [uncompressedEncode.ts](uncompressedEncode.ts): the simplest implementation, which captures nothing in the shape and does no Dictionary Coding of identifiers.
+    This makes for more human readable data as well as a simple reference implementation.
+-   [compressedEncode.ts](compressedEncode.ts): some utilities for actual compressed encoding, but does not contain any advanced strategies (instead they must be provided to it via the `EncoderCache`)
+-   [schemaBasedEncoding.ts](schemaBasedEncoding.ts): a strategy for compressedEncode using schema to infer commonly used shapes.
+
+### Future Optimizations
+
+One nice thing about this encoding scheme is there is room to improve compression without changing the format (and thus without changing the decoder).
+There are two main kinds of optimizations of this type:
+
+-   Improve selection of shapes to be based on more than just schema.
+    For example generate multiple shapes for the same schema, or generate a shape even more specific than the schema if the permitted degrees of freedom are not used.
+-   Improve encoding performance by leveraging existing chunking of the input data.
+    For example uniform chunks could be encoded as nested arrays using their existing data array.
+
+There is also the possibility to performance optimize decode to generate pre-chunked data, using the shapes to infer when uniform should be created,
+and using nested arrays for uniform chunk data without copying them.
+
+Potential optimizations involving breaking format changes:
+
+-   avoid having to put shapes in all the encoded trees, and instead refer to shapes stored elsewhere.
+    This could leverage schema to imply shapes, use shapes referenced earlier in the session (similar to how session identifier allocation is planned to work), or use some other mechanism, such as a shape cache stored in the document or references to blobs containing previously used shapes (possibly from summaries).
+-   binary format
+-   adding more sophisticated shapes: (would also require improvements to shape selection strategy to use these)
+    -   adding delta encoding so shapes can generate sequential values such as node identifier UUIDs implicitly.
+    -   allow use of id compressor to specify UUIDs using shorter formats. Like external shape references, this would make the format not self contained and thus require session state to encode and decode.
+
+## Code Layering
+
+The files in this project are layered (ordered such that dependencies are only take in one direction) as follows:
+
+-   "generic" vs non-generic: "generic" refers to code which does not depend on specific kinds of shapes:
+    format changes which add or remove kinds of encoded shapes should not impact these files.
+-   "format" vs other: "format" files define the actual persisted data format.
+    Note that choices like how the "data" array contents are structured are considered to be defined in the format files as part of the shapes there, despite them not being captured by the schema in those files:
+    these details should be fully defined by documentation in the format files which must not depend on other files.

package/src/feature-libraries/chunked-forest/codec/chunkCodecUtilities.ts

@@ -0,0 +1,183 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert } from "@fluidframework/core-utils";
+import { assertValidIndex } from "../../../util";
+import { TreeChunk } from "../chunk";
+import { FluidSerializableReadOnly, assertAllowedValue } from "../../valueUtilities";
+import { TreeValue } from "../../../core";
+
+/**
+ * Utilities related to chunk encoding and decoding that do not depend on specific chunk types or formats.
+ */
+
+/**
+ * Counts usages of some `T`, then generates tables for replacing those `T` values with small numbers.
+ *
+ * Can be used to deduplicate objects when encoding via {@link https://go-compression.github.io/algorithms/dictionary/ | Dictionary Coding}.
+ */
+export class Counter<T> {
+	private readonly counts: Map<T, number> = new Map();
+	public add(t: T, count = 1) {
+		const old = this.counts.get(t) ?? 0;
+		this.counts.set(t, old + count);
+	}
+
+	/**
+	 * Generate a table which can be used to deduplicate the items added via `add`.
+	 * Table is sorted from most used to least used so that more commonly added items will get smaller indexes.
+	 *
+	 * @param filter - determines which items should be included in the table.
+	 */
+	public buildTable(filter: CounterFilter<T> = () => true): DeduplicationTable<T> {
+		const data: T[] = [...this.counts.keys()];
+		// Sort in descending order by count, giving priority (smaller indexes) to more commonly used values.
+		data.sort((a, b) => (this.counts.get(b) ?? 0) - (this.counts.get(a) ?? 0));
+		// Since the index needed is the output index not the input one, data.filter doesn't quite work here.
+		const filtered: T[] = [];
+		for (const t of data) {
+			const include = filter(t, filtered.length, this.counts.get(t) ?? 0);
+			if (include) {
+				filtered.push(t);
+			}
+		}
+		return {
+			indexToValue: filtered,
+			valueToIndex: new Map(filtered.map((t, index) => [t, index])),
+		};
+	}
+}
+
+/**
+ * Returns true iff a given item `t` (of which there are `count` usages) is worth substituting with the number `value`.
+ */
+export type CounterFilter<T> = (t: T, value: number, count: number) => boolean;
+
+/**
+ * Table for dictionary compression.
+ * See {@link Counter}.
+ */
+export interface DeduplicationTable<T> {
+	/**
+	 * Lookup table derived from indexToValue,
+	 */
+	readonly valueToIndex: ReadonlyMap<T, number>;
+	/**
+	 * Deduplication table: lookup values by their index.
+	 *
+	 * This is the portion of the table that should be included in persisted data.
+	 */
+	readonly indexToValue: readonly T[];
+}
+
+/**
+ * Filter for use with Counter.buildTable that minimizes the size of the produced JSON.
+ *
+ * @remarks
+ * This minimizes the number of UTF-16 code units in the produced JSON.
+ * This is not ideal: minimizing UTF-8 bytes would be better.
+ * For ascii text these are the same, and in worst cases they correlate pretty well (accurate to within a small constant factor).
+ * This is thus good enough for a heuristic.
+ *
+ * Also the use of this filter doesn't take into account the length cost of keeping a value in the table possibly
+ * making some of the entries at higher indexes in the table get a larger number.
+ * For example, if keeping an entry at index 9 saves one character this will keep it,
+ * even if that means the next entry (at index 10) loses more than one character due to 10 being two digits instead of the one digit it would have been at index 9.
+ * This means that this filter is not guaranteed to be optimal, but it should always be quite close.
+ */
+export function jsonMinimizingFilter(s: string, value: number, count: number): boolean {
+	// The most practical way to compute how long s will be with quoting and escaping
+	// is to actually quote and escape it with JSON.stringify:
+	const quotedAndEscaped = JSON.stringify(s);
+	// Account for count instances of value, and one instance of a `,s` which would go in the table.
+	const lengthUsingTable = String(value).length * count + quotedAndEscaped.length + 1;
+	// Account for count instances of s.
+	const lengthWithoutTable = quotedAndEscaped.length * count;
+	// Break ties to not use the table to avoid needing a lookup,
+	// and save table entries for other uses.
+	return lengthUsingTable < lengthWithoutTable;
+}
+
+/**
+ * Read from an array, but error if index is not valid.
+ */
+export function getChecked<T>(data: readonly T[], index: number): T {
+	assertValidIndex(index, data);
+	return data[index];
+}
+
+/**
+ * A readable stream.
+ */
+export interface StreamCursor {
+	/**
+	 * The data to read.
+	 */
+	readonly data: readonly FluidSerializableReadOnly[];
+	/**
+	 * Location in the data.
+	 */
+	offset: number;
+}
+
+/**
+ * Read one item from the stream, advancing the stream offset.
+ */
+export function readStream(stream: StreamCursor): FluidSerializableReadOnly {
+	const content = getChecked(stream.data, stream.offset);
+	stream.offset++;
+	return content;
+}
+
+/**
+ * Read one number from the stream, advancing the stream offset.
+ */
+export function readStreamNumber(stream: StreamCursor): number {
+	const content = readStream(stream);
+	assert(typeof content === "number", 0x730 /* expected number in stream */);
+	return content;
+}
+
+/**
+ * Read one boolean from the stream, advancing the stream offset.
+ */
+export function readStreamBoolean(stream: StreamCursor): boolean {
+	const content = readStream(stream);
+	assert(typeof content === "boolean", 0x731 /* expected boolean in stream */);
+	return content;
+}
+
+/**
+ * Read one TreeValue from the stream, advancing the stream offset.
+ */
+export function readStreamValue(stream: StreamCursor): TreeValue {
+	const content = readStream(stream);
+	assertAllowedValue(content);
+	return content;
+}
+
+/**
+ * Read one nested array from the stream, advancing the stream offset.
+ *
+ * @returns the nested array as a stream.
+ */
+export function readStreamStream(stream: StreamCursor): StreamCursor {
+	const content = readStream(stream);
+	assert(Array.isArray(content), 0x732 /* expected Array in stream */);
+	return { data: content, offset: 0 };
+}
+
+/**
+ * Decodes a chunk within a FieldBatch.
+ */
+export interface ChunkDecoder {
+	/**
+	 * Read from stream, updating the offset.
+	 *
+	 * @returns a TreeChunk made from the data from `stream`.
+	 * This chunk is allowed to reference/take ownership of content it reads from the stream.
+	 */
+	decode(decoders: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk;
+}