@fluidframework/tree 2.53.0 → 2.60.0

package/src/feature-libraries/chunked-forest/codec/schemaBasedEncode.ts

@@ -16,6 +16,7 @@ import {
 	ValueSchema,
 	Multiplicity,
 	identifierFieldKindIdentifier,
+	type FieldKey,
 } from "../../../core/index.js";
 import type { FullSchemaPolicy } from "../../modular-schema/index.js";
 
@@ -28,13 +29,19 @@ import {
 	anyNodeEncoder,
 	asFieldEncoder,
 	compressedEncode,
+	incrementalFieldEncoder,
 } from "./compressedEncode.js";
 import type { FieldBatch } from "./fieldBatch.js";
 import { type EncodedFieldBatch, type EncodedValueShape, SpecialField } from "./format.js";
+import type { IncrementalEncoder } from "./codecs.js";
 import { NodeShapeBasedEncoder } from "./nodeEncoder.js";
 
 /**
  * Encode data from `fieldBatch` in into an `EncodedChunk`.
+ * @remarks
+ * If `incrementalEncoder` is provided,
+ * fields that support incremental encoding will encode their chunks separately via the `incrementalEncoder`.
+ * See {@link IncrementalEncoder} for more details.
  *
  * Optimized for encoded size and encoding performance.
  * TODO: This function should eventually also take in the root FieldSchema to more efficiently compress the nodes.
@@ -44,22 +51,28 @@ export function schemaCompressedEncode(
 	policy: FullSchemaPolicy,
 	fieldBatch: FieldBatch,
 	idCompressor: IIdCompressor,
+	incrementalEncoder?: IncrementalEncoder,
 ): EncodedFieldBatch {
-	return compressedEncode(fieldBatch, buildContext(schema, policy, idCompressor));
+	return compressedEncode(
+		fieldBatch,
+		buildContext(schema, policy, idCompressor, incrementalEncoder),
+	);
 }
 
 export function buildContext(
 	storedSchema: StoredSchemaCollection,
 	policy: FullSchemaPolicy,
 	idCompressor: IIdCompressor,
+	incrementalEncoder?: IncrementalEncoder,
 ): EncoderContext {
 	const context: EncoderContext = new EncoderContext(
 		(fieldBuilder: FieldEncodeBuilder, schemaName: TreeNodeSchemaIdentifier) =>
-			getNodeEncoder(fieldBuilder, storedSchema, schemaName),
+			getNodeEncoder(fieldBuilder, storedSchema, schemaName, incrementalEncoder),
 		(nodeBuilder: NodeEncodeBuilder, fieldSchema: TreeFieldStoredSchema) =>
 			getFieldEncoder(nodeBuilder, fieldSchema, context, storedSchema),
 		policy.fieldKinds,
 		idCompressor,
+		incrementalEncoder,
 	);
 	return context;
 }
@@ -111,6 +124,7 @@ export function getNodeEncoder(
 	fieldBuilder: FieldEncodeBuilder,
 	storedSchema: StoredSchemaCollection,
 	schemaName: TreeNodeSchemaIdentifier,
+	incrementalEncoder?: IncrementalEncoder,
 ): NodeShapeBasedEncoder {
 	const schema =
 		storedSchema.nodeSchema.get(schemaName) ?? fail(0xb53 /* missing node schema */);
@@ -120,9 +134,20 @@ export function getNodeEncoder(
 		// consider moving some optional and sequence fields to extra fields if they are commonly empty
 		// to reduce encoded size.
 
+		const shouldEncodeFieldIncrementallyLocal = (
+			nodeIdentifier: TreeNodeSchemaIdentifier,
+			fieldKey: FieldKey,
+		): boolean =>
+			incrementalEncoder?.shouldEncodeFieldIncrementally(nodeIdentifier, fieldKey) ?? false;
 		const objectNodeFields: KeyedFieldEncoder[] = [];
 		for (const [key, field] of schema.objectNodeFields ?? []) {
-			objectNodeFields.push({ key, encoder: fieldBuilder.fieldEncoderFromSchema(field) });
+			const fieldEncoder = shouldEncodeFieldIncrementallyLocal(schemaName, key)
+				? incrementalFieldEncoder
+				: fieldBuilder.fieldEncoderFromSchema(field);
+			objectNodeFields.push({
+				key,
+				encoder: fieldEncoder,
+			});
 		}
 
 		const shape = new NodeShapeBasedEncoder(schemaName, false, objectNodeFields, undefined);

package/src/feature-libraries/chunked-forest/index.ts

@@ -21,5 +21,7 @@ export {
 	makeFieldBatchCodec,
 	type FieldBatchEncodingContext,
 	fluidVersionToFieldBatchCodecWriteVersion,
+	type IncrementalEncoderDecoder,
+	type ChunkReferenceId,
 } from "./codec/index.js";
 export { emptyChunk } from "./emptyChunk.js";

package/src/feature-libraries/forest-summary/codec.ts

@@ -26,6 +26,9 @@ export function makeForestSummarizerCodec(
 	fieldBatchCodec: FieldBatchCodec,
 ): ForestCodec {
 	const inner = fieldBatchCodec;
+	// TODO: AB#41865
+	// This needs to be updated to support multiple versions.
+	// The second version will be used to enable incremental summarization.
 	return makeVersionedValidatedCodec(options, new Set([1]), Format, {
 		encode: (data: FieldSet, context: FieldBatchEncodingContext): Format => {
 			const keys: FieldKey[] = [];

package/src/feature-libraries/forest-summary/forestSummarizer.ts

@@ -12,7 +12,6 @@ import type {
 	ISummaryTreeWithStats,
 	ITelemetryContext,
 } from "@fluidframework/runtime-definitions/internal";
-import { createSingleBlobSummary } from "@fluidframework/shared-object-base/internal";
 
 import type { CodecWriteOptions } from "../../codec/index.js";
 import {
@@ -24,6 +23,7 @@ import {
 	type ITreeSubscriptionCursor,
 	type RevisionTagCodec,
 	TreeNavigationResult,
+	type TreeNodeSchemaIdentifier,
 	applyDelta,
 	forEachField,
 	makeDetachedFieldIndex,
@@ -33,25 +33,37 @@ import type {
 	SummaryElementParser,
 	SummaryElementStringifier,
 } from "../../shared-tree-core/index.js";
-import { idAllocatorFromMaxId } from "../../util/index.js";
-import { chunkFieldSingle, defaultChunkPolicy } from "../chunked-forest/index.js";
+import { idAllocatorFromMaxId, type JsonCompatible } from "../../util/index.js";
+// eslint-disable-next-line import/no-internal-modules
+import { chunkFieldSingle, defaultChunkPolicy } from "../chunked-forest/chunkTree.js";
 import type { FieldBatchCodec, FieldBatchEncodingContext } from "../chunked-forest/index.js";
 
 import { type ForestCodec, makeForestSummarizerCodec } from "./codec.js";
-import type { Format } from "./format.js";
+import {
+	ForestIncrementalSummaryBehavior,
+	ForestIncrementalSummaryBuilder,
+	forestSummaryContentKey,
+} from "./incrementalSummaryBuilder.js";
+import { TreeCompressionStrategyExtended } from "../treeCompressionUtils.js";
+import type { IFluidHandle } from "@fluidframework/core-interfaces";
+
 /**
- * The storage key for the blob in the summary containing tree data
+ * The key for the tree that contains the overall forest's summary tree.
+ * This tree is added by the parent of the forest summarizer.
+ * See {@link ForestIncrementalSummaryBuilder} for details on the summary structure.
  */
-export const treeBlobKey = "ForestTree";
+export const forestSummaryKey = "Forest";
 
 /**
  * Provides methods for summarizing and loading a forest.
  */
 export class ForestSummarizer implements Summarizable {
-	public readonly key = "Forest";
+	public readonly key = forestSummaryKey;
 
 	private readonly codec: ForestCodec;
 
+	private readonly incrementalSummaryBuilder: ForestIncrementalSummaryBuilder;
+
 	/**
 	 * @param encoderContext - The schema if provided here must be mutated by the caller to keep it up to date.
 	 */
@@ -62,19 +74,45 @@ export class ForestSummarizer implements Summarizable {
 		private readonly encoderContext: FieldBatchEncodingContext,
 		options: CodecWriteOptions,
 		private readonly idCompressor: IIdCompressor,
+		shouldEncodeFieldIncrementally?: (
+			nodeIdentifier: TreeNodeSchemaIdentifier,
+			fieldKey: FieldKey,
+		) => boolean,
 	) {
 		// TODO: this should take in CodecWriteOptions, and use it to pick the write version.
 		this.codec = makeForestSummarizerCodec(options, fieldBatchCodec);
+
+		const shouldEncodeFieldIncrementallyLocal = (
+			nodeIdentifier: TreeNodeSchemaIdentifier,
+			fieldKey: FieldKey,
+		): boolean => shouldEncodeFieldIncrementally?.(nodeIdentifier, fieldKey) ?? false;
+		this.incrementalSummaryBuilder = new ForestIncrementalSummaryBuilder(
+			encoderContext.encodeType ===
+				TreeCompressionStrategyExtended.CompressedIncremental /* enableIncrementalSummary */,
+			(cursor: ITreeCursorSynchronous) => this.forest.chunkField(cursor),
+			shouldEncodeFieldIncrementallyLocal,
+		);
 	}
 
 	/**
-	 * Synchronous monolithic summarization of tree content.
+	 * Summarization of the forest's tree content.
+	 * @returns a summary tree containing the forest's tree content.
+	 * @remarks
+	 * If incremental summary is disabled, all the content will be added to a single summary blob.
+	 * If incremental summary is enabled, the summary will be a tree.
+	 * See {@link ForestIncrementalSummaryBuilder} for details of what this tree looks like.
 	 *
 	 * TODO: when perf matters, this should be replaced with a chunked async version using a binary format.
-	 *
-	 * @returns a snapshot of the forest's tree as a string.
 	 */
-	private getTreeString(stringify: SummaryElementStringifier): string {
+	public summarize(props: {
+		stringify: SummaryElementStringifier;
+		fullTree?: boolean;
+		trackState?: boolean;
+		telemetryContext?: ITelemetryContext;
+		incrementalSummaryContext?: IExperimentalIncrementalSummaryContext;
+	}): ISummaryTreeWithStats {
+		const { stringify, fullTree = false, incrementalSummaryContext } = props;
+
 		const rootCursor = this.forest.getCursorAboveDetachedFields();
 		const fieldMap: Map<FieldKey, ITreeCursorSynchronous & ITreeSubscriptionCursor> =
 			new Map();
@@ -89,55 +127,85 @@ export class ForestSummarizer implements Summarizable {
 			);
 			fieldMap.set(key, innerCursor as ITreeCursorSynchronous & ITreeSubscriptionCursor);
 		});
-		const encoded = this.codec.encode(fieldMap, this.encoderContext);
 
+		// Let the incremental summary builder know that we are starting a new summary.
+		// It returns whether incremental encoding is enabled.
+		const incrementalSummaryBehavior = this.incrementalSummaryBuilder.startSummary({
+			fullTree,
+			incrementalSummaryContext,
+			stringify,
+		});
+		const encoderContext: FieldBatchEncodingContext = {
+			...this.encoderContext,
+			incrementalEncoderDecoder:
+				incrementalSummaryBehavior === ForestIncrementalSummaryBehavior.Incremental
+					? this.incrementalSummaryBuilder
+					: undefined,
+		};
+		const encoded = this.codec.encode(fieldMap, encoderContext);
 		fieldMap.forEach((value) => value.free());
-		return stringify(encoded);
-	}
 
-	public summarize(props: {
-		stringify: SummaryElementStringifier;
-		fullTree?: boolean;
-		trackState?: boolean;
-		telemetryContext?: ITelemetryContext;
-		incrementalSummaryContext?: IExperimentalIncrementalSummaryContext;
-	}): ISummaryTreeWithStats {
-		return createSingleBlobSummary(treeBlobKey, this.getTreeString(props.stringify));
+		return this.incrementalSummaryBuilder.completeSummary({
+			incrementalSummaryContext,
+			forestSummaryContent: stringify(encoded),
+		});
 	}
 
 	public async load(
 		services: IChannelStorageService,
 		parse: SummaryElementParser,
 	): Promise<void> {
-		if (await services.contains(treeBlobKey)) {
-			const treeBuffer = await services.readBlob(treeBlobKey);
+		// The contents of the top-level forest must be present under a summary blob named `forestSummaryContentKey`.
+		// If the summary was generated as `ForestIncrementalSummaryBehavior.SingleBlob`, this blob will contain all
+		// of forest's contents.
+		// If the summary was generated as `ForestIncrementalSummaryBehavior.Incremental`, this blob will contain only
+		// the top-level forest node's contents.
+		// The contents of the incremental chunks will be in separate tree nodes and will be read later during decoding.
+		assert(
+			await services.contains(forestSummaryContentKey),
+			0xc21 /* Forest summary content missing in snapshot */,
+		);
+
+		const readAndParseBlob = async <T extends JsonCompatible<IFluidHandle>>(
+			id: string,
+		): Promise<T> => {
+			const treeBuffer = await services.readBlob(id);
 			const treeBufferString = bufferToString(treeBuffer, "utf8");
-			// TODO: this code is parsing data without an optional validator, this should be defined in a typebox schema as part of the
-			// forest summary format.
-			const fields = this.codec.decode(parse(treeBufferString) as Format, this.encoderContext);
-			const allocator = idAllocatorFromMaxId();
-			const fieldChanges: [FieldKey, DeltaFieldChanges][] = [];
-			const build: DeltaDetachedNodeBuild[] = [];
-			for (const [fieldKey, field] of fields) {
-				const chunked = chunkFieldSingle(field, {
-					policy: defaultChunkPolicy,
-					idCompressor: this.idCompressor,
-				});
-				const buildId = { minor: allocator.allocate(chunked.topLevelLength) };
-				build.push({
-					id: buildId,
-					trees: chunked,
-				});
-				fieldChanges.push([fieldKey, [{ count: chunked.topLevelLength, attach: buildId }]]);
-			}
-
-			assert(this.forest.isEmpty, 0x797 /* forest must be empty */);
-			applyDelta(
-				{ build, fields: new Map(fieldChanges) },
-				undefined,
-				this.forest,
-				makeDetachedFieldIndex("init", this.revisionTagCodec, this.idCompressor),
-			);
+			return parse(treeBufferString) as T;
+		};
+
+		// Load the incremental summary builder so that it can download any incremental chunks in the
+		// snapshot.
+		await this.incrementalSummaryBuilder.load(services, readAndParseBlob);
+
+		// TODO: this code is parsing data without an optional validator, this should be defined in a typebox schema as part of the
+		// forest summary format.
+		const fields = this.codec.decode(await readAndParseBlob(forestSummaryContentKey), {
+			...this.encoderContext,
+			incrementalEncoderDecoder: this.incrementalSummaryBuilder,
+		});
+		const allocator = idAllocatorFromMaxId();
+		const fieldChanges: [FieldKey, DeltaFieldChanges][] = [];
+		const build: DeltaDetachedNodeBuild[] = [];
+		for (const [fieldKey, field] of fields) {
+			const chunked = chunkFieldSingle(field, {
+				policy: defaultChunkPolicy,
+				idCompressor: this.idCompressor,
+			});
+			const buildId = { minor: allocator.allocate(chunked.topLevelLength) };
+			build.push({
+				id: buildId,
+				trees: chunked,
+			});
+			fieldChanges.push([fieldKey, [{ count: chunked.topLevelLength, attach: buildId }]]);
 		}
+
+		assert(this.forest.isEmpty, 0x797 /* forest must be empty */);
+		applyDelta(
+			{ build, fields: new Map(fieldChanges) },
+			undefined,
+			this.forest,
+			makeDetachedFieldIndex("init", this.revisionTagCodec, this.idCompressor),
+		);
 	}
 }