@fluidframework/tree 2.53.1 → 2.61.0-355054

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/tree",
-  "version": "2.53.1",
+  "version": "2.61.0-355054",
   "description": "Distributed tree",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -101,17 +101,17 @@
     "temp-directory": "nyc/.nyc_output"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": "~2.53.1",
-    "@fluidframework/container-runtime": "~2.53.1",
-    "@fluidframework/core-interfaces": "~2.53.1",
-    "@fluidframework/core-utils": "~2.53.1",
-    "@fluidframework/datastore-definitions": "~2.53.1",
-    "@fluidframework/driver-definitions": "~2.53.1",
-    "@fluidframework/id-compressor": "~2.53.1",
-    "@fluidframework/runtime-definitions": "~2.53.1",
-    "@fluidframework/runtime-utils": "~2.53.1",
-    "@fluidframework/shared-object-base": "~2.53.1",
-    "@fluidframework/telemetry-utils": "~2.53.1",
+    "@fluid-internal/client-utils": "2.61.0-355054",
+    "@fluidframework/container-runtime": "2.61.0-355054",
+    "@fluidframework/core-interfaces": "2.61.0-355054",
+    "@fluidframework/core-utils": "2.61.0-355054",
+    "@fluidframework/datastore-definitions": "2.61.0-355054",
+    "@fluidframework/driver-definitions": "2.61.0-355054",
+    "@fluidframework/id-compressor": "2.61.0-355054",
+    "@fluidframework/runtime-definitions": "2.61.0-355054",
+    "@fluidframework/runtime-utils": "2.61.0-355054",
+    "@fluidframework/shared-object-base": "2.61.0-355054",
+    "@fluidframework/telemetry-utils": "2.61.0-355054",
     "@sinclair/typebox": "^0.34.13",
     "@tylerbu/sorted-btree-es6": "^1.8.0",
     "@types/ungap__structured-clone": "^1.2.0",
@@ -121,28 +121,28 @@
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.17.1",
     "@biomejs/biome": "~1.9.3",
-    "@fluid-internal/mocha-test-setup": "~2.53.1",
-    "@fluid-private/stochastic-test-utils": "~2.53.1",
-    "@fluid-private/test-dds-utils": "~2.53.1",
-    "@fluid-private/test-drivers": "~2.53.1",
+    "@fluid-internal/mocha-test-setup": "2.61.0-355054",
+    "@fluid-private/stochastic-test-utils": "2.61.0-355054",
+    "@fluid-private/test-dds-utils": "2.61.0-355054",
+    "@fluid-private/test-drivers": "2.61.0-355054",
     "@fluid-tools/benchmark": "^0.51.0",
-    "@fluid-tools/build-cli": "^0.57.0",
+    "@fluid-tools/build-cli": "^0.58.2",
     "@fluidframework/build-common": "^2.0.3",
-    "@fluidframework/build-tools": "^0.57.0",
-    "@fluidframework/container-definitions": "~2.53.1",
-    "@fluidframework/container-loader": "~2.53.1",
-    "@fluidframework/eslint-config-fluid": "^5.7.4",
-    "@fluidframework/test-runtime-utils": "~2.53.1",
-    "@fluidframework/test-utils": "~2.53.1",
-    "@fluidframework/tree-previous": "npm:@fluidframework/tree@2.53.0",
-    "@microsoft/api-extractor": "7.52.8",
+    "@fluidframework/build-tools": "^0.58.2",
+    "@fluidframework/container-definitions": "2.61.0-355054",
+    "@fluidframework/container-loader": "2.61.0-355054",
+    "@fluidframework/eslint-config-fluid": "^6.0.0",
+    "@fluidframework/test-runtime-utils": "2.61.0-355054",
+    "@fluidframework/test-utils": "2.61.0-355054",
+    "@fluidframework/tree-previous": "npm:@fluidframework/tree@2.60.0",
+    "@microsoft/api-extractor": "7.52.11",
     "@types/diff": "^3.5.1",
     "@types/easy-table": "^0.0.32",
     "@types/mocha": "^10.0.10",
     "@types/node": "^18.19.0",
     "ajv": "^8.12.0",
     "ajv-formats": "^3.0.1",
-    "c8": "^8.0.1",
+    "c8": "^10.1.3",
     "concurrently": "^8.2.1",
     "copyfiles": "^2.4.1",
     "cross-env": "^7.0.3",
@@ -177,8 +177,8 @@
   },
   "scripts": {
     "api": "fluid-build . --task api",
-    "api-extractor:commonjs": "flub generate entrypoints --outDir ./dist",
-    "api-extractor:esnext": "flub generate entrypoints --outDir ./lib --node10TypeCompat",
+    "api-extractor:commonjs": "flub generate entrypoints --outFileLegacyBeta legacy --outDir ./dist",
+    "api-extractor:esnext": "flub generate entrypoints --outFileLegacyBeta legacy --outDir ./lib --node10TypeCompat",
     "bench": "mocha --timeout 999999 --perfMode --parentProcess --fgrep @Benchmark --fgrep @ExecutionTime --reporter @fluid-tools/benchmark/dist/MochaReporter.js",
     "bench:profile": "mocha --v8-prof --v8-logfile=profile.log --v8-no-logfile-per-isolate --timeout 999999 --perfMode --fgrep @Benchmark --fgrep @ExecutionTime --reporter @fluid-tools/benchmark/dist/MochaReporter.js && node --prof-process profile.log > profile.txt && rimraf profile.log && echo See results in profile.txt",
     "build": "fluid-build . --task build",

package/src/codec/codec.ts

@@ -3,12 +3,14 @@
  * Licensed under the MIT License.
  */
 
+import type { ErasedType } from "@fluidframework/core-interfaces/internal";
 import { IsoBuffer, bufferToString } from "@fluid-internal/client-utils";
 import { assert, fail } from "@fluidframework/core-utils/internal";
 import type { Static, TAnySchema, TSchema } from "@sinclair/typebox";
 
 import type { ChangeEncodingContext } from "../core/index.js";
 import type { JsonCompatibleReadOnly } from "../util/index.js";
+import { noopValidator } from "./noopValidator.js";
 
 /**
  * Translates decoded data to encoded data.
@@ -44,6 +46,46 @@ export interface SchemaValidationFunction<Schema extends TSchema> {
 	check(data: unknown): data is Static<Schema>;
 }
 
+/**
+ * A kind of validator for SharedTree's internal data formats.
+ * @remarks
+ * Assuming no data corruption or type confusion, such validation should never fail.
+ * Any client version compatibility issues should instead be detected by the data format versioning which Shared Tree does internally independent of data format validation.
+ * However, persisted data can sometimes be corrupted, bugs can produce invalid data, or users can mix up which data is compatible with which APIs.
+ * In such cases, a format validator can help catch issues.
+ *
+ * Current options are {@link FormatValidatorNoOp} and {@link FormatValidatorBasic}.
+ * @privateRemarks
+ * Implement using {@link toFormatValidator}.
+ * Consume using {@link extractJsonValidator}.
+ *
+ * Exposing this as the stable API entry point (instead of {@link JsonValidator}) means that we avoid leaking the reference to TypeBox to the API surface.
+ * Additionally, if we adopt non JSON formats, we can just update the validators as needed without breaking the API.
+ * This also allows us to avoid stabilizing or documenting how handles interact with JSON validation since that is not exposed through this type.
+ * @sealed @alpha
+ */
+export interface FormatValidator extends ErasedType<"FormatValidator"> {}
+
+/**
+ * A {@link FormatValidator} which does no validation.
+ * @alpha
+ */
+export const FormatValidatorNoOp = toFormatValidator(noopValidator);
+
+/**
+ * Type erase a {@link JsonValidator} to a {@link FormatValidator}.
+ */
+export function toFormatValidator(factory: JsonValidator): FormatValidator {
+	return factory as unknown as FormatValidator;
+}
+
+/**
+ * Un-type-erase the {@link FormatValidator}.
+ */
+export function extractJsonValidator(input: FormatValidator | JsonValidator): JsonValidator {
+	return input as unknown as JsonValidator;
+}
+
 /**
  * JSON schema validator compliant with draft 6 schema. See https://json-schema.org.
  * @alpha @input
@@ -69,19 +111,22 @@ export interface JsonValidator {
  */
 export interface ICodecOptions {
 	/**
-	 * {@link JsonValidator} which SharedTree uses to validate persisted data it reads & writes
+	 * {@link FormatValidator} which SharedTree uses to validate persisted data it reads & writes
 	 * matches the expected encoded format (i.e. the wire format for ops and summaries).
-	 *
-	 * See {@link noopValidator} and {@link typeboxValidator} for out-of-the-box implementations.
+	 * @remarks
+	 * See {@link FormatValidatorNoOp} and {@link FormatValidatorBasic} for out-of-the-box implementations.
 	 *
 	 * This option is not "on-by-default" because JSON schema validation comes with a small but noticeable
 	 * runtime performance cost, and popular schema validation libraries have relatively large bundle size.
 	 *
-	 * SharedTree users are still encouraged to use a non-trivial validator (i.e. not `noopValidator`)
+	 * SharedTree users are still encouraged to use a non-trivial validator (i.e. not `FormatValidatorNoOp`)
 	 * whenever reasonable: it gives better fail-fast behavior when unexpected encoded data is found,
 	 * which reduces the risk of unrecoverable data corruption.
+	 *
+	 * Use of {@link JsonValidator} here is deprecated and will be removed:
+	 * it is recommended to use {@link FormatValidator} instead.
 	 */
-	readonly jsonValidator: JsonValidator;
+	readonly jsonValidator: JsonValidator | FormatValidator;
 }
 
 /**
@@ -326,12 +371,12 @@ export function withSchemaValidation<
 >(
 	schema: EncodedSchema,
 	codec: IJsonCodec<TInMemoryFormat, TEncodedFormat, TValidate, TContext>,
-	validator?: JsonValidator,
+	validator?: JsonValidator | FormatValidator,
 ): IJsonCodec<TInMemoryFormat, TEncodedFormat, TValidate, TContext> {
 	if (!validator) {
 		return codec;
 	}
-	const compiledFormat = validator.compile(schema);
+	const compiledFormat = extractJsonValidator(validator).compile(schema);
 	return {
 		encode: (obj: TInMemoryFormat, context: TContext): TEncodedFormat => {
 			const encoded = codec.encode(obj, context);

package/src/codec/index.ts

@@ -21,6 +21,10 @@ export {
 	withSchemaValidation,
 	FluidClientVersion,
 	currentVersion,
+	toFormatValidator,
+	FormatValidatorNoOp,
+	type FormatValidator,
+	extractJsonValidator,
 } from "./codec.js";
 export {
 	DiscriminatedUnionDispatcher,

package/src/external-utilities/index.ts

@@ -3,4 +3,4 @@
  * Licensed under the MIT License.
  */
 
-export { typeboxValidator } from "./typeboxValidator.js";
+export { typeboxValidator, FormatValidatorBasic } from "./typeboxValidator.js";

package/src/external-utilities/typeboxValidator.ts

@@ -8,7 +8,7 @@ import type { Static, TSchema } from "@sinclair/typebox";
 // eslint-disable-next-line import/no-internal-modules
 import { TypeCompiler } from "@sinclair/typebox/compiler";
 
-import type { JsonValidator } from "../codec/index.js";
+import { toFormatValidator, type JsonValidator } from "../codec/index.js";
 
 /**
  * A {@link JsonValidator} implementation which uses TypeBox's JSON schema validator.
@@ -29,3 +29,9 @@ export const typeboxValidator: JsonValidator = {
 		};
 	},
 };
+
+/**
+ * A {@link FormatValidator} implementation which uses TypeBox's JSON schema validator.
+ * @alpha
+ */
+export const FormatValidatorBasic = toFormatValidator(typeboxValidator);

package/src/feature-libraries/chunked-forest/codec/chunkDecoding.ts

@@ -17,7 +17,7 @@ import type {
 	Value,
 	TreeChunk,
 } from "../../../core/index.js";
-import { assertValidIndex } from "../../../util/index.js";
+import { assertValidIndex, brand } from "../../../util/index.js";
 import { BasicChunk } from "../basicChunk.js";
 import { emptyChunk } from "../emptyChunk.js";
 import { SequenceChunk } from "../sequenceChunk.js";
@@ -41,12 +41,14 @@ import {
 	type EncodedAnyShape,
 	type EncodedChunkShape,
 	type EncodedFieldBatch,
+	type EncodedIncrementalChunkShape,
 	type EncodedInlineArrayShape,
 	type EncodedNestedArrayShape,
 	type EncodedNodeShape,
 	type EncodedValueShape,
 	SpecialField,
 } from "./format.js";
+import type { IncrementalDecoder } from "./codecs.js";
 
 export interface IdDecodingContext {
 	idCompressor: IIdCompressor;
@@ -61,10 +63,11 @@ export interface IdDecodingContext {
 export function decode(
 	chunk: EncodedFieldBatch,
 	idDecodingContext: { idCompressor: IIdCompressor; originatorId: SessionId },
+	incrementalDecoder?: IncrementalDecoder,
 ): TreeChunk[] {
 	return genericDecode(
 		decoderLibrary,
-		new DecoderContext(chunk.identifiers, chunk.shapes, idDecodingContext),
+		new DecoderContext(chunk.identifiers, chunk.shapes, idDecodingContext, incrementalDecoder),
 		chunk,
 		anyDecoder,
 	);
@@ -87,6 +90,9 @@ const decoderLibrary = new DiscriminatedUnionDispatcher<
 	d(shape: EncodedAnyShape): ChunkDecoder {
 		return anyDecoder;
 	},
+	e(shape: EncodedIncrementalChunkShape, cache): ChunkDecoder {
+		return new IncrementalChunkDecoder(cache);
+	},
 });
 
 /**
@@ -228,6 +234,34 @@ export class InlineArrayDecoder implements ChunkDecoder {
 	}
 }
 
+/**
+ * Decoder for {@link EncodedIncrementalChunkShape}s.
+ */
+export class IncrementalChunkDecoder implements ChunkDecoder {
+	public constructor(private readonly cache: DecoderContext<EncodedChunkShape>) {}
+	public decode(_: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk {
+		assert(
+			this.cache.incrementalDecoder !== undefined,
+			0xc27 /* incremental decoder not available for incremental field decoding */,
+		);
+		const chunkReferenceId = readStreamNumber(stream);
+		const batch = this.cache.incrementalDecoder.getEncodedIncrementalChunk(
+			brand(chunkReferenceId),
+		);
+		assert(batch !== undefined, 0xc28 /* Incremental chunk data missing */);
+		// The incremental chunk data is self-describing, i.e., it contain its own shapes list and identifier table.
+		// Use these to create a new decoder context to be used to decode the incremental chunk's data.
+		const context = new DecoderContext(
+			batch.identifiers,
+			batch.shapes,
+			this.cache.idDecodingContext,
+			this.cache.incrementalDecoder,
+		);
+		const chunks = genericDecode(decoderLibrary, context, batch, anyDecoder);
+		return aggregateChunks(chunks);
+	}
+}
+
 /**
  * Decoder for {@link EncodedAnyShape}s.
  */
@@ -300,8 +334,8 @@ export class NodeDecoder implements ChunkDecoder {
 			}
 		}
 
-		for (const field of this.fieldDecoders) {
-			const [key, content] = field(decoders, stream);
+		for (const decoder of this.fieldDecoders) {
+			const [key, content] = decoder(decoders, stream);
 			addField(key, content);
 		}
 

package/src/feature-libraries/chunked-forest/codec/chunkDecodingGeneric.ts

@@ -17,6 +17,7 @@ import {
 } from "./chunkCodecUtilities.js";
 import type { IdDecodingContext } from "./chunkDecoding.js";
 import type { EncodedFieldBatchGeneric, IdentifierOrIndex } from "./formatGeneric.js";
+import type { IncrementalDecoder } from "./codecs.js";
 
 /**
  * General purpose shape based tree decoder which gets its support for specific shapes from the caller.
@@ -50,13 +51,19 @@ export function decode<TEncodedShape extends object, TContext>(
  * Shared data for use in constructing decoders.
  */
 export class DecoderContext<TEncodedShape = unknown> {
-	/**
-	 * @param identifiers - identifier substitution table (use to replace numeric identifier indexes with the actual identifiers from this table).
-	 */
 	public constructor(
+		/**
+		 * Identifier substitution table (use to replace numeric identifier indexes with the actual identifiers from this table).
+		 */
 		public readonly identifiers: readonly string[],
 		public readonly shapes: readonly TEncodedShape[],
 		public readonly idDecodingContext: IdDecodingContext,
+		/**
+		 * To be used to decode incremental chunks, if any.
+		 * @remarks
+		 * See {@link IncrementalDecoder} for more information.
+		 */
+		public readonly incrementalDecoder: IncrementalDecoder | undefined,
 	) {}
 
 	public identifier<T extends string & BrandedType<string, string>>(

package/src/feature-libraries/chunked-forest/codec/codecs.ts

@@ -12,9 +12,24 @@ import {
 	type IJsonCodec,
 	makeVersionedValidatedCodec,
 } from "../../../codec/index.js";
-import { CursorLocationType, type SchemaAndPolicy } from "../../../core/index.js";
-import type { JsonCompatibleReadOnly } from "../../../util/index.js";
-import { TreeCompressionStrategy } from "../../treeCompressionUtils.js";
+import {
+	CursorLocationType,
+	type FieldKey,
+	type ITreeCursorSynchronous,
+	type SchemaAndPolicy,
+	type TreeChunk,
+	type TreeNodeSchemaIdentifier,
+} from "../../../core/index.js";
+import {
+	brandedNumberType,
+	type Brand,
+	type JsonCompatibleReadOnly,
+} from "../../../util/index.js";
+import {
+	TreeCompressionStrategy,
+	TreeCompressionStrategyExtended,
+	type TreeCompressionStrategyPrivate,
+} from "../../treeCompressionUtils.js";
 
 import { decode } from "./chunkDecoding.js";
 import type { FieldBatch } from "./fieldBatch.js";
@@ -22,11 +37,75 @@ import { EncodedFieldBatch, validVersions } from "./format.js";
 import { schemaCompressedEncode } from "./schemaBasedEncode.js";
 import { uncompressedEncode } from "./uncompressedEncode.js";
 
+/**
+ * Reference ID for a chunk that is incrementally encoded.
+ */
+export type ChunkReferenceId = Brand<number, "forest.ChunkReferenceId">;
+const ChunkReferenceId = brandedNumberType<ChunkReferenceId>({ multipleOf: 1, minimum: 0 });
+
+/**
+ * Properties for incremental encoding.
+ * Fields that support incremental encoding will encode their chunks separately by calling `encodeIncrementalField`.
+ * @remarks
+ * This supports features like incremental summarization where the summary from these fields can be re-used if
+ * unchanged between summaries.
+ * Note that each of these chunks that are incrementally encoded is fully self-describing (contain its own shapes
+ * list and identifier table) and does not rely on context from its parent.
+ */
+export interface IncrementalEncoder {
+	/**
+	 * Returns whether a field should be incrementally encoded.
+	 * @param nodeIdentifier - The identifier of the node containing the field.
+	 * @param fieldKey - The key of the field to check.
+	 */
+	shouldEncodeFieldIncrementally(
+		nodeIdentifier: TreeNodeSchemaIdentifier,
+		fieldKey: FieldKey,
+	): boolean;
+	/**
+	 * Called to encode an incremental field at the cursor.
+	 * The chunks for this field are encoded separately from the main buffer.
+	 * @param cursor - The cursor pointing to the field to encode.
+	 * @param chunkEncoder - A function that encodes the contents of the passed chunk in the field.
+	 * @returns The reference IDs of the encoded chunks in the field.
+	 * This is used to retrieve the encoded chunks later.
+	 */
+	encodeIncrementalField(
+		cursor: ITreeCursorSynchronous,
+		chunkEncoder: (chunk: TreeChunk) => EncodedFieldBatch,
+	): ChunkReferenceId[];
+}
+
+/**
+ * Properties for incremental decoding.
+ *
+ * Fields that had their chunks incrementally encoded will retrieve them by calling `getEncodedIncrementalChunk`.
+ * @remarks
+ * See {@link IncrementalEncoder} for more details.
+ */
+export interface IncrementalDecoder {
+	/**
+	 * Called to get the encoded contents of an chunk in an incremental field with the given reference ID.
+	 * @param referenceId - The reference ID of the chunk to retrieve.
+	 * @returns The encoded contents of the chunk.
+	 */
+	getEncodedIncrementalChunk: (referenceId: ChunkReferenceId) => EncodedFieldBatch;
+}
+/**
+ * Combines the properties of {@link IncrementalEncoder} and {@link IncrementalDecoder}.
+ */
+export interface IncrementalEncoderDecoder extends IncrementalEncoder, IncrementalDecoder {}
+
 export interface FieldBatchEncodingContext {
-	readonly encodeType: TreeCompressionStrategy;
+	readonly encodeType: TreeCompressionStrategyPrivate;
 	readonly idCompressor: IIdCompressor;
 	readonly originatorId: SessionId;
 	readonly schema?: SchemaAndPolicy;
+	/**
+	 * An encoder / decoder for encoding and decoding of incremental fields.
+	 * This will be defined if incremental encoding is supported and enabled.
+	 */
+	readonly incrementalEncoderDecoder?: IncrementalEncoderDecoder;
 }
 /**
  * @remarks
@@ -78,6 +157,7 @@ export function makeFieldBatchCodec(
 				case TreeCompressionStrategy.Uncompressed:
 					encoded = uncompressedEncode(data);
 					break;
+				case TreeCompressionStrategyExtended.CompressedIncremental:
 				case TreeCompressionStrategy.Compressed:
 					// eslint-disable-next-line unicorn/prefer-ternary
 					if (context.schema !== undefined) {
@@ -86,6 +166,10 @@ export function makeFieldBatchCodec(
 							context.schema.policy,
 							data,
 							context.idCompressor,
+							// Incremental encoding is only supported for CompressedIncremental.
+							context.encodeType === TreeCompressionStrategyExtended.CompressedIncremental
+								? context.incrementalEncoderDecoder
+								: undefined,
 						);
 					} else {
 						// TODO: consider enabling a somewhat compressed but not schema accelerated encode.
@@ -102,10 +186,14 @@ export function makeFieldBatchCodec(
 		},
 		decode: (data: EncodedFieldBatch, context: FieldBatchEncodingContext): FieldBatch => {
 			// TODO: consider checking data is in schema.
-			return decode(data, {
-				idCompressor: context.idCompressor,
-				originatorId: context.originatorId,
-			}).map((chunk) => chunk.cursor());
+			return decode(
+				data,
+				{
+					idCompressor: context.idCompressor,
+					originatorId: context.originatorId,
+				},
+				context.incrementalEncoderDecoder,
+			).map((chunk) => chunk.cursor());
 		},
 	});
 }