@fluidframework/container-loader 2.63.0-359962 → 2.70.0-360374

package/src/containerStorageAdapter.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { bufferToString, stringToBuffer } from "@fluid-internal/client-utils";
+import { bufferToString } from "@fluid-internal/client-utils";
 import type {
 	ISnapshotTreeWithBlobContents,
 	IContainerStorageService,
@@ -71,6 +71,11 @@ export class ContainerStorageAdapter
 		return this._loadedGroupIdSnapshots;
 	}
 
+	/**
+	 * ArrayBufferLikes  containing blobs from a snapshot
+	 */
+	private readonly blobContents: { [id: string]: ArrayBufferLike } = {};
+
 	/**
 	 * An adapter that ensures we're using detachedBlobStorage up until we connect to a real service, and then
 	 * after connecting to a real service augments it with retry and combined summary tree enforcement.
@@ -84,10 +89,6 @@ export class ContainerStorageAdapter
 	public constructor(
 		detachedBlobStorage: MemoryDetachedBlobStorage | undefined,
 		private readonly logger: ITelemetryLoggerExt,
-		/**
-		 * ArrayBufferLikes or utf8 encoded strings, containing blobs from a snapshot
-		 */
-		private readonly blobContents: { [id: string]: ArrayBufferLike | string } = {},
 		private loadingGroupIdSnapshotsFromPendingState:
 			| Record<string, SerializedSnapshotInfo>
 			| undefined,
@@ -143,9 +144,9 @@ export class ContainerStorageAdapter
 		);
 	}
 
-	public loadSnapshotFromSnapshotBlobs(snapshotBlobs: ISerializableBlobContents): void {
-		for (const [id, value] of Object.entries(snapshotBlobs)) {
-			this.blobContents[id] = value;
+	public cacheSnapshotBlobs(snapshotBlobs: Map<string, ArrayBuffer>): void {
+		for (const [id, value] of snapshotBlobs.entries()) {
+			this.blobContents[id] ??= value;
 		}
 	}
 
@@ -220,10 +221,6 @@ export class ContainerStorageAdapter
 	public async readBlob(id: string): Promise<ArrayBufferLike> {
 		const maybeBlob = this.blobContents[id];
 		if (maybeBlob !== undefined) {
-			if (typeof maybeBlob === "string") {
-				const blob = stringToBuffer(maybeBlob, "utf8");
-				return blob;
-			}
 			return maybeBlob;
 		}
 		return this._storageService.readBlob(id);

package/src/createAndLoadContainerUtils.ts

@@ -9,6 +9,7 @@ import type {
 	IFluidCodeDetails,
 	IContainerPolicies,
 } from "@fluidframework/container-definitions/internal";
+import { LoaderHeader } from "@fluidframework/container-definitions/internal";
 import type {
 	FluidObject,
 	IConfigProviderBase,
@@ -20,10 +21,44 @@ import type {
 	IDocumentServiceFactory,
 	IUrlResolver,
 } from "@fluidframework/driver-definitions/internal";
+import { DriverHeader } from "@fluidframework/driver-definitions/internal";
+import {
+	GenericError,
+	normalizeError,
+	createChildMonitoringContext,
+	mixinMonitoringContext,
+	sessionStorageConfigProvider,
+	PerformanceEvent,
+	isFluidError,
+} from "@fluidframework/telemetry-utils/internal";
+import { v4 as uuid } from "uuid";
 
+import { DebugLogger } from "./debugLogger.js";
 import { createFrozenDocumentServiceFactory } from "./frozenServices.js";
 import { Loader } from "./loader.js";
+import { pkgVersion } from "./packageVersion.js";
 import type { ProtocolHandlerBuilder } from "./protocol.js";
+import type {
+	LoadSummarizerSummaryResult,
+	OnDemandSummaryResults,
+	SummarizeOnDemandResults,
+} from "./summarizerResultTypes.js";
+
+interface OnDemandSummarizeResultsPromises {
+	readonly summarySubmitted: Promise<SummarizeOnDemandResults["summarySubmitted"]>;
+	readonly summaryOpBroadcasted: Promise<SummarizeOnDemandResults["summaryOpBroadcasted"]>;
+}
+
+interface OnDemandSummarizeOptions {
+	readonly reason?: string;
+	readonly retryOnFailure?: boolean;
+	readonly fullTree?: boolean;
+}
+
+interface SummarizerLike {
+	readonly ISummarizer?: SummarizerLike;
+	summarizeOnDemand(options: OnDemandSummarizeOptions): OnDemandSummarizeResultsPromises;
+}
 
 /**
  * Properties necessary for creating and loading a container.
@@ -103,6 +138,15 @@ export interface ILoadExistingContainerProps extends ICreateAndLoadContainerProp
 	readonly pendingLocalState?: string | undefined;
 }
 
+/**
+ * Props used to load summarizer container.
+ * @legacy @alpha
+ */
+export type ILoadSummarizerContainerProps = Omit<
+	ILoadExistingContainerProps,
+	"pendingLocalState"
+>;
+
 /**
  * Props used to create a detached container.
  * @legacy @beta
@@ -200,3 +244,133 @@ export async function loadFrozenContainerFromPendingState(
 		documentServiceFactory: createFrozenDocumentServiceFactory(props.documentServiceFactory),
 	});
 }
+
+/**
+ * Loads a summarizer container with the required headers, triggers an on-demand summary, and then closes it.
+ * Returns success/failure and an optional error for host-side handling.
+ *
+ * @legacy @alpha
+ */
+export async function loadSummarizerContainerAndMakeSummary(
+	loadSummarizerContainerProps: ILoadSummarizerContainerProps,
+): Promise<LoadSummarizerSummaryResult> {
+	const { logger, configProvider, request: originalRequest } = loadSummarizerContainerProps;
+	const telemetryProps = {
+		loaderId: uuid(),
+		loaderVersion: pkgVersion,
+	};
+
+	const subMc = mixinMonitoringContext(
+		DebugLogger.mixinDebugLogger("fluid:telemetry", logger, {
+			all: telemetryProps,
+		}),
+		sessionStorageConfigProvider.value,
+		configProvider,
+	);
+	const mc = createChildMonitoringContext({
+		logger: subMc.logger,
+		namespace: "SummarizerOnDemand",
+	});
+	return PerformanceEvent.timedExecAsync(
+		mc.logger,
+		{ eventName: "SummarizerOnDemandSummary" },
+		async (event) => {
+			const baseHeaders = originalRequest.headers;
+			const request = {
+				...originalRequest,
+				headers: {
+					...baseHeaders,
+					[LoaderHeader.cache]: false,
+					[LoaderHeader.clientDetails]: {
+						capabilities: { interactive: false },
+						type: "summarizer",
+					},
+					[DriverHeader.summarizingClient]: true,
+					[LoaderHeader.reconnect]: false,
+				},
+			};
+
+			const container = await loadExistingContainer({
+				...loadSummarizerContainerProps,
+				request,
+			});
+
+			let summarySubmitted: SummarizeOnDemandResults["summarySubmitted"];
+			let summaryOpBroadcasted: SummarizeOnDemandResults["summaryOpBroadcasted"];
+			try {
+				if (container.getEntryPoint === undefined) {
+					throw new GenericError("container.getEntryPoint() is undefined");
+				}
+				const fluidObject = (await container.getEntryPoint()) as FluidObject<SummarizerLike>;
+				const summarizer = fluidObject?.ISummarizer;
+				if (summarizer === undefined) {
+					throw new GenericError("Summarizer entry point not available");
+				}
+				// Host controlled feature gate for fullTree
+				// Default value will be false
+				const fullTreeGate =
+					mc.config.getBoolean("Fluid.Summarizer.FullTree.OnDemand") === true;
+
+				const summarizeResults: OnDemandSummarizeResultsPromises =
+					summarizer.summarizeOnDemand({
+						reason: "summaryOnRequest",
+						retryOnFailure: true,
+						fullTree: fullTreeGate,
+					});
+				[summarySubmitted, summaryOpBroadcasted] = await Promise.all([
+					summarizeResults.summarySubmitted,
+					summarizeResults.summaryOpBroadcasted,
+				]);
+
+				const summaryResults: OnDemandSummaryResults = {
+					summarySubmitted: summarySubmitted.success,
+					summaryInfo: summarySubmitted.success
+						? {
+								stage: summarySubmitted.data.stage,
+								handle: summaryOpBroadcasted.success
+									? summaryOpBroadcasted.data.summarizeOp.contents.handle
+									: undefined,
+							}
+						: {},
+					summaryOpBroadcasted: summaryOpBroadcasted.success,
+				};
+
+				if (summarySubmitted.success && summaryOpBroadcasted.success) {
+					event.end({
+						success: true,
+						summarySubmitted: true,
+						summaryOpBroadcasted: true,
+					});
+					return {
+						success: true,
+						summaryResults,
+					};
+				}
+
+				const failureError =
+					summarySubmitted.success === false
+						? summarySubmitted.error
+						: summaryOpBroadcasted.success === false
+							? summaryOpBroadcasted.error
+							: new GenericError("On demand summary failed");
+
+				event.end({
+					success: false,
+					summarySubmitted: summarySubmitted.success,
+					summaryOpBroadcasted: summaryOpBroadcasted.success,
+				});
+				return {
+					success: false,
+					error: failureError,
+				};
+			} catch (error) {
+				event.cancel({ success: false }, error);
+				const caughtError = isFluidError(error) ? error : normalizeError(error);
+				return { success: false, error: caughtError };
+			} finally {
+				container.dispose();
+			}
+		},
+		{ start: true, end: true, cancel: "generic" },
+	);
+}

package/src/index.ts

@@ -11,12 +11,19 @@ export {
 	loadExistingContainer,
 	rehydrateDetachedContainer,
 	loadFrozenContainerFromPendingState,
+	loadSummarizerContainerAndMakeSummary,
 	type ICreateAndLoadContainerProps,
 	type ICreateDetachedContainerProps,
 	type ILoadExistingContainerProps,
+	type ILoadSummarizerContainerProps,
 	type IRehydrateDetachedContainerProps,
 	type ILoadFrozenContainerFromPendingStateProps,
 } from "./createAndLoadContainerUtils.js";
+export type {
+	LoadSummarizerSummaryResult,
+	OnDemandSummaryResults,
+	SummaryStage,
+} from "./summarizerResultTypes.js";
 export {
 	type ICodeDetailsLoader,
 	type IFluidModuleWithDetails,

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-loader";
-export const pkgVersion = "2.63.0-359962";
+export const pkgVersion = "2.70.0-360374";

package/src/serializedStateManager.ts

@@ -32,6 +32,7 @@ import {
 
 import {
 	getBlobContentsFromTree,
+	type ContainerStorageAdapter,
 	type ISerializableBlobContents,
 } from "./containerStorageAdapter.js";
 import {
@@ -133,8 +134,8 @@ interface ISnapshotInfo {
 }
 
 export type ISerializedStateManagerDocumentStorageService = Pick<
-	IDocumentStorageService,
-	"getSnapshot" | "getSnapshotTree" | "getVersions" | "readBlob"
+	ContainerStorageAdapter,
+	"getSnapshot" | "getSnapshotTree" | "getVersions" | "readBlob" | "cacheSnapshotBlobs"
 > & {
 	loadedGroupIdSnapshots: Record<string, ISnapshot>;
 };
@@ -300,11 +301,13 @@ export class SerializedStateManager implements IDisposable {
 			return { snapshot, version, attributes };
 		} else {
 			const { baseSnapshot, snapshotBlobs, savedOps } = pendingLocalState;
-			const attributes = await getDocumentAttributes(this.storageAdapter, baseSnapshot);
 			const blobContents = new Map<string, ArrayBuffer>();
 			for (const [id, value] of Object.entries(snapshotBlobs)) {
 				blobContents.set(id, stringToBuffer(value, "utf8"));
 			}
+			this.storageAdapter.cacheSnapshotBlobs(blobContents);
+			const attributes = await getDocumentAttributes(this.storageAdapter, baseSnapshot);
+
 			const snapshot: ISnapshot = {
 				sequenceNumber: attributes.sequenceNumber,
 				snapshotTree: baseSnapshot,

package/src/summarizerResultTypes.ts

@@ -0,0 +1,115 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import type { IErrorBase } from "@fluidframework/core-interfaces";
+import type {
+	ISequencedDocumentMessage,
+	ISummaryAck,
+	ISummaryContent,
+	ISummaryTree,
+	MessageType,
+} from "@fluidframework/driver-definitions/internal";
+
+export const summarizerRequestUrl = "_summarizer";
+
+/**
+ * Stages of summary process.
+ *
+ * Stages:
+ *
+ * 1. "base" - stopped before the summary tree was even generated, and the result only contains the base data
+ *
+ * 2. "generate" - the summary tree was generated, and the result will contain that tree + stats
+ *
+ * 3. "upload" - the summary was uploaded to storage, and the result contains the server-provided handle
+ *
+ * 4. "submit" - the summarize op was submitted, and the result contains the op client sequence number.
+ *
+ * @legacy @alpha
+ */
+export type SummaryStage = "base" | "generate" | "upload" | "submit" | "unknown";
+
+type OnDemandSummaryStageResult<TSuccess> =
+	| {
+			readonly success: true;
+			readonly data: TSuccess;
+	  }
+	| {
+			readonly success: false;
+			readonly error: IErrorBase;
+	  };
+
+interface ISummaryOpMessage extends ISequencedDocumentMessage {
+	type: MessageType.Summarize;
+	contents: ISummaryContent;
+}
+
+interface ISummaryAckMessage extends ISequencedDocumentMessage {
+	type: MessageType.SummaryAck;
+	contents: ISummaryAck;
+}
+
+/**
+ * @internal
+ */
+export interface SummarizeOnDemandResults {
+	readonly summarySubmitted: OnDemandSummaryStageResult<{
+		readonly stage: SummaryStage;
+		readonly summaryTree?: ISummaryTree;
+		readonly handle?: string;
+	}>;
+	readonly summaryOpBroadcasted: OnDemandSummaryStageResult<{
+		readonly broadcastDuration: number;
+		readonly summarizeOp: ISummaryOpMessage;
+	}>;
+	readonly receivedSummaryAckOrNack: OnDemandSummaryStageResult<{
+		readonly summaryAckOp: ISummaryAckMessage;
+		readonly ackNackDuration: number;
+	}>;
+}
+
+/**
+ * Results from an on-demand summary request.
+ * @legacy @alpha
+ */
+export interface OnDemandSummaryResults {
+	/**
+	 * True if summary was generated, uploaded, and submitted.
+	 */
+	readonly summarySubmitted: boolean;
+
+	/**
+	 * Information about the summary that was submitted, if any.
+	 */
+	readonly summaryInfo: {
+		/**
+		 * Stage at which summary process ended.
+		 */
+		readonly stage?: SummaryStage;
+		/**
+		 * Handle of the complete summary.
+		 */
+		readonly handle?: string;
+	};
+
+	/**
+	 * True if summarize op broadcast was observed.
+	 */
+	readonly summaryOpBroadcasted: boolean;
+}
+
+/**
+ * Outcome from {@link loadSummarizerContainerAndMakeSummary}.
+ * @legacy @alpha
+ */
+export type LoadSummarizerSummaryResult =
+	| {
+			readonly success: true;
+			readonly summaryResults: OnDemandSummaryResults;
+	  }
+	| {
+			readonly success: false;
+			readonly error: IErrorBase;
+	  };

package/src/utils.ts

@@ -274,33 +274,34 @@ export function getProtocolSnapshotTree(snapshot: ISnapshotTree): ISnapshotTree
 	return ".protocol" in snapshot.trees ? snapshot.trees[".protocol"] : snapshot;
 }
 
-export const combineSnapshotTreeAndSnapshotBlobs = (
-	baseSnapshot: ISnapshotTree,
-	snapshotBlobs: ISerializableBlobContents,
-): ISnapshotTreeWithBlobContents => {
-	const blobsContents: { [path: string]: ArrayBufferLike } = {};
+export const combineSnapshotTreeAndSnapshotBlobs = ({
+	blobContents,
+	snapshotTree,
+}: Pick<ISnapshot, "blobContents" | "snapshotTree">): ISnapshotTreeWithBlobContents => {
+	const currentTreeBlobs: { [path: string]: ArrayBufferLike } = {};
 
 	// Process blobs in the current level
-	for (const [, id] of Object.entries(baseSnapshot.blobs)) {
-		if (snapshotBlobs[id] !== undefined) {
-			blobsContents[id] = stringToBuffer(snapshotBlobs[id], "utf8");
+	for (const [, id] of Object.entries(snapshotTree.blobs)) {
+		const blob = blobContents.get(id);
+		if (blob !== undefined) {
+			currentTreeBlobs[id] = blob;
 		}
 	}
 
 	// Recursively process trees in the current level
 	const trees: { [path: string]: ISnapshotTreeWithBlobContents } = {};
-	for (const [path, tree] of Object.entries(baseSnapshot.trees)) {
-		trees[path] = combineSnapshotTreeAndSnapshotBlobs(tree, snapshotBlobs);
+	for (const [path, tree] of Object.entries(snapshotTree.trees)) {
+		trees[path] = combineSnapshotTreeAndSnapshotBlobs({ snapshotTree: tree, blobContents });
 	}
 
 	// Create a new snapshot tree with blob contents and processed trees
-	const snapshotTreeWithBlobContents: ISnapshotTreeWithBlobContents = {
-		...baseSnapshot,
-		blobsContents,
+	const snapshot: ISnapshotTreeWithBlobContents = {
+		...snapshotTree,
+		blobsContents: currentTreeBlobs,
 		trees,
 	};
 
-	return snapshotTreeWithBlobContents;
+	return snapshot;
 };
 
 export function isDeltaStreamConnectionForbiddenError(