@fluidframework/routerlicious-driver 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.2.0.153917

package/src/wholeSummaryDocumentStorageService.ts

@@ -4,13 +4,18 @@
  */
 
 import type { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { assert, stringToBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
+import {
+	assert,
+	performance,
+	stringToBuffer,
+	Uint8ArrayToString,
+} from "@fluidframework/common-utils";
+import { getW3CData, promiseRaceWithWinner } from "@fluidframework/driver-base";
 import {
 	IDocumentStorageService,
 	ISummaryContext,
 	IDocumentStorageServicePolicies,
 } from "@fluidframework/driver-definitions";
-import { convertSnapshotAndBlobsToSummaryTree } from "@fluidframework/driver-utils";
 import {
 	ICreateBlobResponse,
 	ISnapshotTree,
@@ -20,14 +25,21 @@ import {
 } from "@fluidframework/protocol-definitions";
 import {
 	convertWholeFlatSummaryToSnapshotTreeAndBlobs,
-	GitManager,
-	ISummaryUploadManager,
-	WholeSummaryUploadManager,
+	INormalizedWholeSummary,
+	IWholeFlatSummary,
 } from "@fluidframework/server-services-client";
 import { PerformanceEvent } from "@fluidframework/telemetry-utils";
 import { ICache, InMemoryCache } from "./cache";
-import { ISnapshotTreeVersion } from "./definitions";
 import { IRouterliciousDriverPolicies } from "./policies";
+import {
+	convertSnapshotAndBlobsToSummaryTree,
+	evalBlobsAndTrees,
+	validateBlobsAndTrees,
+} from "./treeUtils";
+import { GitManager } from "./gitManager";
+import { WholeSummaryUploadManager } from "./wholeSummaryUploadManager";
+import { ISummaryUploadManager } from "./storageContracts";
+import { IR11sResponse } from "./restWrapper";
 
 const latestSnapshotId: string = "latest";
 
@@ -50,7 +62,7 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 		public readonly policies: IDocumentStorageServicePolicies,
 		private readonly driverPolicies?: IRouterliciousDriverPolicies,
 		private readonly blobCache: ICache<ArrayBufferLike> = new InMemoryCache(),
-		private readonly snapshotTreeCache: ICache<ISnapshotTreeVersion> = new InMemoryCache(),
+		private readonly snapshotTreeCache: ICache<INormalizedWholeSummary> = new InMemoryCache(),
 		private readonly noCacheGitManager?: GitManager,
 		private readonly getStorageManager: (
 			disableCache?: boolean,
@@ -73,14 +85,60 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 		// If this is the first versions call for the document, we know we will want the latest summary.
 		// Fetch latest summary, cache it, and return its id.
 		if (this.firstVersionsCall && count === 1) {
+			const normalizedSnapshotContents = await PerformanceEvent.timedExecAsync(
+				this.logger,
+				{
+					eventName: "ObtainSnapshot",
+					versionId: versionId ?? undefined,
+					count,
+					enableDiscovery: this.driverPolicies?.enableDiscovery,
+				},
+				async (event) => {
+					let method: string;
+					const cachedSnapshotP = this.snapshotTreeCache.get(
+						this.getCacheKey(latestSnapshotId),
+					);
+
+					const networkSnapshotP = !this.driverPolicies?.enableDiscovery
+						? this.fetchSnapshotTree(latestSnapshotId, false)
+						: this.fetchSnapshotTree(latestSnapshotId, true);
+
+					const promiseRaceWinner = await promiseRaceWithWinner([
+						cachedSnapshotP.catch(() => undefined),
+						networkSnapshotP.catch(() => undefined),
+					]);
+
+					let retrievedSnapshot = promiseRaceWinner.value;
+					method = promiseRaceWinner.index === 0 ? "cache" : "network";
+
+					if (retrievedSnapshot === undefined) {
+						// if network failed -> wait for cache ( then return network failure)
+						// If cache returned empty or failed -> wait for network (success of failure)
+						if (promiseRaceWinner.index === 1) {
+							retrievedSnapshot = await cachedSnapshotP;
+							method = "cache";
+						}
+						if (retrievedSnapshot === undefined) {
+							retrievedSnapshot = await networkSnapshotP;
+							method = "network";
+						}
+					}
+					event.end({
+						method,
+					});
+					return retrievedSnapshot;
+				},
+			);
+
+			const _id = await this.initializeFromSnapshot(
+				normalizedSnapshotContents,
+				latestSnapshotId,
+			);
 			this.firstVersionsCall = false;
-			const { id: _id, snapshotTree } = !this.driverPolicies?.enableDiscovery
-				? await this.fetchAndCacheSnapshotTree(latestSnapshotId, false)
-				: await this.fetchAndCacheSnapshotTree(latestSnapshotId, true);
 			return [
 				{
 					id: _id,
-					treeId: snapshotTree.id!,
+					treeId: normalizedSnapshotContents.snapshotTree.id!,
 				},
 			];
 		}
@@ -96,7 +154,7 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 			},
 			async () => {
 				const manager = await this.getStorageManager();
-				return manager.getCommits(id, count);
+				return (await manager.getCommits(id, count)).content;
 			},
 		);
 		return commits.map((commit) => ({
@@ -117,7 +175,13 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 			requestVersion = versions[0];
 		}
 
-		return (await this.fetchAndCacheSnapshotTree(requestVersion.id)).snapshotTree;
+		const normalizedWholeSnapshot = await this.snapshotTreeCache.get(
+			this.getCacheKey(requestVersion.id),
+		);
+		if (normalizedWholeSnapshot !== undefined) {
+			return normalizedWholeSnapshot.snapshotTree;
+		}
+		return (await this.fetchSnapshotTree(requestVersion.id)).snapshotTree;
 	}
 
 	public async readBlob(blobId: string): Promise<ArrayBufferLike> {
@@ -134,7 +198,7 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 			},
 			async (event) => {
 				const manager = await this.getStorageManager();
-				const response = await manager.getBlob(blobId);
+				const response = (await manager.getBlob(blobId)).content;
 				event.end({
 					size: response.size,
 				});
@@ -183,9 +247,9 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 				const manager = await this.getStorageManager();
 				const response = await manager.getSummary(summaryHandle.handle);
 				event.end({
-					size: response.trees[0]?.entries.length,
+					size: response.content.trees[0]?.entries.length,
 				});
-				return response;
+				return response.content;
 			},
 		);
 
@@ -208,7 +272,7 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 				const manager = await this.getStorageManager();
 				const response = await manager
 					.createBlob(Uint8ArrayToString(uint8ArrayFile, "base64"), "base64")
-					.then((r) => ({ id: r.sha, url: r.url }));
+					.then((r) => ({ id: r.content.sha, url: r.content.url }));
 				event.end({
 					blobId: response.id,
 				});
@@ -217,21 +281,11 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 		);
 	}
 
-	private async fetchAndCacheSnapshotTree(
+	private async fetchSnapshotTree(
 		versionId: string,
 		disableCache?: boolean,
-	): Promise<ISnapshotTreeVersion> {
-		const cachedSnapshotTreeVersion = await this.snapshotTreeCache.get(
-			this.getCacheKey(versionId),
-		);
-		if (cachedSnapshotTreeVersion !== undefined) {
-			return {
-				id: cachedSnapshotTreeVersion.id,
-				snapshotTree: cachedSnapshotTreeVersion.snapshotTree,
-			};
-		}
-
-		const wholeFlatSummary = await PerformanceEvent.timedExecAsync(
+	): Promise<INormalizedWholeSummary> {
+		const normalizedWholeSummary = await PerformanceEvent.timedExecAsync(
 			this.logger,
 			{
 				eventName: "getWholeFlatSummary",
@@ -239,40 +293,58 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 			},
 			async (event) => {
 				const manager = await this.getStorageManager(disableCache);
-				const response = await manager.getSummary(versionId);
+				const response: IR11sResponse<IWholeFlatSummary> = await manager.getSummary(
+					versionId,
+				);
+				const start = performance.now();
+				const snapshot: INormalizedWholeSummary =
+					convertWholeFlatSummaryToSnapshotTreeAndBlobs(response.content);
+				const snapshotConversionTime = performance.now() - start;
+				validateBlobsAndTrees(snapshot.snapshotTree);
+				const { trees, numBlobs, encodedBlobsSize } = evalBlobsAndTrees(snapshot);
+
 				event.end({
-					size: response.trees[0]?.entries.length,
+					size: response.content.trees[0]?.entries.length,
+					trees,
+					blobs: numBlobs,
+					encodedBlobsSize,
+					...response.propsToLog,
+					snapshotConversionTime,
+					...getW3CData(response.requestUrl, "xmlhttprequest"),
 				});
-				return response;
+				return snapshot;
 			},
 		);
-		const normalizedWholeSummary =
-			convertWholeFlatSummaryToSnapshotTreeAndBlobs(wholeFlatSummary);
-		const wholeFlatSummaryId: string = wholeFlatSummary.id;
-		const snapshotTreeId = normalizedWholeSummary.snapshotTree.id;
-		assert(snapshotTreeId !== undefined, 0x275 /* "Root tree should contain the id" */);
-		const snapshotTreeVersion = {
-			id: wholeFlatSummaryId,
-			snapshotTree: normalizedWholeSummary.snapshotTree,
-		};
 
+		return normalizedWholeSummary;
+	}
+
+	private async initializeFromSnapshot(
+		normalizedWholeSummary: INormalizedWholeSummary,
+		versionId: string | null,
+	): Promise<string> {
+		const wholeFlatSummaryId = normalizedWholeSummary.snapshotTree.id;
+		assert(wholeFlatSummaryId !== undefined, 0x275 /* "Root tree should contain the id" */);
 		const cachePs: Promise<any>[] = [
-			this.snapshotTreeCache.put(this.getCacheKey(snapshotTreeId), snapshotTreeVersion),
+			this.snapshotTreeCache.put(
+				this.getCacheKey(wholeFlatSummaryId),
+				normalizedWholeSummary,
+			),
 			this.initBlobCache(normalizedWholeSummary.blobs),
 		];
-		if (snapshotTreeId !== versionId) {
+		if (wholeFlatSummaryId !== versionId && versionId !== null) {
 			// versionId could be "latest". When summarizer checks cache for "latest", we want it to be available.
 			// TODO: For in-memory cache, <latest,snapshotTree> will be a shared pointer with <snapshotId,snapshotTree>,
 			// However, for something like Redis, this will cache the same value twice. Alternatively, could we simply
 			// cache with versionId?
 			cachePs.push(
-				this.snapshotTreeCache.put(this.getCacheKey(versionId), snapshotTreeVersion),
+				this.snapshotTreeCache.put(this.getCacheKey(versionId), normalizedWholeSummary),
 			);
 		}
 
 		await Promise.all(cachePs);
 
-		return snapshotTreeVersion;
+		return wholeFlatSummaryId;
 	}
 
 	private async initBlobCache(blobs: Map<string, ArrayBuffer>): Promise<void> {

package/src/wholeSummaryUploadManager.ts

@@ -0,0 +1,64 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import {
+	ISummaryTree,
+	IWholeSummaryPayload,
+	IWholeSummaryPayloadType,
+	convertSummaryTreeToWholeSummaryTree,
+} from "@fluidframework/server-services-client";
+import { IGitManager, ISummaryUploadManager } from "./storageContracts";
+
+/**
+ * Converts summary to snapshot tree and uploads with single snaphot tree payload.
+ */
+export class WholeSummaryUploadManager implements ISummaryUploadManager {
+	constructor(private readonly manager: IGitManager) {}
+
+	public async writeSummaryTree(
+		summaryTree: ISummaryTree,
+		parentHandle: string | undefined,
+		summaryType: IWholeSummaryPayloadType,
+		sequenceNumber: number = 0,
+		initial: boolean = false,
+	): Promise<string> {
+		const id = await this.writeSummaryTreeCore(
+			parentHandle,
+			summaryTree,
+			summaryType,
+			sequenceNumber,
+			initial,
+		);
+		if (!id) {
+			throw new Error(`Failed to write summary tree`);
+		}
+		return id;
+	}
+
+	private async writeSummaryTreeCore(
+		parentHandle: string | undefined,
+		tree: ISummaryTree,
+		type: IWholeSummaryPayloadType,
+		sequenceNumber: number,
+		initial: boolean,
+	): Promise<string> {
+		const snapshotTree = convertSummaryTreeToWholeSummaryTree(
+			parentHandle,
+			tree,
+			"",
+			type === "channel" ? ".app" : "",
+		);
+		const snapshotPayload: IWholeSummaryPayload = {
+			entries: snapshotTree.entries ?? [],
+			message: "",
+			sequenceNumber,
+			type,
+		};
+
+		return this.manager
+			.createSummary(snapshotPayload, initial)
+			.then((response) => response.content.id);
+	}
+}