@fluidframework/routerlicious-driver 2.0.0-dev.4.1.0.148229 → 2.0.0-dev.4.3.0.157531

package/src/storageContracts.ts

@@ -0,0 +1,63 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as git from "@fluidframework/gitresources";
+import * as api from "@fluidframework/protocol-definitions";
+import {
+	IWholeFlatSummary,
+	IWholeSummaryPayload,
+	IWholeSummaryPayloadType,
+	IWriteSummaryResponse,
+} from "@fluidframework/server-services-client";
+import { IR11sResponse } from "./restWrapper";
+
+/**
+ * Interface to a generic Git provider
+ */
+export interface IHistorian {
+	getBlob(sha: string): Promise<IR11sResponse<git.IBlob>>;
+	createBlob(blob: git.ICreateBlobParams): Promise<IR11sResponse<git.ICreateBlobResponse>>;
+	getCommits(sha: string, count: number): Promise<IR11sResponse<git.ICommitDetails[]>>;
+	createTree(tree: git.ICreateTreeParams): Promise<IR11sResponse<git.ITree>>;
+	getTree(sha: string, recursive: boolean): Promise<IR11sResponse<git.ITree>>;
+	createSummary(
+		summary: IWholeSummaryPayload,
+		initial?: boolean,
+	): Promise<IR11sResponse<IWriteSummaryResponse>>;
+	getSummary(sha: string): Promise<IR11sResponse<IWholeFlatSummary>>;
+}
+
+export interface IGitManager {
+	getCommits(sha: string, count: number): Promise<IR11sResponse<git.ICommitDetails[]>>;
+	getTree(root: string, recursive: boolean): Promise<IR11sResponse<git.ITree>>;
+	getBlob(sha: string): Promise<IR11sResponse<git.IBlob>>;
+	createBlob(content: string, encoding: string): Promise<IR11sResponse<git.ICreateBlobResponse>>;
+	createGitTree(params: git.ICreateTreeParams): Promise<IR11sResponse<git.ITree>>;
+	createSummary(
+		summary: IWholeSummaryPayload,
+		initial?: boolean,
+	): Promise<IR11sResponse<IWriteSummaryResponse>>;
+	getSummary(sha: string): Promise<IR11sResponse<IWholeFlatSummary>>;
+}
+
+/**
+ * Uploads a summary to storage.
+ */
+export interface ISummaryUploadManager {
+	/**
+	 * Writes summary tree to storage.
+	 * @param summaryTree - Summary tree to write to storage
+	 * @param parentHandle - Parent summary acked handle (if available from summary ack)
+	 * @param summaryType - type of summary being uploaded
+	 * @param sequenceNumber - optional reference sequence number of the summary
+	 * @returns Id of created tree as a string.
+	 */
+	writeSummaryTree(
+		summaryTree: api.ISummaryTree,
+		parentHandle: string,
+		summaryType: IWholeSummaryPayloadType,
+		sequenceNumber?: number,
+	): Promise<string>;
+}

package/src/summaryTreeUploadManager.ts

@@ -0,0 +1,160 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import {
+	assert,
+	gitHashFile,
+	IsoBuffer,
+	Uint8ArrayToString,
+	unreachableCase,
+} from "@fluidframework/common-utils";
+import { ICreateTreeEntry } from "@fluidframework/gitresources";
+import { getGitMode, getGitType } from "@fluidframework/protocol-base";
+import {
+	ISnapshotTreeEx,
+	ISummaryTree,
+	SummaryObject,
+	SummaryType,
+} from "@fluidframework/protocol-definitions";
+import { IWholeSummaryPayloadType } from "@fluidframework/server-services-client";
+import { IGitManager, ISummaryUploadManager } from "./storageContracts";
+
+/**
+ * Recursively writes summary tree as individual summary blobs.
+ */
+export class SummaryTreeUploadManager implements ISummaryUploadManager {
+	constructor(
+		private readonly manager: IGitManager,
+		private readonly blobsShaCache: Map<string, string>,
+		private readonly getPreviousFullSnapshot: (
+			parentHandle: string,
+		) => Promise<ISnapshotTreeEx | null | undefined>,
+	) {}
+
+	public async writeSummaryTree(
+		summaryTree: ISummaryTree,
+		parentHandle: string,
+		summaryType: IWholeSummaryPayloadType,
+		sequenceNumber?: number,
+		initial?: boolean,
+	): Promise<string> {
+		const previousFullSnapshot = await this.getPreviousFullSnapshot(parentHandle);
+		return this.writeSummaryTreeCore(summaryTree, previousFullSnapshot ?? undefined);
+	}
+
+	private async writeSummaryTreeCore(
+		summaryTree: ISummaryTree,
+		previousFullSnapshot: ISnapshotTreeEx | undefined,
+	): Promise<string> {
+		const entries = await Promise.all(
+			Object.keys(summaryTree.tree).map(async (key) => {
+				const entry = summaryTree.tree[key];
+				const pathHandle = await this.writeSummaryTreeObject(entry, previousFullSnapshot);
+				const treeEntry: ICreateTreeEntry = {
+					mode: getGitMode(entry),
+					path: encodeURIComponent(key),
+					sha: pathHandle,
+					type: getGitType(entry),
+				};
+				return treeEntry;
+			}),
+		);
+
+		const treeHandle = (await this.manager.createGitTree({ tree: entries })).content;
+		return treeHandle.sha;
+	}
+
+	private async writeSummaryTreeObject(
+		object: SummaryObject,
+		previousFullSnapshot: ISnapshotTreeEx | undefined,
+	): Promise<string> {
+		switch (object.type) {
+			case SummaryType.Blob: {
+				return this.writeSummaryBlob(object.content);
+			}
+			case SummaryType.Handle: {
+				if (previousFullSnapshot === undefined) {
+					throw Error("Parent summary does not exist to reference by handle.");
+				}
+				return this.getIdFromPath(object.handleType, object.handle, previousFullSnapshot);
+			}
+			case SummaryType.Tree: {
+				return this.writeSummaryTreeCore(object, previousFullSnapshot);
+			}
+			case SummaryType.Attachment: {
+				return object.id;
+			}
+
+			default:
+				unreachableCase(object, `Unknown type: ${(object as any).type}`);
+		}
+	}
+
+	private async writeSummaryBlob(content: string | Uint8Array): Promise<string> {
+		const { parsedContent, encoding } =
+			typeof content === "string"
+				? { parsedContent: content, encoding: "utf-8" }
+				: { parsedContent: Uint8ArrayToString(content, "base64"), encoding: "base64" };
+
+		// The gitHashFile would return the same hash as returned by the server as blob.sha
+		const hash = await gitHashFile(IsoBuffer.from(parsedContent, encoding));
+		if (!this.blobsShaCache.has(hash)) {
+			this.blobsShaCache.set(hash, "");
+			const blob = (await this.manager.createBlob(parsedContent, encoding)).content;
+			assert(hash === blob.sha, 0x0b6 /* "Blob.sha and hash do not match!!" */);
+		}
+		return hash;
+	}
+
+	private getIdFromPath(
+		handleType: SummaryType,
+		handlePath: string,
+		previousFullSnapshot: ISnapshotTreeEx,
+	): string {
+		const path = handlePath.split("/").map((part) => decodeURIComponent(part));
+		if (path[0] === "") {
+			// root of tree should be unnamed
+			path.shift();
+		}
+		if (path.length === 0) {
+			return previousFullSnapshot.id;
+		}
+
+		return this.getIdFromPathCore(handleType, path, previousFullSnapshot);
+	}
+
+	private getIdFromPathCore(
+		handleType: SummaryType,
+		path: string[],
+		/** Previous snapshot, subtree relative to this path part */
+		previousSnapshot: ISnapshotTreeEx,
+	): string {
+		assert(path.length > 0, 0x0b3 /* "Expected at least 1 path part" */);
+		const key = path[0];
+		if (path.length === 1) {
+			switch (handleType) {
+				case SummaryType.Blob: {
+					const tryId = previousSnapshot.blobs[key];
+					assert(
+						!!tryId,
+						0x0b4 /* "Parent summary does not have blob handle for specified path." */,
+					);
+					return tryId;
+				}
+				case SummaryType.Tree: {
+					const tryId = previousSnapshot.trees[key]?.id;
+					assert(
+						!!tryId,
+						0x0b5 /* "Parent summary does not have tree handle for specified path." */,
+					);
+					return tryId;
+				}
+				default:
+					throw Error(`Unexpected handle summary object type: "${handleType}".`);
+			}
+		}
+		return this.getIdFromPathCore(handleType, path.slice(1), previousSnapshot.trees[key]);
+	}
+}

package/src/treeUtils.ts

@@ -10,6 +10,7 @@ import {
 	ISummaryTree,
 	SummaryObject,
 } from "@fluidframework/protocol-definitions";
+import { INormalizedWholeSummary } from "./contracts";
 
 /**
  * Summary tree assembler props
@@ -105,3 +106,32 @@ export function convertSnapshotAndBlobsToSummaryTree(
 	}
 	return assembler.summary;
 }
+
+export function evalBlobsAndTrees(snapshot: INormalizedWholeSummary) {
+	const trees = countTreesInSnapshotTree(snapshot.snapshotTree);
+	const numBlobs = snapshot.blobs.size;
+	let encodedBlobsSize = 0;
+	for (const [_, blobContent] of snapshot.blobs) {
+		encodedBlobsSize += blobContent.byteLength;
+	}
+	return { trees, numBlobs, encodedBlobsSize };
+}
+
+export function validateBlobsAndTrees(snapshot: ISnapshotTree) {
+	assert(
+		snapshot.trees !== undefined,
+		0x5d0 /* Returned r11s snapshot is malformed. No trees! */,
+	);
+	assert(
+		snapshot.blobs !== undefined,
+		0x5d1 /* Returned r11s snapshot is malformed. No blobs! */,
+	);
+}
+
+function countTreesInSnapshotTree(snapshotTree: ISnapshotTree): number {
+	let numTrees = 0;
+	for (const [_, tree] of Object.entries(snapshotTree.trees)) {
+		numTrees += 1 + countTreesInSnapshotTree(tree);
+	}
+	return numTrees;
+}

package/src/wholeSummaryDocumentStorageService.ts

@@ -4,7 +4,13 @@
  */
 
 import type { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { assert, stringToBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
+import {
+	assert,
+	performance,
+	stringToBuffer,
+	Uint8ArrayToString,
+} from "@fluidframework/common-utils";
+import { getW3CData, promiseRaceWithWinner } from "@fluidframework/driver-base";
 import {
 	IDocumentStorageService,
 	ISummaryContext,
@@ -17,17 +23,21 @@ import {
 	ISummaryTree,
 	IVersion,
 } from "@fluidframework/protocol-definitions";
-import {
-	convertWholeFlatSummaryToSnapshotTreeAndBlobs,
-	GitManager,
-	ISummaryUploadManager,
-	WholeSummaryUploadManager,
-} from "@fluidframework/server-services-client";
+import { IWholeFlatSummary } from "@fluidframework/server-services-client";
 import { PerformanceEvent } from "@fluidframework/telemetry-utils";
 import { ICache, InMemoryCache } from "./cache";
-import { ISnapshotTreeVersion } from "./definitions";
 import { IRouterliciousDriverPolicies } from "./policies";
-import { convertSnapshotAndBlobsToSummaryTree } from "./treeUtils";
+import {
+	convertSnapshotAndBlobsToSummaryTree,
+	evalBlobsAndTrees,
+	validateBlobsAndTrees,
+} from "./treeUtils";
+import { GitManager } from "./gitManager";
+import { WholeSummaryUploadManager } from "./wholeSummaryUploadManager";
+import { ISummaryUploadManager } from "./storageContracts";
+import { IR11sResponse } from "./restWrapper";
+import { INormalizedWholeSummary } from "./contracts";
+import { convertWholeFlatSummaryToSnapshotTreeAndBlobs } from "./r11sSnapshotParser";
 
 const latestSnapshotId: string = "latest";
 
@@ -50,7 +60,7 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 		public readonly policies: IDocumentStorageServicePolicies,
 		private readonly driverPolicies?: IRouterliciousDriverPolicies,
 		private readonly blobCache: ICache<ArrayBufferLike> = new InMemoryCache(),
-		private readonly snapshotTreeCache: ICache<ISnapshotTreeVersion> = new InMemoryCache(),
+		private readonly snapshotTreeCache: ICache<INormalizedWholeSummary> = new InMemoryCache(),
 		private readonly noCacheGitManager?: GitManager,
 		private readonly getStorageManager: (
 			disableCache?: boolean,
@@ -73,14 +83,57 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 		// If this is the first versions call for the document, we know we will want the latest summary.
 		// Fetch latest summary, cache it, and return its id.
 		if (this.firstVersionsCall && count === 1) {
+			const normalizedSnapshotContents = await PerformanceEvent.timedExecAsync(
+				this.logger,
+				{
+					eventName: "ObtainSnapshot",
+					versionId: versionId ?? undefined,
+					count,
+					enableDiscovery: this.driverPolicies?.enableDiscovery,
+				},
+				async (event) => {
+					let method: string;
+					const cachedSnapshotP = this.snapshotTreeCache.get(
+						this.getCacheKey(latestSnapshotId),
+					);
+
+					const networkSnapshotP = !this.driverPolicies?.enableDiscovery
+						? this.fetchSnapshotTree(latestSnapshotId, false, "getVersions")
+						: this.fetchSnapshotTree(latestSnapshotId, true, "getVersions");
+
+					const promiseRaceWinner = await promiseRaceWithWinner([
+						cachedSnapshotP.catch(() => undefined),
+						networkSnapshotP.catch(() => undefined),
+					]);
+
+					let retrievedSnapshot = promiseRaceWinner.value;
+					method = promiseRaceWinner.index === 0 ? "cache" : "network";
+
+					if (retrievedSnapshot === undefined) {
+						// if network failed -> wait for cache ( then return network failure)
+						// If cache returned empty or failed -> wait for network (success of failure)
+						if (promiseRaceWinner.index === 1) {
+							retrievedSnapshot = await cachedSnapshotP;
+							method = "cache";
+						}
+						if (retrievedSnapshot === undefined) {
+							retrievedSnapshot = await networkSnapshotP;
+							method = "network";
+						}
+					}
+					event.end({
+						method,
+					});
+					return retrievedSnapshot;
+				},
+			);
+
+			const _id = await this.initializeFromSnapshot(normalizedSnapshotContents);
 			this.firstVersionsCall = false;
-			const { id: _id, snapshotTree } = !this.driverPolicies?.enableDiscovery
-				? await this.fetchAndCacheSnapshotTree(latestSnapshotId, false)
-				: await this.fetchAndCacheSnapshotTree(latestSnapshotId, true);
 			return [
 				{
 					id: _id,
-					treeId: snapshotTree.id!,
+					treeId: normalizedSnapshotContents.snapshotTree.id!,
 				},
 			];
 		}
@@ -96,7 +149,7 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 			},
 			async () => {
 				const manager = await this.getStorageManager();
-				return manager.getCommits(id, count);
+				return (await manager.getCommits(id, count)).content;
 			},
 		);
 		return commits.map((commit) => ({
@@ -117,7 +170,14 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 			requestVersion = versions[0];
 		}
 
-		return (await this.fetchAndCacheSnapshotTree(requestVersion.id)).snapshotTree;
+		const normalizedWholeSnapshot = await this.snapshotTreeCache.get(
+			this.getCacheKey(requestVersion.id),
+		);
+		if (normalizedWholeSnapshot !== undefined) {
+			return normalizedWholeSnapshot.snapshotTree;
+		}
+		return (await this.fetchSnapshotTree(requestVersion.id, undefined, "getSnapshotTree"))
+			.snapshotTree;
 	}
 
 	public async readBlob(blobId: string): Promise<ArrayBufferLike> {
@@ -134,7 +194,7 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 			},
 			async (event) => {
 				const manager = await this.getStorageManager();
-				const response = await manager.getBlob(blobId);
+				const response = (await manager.getBlob(blobId)).content;
 				event.end({
 					size: response.size,
 				});
@@ -183,9 +243,9 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 				const manager = await this.getStorageManager();
 				const response = await manager.getSummary(summaryHandle.handle);
 				event.end({
-					size: response.trees[0]?.entries.length,
+					size: response.content.trees[0]?.entries.length,
 				});
-				return response;
+				return response.content;
 			},
 		);
 
@@ -208,7 +268,7 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 				const manager = await this.getStorageManager();
 				const response = await manager
 					.createBlob(Uint8ArrayToString(uint8ArrayFile, "base64"), "base64")
-					.then((r) => ({ id: r.sha, url: r.url }));
+					.then((r) => ({ id: r.content.sha, url: r.content.url }));
 				event.end({
 					blobId: response.id,
 				});
@@ -217,64 +277,63 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
 		);
 	}
 
-	private async fetchAndCacheSnapshotTree(
+	private async fetchSnapshotTree(
 		versionId: string,
 		disableCache?: boolean,
-	): Promise<ISnapshotTreeVersion> {
-		const cachedSnapshotTreeVersion = await this.snapshotTreeCache.get(
-			this.getCacheKey(versionId),
-		);
-		if (cachedSnapshotTreeVersion !== undefined) {
-			return {
-				id: cachedSnapshotTreeVersion.id,
-				snapshotTree: cachedSnapshotTreeVersion.snapshotTree,
-			};
-		}
-
-		const wholeFlatSummary = await PerformanceEvent.timedExecAsync(
+		scenarioName?: string,
+	): Promise<INormalizedWholeSummary> {
+		const normalizedWholeSummary = await PerformanceEvent.timedExecAsync(
 			this.logger,
 			{
 				eventName: "getWholeFlatSummary",
 				treeId: versionId,
+				scenarioName,
 			},
 			async (event) => {
 				const manager = await this.getStorageManager(disableCache);
-				const response = await manager.getSummary(versionId);
+				const response: IR11sResponse<IWholeFlatSummary> = await manager.getSummary(
+					versionId,
+				);
+				const start = performance.now();
+				const snapshot: INormalizedWholeSummary =
+					convertWholeFlatSummaryToSnapshotTreeAndBlobs(response.content);
+				const snapshotConversionTime = performance.now() - start;
+				validateBlobsAndTrees(snapshot.snapshotTree);
+				const { trees, numBlobs, encodedBlobsSize } = evalBlobsAndTrees(snapshot);
+
 				event.end({
-					size: response.trees[0]?.entries.length,
+					size: response.content.trees[0]?.entries.length,
+					trees,
+					blobs: numBlobs,
+					encodedBlobsSize,
+					...response.propsToLog,
+					snapshotConversionTime,
+					...getW3CData(response.requestUrl, "xmlhttprequest"),
 				});
-				return response;
+				return snapshot;
 			},
 		);
-		const normalizedWholeSummary =
-			convertWholeFlatSummaryToSnapshotTreeAndBlobs(wholeFlatSummary);
-		assert(
-			normalizedWholeSummary.snapshotTree.id !== undefined,
-			0x275 /* "Root tree should contain the id" */,
-		);
-		const wholeFlatSummaryId: string = wholeFlatSummary.id;
-		const snapshotTreeVersion = {
-			id: wholeFlatSummaryId,
-			snapshotTree: normalizedWholeSummary.snapshotTree,
-		};
 
+		// Also add the result into the cache.
+		await this.snapshotTreeCache
+			.put(this.getCacheKey(versionId), normalizedWholeSummary)
+			.catch(() => undefined);
+		return normalizedWholeSummary;
+	}
+
+	private async initializeFromSnapshot(
+		normalizedWholeSummary: INormalizedWholeSummary,
+	): Promise<string> {
+		const snapshotId = normalizedWholeSummary.id;
+		assert(snapshotId !== undefined, 0x275 /* "Root tree should contain the id" */);
 		const cachePs: Promise<any>[] = [
-			this.snapshotTreeCache.put(this.getCacheKey(wholeFlatSummaryId), snapshotTreeVersion),
+			this.snapshotTreeCache.put(this.getCacheKey(snapshotId), normalizedWholeSummary),
 			this.initBlobCache(normalizedWholeSummary.blobs),
 		];
-		if (wholeFlatSummaryId !== versionId) {
-			// versionId could be "latest". When summarizer checks cache for "latest", we want it to be available.
-			// TODO: For in-memory cache, <latest,snapshotTree> will be a shared pointer with <snapshotId,snapshotTree>,
-			// However, for something like Redis, this will cache the same value twice. Alternatively, could we simply
-			// cache with versionId?
-			cachePs.push(
-				this.snapshotTreeCache.put(this.getCacheKey(versionId), snapshotTreeVersion),
-			);
-		}
 
 		await Promise.all(cachePs);
 
-		return snapshotTreeVersion;
+		return snapshotId;
 	}
 
 	private async initBlobCache(blobs: Map<string, ArrayBuffer>): Promise<void> {

package/src/wholeSummaryUploadManager.ts

@@ -0,0 +1,64 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import {
+	ISummaryTree,
+	IWholeSummaryPayload,
+	IWholeSummaryPayloadType,
+	convertSummaryTreeToWholeSummaryTree,
+} from "@fluidframework/server-services-client";
+import { IGitManager, ISummaryUploadManager } from "./storageContracts";
+
+/**
+ * Converts summary to snapshot tree and uploads with single snaphot tree payload.
+ */
+export class WholeSummaryUploadManager implements ISummaryUploadManager {
+	constructor(private readonly manager: IGitManager) {}
+
+	public async writeSummaryTree(
+		summaryTree: ISummaryTree,
+		parentHandle: string | undefined,
+		summaryType: IWholeSummaryPayloadType,
+		sequenceNumber: number = 0,
+		initial: boolean = false,
+	): Promise<string> {
+		const id = await this.writeSummaryTreeCore(
+			parentHandle,
+			summaryTree,
+			summaryType,
+			sequenceNumber,
+			initial,
+		);
+		if (!id) {
+			throw new Error(`Failed to write summary tree`);
+		}
+		return id;
+	}
+
+	private async writeSummaryTreeCore(
+		parentHandle: string | undefined,
+		tree: ISummaryTree,
+		type: IWholeSummaryPayloadType,
+		sequenceNumber: number,
+		initial: boolean,
+	): Promise<string> {
+		const snapshotTree = convertSummaryTreeToWholeSummaryTree(
+			parentHandle,
+			tree,
+			"",
+			type === "channel" ? ".app" : "",
+		);
+		const snapshotPayload: IWholeSummaryPayload = {
+			entries: snapshotTree.entries ?? [],
+			message: "",
+			sequenceNumber,
+			type,
+		};
+
+		return this.manager
+			.createSummary(snapshotPayload, initial)
+			.then((response) => response.content.id);
+	}
+}