@fluidframework/routerlicious-driver 1.4.0-115997 → 2.0.0-dev-rc.1.0.0.224419

package/src/shreddedSummaryDocumentStorageService.ts

@@ -3,34 +3,33 @@
  * Licensed under the MIT License.
  */
 
-import type { ITelemetryLogger } from "@fluidframework/common-definitions";
 import {
-    stringToBuffer,
-    Uint8ArrayToString,
-} from "@fluidframework/common-utils";
+	ITelemetryLoggerExt,
+	MonitoringContext,
+	PerformanceEvent,
+	createChildMonitoringContext,
+} from "@fluidframework/telemetry-utils";
+import { stringToBuffer, Uint8ArrayToString } from "@fluid-internal/client-utils";
 import {
-    IDocumentStorageService,
-    ISummaryContext,
-    IDocumentStorageServicePolicies,
+	IDocumentStorageService,
+	ISummaryContext,
+	IDocumentStorageServicePolicies,
 } from "@fluidframework/driver-definitions";
-import { buildHierarchy } from "@fluidframework/protocol-base";
+import { buildGitTreeHierarchy } from "@fluidframework/protocol-base";
 import {
-    ICreateBlobResponse,
-    ISnapshotTreeEx,
-    ISummaryHandle,
-    ISummaryTree,
-    IVersion,
+	ICreateBlobResponse,
+	ISnapshotTreeEx,
+	ISummaryHandle,
+	ISummaryTree,
+	IVersion,
 } from "@fluidframework/protocol-definitions";
-import {
-    GitManager,
-    ISummaryUploadManager,
-    SummaryTreeUploadManager,
-} from "@fluidframework/server-services-client";
-import { PerformanceEvent } from "@fluidframework/telemetry-utils";
 import { IRouterliciousDriverPolicies } from "./policies";
 import { ICache, InMemoryCache } from "./cache";
 import { RetriableGitManager } from "./retriableGitManager";
 import { ISnapshotTreeVersion } from "./definitions";
+import { GitManager } from "./gitManager";
+import { ISummaryUploadManager } from "./storageContracts";
+import { SummaryTreeUploadManager } from "./summaryTreeUploadManager";
 
 const isNode = typeof window === "undefined";
 
@@ -40,180 +39,200 @@ const isNode = typeof window === "undefined";
  * Downloads summaries piece-by-piece on-demand, or up-front when prefetch is enabled.
  */
 export class ShreddedSummaryDocumentStorageService implements IDocumentStorageService {
-    // The values of this cache is useless. We only need the keys. So we are always putting
-    // empty strings as values.
-    protected readonly blobsShaCache = new Map<string, string>();
-    private readonly blobCache: ICache<ArrayBufferLike> | undefined;
-    private readonly snapshotTreeCache: ICache<ISnapshotTreeVersion> | undefined;
-
-    public get repositoryUrl(): string {
-        return "";
-    }
-
-    private async getSummaryUploadManager(): Promise<ISummaryUploadManager> {
-        const manager = await this.getStorageManager();
-        return new SummaryTreeUploadManager(
-            new RetriableGitManager(manager, this.logger),
-            this.blobsShaCache,
-            this.getPreviousFullSnapshot.bind(this),
-        );
-    }
-
-    constructor(
-        protected readonly id: string,
-        protected readonly manager: GitManager,
-        protected readonly logger: ITelemetryLogger,
-        public readonly policies: IDocumentStorageServicePolicies = {},
-        driverPolicies?: IRouterliciousDriverPolicies,
-        blobCache?: ICache<ArrayBufferLike>,
-        snapshotTreeCache?: ICache<ISnapshotTreeVersion>,
-        private readonly getStorageManager: (disableCache?: boolean) => Promise<GitManager> = async () => this.manager,
-    ) {
-        if (driverPolicies?.enableRestLess === true || isNode) {
-            this.blobCache = blobCache ?? new InMemoryCache();
-            this.snapshotTreeCache = snapshotTreeCache ?? new InMemoryCache();
-        }
-    }
-
-    public async getVersions(versionId: string | null, count: number): Promise<IVersion[]> {
-        const id = versionId ? versionId : this.id;
-        const commits = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "getVersions",
-                versionId: id,
-                count,
-            },
-            async () => {
-                const manager = await this.getStorageManager();
-                return manager.getCommits(id, count);
-            },
-        );
-        return commits.map((commit) => ({
-            date: commit.commit.author.date,
-            id: commit.sha,
-            treeId: commit.commit.tree.sha,
-        }));
-    }
-
-    public async getSnapshotTree(version?: IVersion): Promise<ISnapshotTreeEx | null> {
-        let requestVersion = version;
-        if (!requestVersion) {
-            const versions = await this.getVersions(this.id, 1);
-            if (versions.length === 0) {
-                return null;
-            }
-
-            requestVersion = versions[0];
-        }
-
-        const cachedSnapshotTree = await this.snapshotTreeCache?.get(this.getCacheKey(requestVersion.treeId));
-        if (cachedSnapshotTree) {
-            return cachedSnapshotTree.snapshotTree as ISnapshotTreeEx;
-        }
-
-        const rawTree = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "getSnapshotTree",
-                treeId: requestVersion.treeId,
-            },
-            async (event) => {
-                const manager = await this.getStorageManager();
-                const response = await manager.getTree(requestVersion!.treeId);
-                event.end({
-                    size: response.tree.length,
-                });
-                return response;
-            },
-        );
-        const tree = buildHierarchy(rawTree, this.blobsShaCache, true);
-        await this.snapshotTreeCache?.put(
-            this.getCacheKey(tree.id),
-            { id: requestVersion.id, snapshotTree: tree },
-        );
-        return tree;
-    }
-
-    public async readBlob(blobId: string): Promise<ArrayBufferLike> {
-        const cachedBlob = await this.blobCache?.get(this.getCacheKey(blobId));
-        if (cachedBlob) {
-            return cachedBlob;
-        }
-
-        const value = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "readBlob",
-                blobId,
-            },
-            async (event) => {
-                const manager = await this.getStorageManager();
-                const response = await manager.getBlob(blobId);
-                event.end({
-                    size: response.size,
-                });
-                return response;
-            },
-        );
-        this.blobsShaCache.set(value.sha, "");
-        const bufferContent = stringToBuffer(value.content, value.encoding);
-        await this.blobCache?.put(this.getCacheKey(value.sha), bufferContent);
-        return bufferContent;
-    }
-
-    public async uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise<string> {
-        const summaryHandle = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "uploadSummaryWithContext",
-            },
-            async () => {
-                const summaryUploadManager = await this.getSummaryUploadManager();
-                return summaryUploadManager.writeSummaryTree(summary, context.ackHandle ?? "", "channel");
-            },
-        );
-        return summaryHandle;
-    }
-
-    public async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {
-        throw new Error("NOT IMPLEMENTED!");
-    }
-
-    public async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {
-        const uint8ArrayFile = new Uint8Array(file);
-        return PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "createBlob",
-                size: uint8ArrayFile.length,
-            },
-            async (event) => {
-                const manager = await this.getStorageManager();
-                const response = await manager.createBlob(
-                    Uint8ArrayToString(
-                        uint8ArrayFile, "base64"),
-                    "base64").then((r) => ({ id: r.sha, url: r.url }));
-                event.end({
-                    blobId: response.id,
-                });
-                return response;
-            },
-        );
-    }
-
-    private async getPreviousFullSnapshot(parentHandle: string): Promise<ISnapshotTreeEx | null | undefined> {
-        return parentHandle
-            ? this.getVersions(parentHandle, 1)
-                .then(async (versions) => {
-                    // Clear the cache as the getSnapshotTree call will fill the cache.
-                    this.blobsShaCache.clear();
-                    return this.getSnapshotTree(versions[0]);
-                })
-            : undefined;
-    }
-
-    private getCacheKey(blobId: string): string {
-        return `${this.id}:${blobId}`;
-    }
+	private readonly mc: MonitoringContext;
+	// The values of this cache is useless. We only need the keys. So we are always putting
+	// empty strings as values.
+	protected readonly blobsShaCache = new Map<string, string>();
+	private readonly blobCache: ICache<ArrayBufferLike> | undefined;
+	private readonly snapshotTreeCache: ICache<ISnapshotTreeVersion> | undefined;
+
+	public readonly repositoryUrl = "";
+
+	private async getSummaryUploadManager(): Promise<ISummaryUploadManager> {
+		const manager = await this.getStorageManager();
+		return new SummaryTreeUploadManager(
+			new RetriableGitManager(manager, this.logger),
+			this.blobsShaCache,
+			this.getPreviousFullSnapshot.bind(this),
+		);
+	}
+
+	constructor(
+		protected readonly id: string,
+		protected readonly manager: GitManager,
+		protected readonly logger: ITelemetryLoggerExt,
+		public readonly policies: IDocumentStorageServicePolicies,
+		driverPolicies?: IRouterliciousDriverPolicies,
+		blobCache?: ICache<ArrayBufferLike>,
+		snapshotTreeCache?: ICache<ISnapshotTreeVersion>,
+		private readonly getStorageManager: (
+			disableCache?: boolean,
+		) => Promise<GitManager> = async () => this.manager,
+	) {
+		if (driverPolicies?.enableRestLess === true || isNode) {
+			this.blobCache = blobCache ?? new InMemoryCache();
+			this.snapshotTreeCache = snapshotTreeCache ?? new InMemoryCache();
+		}
+
+		this.mc = createChildMonitoringContext({
+			logger,
+		});
+	}
+
+	public async getVersions(versionId: string | null, count: number): Promise<IVersion[]> {
+		const id = versionId ? versionId : this.id;
+		const commits = await PerformanceEvent.timedExecAsync(
+			this.logger,
+			{
+				eventName: "getVersions",
+				versionId: id,
+				count,
+			},
+			async () => {
+				const manager = await this.getStorageManager();
+				return (await manager.getCommits(id, count)).content;
+			},
+		);
+		return commits.map((commit) => ({
+			date: commit.commit.author.date,
+			id: commit.sha,
+			treeId: commit.commit.tree.sha,
+		}));
+	}
+
+	public async getSnapshotTree(version?: IVersion): Promise<ISnapshotTreeEx | null> {
+		let requestVersion = version;
+		if (!requestVersion) {
+			const versions = await this.getVersions(this.id, 1);
+			if (versions.length === 0) {
+				return null;
+			}
+
+			requestVersion = versions[0];
+		}
+
+		const cachedSnapshotTree = await this.snapshotTreeCache?.get(
+			this.getCacheKey(requestVersion.treeId),
+		);
+		if (cachedSnapshotTree) {
+			return cachedSnapshotTree.snapshotTree as ISnapshotTreeEx;
+		}
+
+		const rawTree = await PerformanceEvent.timedExecAsync(
+			this.logger,
+			{
+				eventName: "getSnapshotTree",
+				treeId: requestVersion.treeId,
+			},
+			async (event) => {
+				const manager = await this.getStorageManager();
+				const response = (await manager.getTree(requestVersion!.treeId)).content;
+				event.end({
+					size: response.tree.length,
+				});
+				return response;
+			},
+		);
+		const tree = buildGitTreeHierarchy(rawTree, this.blobsShaCache, true);
+		await this.snapshotTreeCache?.put(this.getCacheKey(tree.id), {
+			id: requestVersion.id,
+			snapshotTree: tree,
+		});
+		return tree;
+	}
+
+	public async readBlob(blobId: string): Promise<ArrayBufferLike> {
+		const cachedBlob = await this.blobCache?.get(this.getCacheKey(blobId));
+		if (cachedBlob) {
+			return cachedBlob;
+		}
+
+		const value = await PerformanceEvent.timedExecAsync(
+			this.logger,
+			{
+				eventName: "readBlob",
+				blobId,
+			},
+			async (event) => {
+				const manager = await this.getStorageManager();
+				const response = (await manager.getBlob(blobId)).content;
+				event.end({
+					size: response.size,
+				});
+				return response;
+			},
+			undefined, // workers
+			undefined, // recordHeapSize
+			this.mc.config.getNumber("Fluid.Driver.ReadBlobTelemetrySampling"),
+		);
+		this.blobsShaCache.set(value.sha, "");
+		const bufferContent = stringToBuffer(value.content, value.encoding);
+		await this.blobCache?.put(this.getCacheKey(value.sha), bufferContent);
+		return bufferContent;
+	}
+
+	public async uploadSummaryWithContext(
+		summary: ISummaryTree,
+		context: ISummaryContext,
+	): Promise<string> {
+		const summaryHandle = await PerformanceEvent.timedExecAsync(
+			this.logger,
+			{
+				eventName: "uploadSummaryWithContext",
+				proposalHandle: context.proposalHandle,
+				ackHandle: context.ackHandle,
+				referenceSequenceNumber: context.referenceSequenceNumber,
+			},
+			async () => {
+				const summaryUploadManager = await this.getSummaryUploadManager();
+				return summaryUploadManager.writeSummaryTree(
+					summary,
+					context.ackHandle ?? "",
+					"channel",
+				);
+			},
+		);
+		return summaryHandle;
+	}
+
+	public async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {
+		throw new Error("NOT IMPLEMENTED!");
+	}
+
+	public async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {
+		const uint8ArrayFile = new Uint8Array(file);
+		return PerformanceEvent.timedExecAsync(
+			this.logger,
+			{
+				eventName: "createBlob",
+				size: uint8ArrayFile.length,
+			},
+			async (event) => {
+				const manager = await this.getStorageManager();
+				const response = await manager
+					.createBlob(Uint8ArrayToString(uint8ArrayFile, "base64"), "base64")
+					.then((r) => ({ id: r.content.sha, url: r.content.url }));
+				event.end({
+					blobId: response.id,
+				});
+				return response;
+			},
+		);
+	}
+
+	private async getPreviousFullSnapshot(
+		parentHandle: string,
+	): Promise<ISnapshotTreeEx | null | undefined> {
+		return parentHandle
+			? this.getVersions(parentHandle, 1).then(async (versions) => {
+					// Clear the cache as the getSnapshotTree call will fill the cache.
+					this.blobsShaCache.clear();
+					return this.getSnapshotTree(versions[0]);
+			  })
+			: undefined;
+	}
+
+	private getCacheKey(blobId: string): string {
+		return `${this.id}:${blobId}`;
+	}
 }

package/src/storageContracts.ts

@@ -0,0 +1,63 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as git from "@fluidframework/gitresources";
+import * as api from "@fluidframework/protocol-definitions";
+import {
+	IWholeSummaryPayload,
+	IWholeSummaryPayloadType,
+	IWriteSummaryResponse,
+} from "@fluidframework/server-services-client";
+import { IR11sResponse } from "./restWrapper";
+import { IWholeFlatSnapshot } from "./contracts";
+
+/**
+ * Interface to a generic Git provider
+ */
+export interface IHistorian {
+	getBlob(sha: string): Promise<IR11sResponse<git.IBlob>>;
+	createBlob(blob: git.ICreateBlobParams): Promise<IR11sResponse<git.ICreateBlobResponse>>;
+	getCommits(sha: string, count: number): Promise<IR11sResponse<git.ICommitDetails[]>>;
+	createTree(tree: git.ICreateTreeParams): Promise<IR11sResponse<git.ITree>>;
+	getTree(sha: string, recursive: boolean): Promise<IR11sResponse<git.ITree>>;
+	createSummary(
+		summary: IWholeSummaryPayload,
+		initial?: boolean,
+	): Promise<IR11sResponse<IWriteSummaryResponse>>;
+	getSnapshot(sha: string): Promise<IR11sResponse<IWholeFlatSnapshot>>;
+}
+
+export interface IGitManager {
+	getCommits(sha: string, count: number): Promise<IR11sResponse<git.ICommitDetails[]>>;
+	getTree(root: string, recursive: boolean): Promise<IR11sResponse<git.ITree>>;
+	getBlob(sha: string): Promise<IR11sResponse<git.IBlob>>;
+	createBlob(content: string, encoding: string): Promise<IR11sResponse<git.ICreateBlobResponse>>;
+	createGitTree(params: git.ICreateTreeParams): Promise<IR11sResponse<git.ITree>>;
+	createSummary(
+		summary: IWholeSummaryPayload,
+		initial?: boolean,
+	): Promise<IR11sResponse<IWriteSummaryResponse>>;
+	getSnapshot(sha: string): Promise<IR11sResponse<IWholeFlatSnapshot>>;
+}
+
+/**
+ * Uploads a summary to storage.
+ */
+export interface ISummaryUploadManager {
+	/**
+	 * Writes summary tree to storage.
+	 * @param summaryTree - Summary tree to write to storage
+	 * @param parentHandle - Parent summary acked handle (if available from summary ack)
+	 * @param summaryType - type of summary being uploaded
+	 * @param sequenceNumber - optional reference sequence number of the summary
+	 * @returns Id of created tree as a string.
+	 */
+	writeSummaryTree(
+		summaryTree: api.ISummaryTree,
+		parentHandle: string,
+		summaryType: IWholeSummaryPayloadType,
+		sequenceNumber?: number,
+	): Promise<string>;
+}

package/src/summaryTreeUploadManager.ts

@@ -0,0 +1,155 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { gitHashFile, IsoBuffer, Uint8ArrayToString } from "@fluid-internal/client-utils";
+import { assert, unreachableCase } from "@fluidframework/core-utils";
+import { ICreateTreeEntry } from "@fluidframework/gitresources";
+import { getGitMode, getGitType } from "@fluidframework/protocol-base";
+import {
+	ISnapshotTreeEx,
+	ISummaryTree,
+	SummaryObject,
+	SummaryType,
+} from "@fluidframework/protocol-definitions";
+import { IWholeSummaryPayloadType } from "@fluidframework/server-services-client";
+import { IGitManager, ISummaryUploadManager } from "./storageContracts";
+
+/**
+ * Recursively writes summary tree as individual summary blobs.
+ */
+export class SummaryTreeUploadManager implements ISummaryUploadManager {
+	constructor(
+		private readonly manager: IGitManager,
+		private readonly blobsShaCache: Map<string, string>,
+		private readonly getPreviousFullSnapshot: (
+			parentHandle: string,
+		) => Promise<ISnapshotTreeEx | null | undefined>,
+	) {}
+
+	public async writeSummaryTree(
+		summaryTree: ISummaryTree,
+		parentHandle: string,
+		summaryType: IWholeSummaryPayloadType,
+		sequenceNumber?: number,
+		initial?: boolean,
+	): Promise<string> {
+		const previousFullSnapshot = await this.getPreviousFullSnapshot(parentHandle);
+		return this.writeSummaryTreeCore(summaryTree, previousFullSnapshot ?? undefined);
+	}
+
+	private async writeSummaryTreeCore(
+		summaryTree: ISummaryTree,
+		previousFullSnapshot: ISnapshotTreeEx | undefined,
+	): Promise<string> {
+		const entries = await Promise.all(
+			Object.keys(summaryTree.tree).map(async (key) => {
+				const entry = summaryTree.tree[key];
+				const pathHandle = await this.writeSummaryTreeObject(entry, previousFullSnapshot);
+				const treeEntry: ICreateTreeEntry = {
+					mode: getGitMode(entry),
+					path: encodeURIComponent(key),
+					sha: pathHandle,
+					type: getGitType(entry),
+				};
+				return treeEntry;
+			}),
+		);
+
+		const treeHandle = (await this.manager.createGitTree({ tree: entries })).content;
+		return treeHandle.sha;
+	}
+
+	private async writeSummaryTreeObject(
+		object: SummaryObject,
+		previousFullSnapshot: ISnapshotTreeEx | undefined,
+	): Promise<string> {
+		switch (object.type) {
+			case SummaryType.Blob: {
+				return this.writeSummaryBlob(object.content);
+			}
+			case SummaryType.Handle: {
+				if (previousFullSnapshot === undefined) {
+					throw Error("Parent summary does not exist to reference by handle.");
+				}
+				return this.getIdFromPath(object.handleType, object.handle, previousFullSnapshot);
+			}
+			case SummaryType.Tree: {
+				return this.writeSummaryTreeCore(object, previousFullSnapshot);
+			}
+			case SummaryType.Attachment: {
+				return object.id;
+			}
+
+			default:
+				unreachableCase(object, `Unknown type: ${(object as any).type}`);
+		}
+	}
+
+	private async writeSummaryBlob(content: string | Uint8Array): Promise<string> {
+		const { parsedContent, encoding } =
+			typeof content === "string"
+				? { parsedContent: content, encoding: "utf-8" }
+				: { parsedContent: Uint8ArrayToString(content, "base64"), encoding: "base64" };
+
+		// The gitHashFile would return the same hash as returned by the server as blob.sha
+		const hash = await gitHashFile(IsoBuffer.from(parsedContent, encoding));
+		if (!this.blobsShaCache.has(hash)) {
+			this.blobsShaCache.set(hash, "");
+			const blob = (await this.manager.createBlob(parsedContent, encoding)).content;
+			assert(hash === blob.sha, 0x0b6 /* "Blob.sha and hash do not match!!" */);
+		}
+		return hash;
+	}
+
+	private getIdFromPath(
+		handleType: SummaryType,
+		handlePath: string,
+		previousFullSnapshot: ISnapshotTreeEx,
+	): string {
+		const path = handlePath.split("/").map((part) => decodeURIComponent(part));
+		if (path[0] === "") {
+			// root of tree should be unnamed
+			path.shift();
+		}
+		if (path.length === 0) {
+			return previousFullSnapshot.id;
+		}
+
+		return this.getIdFromPathCore(handleType, path, previousFullSnapshot);
+	}
+
+	private getIdFromPathCore(
+		handleType: SummaryType,
+		path: string[],
+		/** Previous snapshot, subtree relative to this path part */
+		previousSnapshot: ISnapshotTreeEx,
+	): string {
+		assert(path.length > 0, 0x0b3 /* "Expected at least 1 path part" */);
+		const key = path[0];
+		if (path.length === 1) {
+			switch (handleType) {
+				case SummaryType.Blob: {
+					const tryId = previousSnapshot.blobs[key];
+					assert(
+						!!tryId,
+						0x0b4 /* "Parent summary does not have blob handle for specified path." */,
+					);
+					return tryId;
+				}
+				case SummaryType.Tree: {
+					const tryId = previousSnapshot.trees[key]?.id;
+					assert(
+						!!tryId,
+						0x0b5 /* "Parent summary does not have tree handle for specified path." */,
+					);
+					return tryId;
+				}
+				default:
+					throw Error(`Unexpected handle summary object type: "${handleType}".`);
+			}
+		}
+		return this.getIdFromPathCore(handleType, path.slice(1), previousSnapshot.trees[key]);
+	}
+}