@fluidframework/routerlicious-driver 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.2.0.153917

package/src/retriableGitManager.ts

@@ -4,15 +4,15 @@
  */
 
 import type * as git from "@fluidframework/gitresources";
-import type * as protocol from "@fluidframework/protocol-definitions";
 import {
-	IGitManager,
 	IWholeFlatSummary,
 	IWholeSummaryPayload,
 	IWriteSummaryResponse,
 } from "@fluidframework/server-services-client";
 import { runWithRetry } from "@fluidframework/driver-utils";
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { IGitManager } from "./storageContracts";
+import { IR11sResponse } from "./restWrapper";
 
 export class RetriableGitManager implements IGitManager {
 	constructor(
@@ -20,135 +20,57 @@ export class RetriableGitManager implements IGitManager {
 		private readonly logger: ITelemetryLogger,
 	) {}
 
-	public async getHeader(id: string, sha: string): Promise<protocol.ISnapshotTree> {
-		return this.runWithRetry(
-			async () => this.internalGitManager.getHeader(id, sha),
-			"gitManager_getHeader",
-		);
-	}
-
-	public async getFullTree(sha: string): Promise<any> {
-		return this.runWithRetry(
-			async () => this.internalGitManager.getFullTree(sha),
-			"gitManager_getFullTree",
-		);
-	}
-
-	public async getCommit(sha: string): Promise<git.ICommit> {
-		return this.runWithRetry(
-			async () => this.internalGitManager.getCommit(sha),
-			"gitManager_getCommit",
-		);
-	}
-
-	public async getCommits(sha: string, count: number): Promise<git.ICommitDetails[]> {
+	public async getCommits(
+		sha: string,
+		count: number,
+	): Promise<IR11sResponse<git.ICommitDetails[]>> {
 		return this.runWithRetry(
 			async () => this.internalGitManager.getCommits(sha, count),
 			"gitManager_getCommits",
 		);
 	}
 
-	public async getTree(root: string, recursive: boolean): Promise<git.ITree> {
+	public async getTree(root: string, recursive: boolean): Promise<IR11sResponse<git.ITree>> {
 		return this.runWithRetry(
 			async () => this.internalGitManager.getTree(root, recursive),
 			"gitManager_getTree",
 		);
 	}
 
-	public async getBlob(sha: string): Promise<git.IBlob> {
+	public async getBlob(sha: string): Promise<IR11sResponse<git.IBlob>> {
 		return this.runWithRetry(
 			async () => this.internalGitManager.getBlob(sha),
 			"gitManager_getBlob",
 		);
 	}
 
-	public getRawUrl(sha: string): string {
-		return this.internalGitManager.getRawUrl(sha);
-	}
-
-	public async getContent(commit: string, path: string): Promise<git.IBlob> {
-		return this.runWithRetry(
-			async () => this.internalGitManager.getContent(commit, path),
-			"gitManager_getContent",
-		);
-	}
-
-	public async createBlob(content: string, encoding: string): Promise<git.ICreateBlobResponse> {
+	public async createBlob(
+		content: string,
+		encoding: string,
+	): Promise<IR11sResponse<git.ICreateBlobResponse>> {
 		return this.runWithRetry(
 			async () => this.internalGitManager.createBlob(content, encoding),
 			"gitManager_createBlob",
 		);
 	}
 
-	public async createGitTree(params: git.ICreateTreeParams): Promise<git.ITree> {
+	public async createGitTree(params: git.ICreateTreeParams): Promise<IR11sResponse<git.ITree>> {
 		return this.runWithRetry(
 			async () => this.internalGitManager.createGitTree(params),
 			"gitManager_createGitTree",
 		);
 	}
 
-	public async createTree(files: protocol.ITree): Promise<git.ITree> {
-		return this.runWithRetry(
-			async () => this.internalGitManager.createTree(files),
-			"gitManager_createTree",
-		);
-	}
-
-	public async createCommit(commit: git.ICreateCommitParams): Promise<git.ICommit> {
-		return this.runWithRetry(
-			async () => this.internalGitManager.createCommit(commit),
-			"gitManager_createCommit",
-		);
-	}
-
-	public async getRef(ref: string): Promise<git.IRef> {
-		return this.runWithRetry(
-			async () => this.internalGitManager.getRef(ref),
-			"gitManager_getRef",
-		);
-	}
-
-	public async createRef(branch: string, sha: string): Promise<git.IRef> {
-		return this.runWithRetry(
-			async () => this.internalGitManager.createRef(branch, sha),
-			"gitManager_createRef",
-		);
-	}
-
-	public async upsertRef(branch: string, commitSha: string): Promise<git.IRef> {
-		return this.runWithRetry(
-			async () => this.internalGitManager.upsertRef(branch, commitSha),
-			"gitManager_upsertRef",
-		);
-	}
-
-	public async write(
-		branch: string,
-		inputTree: protocol.ITree,
-		parents: string[],
-		message: string,
-	): Promise<git.ICommit> {
-		return this.runWithRetry(
-			async () => this.internalGitManager.write(branch, inputTree, parents, message),
-			"gitManager_write",
-		);
-	}
-
-	public async createSummary(summary: IWholeSummaryPayload): Promise<IWriteSummaryResponse> {
+	public async createSummary(
+		summary: IWholeSummaryPayload,
+	): Promise<IR11sResponse<IWriteSummaryResponse>> {
 		return this.runWithRetry(
 			async () => this.internalGitManager.createSummary(summary),
 			"gitManager_createSummary",
 		);
 	}
 
-	public async deleteSummary(softDelete: boolean): Promise<void> {
-		return this.runWithRetry(
-			async () => this.internalGitManager.deleteSummary(softDelete),
-			"gitManager_deleteSummary",
-		);
-	}
-
-	public async getSummary(sha: string): Promise<IWholeFlatSummary> {
+	public async getSummary(sha: string): Promise<IR11sResponse<IWholeFlatSummary>> {
 		return this.runWithRetry(
 			async () => this.internalGitManager.getSummary(sha),
 			"gitManager_getSummary",

package/src/shreddedSummaryDocumentStorageService.ts

@@ -18,16 +18,14 @@ import {
 	ISummaryTree,
 	IVersion,
 } from "@fluidframework/protocol-definitions";
-import {
-	GitManager,
-	ISummaryUploadManager,
-	SummaryTreeUploadManager,
-} from "@fluidframework/server-services-client";
 import { PerformanceEvent } from "@fluidframework/telemetry-utils";
 import { IRouterliciousDriverPolicies } from "./policies";
 import { ICache, InMemoryCache } from "./cache";
 import { RetriableGitManager } from "./retriableGitManager";
 import { ISnapshotTreeVersion } from "./definitions";
+import { GitManager } from "./gitManager";
+import { ISummaryUploadManager } from "./storageContracts";
+import { SummaryTreeUploadManager } from "./summaryTreeUploadManager";
 
 const isNode = typeof window === "undefined";
 
@@ -85,7 +83,7 @@ export class ShreddedSummaryDocumentStorageService implements IDocumentStorageSe
 			},
 			async () => {
 				const manager = await this.getStorageManager();
-				return manager.getCommits(id, count);
+				return (await manager.getCommits(id, count)).content;
 			},
 		);
 		return commits.map((commit) => ({
@@ -121,7 +119,7 @@ export class ShreddedSummaryDocumentStorageService implements IDocumentStorageSe
 			},
 			async (event) => {
 				const manager = await this.getStorageManager();
-				const response = await manager.getTree(requestVersion!.treeId);
+				const response = (await manager.getTree(requestVersion!.treeId)).content;
 				event.end({
 					size: response.tree.length,
 				});
@@ -150,7 +148,7 @@ export class ShreddedSummaryDocumentStorageService implements IDocumentStorageSe
 			},
 			async (event) => {
 				const manager = await this.getStorageManager();
-				const response = await manager.getBlob(blobId);
+				const response = (await manager.getBlob(blobId)).content;
 				event.end({
 					size: response.size,
 				});
@@ -203,7 +201,7 @@ export class ShreddedSummaryDocumentStorageService implements IDocumentStorageSe
 				const manager = await this.getStorageManager();
 				const response = await manager
 					.createBlob(Uint8ArrayToString(uint8ArrayFile, "base64"), "base64")
-					.then((r) => ({ id: r.sha, url: r.url }));
+					.then((r) => ({ id: r.content.sha, url: r.content.url }));
 				event.end({
 					blobId: response.id,
 				});

package/src/storageContracts.ts

@@ -0,0 +1,63 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as git from "@fluidframework/gitresources";
+import * as api from "@fluidframework/protocol-definitions";
+import {
+	IWholeFlatSummary,
+	IWholeSummaryPayload,
+	IWholeSummaryPayloadType,
+	IWriteSummaryResponse,
+} from "@fluidframework/server-services-client";
+import { IR11sResponse } from "./restWrapper";
+
+/**
+ * Interface to a generic Git provider
+ */
+export interface IHistorian {
+	getBlob(sha: string): Promise<IR11sResponse<git.IBlob>>;
+	createBlob(blob: git.ICreateBlobParams): Promise<IR11sResponse<git.ICreateBlobResponse>>;
+	getCommits(sha: string, count: number): Promise<IR11sResponse<git.ICommitDetails[]>>;
+	createTree(tree: git.ICreateTreeParams): Promise<IR11sResponse<git.ITree>>;
+	getTree(sha: string, recursive: boolean): Promise<IR11sResponse<git.ITree>>;
+	createSummary(
+		summary: IWholeSummaryPayload,
+		initial?: boolean,
+	): Promise<IR11sResponse<IWriteSummaryResponse>>;
+	getSummary(sha: string): Promise<IR11sResponse<IWholeFlatSummary>>;
+}
+
+export interface IGitManager {
+	getCommits(sha: string, count: number): Promise<IR11sResponse<git.ICommitDetails[]>>;
+	getTree(root: string, recursive: boolean): Promise<IR11sResponse<git.ITree>>;
+	getBlob(sha: string): Promise<IR11sResponse<git.IBlob>>;
+	createBlob(content: string, encoding: string): Promise<IR11sResponse<git.ICreateBlobResponse>>;
+	createGitTree(params: git.ICreateTreeParams): Promise<IR11sResponse<git.ITree>>;
+	createSummary(
+		summary: IWholeSummaryPayload,
+		initial?: boolean,
+	): Promise<IR11sResponse<IWriteSummaryResponse>>;
+	getSummary(sha: string): Promise<IR11sResponse<IWholeFlatSummary>>;
+}
+
+/**
+ * Uploads a summary to storage.
+ */
+export interface ISummaryUploadManager {
+	/**
+	 * Writes summary tree to storage.
+	 * @param summaryTree - Summary tree to write to storage
+	 * @param parentHandle - Parent summary acked handle (if available from summary ack)
+	 * @param summaryType - type of summary being uploaded
+	 * @param sequenceNumber - optional reference sequence number of the summary
+	 * @returns Id of created tree as a string.
+	 */
+	writeSummaryTree(
+		summaryTree: api.ISummaryTree,
+		parentHandle: string,
+		summaryType: IWholeSummaryPayloadType,
+		sequenceNumber?: number,
+	): Promise<string>;
+}

package/src/summaryTreeUploadManager.ts

@@ -0,0 +1,160 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import {
+	assert,
+	gitHashFile,
+	IsoBuffer,
+	Uint8ArrayToString,
+	unreachableCase,
+} from "@fluidframework/common-utils";
+import { ICreateTreeEntry } from "@fluidframework/gitresources";
+import { getGitMode, getGitType } from "@fluidframework/protocol-base";
+import {
+	ISnapshotTreeEx,
+	ISummaryTree,
+	SummaryObject,
+	SummaryType,
+} from "@fluidframework/protocol-definitions";
+import { IWholeSummaryPayloadType } from "@fluidframework/server-services-client";
+import { IGitManager, ISummaryUploadManager } from "./storageContracts";
+
+/**
+ * Recursively writes summary tree as individual summary blobs.
+ */
+export class SummaryTreeUploadManager implements ISummaryUploadManager {
+	constructor(
+		private readonly manager: IGitManager,
+		private readonly blobsShaCache: Map<string, string>,
+		private readonly getPreviousFullSnapshot: (
+			parentHandle: string,
+		) => Promise<ISnapshotTreeEx | null | undefined>,
+	) {}
+
+	public async writeSummaryTree(
+		summaryTree: ISummaryTree,
+		parentHandle: string,
+		summaryType: IWholeSummaryPayloadType,
+		sequenceNumber?: number,
+		initial?: boolean,
+	): Promise<string> {
+		const previousFullSnapshot = await this.getPreviousFullSnapshot(parentHandle);
+		return this.writeSummaryTreeCore(summaryTree, previousFullSnapshot ?? undefined);
+	}
+
+	private async writeSummaryTreeCore(
+		summaryTree: ISummaryTree,
+		previousFullSnapshot: ISnapshotTreeEx | undefined,
+	): Promise<string> {
+		const entries = await Promise.all(
+			Object.keys(summaryTree.tree).map(async (key) => {
+				const entry = summaryTree.tree[key];
+				const pathHandle = await this.writeSummaryTreeObject(entry, previousFullSnapshot);
+				const treeEntry: ICreateTreeEntry = {
+					mode: getGitMode(entry),
+					path: encodeURIComponent(key),
+					sha: pathHandle,
+					type: getGitType(entry),
+				};
+				return treeEntry;
+			}),
+		);
+
+		const treeHandle = (await this.manager.createGitTree({ tree: entries })).content;
+		return treeHandle.sha;
+	}
+
+	private async writeSummaryTreeObject(
+		object: SummaryObject,
+		previousFullSnapshot: ISnapshotTreeEx | undefined,
+	): Promise<string> {
+		switch (object.type) {
+			case SummaryType.Blob: {
+				return this.writeSummaryBlob(object.content);
+			}
+			case SummaryType.Handle: {
+				if (previousFullSnapshot === undefined) {
+					throw Error("Parent summary does not exist to reference by handle.");
+				}
+				return this.getIdFromPath(object.handleType, object.handle, previousFullSnapshot);
+			}
+			case SummaryType.Tree: {
+				return this.writeSummaryTreeCore(object, previousFullSnapshot);
+			}
+			case SummaryType.Attachment: {
+				return object.id;
+			}
+
+			default:
+				unreachableCase(object, `Unknown type: ${(object as any).type}`);
+		}
+	}
+
+	private async writeSummaryBlob(content: string | Uint8Array): Promise<string> {
+		const { parsedContent, encoding } =
+			typeof content === "string"
+				? { parsedContent: content, encoding: "utf-8" }
+				: { parsedContent: Uint8ArrayToString(content, "base64"), encoding: "base64" };
+
+		// The gitHashFile would return the same hash as returned by the server as blob.sha
+		const hash = await gitHashFile(IsoBuffer.from(parsedContent, encoding));
+		if (!this.blobsShaCache.has(hash)) {
+			this.blobsShaCache.set(hash, "");
+			const blob = (await this.manager.createBlob(parsedContent, encoding)).content;
+			assert(hash === blob.sha, 0x0b6 /* "Blob.sha and hash do not match!!" */);
+		}
+		return hash;
+	}
+
+	private getIdFromPath(
+		handleType: SummaryType,
+		handlePath: string,
+		previousFullSnapshot: ISnapshotTreeEx,
+	): string {
+		const path = handlePath.split("/").map((part) => decodeURIComponent(part));
+		if (path[0] === "") {
+			// root of tree should be unnamed
+			path.shift();
+		}
+		if (path.length === 0) {
+			return previousFullSnapshot.id;
+		}
+
+		return this.getIdFromPathCore(handleType, path, previousFullSnapshot);
+	}
+
+	private getIdFromPathCore(
+		handleType: SummaryType,
+		path: string[],
+		/** Previous snapshot, subtree relative to this path part */
+		previousSnapshot: ISnapshotTreeEx,
+	): string {
+		assert(path.length > 0, 0x0b3 /* "Expected at least 1 path part" */);
+		const key = path[0];
+		if (path.length === 1) {
+			switch (handleType) {
+				case SummaryType.Blob: {
+					const tryId = previousSnapshot.blobs[key];
+					assert(
+						!!tryId,
+						0x0b4 /* "Parent summary does not have blob handle for specified path." */,
+					);
+					return tryId;
+				}
+				case SummaryType.Tree: {
+					const tryId = previousSnapshot.trees[key]?.id;
+					assert(
+						!!tryId,
+						0x0b5 /* "Parent summary does not have tree handle for specified path." */,
+					);
+					return tryId;
+				}
+				default:
+					throw Error(`Unexpected handle summary object type: "${handleType}".`);
+			}
+		}
+		return this.getIdFromPathCore(handleType, path.slice(1), previousSnapshot.trees[key]);
+	}
+}

package/src/tokens.ts

@@ -6,9 +6,13 @@
 import { ITokenClaims } from "@fluidframework/protocol-definitions";
 
 /**
- * The ITokenService abstracts the discovery of claims contained within a token
+ * Abstracts the discovery of claims contained within a token.
  */
 export interface ITokenService {
+	/**
+	 * Extracts the {@link @fluidframework/protocol-definitions#ITokenClaims | token claims} from the provided
+	 * {@link https://jwt.io/introduction/ | JSON Web Token (JWT)} string representation.
+	 */
 	extractClaims(token: string): ITokenClaims;
 }
 
@@ -19,8 +23,9 @@ export interface ITokenResponse {
 	jwt: string;
 
 	/**
-	 * Flag indicating whether token was obtained from local cache.
-	 * Undefined indicates that the source of the token could not be determined.
+	 * A flag indicating whether token was obtained from local cache.
+	 *
+	 * @remarks `undefined` indicates that the source of the token could not be determined.
 	 */
 	fromCache?: boolean;
 }
@@ -32,11 +37,16 @@ export interface ITokenResponse {
 export interface ITokenProvider {
 	/**
 	 * Fetches the orderer token from host.
+	 *
 	 * @param tenantId - Tenant ID.
 	 * @param documentId - Optional. Document ID is only required for document-scoped requests.
 	 * @param refresh - Optional flag indicating whether token fetch must bypass local cache.
-	 * @returns TokenResponse object representing token value along with flag indicating
-	 * whether token came from cache.
+	 * This likely indicates that some previous request failed authorization due to an expired token,
+	 * and so a fresh token is required.
+	 *
+	 * Default: `false`.
+	 *
+	 * NOTE: This parameter will be made required in the future.
 	 */
 	fetchOrdererToken(
 		tenantId: string,
@@ -46,11 +56,16 @@ export interface ITokenProvider {
 
 	/**
 	 * Fetches the storage token from host.
+	 *
 	 * @param tenantId - Tenant ID.
 	 * @param documentId - Document ID.
 	 * @param refresh - Optional flag indicating whether token fetch must bypass local cache.
-	 * @returns TokenResponse object representing token value along with flag indicating
-	 * whether token came from cache.
+	 * This likely indicates that some previous request failed authorization due to an expired token,
+	 * and so a fresh token is required.
+	 *
+	 * Default: `false`.
+	 *
+	 * NOTE: This parameter will be made required in the future.
 	 */
 	fetchStorageToken(
 		tenantId: string,
@@ -64,7 +79,9 @@ export interface ITokenProvider {
 	 * created it.
 	 *
 	 * @remarks Notes:
+	 *
 	 * * Using the callback may have performance impact on the document creation process.
+	 *
 	 * * Any exceptions thrown in the callback would fail the creation workflow
 	 * (see {@link RouterliciousDocumentServiceFactory.createContainer} for more details).
 	 *

package/src/treeUtils.ts

@@ -0,0 +1,137 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert, IsoBuffer } from "@fluidframework/common-utils";
+import {
+	SummaryType,
+	ISnapshotTree,
+	ISummaryTree,
+	SummaryObject,
+} from "@fluidframework/protocol-definitions";
+import { INormalizedWholeSummary } from "@fluidframework/server-services-client";
+
+/**
+ * Summary tree assembler props
+ */
+export interface ISummaryTreeAssemblerProps {
+	/**
+	 * Indicates that this tree is unreferenced. If this is not present, the tree is considered referenced.
+	 */
+	unreferenced?: true;
+}
+
+/**
+ * Summary tree assembler (without stats collection).
+ */
+export class SummaryTreeAssembler {
+	private attachmentCounter: number = 0;
+	private readonly summaryTree: { [path: string]: SummaryObject } = {};
+
+	constructor(private readonly props?: ISummaryTreeAssemblerProps) {}
+
+	/**
+	 * Get final summary
+	 */
+	public get summary(): ISummaryTree {
+		return {
+			type: SummaryType.Tree,
+			tree: { ...this.summaryTree },
+			unreferenced: this.props?.unreferenced,
+		};
+	}
+
+	/**
+	 * Add blob to summary
+	 */
+	public addBlob(key: string, content: string | Uint8Array): void {
+		this.summaryTree[key] = {
+			type: SummaryType.Blob,
+			content,
+		};
+	}
+
+	/**
+	 * Add handle to summary
+	 */
+	public addHandle(
+		key: string,
+		handleType: SummaryType.Tree | SummaryType.Blob | SummaryType.Attachment,
+		handle: string,
+	): void {
+		this.summaryTree[key] = {
+			type: SummaryType.Handle,
+			handleType,
+			handle,
+		};
+	}
+
+	/**
+	 * Add tree to summary
+	 */
+	public addTree(key: string, summary: ISummaryTree): void {
+		this.summaryTree[key] = summary;
+	}
+
+	/**
+	 * Add attachment to summary
+	 */
+	public addAttachment(id: string) {
+		this.summaryTree[this.attachmentCounter++] = { id, type: SummaryType.Attachment };
+	}
+}
+
+/**
+ * Helper function that converts ISnapshotTree and blobs to ISummaryTree
+ * @param snapshot - Source snapshot tree
+ * @param blobs - Blobs cache
+ * @returns Converted snapshot in ISummaryTree format
+ */
+export function convertSnapshotAndBlobsToSummaryTree(
+	snapshot: ISnapshotTree,
+	blobs: Map<string, ArrayBuffer>,
+): ISummaryTree {
+	const assembler = new SummaryTreeAssembler({
+		unreferenced: snapshot.unreferenced,
+	});
+	for (const [path, id] of Object.entries(snapshot.blobs)) {
+		const blob = blobs.get(id);
+		assert(blob !== undefined, 0x2dd /* "Cannot find blob for a given id" */);
+		assembler.addBlob(path, IsoBuffer.from(blob).toString("utf-8"));
+	}
+	for (const [key, tree] of Object.entries(snapshot.trees)) {
+		const subtree = convertSnapshotAndBlobsToSummaryTree(tree, blobs);
+		assembler.addTree(key, subtree);
+	}
+	return assembler.summary;
+}
+
+export function evalBlobsAndTrees(snapshot: INormalizedWholeSummary) {
+	const trees = countTreesInSnapshotTree(snapshot.snapshotTree);
+	const numBlobs = snapshot.blobs.size;
+	let encodedBlobsSize = 0;
+	for (const [_, blobContent] of snapshot.blobs) {
+		encodedBlobsSize += blobContent.byteLength;
+	}
+	return { trees, numBlobs, encodedBlobsSize };
+}
+
+export function validateBlobsAndTrees(snapshot: ISnapshotTree) {
+	assert(
+		snapshot.trees !== undefined,
+		0x5d0 /* Returned r11s snapshot is malformed. No trees! */,
+	);
+	assert(
+		snapshot.blobs !== undefined,
+		0x5d1 /* Returned r11s snapshot is malformed. No blobs! */,
+	);
+}
+
+function countTreesInSnapshotTree(snapshotTree: ISnapshotTree): number {
+	let numTrees = 0;
+	for (const [_, tree] of Object.entries(snapshotTree.trees)) {
+		numTrees += 1 + countTreesInSnapshotTree(tree);
+	}
+	return numTrees;
+}