@fluidframework/routerlicious-driver 2.0.0-dev.4.1.0.148229 → 2.0.0-dev.4.3.0.157531

package/src/documentServiceFactory.ts

@@ -4,6 +4,7 @@
  */
 
 import { assert } from "@fluidframework/common-utils";
+import { getW3CData } from "@fluidframework/driver-base";
 import {
 	FiveDaysMs,
 	IDocumentService,
@@ -33,6 +34,7 @@ import { parseFluidUrl, replaceDocumentIdInPath, getDiscoveredFluidResolvedUrl }
 import { ICache, InMemoryCache, NullCache } from "./cache";
 import { pkgVersion as driverVersion } from "./packageVersion";
 import { ISnapshotTreeVersion } from "./definitions";
+import { INormalizedWholeSummary } from "./contracts";
 
 const maximumSnapshotCacheDurationMs: FiveDaysMs = 432_000_000; // 5 days in ms
 
@@ -54,7 +56,8 @@ const defaultRouterliciousDriverPolicies: IRouterliciousDriverPolicies = {
 export class RouterliciousDocumentServiceFactory implements IDocumentServiceFactory {
 	private readonly driverPolicies: IRouterliciousDriverPolicies;
 	private readonly blobCache: ICache<ArrayBufferLike>;
-	private readonly snapshotTreeCache: ICache<ISnapshotTreeVersion>;
+	private readonly wholeSnapshotTreeCache: ICache<INormalizedWholeSummary> = new NullCache();
+	private readonly shreddedSummaryTreeCache: ICache<ISnapshotTreeVersion> = new NullCache();
 
 	constructor(
 		private readonly tokenProvider: ITokenProvider,
@@ -68,9 +71,17 @@ export class RouterliciousDocumentServiceFactory implements IDocumentServiceFact
 			...driverPolicies,
 		};
 		this.blobCache = new InMemoryCache<ArrayBufferLike>();
-		this.snapshotTreeCache = this.driverPolicies.enableInternalSummaryCaching
-			? new InMemoryCache<ISnapshotTreeVersion>(snapshotCacheExpiryMs)
-			: new NullCache<ISnapshotTreeVersion>();
+		if (this.driverPolicies.enableInternalSummaryCaching) {
+			if (this.driverPolicies.enableWholeSummaryUpload) {
+				this.wholeSnapshotTreeCache = new InMemoryCache<INormalizedWholeSummary>(
+					snapshotCacheExpiryMs,
+				);
+			} else {
+				this.shreddedSummaryTreeCache = new InMemoryCache<ISnapshotTreeVersion>(
+					snapshotCacheExpiryMs,
+				);
+			}
+		}
 	}
 
 	/**
@@ -128,15 +139,17 @@ export class RouterliciousDocumentServiceFactory implements IDocumentServiceFact
 			},
 			async (event) => {
 				// @TODO: Remove returned "string" type when removing back-compat code
-				const postRes = await ordererRestWrapper.post<
-					{ id: string; token?: string; session?: ISession } | string
-				>(`/documents/${tenantId}`, {
-					summary: convertSummaryToCreateNewSummary(appSummary),
-					sequenceNumber: documentAttributes.sequenceNumber,
-					values: quorumValues,
-					enableDiscovery: this.driverPolicies.enableDiscovery,
-					generateToken: this.tokenProvider.documentPostCreateCallback !== undefined,
-				});
+				const postRes = (
+					await ordererRestWrapper.post<
+						{ id: string; token?: string; session?: ISession } | string
+					>(`/documents/${tenantId}`, {
+						summary: convertSummaryToCreateNewSummary(appSummary),
+						sequenceNumber: documentAttributes.sequenceNumber,
+						values: quorumValues,
+						enableDiscovery: this.driverPolicies.enableDiscovery,
+						generateToken: this.tokenProvider.documentPostCreateCallback !== undefined,
+					})
+				).content;
 
 				event.end({
 					docId: typeof postRes === "string" ? postRes : postRes.id,
@@ -253,11 +266,16 @@ export class RouterliciousDocumentServiceFactory implements IDocumentServiceFact
 					eventName: "DiscoverSession",
 					docId: documentId,
 				},
-				async () => {
+				async (event) => {
 					// The service responds with the current document session associated with the container.
-					return ordererRestWrapper.get<ISession>(
+					const response = await ordererRestWrapper.get<ISession>(
 						`${resolvedUrl.endpoints.ordererUrl}/documents/${tenantId}/session/${documentId}`,
 					);
+					event.end({
+						...response.propsToLog,
+						...getW3CData(response.requestUrl, "xmlhttprequest"),
+					});
+					return response.content;
 				},
 			);
 			return getDiscoveredFluidResolvedUrl(resolvedUrl, discoveredSession);
@@ -299,7 +317,8 @@ export class RouterliciousDocumentServiceFactory implements IDocumentServiceFact
 			documentStorageServicePolicies,
 			this.driverPolicies,
 			this.blobCache,
-			this.snapshotTreeCache,
+			this.wholeSnapshotTreeCache,
+			this.shreddedSummaryTreeCache,
 			discoverFluidResolvedUrl,
 		);
 	}

package/src/documentStorageService.ts

@@ -10,7 +10,6 @@ import {
 	LoaderCachingPolicy,
 } from "@fluidframework/driver-definitions";
 import { ISnapshotTree, IVersion } from "@fluidframework/protocol-definitions";
-import { GitManager } from "@fluidframework/server-services-client";
 import {
 	DocumentStorageServiceProxy,
 	PrefetchDocumentStorageService,
@@ -19,7 +18,9 @@ import { IRouterliciousDriverPolicies } from "./policies";
 import { ICache } from "./cache";
 import { WholeSummaryDocumentStorageService } from "./wholeSummaryDocumentStorageService";
 import { ShreddedSummaryDocumentStorageService } from "./shreddedSummaryDocumentStorageService";
+import { GitManager } from "./gitManager";
 import { ISnapshotTreeVersion } from "./definitions";
+import { INormalizedWholeSummary } from "./contracts";
 
 export class DocumentStorageService extends DocumentStorageServiceProxy {
 	private _logTailSha: string | undefined = undefined;
@@ -35,7 +36,8 @@ export class DocumentStorageService extends DocumentStorageServiceProxy {
 		policies: IDocumentStorageServicePolicies,
 		driverPolicies?: IRouterliciousDriverPolicies,
 		blobCache?: ICache<ArrayBufferLike>,
-		snapshotTreeCache?: ICache<ISnapshotTreeVersion>,
+		snapshotTreeCache?: ICache<INormalizedWholeSummary>,
+		shreddedSummaryTreeCache?: ICache<ISnapshotTreeVersion>,
 		noCacheGitManager?: GitManager,
 		getStorageManager?: (disableCache?: boolean) => Promise<GitManager>,
 	): IDocumentStorageService {
@@ -58,7 +60,7 @@ export class DocumentStorageService extends DocumentStorageServiceProxy {
 					policies,
 					driverPolicies,
 					blobCache,
-					snapshotTreeCache,
+					shreddedSummaryTreeCache,
 					getStorageManager,
 			  );
 		// TODO: worth prefetching latest summary making version + snapshot call with WholeSummary storage?
@@ -78,7 +80,8 @@ export class DocumentStorageService extends DocumentStorageServiceProxy {
 		policies: IDocumentStorageServicePolicies,
 		driverPolicies?: IRouterliciousDriverPolicies,
 		blobCache?: ICache<ArrayBufferLike>,
-		snapshotTreeCache?: ICache<ISnapshotTreeVersion>,
+		snapshotTreeCache?: ICache<INormalizedWholeSummary>,
+		shreddedSummaryTreeCache?: ICache<ISnapshotTreeVersion>,
 		public noCacheGitManager?: GitManager,
 		getStorageManager?: (disableCache?: boolean) => Promise<GitManager>,
 	) {
@@ -91,6 +94,7 @@ export class DocumentStorageService extends DocumentStorageServiceProxy {
 				driverPolicies,
 				blobCache,
 				snapshotTreeCache,
+				shreddedSummaryTreeCache,
 				noCacheGitManager,
 				getStorageManager,
 			),

package/src/gitManager.ts

@@ -0,0 +1,116 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as resources from "@fluidframework/gitresources";
+import {
+	IWholeFlatSummary,
+	IWholeSummaryPayload,
+	IWriteSummaryResponse,
+} from "@fluidframework/server-services-client";
+import { IGitManager, IHistorian } from "./storageContracts";
+import { IR11sResponse, createR11sResponseFromContent } from "./restWrapper";
+
+export class GitManager implements IGitManager {
+	private readonly blobCache = new Map<string, resources.IBlob>();
+	private readonly commitCache = new Map<string, resources.ICommit>();
+	private readonly treeCache = new Map<string, resources.ITree>();
+	private readonly refCache = new Map<string, string>();
+
+	constructor(private readonly historian: IHistorian) {}
+
+	/**
+	 * Reads the object with the given ID. We defer to the client implementation to do the actual read.
+	 */
+	public async getCommits(
+		shaOrRef: string,
+		count: number,
+	): Promise<IR11sResponse<resources.ICommitDetails[]>> {
+		let sha: string | undefined = shaOrRef;
+
+		// See if the sha is really a ref and convert
+		if (this.refCache.has(shaOrRef)) {
+			sha = this.refCache.get(shaOrRef);
+
+			// Delete refcache after first use
+			this.refCache.delete(shaOrRef);
+
+			// If null is stored for the ref then there are no commits - return an empty array
+			if (!sha) {
+				return createR11sResponseFromContent([]);
+			}
+		}
+
+		// See if the commit sha is hashed and return it if so
+		const commit = this.commitCache.get(sha);
+		if (commit !== undefined) {
+			return createR11sResponseFromContent([
+				{
+					commit: {
+						author: commit.author,
+						committer: commit.committer,
+						message: commit.message,
+						tree: commit.tree,
+						url: commit.url,
+					},
+					parents: commit.parents,
+					sha: commit.sha,
+					url: commit.url,
+				},
+			]);
+		}
+
+		// Otherwise fall back to the historian
+		return this.historian.getCommits(sha, count);
+	}
+
+	/**
+	 * Reads the object with the given ID. We defer to the client implementation to do the actual read.
+	 */
+	public async getTree(root: string, recursive = true): Promise<IR11sResponse<resources.ITree>> {
+		const tree = this.treeCache.get(root);
+		if (tree !== undefined) {
+			return createR11sResponseFromContent(tree);
+		}
+
+		return this.historian.getTree(root, recursive);
+	}
+
+	public async getBlob(sha: string): Promise<IR11sResponse<resources.IBlob>> {
+		const blob = this.blobCache.get(sha);
+		if (blob !== undefined) {
+			return createR11sResponseFromContent(blob);
+		}
+		return this.historian.getBlob(sha);
+	}
+
+	public async createBlob(
+		content: string,
+		encoding: "utf-8" | "base64",
+	): Promise<IR11sResponse<resources.ICreateBlobResponse>> {
+		const blob: resources.ICreateBlobParams = {
+			content,
+			encoding,
+		};
+		return this.historian.createBlob(blob);
+	}
+
+	public async createGitTree(
+		params: resources.ICreateTreeParams,
+	): Promise<IR11sResponse<resources.ITree>> {
+		const treeP = this.historian.createTree(params);
+		return treeP;
+	}
+
+	public async createSummary(
+		summary: IWholeSummaryPayload,
+		initial: boolean = false,
+	): Promise<IR11sResponse<IWriteSummaryResponse>> {
+		return this.historian.createSummary(summary, initial);
+	}
+
+	public async getSummary(sha: string): Promise<IR11sResponse<IWholeFlatSummary>> {
+		return this.historian.getSummary(sha);
+	}
+}

package/src/historian.ts

@@ -0,0 +1,121 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { fromUtf8ToBase64 } from "@fluidframework/common-utils";
+import * as git from "@fluidframework/gitresources";
+import {
+	IWholeFlatSummary,
+	IWholeSummaryPayload,
+	IWriteSummaryResponse,
+} from "@fluidframework/server-services-client";
+import { QueryStringType, RestWrapper } from "./restWrapperBase";
+import { IR11sResponse } from "./restWrapper";
+import { IHistorian } from "./storageContracts";
+
+export interface ICredentials {
+	user: string;
+	password: string;
+}
+
+export const getAuthorizationTokenFromCredentials = (credentials: ICredentials): string =>
+	`Basic ${fromUtf8ToBase64(`${credentials.user}:${credentials.password}`)}`;
+
+/**
+ * Implementation of the IHistorian interface that calls out to a REST interface
+ */
+export class Historian implements IHistorian {
+	private readonly defaultQueryString: QueryStringType = {};
+	private readonly cacheBust: boolean;
+
+	constructor(
+		private readonly historianApi: boolean,
+		disableCache: boolean,
+		private readonly restWrapper: RestWrapper,
+	) {
+		if (disableCache && this.historianApi) {
+			this.defaultQueryString.disableCache = disableCache;
+			this.cacheBust = false;
+		} else {
+			this.cacheBust = disableCache;
+		}
+	}
+
+	public async getBlob(sha: string): Promise<IR11sResponse<git.IBlob>> {
+		return this.restWrapper.get<git.IBlob>(
+			`/git/blobs/${encodeURIComponent(sha)}`,
+			this.getQueryString(),
+		);
+	}
+
+	public async createBlob(
+		blob: git.ICreateBlobParams,
+	): Promise<IR11sResponse<git.ICreateBlobResponse>> {
+		return this.restWrapper.post<git.ICreateBlobResponse>(
+			`/git/blobs`,
+			blob,
+			this.getQueryString(),
+		);
+	}
+
+	public async getCommits(
+		sha: string,
+		count: number,
+	): Promise<IR11sResponse<git.ICommitDetails[]>> {
+		return this.restWrapper
+			.get<git.ICommitDetails[]>(`/commits`, this.getQueryString({ count, sha }))
+			.catch((error) =>
+				error.statusCode === 400 || error.statusCode === 404
+					? {
+							content: [],
+							headers: new Map(),
+							propsToLog: {},
+							requestUrl: "",
+					  }
+					: Promise.reject<IR11sResponse<git.ICommitDetails[]>>(error),
+			);
+	}
+
+	public async createTree(tree: git.ICreateTreeParams): Promise<IR11sResponse<git.ITree>> {
+		return this.restWrapper.post<git.ITree>(`/git/trees`, tree, this.getQueryString());
+	}
+
+	public async getTree(sha: string, recursive: boolean): Promise<IR11sResponse<git.ITree>> {
+		return this.restWrapper.get<git.ITree>(
+			`/git/trees/${encodeURIComponent(sha)}`,
+			this.getQueryString({ recursive: recursive ? 1 : 0 }),
+		);
+	}
+	public async createSummary(
+		summary: IWholeSummaryPayload,
+		initial?: boolean,
+	): Promise<IR11sResponse<IWriteSummaryResponse>> {
+		return this.restWrapper.post<IWriteSummaryResponse>(
+			`/git/summaries`,
+			summary,
+			this.getQueryString(initial !== undefined ? { initial } : undefined),
+		);
+	}
+
+	public async getSummary(sha: string): Promise<IR11sResponse<IWholeFlatSummary>> {
+		return this.restWrapper.get<IWholeFlatSummary>(
+			`/git/summaries/${sha}`,
+			this.getQueryString(),
+		);
+	}
+
+	private getQueryString(queryString?: QueryStringType): QueryStringType {
+		if (this.cacheBust) {
+			return {
+				cacheBust: Date.now(),
+				...this.defaultQueryString,
+				...queryString,
+			};
+		}
+		return {
+			...this.defaultQueryString,
+			...queryString,
+		};
+	}
+}

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/routerlicious-driver";
-export const pkgVersion = "2.0.0-dev.4.1.0.148229";
+export const pkgVersion = "2.0.0-dev.4.3.0.157531";

package/src/r11sSnapshotParser.ts

@@ -0,0 +1,83 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ISnapshotTree } from "@fluidframework/protocol-definitions";
+import { IWholeFlatSummary, IWholeFlatSummaryTree } from "@fluidframework/server-services-client";
+import { stringToBuffer } from "@fluidframework/common-utils";
+import { INormalizedWholeSummary } from "./contracts";
+
+/**
+ * Build a tree heirarchy from a flat tree.
+ *
+ * @param flatTree - a flat tree
+ * @param treePrefixToRemove - tree prefix to strip
+ * @returns the heirarchical tree
+ */
+function buildHierarchy(
+	flatTree: IWholeFlatSummaryTree,
+	treePrefixToRemove: string,
+): ISnapshotTree {
+	const lookup: { [path: string]: ISnapshotTree } = {};
+	// Root tree id will be used to determine which version was downloaded.
+	const root: ISnapshotTree = { id: flatTree.id, blobs: {}, trees: {} };
+	lookup[""] = root;
+
+	for (const entry of flatTree.entries) {
+		// Strip the `treePrefixToRemove` path from tree entries such that they are stored under root.
+		const entryPath = entry.path.replace(new RegExp(`^${treePrefixToRemove}/`), "");
+		const lastIndex = entryPath.lastIndexOf("/");
+		const entryPathDir = entryPath.slice(0, Math.max(0, lastIndex));
+		const entryPathBase = entryPath.slice(lastIndex + 1);
+
+		// The flat output is breadth-first so we can assume we see tree nodes prior to their contents
+		const node = lookup[entryPathDir];
+
+		// Add in either the blob or tree
+		if (entry.type === "tree") {
+			const newTree: ISnapshotTree = {
+				blobs: {},
+				trees: {},
+				unreferenced: entry.unreferenced,
+			};
+			node.trees[decodeURIComponent(entryPathBase)] = newTree;
+			lookup[entryPath] = newTree;
+		} else if (entry.type === "blob") {
+			node.blobs[decodeURIComponent(entryPathBase)] = entry.id;
+		} else {
+			throw new Error(`Unknown entry type!!`);
+		}
+	}
+
+	return root;
+}
+
+/**
+ * Converts existing IWholeFlatSummary to snapshot tree, blob array, and sequence number.
+ *
+ * @param flatSummary - flat summary
+ * @param treePrefixToRemove - tree prefix to strip. By default we are stripping ".app" prefix
+ * @returns snapshot tree, blob array, and sequence number
+ */
+export function convertWholeFlatSummaryToSnapshotTreeAndBlobs(
+	flatSummary: IWholeFlatSummary,
+	treePrefixToRemove: string = ".app",
+): INormalizedWholeSummary {
+	const blobs = new Map<string, ArrayBuffer>();
+	if (flatSummary.blobs) {
+		flatSummary.blobs.forEach((blob) => {
+			blobs.set(blob.id, stringToBuffer(blob.content, blob.encoding ?? "utf-8"));
+		});
+	}
+	const flatSummaryTree = flatSummary.trees?.[0];
+	const sequenceNumber = flatSummaryTree?.sequenceNumber;
+	const snapshotTree = buildHierarchy(flatSummaryTree, treePrefixToRemove);
+
+	return {
+		blobs,
+		snapshotTree,
+		sequenceNumber,
+		id: flatSummary.id,
+	};
+}