@fluidframework/routerlicious-driver 0.51.3 → 0.53.0-46105

package/lib/wholeSummaryDocumentStorageService.js

@@ -0,0 +1,150 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { assert, stringToBuffer, Uint8ArrayToString, } from "@fluidframework/common-utils";
+import { convertWholeFlatSummaryToSnapshotTreeAndBlobs, WholeSummaryUploadManager, } from "@fluidframework/server-services-client";
+import { PerformanceEvent } from "@fluidframework/telemetry-utils";
+import { InMemoryCache } from "./cache";
+const latestSnapshotId = "latest";
+export class WholeSummaryDocumentStorageService {
+    constructor(id, manager, logger, policies = {}, blobCache = new InMemoryCache(), snapshotTreeCache = new InMemoryCache()) {
+        this.id = id;
+        this.manager = manager;
+        this.logger = logger;
+        this.policies = policies;
+        this.blobCache = blobCache;
+        this.snapshotTreeCache = snapshotTreeCache;
+        this.firstVersionsCall = true;
+        this.summaryUploadManager = new WholeSummaryUploadManager(manager);
+    }
+    get repositoryUrl() {
+        return "";
+    }
+    async getVersions(versionId, count) {
+        if (versionId !== this.id && versionId !== null) {
+            // Blobs in this scenario will never have multiple versions, so return blobId as is
+            return [{
+                    id: versionId,
+                    treeId: undefined,
+                }];
+        }
+        // If this is the first versions call for the document, we know we will want the latest summary.
+        // Fetch latest summary, cache it, and return its id.
+        if (this.firstVersionsCall && count === 1) {
+            this.firstVersionsCall = false;
+            return [{
+                    id: (await this.fetchAndCacheSnapshotTree(latestSnapshotId)).id,
+                    treeId: undefined,
+                }];
+        }
+        // Otherwise, get the latest version of the document as normal.
+        const id = versionId ? versionId : this.id;
+        const commits = await PerformanceEvent.timedExecAsync(this.logger, {
+            eventName: "getVersions",
+            versionId: id,
+            count,
+        }, async () => this.manager.getCommits(id, count));
+        return commits.map((commit) => ({
+            date: commit.commit.author.date,
+            id: commit.sha,
+            treeId: undefined,
+        }));
+    }
+    async getSnapshotTree(version) {
+        let requestVersion = version;
+        if (!requestVersion) {
+            const versions = await this.getVersions(this.id, 1);
+            if (versions.length === 0) {
+                return null;
+            }
+            requestVersion = versions[0];
+        }
+        return (await this.fetchAndCacheSnapshotTree(requestVersion.id)).snapshotTree;
+    }
+    async readBlob(blobId) {
+        const cachedBlob = await this.blobCache.get(blobId);
+        if (cachedBlob !== undefined) {
+            return cachedBlob;
+        }
+        const blob = await PerformanceEvent.timedExecAsync(this.logger, {
+            eventName: "readBlob",
+            blobId,
+        }, async (event) => {
+            const response = await this.manager.getBlob(blobId);
+            event.end({
+                size: response.size,
+            });
+            return response;
+        });
+        const bufferValue = stringToBuffer(blob.content, blob.encoding);
+        await this.blobCache.put(blob.sha, bufferValue);
+        return bufferValue;
+    }
+    async uploadSummaryWithContext(summary, context) {
+        const summaryHandle = await PerformanceEvent.timedExecAsync(this.logger, {
+            eventName: "uploadSummaryWithContext",
+        }, async () => { var _a; return this.summaryUploadManager.writeSummaryTree(summary, (_a = context.ackHandle) !== null && _a !== void 0 ? _a : "", "channel"); });
+        return summaryHandle;
+    }
+    async downloadSummary(handle) {
+        throw new Error("NOT IMPLEMENTED!");
+    }
+    async write(tree, parents, message, ref) {
+        throw new Error("NOT IMPLEMENTED!");
+    }
+    async createBlob(file) {
+        const uint8ArrayFile = new Uint8Array(file);
+        return PerformanceEvent.timedExecAsync(this.logger, {
+            eventName: "createBlob",
+            size: uint8ArrayFile.length,
+        }, async (event) => {
+            const response = await this.manager.createBlob(Uint8ArrayToString(uint8ArrayFile, "base64"), "base64").then((r) => ({ id: r.sha, url: r.url }));
+            event.end({
+                blobId: response.id,
+            });
+            return response;
+        });
+    }
+    async fetchAndCacheSnapshotTree(versionId) {
+        const cachedSnapshotTree = await this.snapshotTreeCache.get(versionId);
+        if (cachedSnapshotTree !== undefined) {
+            return { id: cachedSnapshotTree.id, snapshotTree: cachedSnapshotTree };
+        }
+        const wholeFlatSummary = await PerformanceEvent.timedExecAsync(this.logger, {
+            eventName: "getWholeFlatSummary",
+            treeId: versionId,
+        }, async (event) => {
+            var _a;
+            const response = await this.manager.getSummary(versionId);
+            event.end({
+                size: (_a = response.trees[0]) === null || _a === void 0 ? void 0 : _a.entries.length,
+            });
+            return response;
+        });
+        const normalizedWholeSummary = convertWholeFlatSummaryToSnapshotTreeAndBlobs(wholeFlatSummary);
+        const snapshotId = normalizedWholeSummary.snapshotTree.id;
+        assert(snapshotId !== undefined, 0x275 /* "Root tree should contain the id" */);
+        const cachePs = [
+            this.snapshotTreeCache.put(snapshotId, normalizedWholeSummary.snapshotTree),
+            this.initBlobCache(normalizedWholeSummary.blobs),
+        ];
+        if (snapshotId !== versionId) {
+            // versionId could be "latest". When summarizer checks cache for "latest", we want it to be available.
+            // TODO: For in-memory cache, <latest,snapshotTree> will be a shared pointer with <snapshotId,snapshotTree>,
+            // However, for something like Redis, this will cache the same value twice. Alternatively, could we simply
+            // cache with versionId?
+            cachePs.push(this.snapshotTreeCache.put(versionId, normalizedWholeSummary.snapshotTree));
+        }
+        await Promise.all(cachePs);
+        return { id: snapshotId, snapshotTree: normalizedWholeSummary.snapshotTree };
+    }
+    async initBlobCache(blobs) {
+        const blobCachePutPs = [];
+        blobs.forEach((value, id) => {
+            blobCachePutPs.push(this.blobCache.put(id, value));
+        });
+        await Promise.all(blobCachePutPs);
+    }
+}
+//# sourceMappingURL=wholeSummaryDocumentStorageService.js.map

package/lib/wholeSummaryDocumentStorageService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wholeSummaryDocumentStorageService.js","sourceRoot":"","sources":["../src/wholeSummaryDocumentStorageService.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EACH,MAAM,EACN,cAAc,EACd,kBAAkB,GACrB,MAAM,8BAA8B,CAAC;AActC,OAAO,EACH,6CAA6C,EAG7C,yBAAyB,GAC5B,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAU,aAAa,EAAE,MAAM,SAAS,CAAC;AAEhD,MAAM,gBAAgB,GAAW,QAAQ,CAAC;AAE1C,MAAM,OAAO,kCAAkC;IAQ3C,YACuB,EAAU,EACV,OAAmB,EACnB,MAAwB,EAC3B,WAA4C,EAAE,EAC7C,YAAqC,IAAI,aAAa,EAAE,EACxD,oBAA2C,IAAI,aAAa,EAAE;QAL5D,OAAE,GAAF,EAAE,CAAQ;QACV,YAAO,GAAP,OAAO,CAAY;QACnB,WAAM,GAAN,MAAM,CAAkB;QAC3B,aAAQ,GAAR,QAAQ,CAAsC;QAC7C,cAAS,GAAT,SAAS,CAA+C;QACxD,sBAAiB,GAAjB,iBAAiB,CAA6C;QAZ3E,sBAAiB,GAAY,IAAI,CAAC;QAatC,IAAI,CAAC,oBAAoB,GAAG,IAAI,yBAAyB,CAAC,OAAO,CAAC,CAAC;IACvE,CAAC;IAZD,IAAW,aAAa;QACpB,OAAO,EAAE,CAAC;IACd,CAAC;IAYM,KAAK,CAAC,WAAW,CAAC,SAAwB,EAAE,KAAa;QAC5D,IAAI,SAAS,KAAK,IAAI,CAAC,EAAE,IAAI,SAAS,KAAK,IAAI,EAAE;YAC7C,mFAAmF;YACnF,OAAO,CAAC;oBACJ,EAAE,EAAE,SAAS;oBACb,MAAM,EAAE,SAAU;iBACrB,CAAC,CAAC;SACN;QACD,gGAAgG;QAChG,qDAAqD;QACrD,IAAI,IAAI,CAAC,iBAAiB,IAAI,KAAK,KAAK,CAAC,EAAE;YACvC,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;YAC/B,OAAO,CAAC;oBACJ,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE;oBAC/D,MAAM,EAAE,SAAU;iBACrB,CAAC,CAAC;SACN;QAED,+DAA+D;QAC/D,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,cAAc,CACjD,IAAI,CAAC,MAAM,EACX;YACI,SAAS,EAAE,aAAa;YACxB,SAAS,EAAE,EAAE;YACb,KAAK;SACR,EACD,KAAK,IAAI,EAAE,CAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,KAAK,CAAC,CAClD,CAAC;QACF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAC5B,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI;YAC/B,EAAE,EAAE,MAAM,CAAC,GAAG;YACd,MAAM,EAAE,SAAU;SACrB,CAAC,CAAC,CAAC;IACR,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,OAAkB;QAC3C,IAAI,cAAc,GAAG,OAAO,CAAC;QAC7B,IAAI,CAAC,cAAc,EAAE;YACjB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YACpD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;gBACvB,OAAO,IAAI,CAAC;aACf;YAED,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;SAChC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,yBAAyB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;IAClF,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,MAAc;QAChC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,UAAU,KAAK,SAAS,EAAE;YAC1B,OAAO,UAAU,CAAC;SACrB;QAED,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,cAAc,CAC9C,IAAI,CAAC,MAAM,EACX;YACI,SAAS,EAAE,UAAU;YACrB,MAAM;SACT,EACD,KAAK,EAAE,KAAK,EAAE,EAAE;YACZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACpD,KAAK,CAAC,GAAG,CAAC;gBACN,IAAI,EAAE,QAAQ,CAAC,IAAI;aACtB,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC;QACpB,CAAC,CACJ,CAAC;QACF,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEhE,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAEhD,OAAO,WAAW,CAAC;IACvB,CAAC;IAEM,KAAK,CAAC,wBAAwB,CAAC,OAAqB,EAAE,OAAwB;QACjF,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,cAAc,CACvD,IAAI,CAAC,MAAM,EACX;YACI,SAAS,EAAE,0BAA0B;SACxC,EACD,KAAK,IAAI,EAAE,WAAC,OAAA,IAAI,CAAC,oBAAoB,CAAC,gBAAgB,CAAC,OAAO,QAAE,OAAO,CAAC,SAAS,mCAAI,EAAE,EAAE,SAAS,CAAC,CAAA,EAAA,CACtG,CAAC;QACF,OAAO,aAAa,CAAC;IACzB,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,MAAsB;QAC/C,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACxC,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,IAAW,EAAE,OAAiB,EAAE,OAAe,EAAE,GAAW;QAC3E,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACxC,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,IAAqB;QACzC,MAAM,cAAc,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;QAC5C,OAAO,gBAAgB,CAAC,cAAc,CAClC,IAAI,CAAC,MAAM,EACX;YACI,SAAS,EAAE,YAAY;YACvB,IAAI,EAAE,cAAc,CAAC,MAAM;SAC9B,EACD,KAAK,EAAE,KAAK,EAAE,EAAE;YACZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAC1C,kBAAkB,CACd,cAAc,EAAE,QAAQ,CAAC,EAC7B,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACvD,KAAK,CAAC,GAAG,CAAC;gBACN,MAAM,EAAE,QAAQ,CAAC,EAAE;aACtB,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC;QACpB,CAAC,CACJ,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,SAAiB;QACrD,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvE,IAAI,kBAAkB,KAAK,SAAS,EAAE;YAClC,OAAO,EAAE,EAAE,EAAE,kBAAkB,CAAC,EAAG,EAAE,YAAY,EAAE,kBAAkB,EAAE,CAAC;SAC3E;QAED,MAAM,gBAAgB,GAAG,MAAM,gBAAgB,CAAC,cAAc,CAC1D,IAAI,CAAC,MAAM,EACX;YACI,SAAS,EAAE,qBAAqB;YAChC,MAAM,EAAE,SAAS;SACpB,EACD,KAAK,EAAE,KAAK,EAAE,EAAE;;YACZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAC1D,KAAK,CAAC,GAAG,CAAC;gBACN,IAAI,QAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,0CAAE,OAAO,CAAC,MAAM;aAC1C,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC;QACpB,CAAC,CACJ,CAAC;QACF,MAAM,sBAAsB,GAAG,6CAA6C,CAAC,gBAAgB,CAAC,CAAC;QAC/F,MAAM,UAAU,GAAG,sBAAsB,CAAC,YAAY,CAAC,EAAE,CAAC;QAC1D,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEhF,MAAM,OAAO,GAAmB;YAC5B,IAAI,CAAC,iBAAiB,CAAC,GAAG,CACtB,UAAU,EACV,sBAAsB,CAAC,YAAY,CACtC;YACD,IAAI,CAAC,aAAa,CAAC,sBAAsB,CAAC,KAAK,CAAC;SACnD,CAAC;QACF,IAAI,UAAU,KAAK,SAAS,EAAE;YAC1B,sGAAsG;YACtG,4GAA4G;YAC5G,0GAA0G;YAC1G,wBAAwB;YACxB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CACnC,SAAS,EACT,sBAAsB,CAAC,YAAY,CACtC,CAAC,CAAC;SACN;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE3B,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,sBAAsB,CAAC,YAAY,EAAC,CAAC;IAChF,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,KAA+B;QACvD,MAAM,cAAc,GAAoB,EAAE,CAAC;QAC3C,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACxB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACtC,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport type { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport {\n    assert,\n    stringToBuffer,\n    Uint8ArrayToString,\n} from \"@fluidframework/common-utils\";\nimport {\n    IDocumentStorageService,\n    ISummaryContext,\n    IDocumentStorageServicePolicies,\n } from \"@fluidframework/driver-definitions\";\nimport {\n    ICreateBlobResponse,\n    ISnapshotTree,\n    ISummaryHandle,\n    ISummaryTree,\n    ITree,\n    IVersion,\n} from \"@fluidframework/protocol-definitions\";\nimport {\n    convertWholeFlatSummaryToSnapshotTreeAndBlobs,\n    GitManager,\n    ISummaryUploadManager,\n    WholeSummaryUploadManager,\n} from \"@fluidframework/server-services-client\";\nimport { PerformanceEvent } from \"@fluidframework/telemetry-utils\";\nimport { ICache, InMemoryCache } from \"./cache\";\n\nconst latestSnapshotId: string = \"latest\";\n\nexport class WholeSummaryDocumentStorageService implements IDocumentStorageService {\n    private readonly summaryUploadManager: ISummaryUploadManager;\n    private firstVersionsCall: boolean = true;\n\n    public get repositoryUrl(): string {\n        return \"\";\n    }\n\n    constructor(\n        protected readonly id: string,\n        protected readonly manager: GitManager,\n        protected readonly logger: ITelemetryLogger,\n        public readonly policies: IDocumentStorageServicePolicies = {},\n        private readonly blobCache: ICache<ArrayBufferLike> = new InMemoryCache(),\n        private readonly snapshotTreeCache: ICache<ISnapshotTree> = new InMemoryCache()) {\n        this.summaryUploadManager = new WholeSummaryUploadManager(manager);\n    }\n\n    public async getVersions(versionId: string | null, count: number): Promise<IVersion[]> {\n        if (versionId !== this.id && versionId !== null) {\n            // Blobs in this scenario will never have multiple versions, so return blobId as is\n            return [{\n                id: versionId,\n                treeId: undefined!,\n            }];\n        }\n        // If this is the first versions call for the document, we know we will want the latest summary.\n        // Fetch latest summary, cache it, and return its id.\n        if (this.firstVersionsCall && count === 1) {\n            this.firstVersionsCall = false;\n            return [{\n                id: (await this.fetchAndCacheSnapshotTree(latestSnapshotId)).id,\n                treeId: undefined!,\n            }];\n        }\n\n        // Otherwise, get the latest version of the document as normal.\n        const id = versionId ? versionId : this.id;\n        const commits = await PerformanceEvent.timedExecAsync(\n            this.logger,\n            {\n                eventName: \"getVersions\",\n                versionId: id,\n                count,\n            },\n            async () =>  this.manager.getCommits(id, count),\n        );\n        return commits.map((commit) => ({\n            date: commit.commit.author.date,\n            id: commit.sha,\n            treeId: undefined!,\n        }));\n    }\n\n    public async getSnapshotTree(version?: IVersion): Promise<ISnapshotTree | null> {\n        let requestVersion = version;\n        if (!requestVersion) {\n            const versions = await this.getVersions(this.id, 1);\n            if (versions.length === 0) {\n                return null;\n            }\n\n            requestVersion = versions[0];\n        }\n\n        return (await this.fetchAndCacheSnapshotTree(requestVersion.id)).snapshotTree;\n    }\n\n    public async readBlob(blobId: string): Promise<ArrayBufferLike> {\n        const cachedBlob = await this.blobCache.get(blobId);\n        if (cachedBlob !== undefined) {\n            return cachedBlob;\n        }\n\n        const blob = await PerformanceEvent.timedExecAsync(\n            this.logger,\n            {\n                eventName: \"readBlob\",\n                blobId,\n            },\n            async (event) => {\n                const response = await this.manager.getBlob(blobId);\n                event.end({\n                    size: response.size,\n                });\n                return response;\n            },\n        );\n        const bufferValue = stringToBuffer(blob.content, blob.encoding);\n\n        await this.blobCache.put(blob.sha, bufferValue);\n\n        return bufferValue;\n    }\n\n    public async uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise<string> {\n        const summaryHandle = await PerformanceEvent.timedExecAsync(\n            this.logger,\n            {\n                eventName: \"uploadSummaryWithContext\",\n            },\n            async () => this.summaryUploadManager.writeSummaryTree(summary, context.ackHandle ?? \"\", \"channel\"),\n        );\n        return summaryHandle;\n    }\n\n    public async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {\n        throw new Error(\"NOT IMPLEMENTED!\");\n    }\n\n    public async write(tree: ITree, parents: string[], message: string, ref: string): Promise<IVersion> {\n        throw new Error(\"NOT IMPLEMENTED!\");\n    }\n\n    public async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {\n        const uint8ArrayFile = new Uint8Array(file);\n        return PerformanceEvent.timedExecAsync(\n            this.logger,\n            {\n                eventName: \"createBlob\",\n                size: uint8ArrayFile.length,\n            },\n            async (event) => {\n                const response = await this.manager.createBlob(\n                    Uint8ArrayToString(\n                        uint8ArrayFile, \"base64\"),\n                    \"base64\").then((r) => ({ id: r.sha, url: r.url }));\n                event.end({\n                    blobId: response.id,\n                });\n                return response;\n            },\n        );\n    }\n\n    private async fetchAndCacheSnapshotTree(versionId: string): Promise<{ id: string, snapshotTree: ISnapshotTree }> {\n        const cachedSnapshotTree = await this.snapshotTreeCache.get(versionId);\n        if (cachedSnapshotTree !== undefined) {\n            return { id: cachedSnapshotTree.id!, snapshotTree: cachedSnapshotTree };\n        }\n\n        const wholeFlatSummary = await PerformanceEvent.timedExecAsync(\n            this.logger,\n            {\n                eventName: \"getWholeFlatSummary\",\n                treeId: versionId,\n            },\n            async (event) => {\n                const response = await this.manager.getSummary(versionId);\n                event.end({\n                    size: response.trees[0]?.entries.length,\n                });\n                return response;\n            },\n        );\n        const normalizedWholeSummary = convertWholeFlatSummaryToSnapshotTreeAndBlobs(wholeFlatSummary);\n        const snapshotId = normalizedWholeSummary.snapshotTree.id;\n        assert(snapshotId !== undefined, 0x275 /* \"Root tree should contain the id\" */);\n\n        const cachePs: Promise<any>[] = [\n            this.snapshotTreeCache.put(\n                snapshotId,\n                normalizedWholeSummary.snapshotTree,\n            ),\n            this.initBlobCache(normalizedWholeSummary.blobs),\n        ];\n        if (snapshotId !== versionId) {\n            // versionId could be \"latest\". When summarizer checks cache for \"latest\", we want it to be available.\n            // TODO: For in-memory cache, <latest,snapshotTree> will be a shared pointer with <snapshotId,snapshotTree>,\n            // However, for something like Redis, this will cache the same value twice. Alternatively, could we simply\n            // cache with versionId?\n            cachePs.push(this.snapshotTreeCache.put(\n                versionId,\n                normalizedWholeSummary.snapshotTree,\n            ));\n        }\n\n        await Promise.all(cachePs);\n\n        return { id: snapshotId, snapshotTree: normalizedWholeSummary.snapshotTree};\n    }\n\n    private async initBlobCache(blobs: Map<string, ArrayBuffer>): Promise<void> {\n        const blobCachePutPs: Promise<void>[] = [];\n        blobs.forEach((value, id) => {\n            blobCachePutPs.push(this.blobCache.put(id, value));\n        });\n        await Promise.all(blobCachePutPs);\n    }\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/routerlicious-driver",
-  "version": "0.51.3",
+  "version": "0.53.0-46105",
   "description": "Socket.IO + Git implementation of Fluid service API",
   "homepage": "https://fluidframework.com",
   "repository": "https://github.com/microsoft/FluidFramework",
@@ -57,15 +57,15 @@
   "dependencies": {
     "@fluidframework/common-definitions": "^0.20.1",
     "@fluidframework/common-utils": "^0.32.1",
-    "@fluidframework/driver-base": "^0.51.3",
-    "@fluidframework/driver-definitions": "^0.41.0",
-    "@fluidframework/driver-utils": "^0.51.3",
-    "@fluidframework/gitresources": "^0.1033.0",
-    "@fluidframework/protocol-base": "^0.1033.0",
-    "@fluidframework/protocol-definitions": "^0.1025.0",
-    "@fluidframework/server-services-client": "^0.1033.0",
-    "@fluidframework/telemetry-utils": "^0.51.3",
-    "axios": "^0.21.1",
+    "@fluidframework/driver-base": "0.53.0-46105",
+    "@fluidframework/driver-definitions": "^0.43.0-0",
+    "@fluidframework/driver-utils": "0.53.0-46105",
+    "@fluidframework/gitresources": "^0.1034.0",
+    "@fluidframework/protocol-base": "^0.1034.0",
+    "@fluidframework/protocol-definitions": "^0.1026.0",
+    "@fluidframework/server-services-client": "^0.1034.0",
+    "@fluidframework/telemetry-utils": "0.53.0-46105",
+    "axios": "^0.21.2",
     "json-stringify-safe": "5.0.1",
     "socket.io-client": "^2.4.0",
     "url-parse": "^1.5.3",
@@ -74,7 +74,7 @@
   "devDependencies": {
     "@fluidframework/build-common": "^0.23.0",
     "@fluidframework/eslint-config-fluid": "^0.24.0",
-    "@fluidframework/mocha-test-setup": "^0.51.3",
+    "@fluidframework/mocha-test-setup": "0.53.0-46105",
     "@microsoft/api-extractor": "^7.16.1",
     "@types/mocha": "^8.2.2",
     "@types/nock": "^9.3.0",

package/src/deltaStorageService.ts

@@ -36,6 +36,7 @@ export class DocumentDeltaStorageService implements IDocumentDeltaStorageService
         to: number | undefined,
         abortSignal?: AbortSignal,
         cachedOnly?: boolean,
+        fetchReason?: string,
     ): IStream<ISequencedDocumentMessage[]>
     {
         if (cachedOnly) {
@@ -51,6 +52,7 @@ export class DocumentDeltaStorageService implements IDocumentDeltaStorageService
             MaxBatchDeltas,
             new TelemetryNullLogger(),
             abortSignal,
+            fetchReason,
         );
     }
 

package/src/documentService.ts

@@ -49,6 +49,10 @@ export class DocumentService implements api.IDocumentService {
      * @returns returns the document storage service for routerlicious driver.
      */
     public async connectToStorage(): Promise<api.IDocumentStorageService> {
+        if (this.documentStorageService !== undefined) {
+            return this.documentStorageService;
+        }
+
         if (this.gitUrl === undefined) {
             return new NullBlobStorageService();
         }
@@ -93,6 +97,7 @@ export class DocumentService implements api.IDocumentService {
      * @returns returns the document delta storage service for routerlicious driver.
      */
     public async connectToDeltaStorage(): Promise<api.IDocumentDeltaStorageService> {
+        await this.connectToStorage();
         assert(!!this.documentStorageService, 0x0b1 /* "Storage service not initialized" */);
 
         const rateLimiter = new RateLimiter(this.driverPolicies.maxConcurrentOrdererRequests);

package/src/documentStorageService.ts

@@ -4,392 +4,23 @@
  */
 
 import type { ITelemetryLogger } from "@fluidframework/common-definitions";
-import {
-    assert,
-    stringToBuffer,
-    Uint8ArrayToString,
-} from "@fluidframework/common-utils";
 import {
     IDocumentStorageService,
-    ISummaryContext,
     IDocumentStorageServicePolicies,
     LoaderCachingPolicy,
  } from "@fluidframework/driver-definitions";
-import { buildHierarchy } from "@fluidframework/protocol-base";
 import {
-    ICreateBlobResponse,
     ISnapshotTree,
-    ISnapshotTreeEx,
-    ISummaryHandle,
-    ISummaryTree,
-    ITree,
     IVersion,
 } from "@fluidframework/protocol-definitions";
 import {
-    convertWholeFlatSummaryToSnapshotTreeAndBlobs,
     GitManager,
-    ISummaryUploadManager,
-    SummaryTreeUploadManager,
-    WholeSummaryUploadManager,
 } from "@fluidframework/server-services-client";
-import { PerformanceEvent } from "@fluidframework/telemetry-utils";
 import { DocumentStorageServiceProxy, PrefetchDocumentStorageService } from "@fluidframework/driver-utils";
-import { RetriableGitManager } from "./retriableGitManager";
 import { IRouterliciousDriverPolicies } from "./policies";
-import { ICache, InMemoryCache } from "./cache";
-
-/**
- * Document access to underlying storage for routerlicious driver.
- * Uploads summaries piece-by-piece traversing the tree recursively.
- * Downloads summaries
- */
-class ShreddedSummaryDocumentStorageService implements IDocumentStorageService {
-    // The values of this cache is useless. We only need the keys. So we are always putting
-    // empty strings as values.
-    protected readonly blobsShaCache = new Map<string, string>();
-    private readonly summaryUploadManager: ISummaryUploadManager;
-
-    public get repositoryUrl(): string {
-        return "";
-    }
-
-    constructor(
-        protected readonly id: string,
-        protected readonly manager: GitManager,
-        protected readonly logger: ITelemetryLogger,
-        public readonly policies: IDocumentStorageServicePolicies = {}) {
-        this.summaryUploadManager = new SummaryTreeUploadManager(
-                new RetriableGitManager(manager, logger),
-                this.blobsShaCache,
-                this.getPreviousFullSnapshot.bind(this),
-            );
-    }
-
-    public async getVersions(versionId: string, count: number): Promise<IVersion[]> {
-        const id = versionId ? versionId : this.id;
-        const commits = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "getVersions",
-                versionId: id,
-                count,
-            },
-            async () =>  this.manager.getCommits(id, count),
-        );
-        return commits.map((commit) => ({
-            date: commit.commit.author.date,
-            id: commit.sha,
-            treeId: commit.commit.tree.sha,
-        }));
-    }
-
-    public async getSnapshotTree(version?: IVersion): Promise<ISnapshotTreeEx | null> {
-        let requestVersion = version;
-        if (!requestVersion) {
-            const versions = await this.getVersions(this.id, 1);
-            if (versions.length === 0) {
-                return null;
-            }
-
-            requestVersion = versions[0];
-        }
-
-        const rawTree = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "getSnapshotTree",
-                treeId: requestVersion.treeId,
-            },
-            async (event) => {
-                const response = await this.manager.getTree(requestVersion!.treeId);
-                event.end({
-                    size: response.tree.length,
-                });
-                return response;
-            },
-        );
-        const tree = buildHierarchy(rawTree, this.blobsShaCache, true);
-        return tree;
-    }
-
-    public async readBlob(blobId: string): Promise<ArrayBufferLike> {
-        const value = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "readBlob",
-                blobId,
-            },
-            async (event) => {
-                const response = await this.manager.getBlob(blobId);
-                event.end({
-                    size: response.size,
-                });
-                return response;
-            },
-        );
-        this.blobsShaCache.set(value.sha, "");
-        return stringToBuffer(value.content, value.encoding);
-    }
-
-    public async write(tree: ITree, parents: string[], message: string, ref: string): Promise<IVersion> {
-        const branch = ref ? `datastores/${this.id}/${ref}` : this.id;
-        const commit = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "write",
-                id: branch,
-            },
-            async () => this.manager.write(branch, tree, parents, message),
-        );
-        return { date: commit.committer.date, id: commit.sha, treeId: commit.tree.sha };
-    }
-
-    public async uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise<string> {
-        const summaryHandle = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "uploadSummaryWithContext",
-            },
-            async () => this.summaryUploadManager.writeSummaryTree(summary, context.ackHandle ?? "", "channel"),
-        );
-        return summaryHandle;
-    }
-
-    public async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {
-        throw new Error("NOT IMPLEMENTED!");
-    }
-
-    public async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {
-        const uint8ArrayFile = new Uint8Array(file);
-        return PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "createBlob",
-                size: uint8ArrayFile.length,
-            },
-            async (event) => {
-                const response = await this.manager.createBlob(
-                    Uint8ArrayToString(
-                        uint8ArrayFile, "base64"),
-                    "base64").then((r) => ({ id: r.sha, url: r.url }));
-                event.end({
-                    blobId: response.id,
-                });
-                return response;
-            },
-        );
-    }
-
-    private async getPreviousFullSnapshot(parentHandle: string): Promise<ISnapshotTreeEx | null | undefined> {
-        return parentHandle
-            ? this.getVersions(parentHandle, 1)
-                .then(async (versions) => {
-                    // Clear the cache as the getSnapshotTree call will fill the cache.
-                    this.blobsShaCache.clear();
-                    return this.getSnapshotTree(versions[0]);
-                })
-            : undefined;
-    }
-}
-
-const latestSnapshotId: string = "latest";
-
-class WholeSummaryDocumentStorageService implements IDocumentStorageService {
-    private readonly summaryUploadManager: ISummaryUploadManager;
-    private firstVersionsCall: boolean = true;
-
-    public get repositoryUrl(): string {
-        return "";
-    }
-
-    constructor(
-        protected readonly id: string,
-        protected readonly manager: GitManager,
-        protected readonly logger: ITelemetryLogger,
-        public readonly policies: IDocumentStorageServicePolicies = {},
-        private readonly blobCache: ICache<ArrayBufferLike> = new InMemoryCache(),
-        private readonly snapshotTreeCache: ICache<ISnapshotTree> = new InMemoryCache()) {
-        this.summaryUploadManager = new WholeSummaryUploadManager(manager);
-    }
-
-    public async getVersions(versionId: string | null, count: number): Promise<IVersion[]> {
-        if (versionId !== this.id && versionId !== null) {
-            // Blobs in this scenario will never have multiple versions, so return blobId as is
-            return [{
-                id: versionId,
-                treeId: undefined!,
-            }];
-        }
-        // If this is the first versions call for the document, we know we will want the latest summary.
-        // Fetch latest summary, cache it, and return its id.
-        if (this.firstVersionsCall && count === 1) {
-            this.firstVersionsCall = false;
-            return [{
-                id: (await this.fetchAndCacheSnapshotTree(latestSnapshotId)).id,
-                treeId: undefined!,
-            }];
-        }
-
-        // Otherwise, get the latest version of the document as normal.
-        const id = versionId ? versionId : this.id;
-        const commits = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "getVersions",
-                versionId: id,
-                count,
-            },
-            async () =>  this.manager.getCommits(id, count),
-        );
-        return commits.map((commit) => ({
-            date: commit.commit.author.date,
-            id: commit.sha,
-            treeId: undefined!,
-        }));
-    }
-
-    public async getSnapshotTree(version?: IVersion): Promise<ISnapshotTree | null> {
-        let requestVersion = version;
-        if (!requestVersion) {
-            const versions = await this.getVersions(this.id, 1);
-            if (versions.length === 0) {
-                return null;
-            }
-
-            requestVersion = versions[0];
-        }
-
-        return (await this.fetchAndCacheSnapshotTree(requestVersion.id)).snapshotTree;
-    }
-
-    public async readBlob(blobId: string): Promise<ArrayBufferLike> {
-        const cachedBlob = await this.blobCache.get(blobId);
-        if (cachedBlob !== undefined) {
-            return cachedBlob;
-        }
-
-        const blob = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "readBlob",
-                blobId,
-            },
-            async (event) => {
-                const response = await this.manager.getBlob(blobId);
-                event.end({
-                    size: response.size,
-                });
-                return response;
-            },
-        );
-        const bufferValue = stringToBuffer(blob.content, blob.encoding);
-
-        await this.blobCache.put(blob.sha, bufferValue);
-
-        return bufferValue;
-    }
-
-    public async uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise<string> {
-        const summaryHandle = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "uploadSummaryWithContext",
-            },
-            async () => this.summaryUploadManager.writeSummaryTree(summary, context.ackHandle ?? "", "channel"),
-        );
-        return summaryHandle;
-    }
-
-    public async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {
-        throw new Error("NOT IMPLEMENTED!");
-    }
-
-    public async write(tree: ITree, parents: string[], message: string, ref: string): Promise<IVersion> {
-        throw new Error("NOT IMPLEMENTED!");
-    }
-
-    public async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {
-        const uint8ArrayFile = new Uint8Array(file);
-        return PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "createBlob",
-                size: uint8ArrayFile.length,
-            },
-            async (event) => {
-                const response = await this.manager.createBlob(
-                    Uint8ArrayToString(
-                        uint8ArrayFile, "base64"),
-                    "base64").then((r) => ({ id: r.sha, url: r.url }));
-                event.end({
-                    blobId: response.id,
-                });
-                return response;
-            },
-        );
-    }
-
-    private async fetchAndCacheSnapshotTree(versionId: string): Promise<{ id: string, snapshotTree: ISnapshotTree }> {
-        const cachedSnapshotTree = await this.snapshotTreeCache.get(versionId);
-        if (cachedSnapshotTree !== undefined) {
-            return { id: cachedSnapshotTree.id!, snapshotTree: cachedSnapshotTree };
-        }
-
-        const wholeFlatSummary = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "getWholeFlatSummary",
-                treeId: versionId,
-            },
-            async (event) => {
-                const response = await this.manager.getSummary(versionId);
-                event.end({
-                    size: response.trees[0]?.entries.length,
-                });
-                return response;
-            },
-        );
-        const normalizedWholeSummary = convertWholeFlatSummaryToSnapshotTreeAndBlobs(wholeFlatSummary);
-        const snapshotId = normalizedWholeSummary.snapshotTree.id;
-        assert(snapshotId !== undefined, 0x275 /* "Root tree should contain the id" */);
-
-        const cachePs: Promise<any>[] = [
-            this.snapshotTreeCache.put(
-                snapshotId,
-                normalizedWholeSummary.snapshotTree,
-            ),
-            this.initBlobCache(normalizedWholeSummary.blobs),
-        ];
-        if (snapshotId !== versionId) {
-            // versionId could be "latest". When summarizer checks cache for "latest", we want it to be available.
-            // TODO: For in-memory cache, <latest,snapshotTree> will be a shared pointer with <snapshotId,snapshotTree>,
-            // However, for something like Redis, this will cache the same value twice. Alternatively, could we simply
-            // cache with versionId?
-            cachePs.push(this.snapshotTreeCache.put(
-                versionId,
-                normalizedWholeSummary.snapshotTree,
-            ));
-        }
-
-        await Promise.all([
-            this.snapshotTreeCache.put(
-                snapshotId,
-                normalizedWholeSummary.snapshotTree,
-            ),
-            this.initBlobCache(normalizedWholeSummary.blobs),
-        ]);
-
-        return { id: snapshotId, snapshotTree: normalizedWholeSummary.snapshotTree};
-    }
-
-    private async initBlobCache(blobs: Map<string, ArrayBuffer>): Promise<void> {
-        const blobCachePutPs: Promise<void>[] = [];
-        blobs.forEach((value, id) => {
-            blobCachePutPs.push(this.blobCache.put(id, value));
-        });
-        await Promise.all(blobCachePutPs);
-    }
-}
+import { ICache } from "./cache";
+import { WholeSummaryDocumentStorageService } from "./wholeSummaryDocumentStorageService";
+import { ShreddedSummaryDocumentStorageService } from "./shreddedSummaryDocumentStorageService";
 
 export class DocumentStorageService extends DocumentStorageServiceProxy {
     private _logTailSha: string | undefined = undefined;
@@ -407,8 +38,23 @@ export class DocumentStorageService extends DocumentStorageServiceProxy {
         blobCache?: ICache<ArrayBufferLike>,
         snapshotTreeCache?: ICache<ISnapshotTree>): IDocumentStorageService {
         const storageService = driverPolicies?.enableWholeSummaryUpload ?
-            new WholeSummaryDocumentStorageService(id, manager, logger, policies, blobCache, snapshotTreeCache) :
-            new ShreddedSummaryDocumentStorageService(id, manager, logger, policies);
+            new WholeSummaryDocumentStorageService(
+                id,
+                manager,
+                logger,
+                policies,
+                blobCache,
+                snapshotTreeCache,
+            ) :
+            new ShreddedSummaryDocumentStorageService(
+                id,
+                manager,
+                logger,
+                policies,
+                driverPolicies,
+                blobCache,
+                snapshotTreeCache,
+            );
         // TODO: worth prefetching latest summary making version + snapshot call with WholeSummary storage?
         if (!driverPolicies?.enableWholeSummaryUpload && policies.caching === LoaderCachingPolicy.Prefetch) {
             return new PrefetchDocumentStorageService(storageService);