@fluidframework/odsp-driver 1.4.0-121020 → 2.0.0-dev-rc.1.0.0.225277

package/lib/{odspDeltaStorageService.js → odspDeltaStorageService.mjs}

@@ -2,11 +2,12 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { default as AbortController } from "abort-controller";
 import { v4 as uuid } from "uuid";
-import { assert } from "@fluidframework/common-utils";
-import { requestOps, streamObserver, } from "@fluidframework/driver-utils";
-import { getWithRetryForTokenRefresh } from "./odspUtils";
+import { validateMessages } from "@fluidframework/driver-base";
+import { PerformanceEvent } from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { requestOps, streamObserver } from "@fluidframework/driver-utils";
+import { getWithRetryForTokenRefresh } from "./odspUtils.mjs";
 /**
  * Provides access to the underlying delta storage on the server for sharepoint driver.
  */
@@ -18,7 +19,7 @@ export class OdspDeltaStorageService {
         this.logger = logger;
     }
     /**
-     * Retrieves ops from cache
+     * Retrieves ops from storage
      * @param from - inclusive
      * @param to - exclusive
      * @param telemetryProps - properties to add when issuing telemetry events
@@ -31,42 +32,51 @@ export class OdspDeltaStorageService {
             // Thus it needs to be done before we call getStorageToken() to reduce extra calls
             const baseUrl = this.buildUrl(from, to);
             const storageToken = await this.getStorageToken(options, "DeltaStorage");
-            const formBoundary = uuid();
-            let postBody = `--${formBoundary}\r\n`;
-            postBody += `Authorization: Bearer ${storageToken}\r\n`;
-            postBody += `X-HTTP-Method-Override: GET\r\n`;
-            postBody += `_post: 1\r\n`;
-            postBody += `\r\n--${formBoundary}--`;
-            const headers = {
-                "Content-Type": `multipart/form-data;boundary=${formBoundary}`,
-            };
-            // Some request take a long time (1-2 minutes) to complete, where telemetry shows very small amount
-            // of time spent on server, and usually small payload sizes. I.e. all the time is spent somewhere in
-            // networking. Even bigger problem - a lot of requests timeout (based on cursory look - after 1-2 minutes)
-            // So adding some timeout to ensure we retry again in hope of faster success.
-            // Please see https://github.com/microsoft/FluidFramework/issues/6997 for details.
-            const abort = new AbortController();
-            const timer = setTimeout(() => abort.abort(), 30000);
-            const response = await this.epochTracker.fetchAndParseAsJSON(baseUrl, {
-                headers,
-                body: postBody,
-                method: "POST",
-                signal: abort.signal,
-            }, "ops", true, scenarioName);
-            clearTimeout(timer);
-            const deltaStorageResponse = response.content;
-            let messages;
-            if (deltaStorageResponse.value.length > 0 && "op" in deltaStorageResponse.value[0]) {
-                messages = deltaStorageResponse.value.map((operation) => operation.op);
-            }
-            else {
-                messages = deltaStorageResponse.value;
-            }
-            this.logger.sendPerformanceEvent(Object.assign(Object.assign(Object.assign(Object.assign({ eventName: "OpsFetch", headers: Object.keys(headers).length !== 0 ? true : undefined, length: messages.length, duration: response.duration }, response.propsToLog), { attempts: options.refresh ? 2 : 1, from,
-                to }), telemetryProps), { reason: scenarioName }));
-            // It is assumed that server always returns all the ops that it has in the range that was requested.
-            // This may change in the future, if so, we need to adjust and receive "end" value from server in such case.
-            return { messages, partialResult: false };
+            return PerformanceEvent.timedExecAsync(this.logger, {
+                eventName: "OpsFetch",
+                attempts: options.refresh ? 2 : 1,
+                from,
+                to,
+                ...telemetryProps,
+                reason: scenarioName,
+            }, async (event) => {
+                const formBoundary = uuid();
+                let postBody = `--${formBoundary}\r\n`;
+                postBody += `Authorization: Bearer ${storageToken}\r\n`;
+                postBody += `X-HTTP-Method-Override: GET\r\n`;
+                postBody += `_post: 1\r\n`;
+                postBody += `\r\n--${formBoundary}--`;
+                const headers = {
+                    "Content-Type": `multipart/form-data;boundary=${formBoundary}`,
+                };
+                // Some request take a long time (1-2 minutes) to complete, where telemetry shows very small amount
+                // of time spent on server, and usually small payload sizes. I.e. all the time is spent somewhere in
+                // networking. Even bigger problem - a lot of requests timeout (based on cursory look - after 1-2 minutes)
+                // So adding some timeout to ensure we retry again in hope of faster success.
+                // Please see https://github.com/microsoft/FluidFramework/issues/6997 for details.
+                const abort = new AbortController();
+                const timer = setTimeout(() => abort.abort(), 30000);
+                const response = await this.epochTracker.fetchAndParseAsJSON(baseUrl, {
+                    headers,
+                    body: postBody,
+                    method: "POST",
+                    signal: abort.signal,
+                }, "ops", true, scenarioName);
+                clearTimeout(timer);
+                const deltaStorageResponse = response.content;
+                const messages = deltaStorageResponse.value.length > 0 &&
+                    "op" in deltaStorageResponse.value[0]
+                    ? deltaStorageResponse.value.map((operation) => operation.op)
+                    : deltaStorageResponse.value;
+                event.end({
+                    headers: Object.keys(headers).length !== 0 ? true : undefined,
+                    length: messages.length,
+                    ...response.propsToLog,
+                });
+                // It is assumed that server always returns all the ops that it has in the range that was requested.
+                // This may change in the future, if so, we need to adjust and receive "end" value from server in such case.
+                return { messages, partialResult: false };
+            });
         });
     }
     buildUrl(from, to) {
@@ -76,7 +86,7 @@ export class OdspDeltaStorageService {
     }
 }
 export class OdspDeltaStorageWithCache {
-    constructor(snapshotOps, logger, batchSize, concurrency, getFromStorage, getCached, requestFromSocket, opsReceived) {
+    constructor(snapshotOps, logger, batchSize, concurrency, getFromStorage, getCached, requestFromSocket, opsReceived, storageManagerGetter) {
         this.snapshotOps = snapshotOps;
         this.logger = logger;
         this.batchSize = batchSize;
@@ -85,23 +95,8 @@ export class OdspDeltaStorageWithCache {
         this.getCached = getCached;
         this.requestFromSocket = requestFromSocket;
         this.opsReceived = opsReceived;
-        this.firstCacheMiss = Number.MAX_SAFE_INTEGER;
-    }
-    validateMessages(reason, messages, from) {
-        if (messages.length !== 0) {
-            const start = messages[0].sequenceNumber;
-            const length = messages.length;
-            const last = messages[length - 1].sequenceNumber;
-            if (start !== from) {
-                this.logger.sendErrorEvent({ eventName: "OpsFetchViolation", reason, from, start, last, length });
-                messages.length = 0;
-            }
-            if (last + 1 !== from + length) {
-                this.logger.sendErrorEvent({ eventName: "OpsFetchViolation", reason, from, start, last, length });
-                // we can do better here by finding consecutive sub-block and return it
-                messages.length = 0;
-            }
-        }
+        this.storageManagerGetter = storageManagerGetter;
+        this.useCacheForOps = true;
     }
     fetchMessages(fromTotal, toTotal, abortSignal, cachedOnly, fetchReason) {
         // We do not control what's in the cache. Current API assumes that fetchMessages() keeps banging on
@@ -109,16 +104,20 @@ export class OdspDeltaStorageWithCache {
         // cache and it's not there.
         // Better implementation would be to return only what we have in cache, but that also breaks API
         assert(!cachedOnly || toTotal === undefined, 0x1e3);
+        // Don't use cache for ops is snapshot is fetched from network or if it was not fetched at all.
+        this.useCacheForOps =
+            this.useCacheForOps &&
+                this.storageManagerGetter()?.isFirstSnapshotFromNetwork === false;
         let opsFromSnapshot = 0;
         let opsFromCache = 0;
         let opsFromStorage = 0;
         const requestCallback = async (from, to, telemetryProps) => {
             if (this.snapshotOps !== undefined && this.snapshotOps.length !== 0) {
                 const messages = this.snapshotOps.filter((op) => op.sequenceNumber >= from && op.sequenceNumber < to);
-                this.validateMessages("cached", messages, from);
+                validateMessages("cached", messages, from, this.logger);
                 if (messages.length > 0 && messages[0].sequenceNumber === from) {
                     this.snapshotOps = this.snapshotOps.filter((op) => op.sequenceNumber >= to);
-                    opsFromSnapshot = messages.length;
+                    opsFromSnapshot += messages.length;
                     return { messages, partialResult: true };
                 }
                 this.snapshotOps = undefined;
@@ -126,10 +125,13 @@ export class OdspDeltaStorageWithCache {
             // Kick out request to PUSH for ops if it has them
             this.requestFromSocket(from, to);
             // Cache in normal flow is continuous. Once there is a miss, stop consulting cache.
-            // This saves a bit of processing time
-            if (from < this.firstCacheMiss) {
+            // This saves a bit of processing time.
+            if (this.useCacheForOps) {
                 const messagesFromCache = await this.getCached(from, to);
-                this.validateMessages("cached", messagesFromCache, from);
+                validateMessages("cached", messagesFromCache, from, this.logger);
+                // Set the firstCacheMiss as true in case we didn't get all the ops.
+                // This will save an extra cache read on "DocumentOpen" or "PostDocumentOpen".
+                this.useCacheForOps = from + messagesFromCache.length >= to;
                 if (messagesFromCache.length !== 0) {
                     opsFromCache += messagesFromCache.length;
                     return {
@@ -137,13 +139,12 @@ export class OdspDeltaStorageWithCache {
                         partialResult: true,
                     };
                 }
-                this.firstCacheMiss = Math.min(this.firstCacheMiss, from);
             }
             if (cachedOnly) {
                 return { messages: [], partialResult: false };
             }
             const ops = await this.getFromStorage(from, to, telemetryProps, fetchReason);
-            this.validateMessages("storage", ops.messages, from);
+            validateMessages("storage", ops.messages, from, this.logger);
             opsFromStorage += ops.messages.length;
             this.opsReceived(ops.messages);
             return ops;
@@ -151,7 +152,7 @@ export class OdspDeltaStorageWithCache {
         const stream = requestOps(async (from, to, telemetryProps) => {
             const result = await requestCallback(from, to, telemetryProps);
             // Catch all case, just in case
-            this.validateMessages("catch all", result.messages, from);
+            validateMessages("catch all", result.messages, from, this.logger);
             return result;
         }, 
         // Staging: starting with no concurrency, listening for feedback first.
@@ -166,9 +167,10 @@ export class OdspDeltaStorageWithCache {
                     opsFromSnapshot,
                     opsFromCache,
                     opsFromStorage,
+                    reason: fetchReason,
                 });
             }
         });
     }
 }
-//# sourceMappingURL=odspDeltaStorageService.js.map
+//# sourceMappingURL=odspDeltaStorageService.mjs.map

package/lib/odspDeltaStorageService.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"odspDeltaStorageService.mjs","sourceRoot":"","sources":["../src/odspDeltaStorageService.ts"],"names":[],"mappings":"AAAA;;;GAGG;OAEI,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM;OAE1B,EAAE,gBAAgB,EAAE,MAAM,6BAA6B;OACvD,EAAuB,gBAAgB,EAAE,MAAM,iCAAiC;OAChF,EAAE,MAAM,EAAE,MAAM,4BAA4B;OAO5C,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,8BAA8B;OAGlE,EAAE,2BAA2B,EAAE;AAGtC;;GAEG;AACH,MAAM,OAAO,uBAAuB;IACnC,YACkB,YAAoB,EACpB,eAAgD,EAChD,YAA0B,EAC1B,MAA2B;QAH3B,iBAAY,GAAZ,YAAY,CAAQ;QACpB,oBAAe,GAAf,eAAe,CAAiC;QAChD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,WAAM,GAAN,MAAM,CAAqB;IAC1C,CAAC;IAEJ;;;;;;;OAOG;IACI,KAAK,CAAC,GAAG,CACf,IAAY,EACZ,EAAU,EACV,cAAoC,EACpC,YAAqB;QAErB,OAAO,2BAA2B,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACpD,gFAAgF;YAChF,kFAAkF;YAClF,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACxC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;YAEzE,OAAO,gBAAgB,CAAC,cAAc,CACrC,IAAI,CAAC,MAAM,EACX;gBACC,SAAS,EAAE,UAAU;gBACrB,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjC,IAAI;gBACJ,EAAE;gBACF,GAAG,cAAc;gBACjB,MAAM,EAAE,YAAY;aACpB,EACD,KAAK,EAAE,KAAK,EAAE,EAAE;gBACf,MAAM,YAAY,GAAG,IAAI,EAAE,CAAC;gBAC5B,IAAI,QAAQ,GAAG,KAAK,YAAY,MAAM,CAAC;gBACvC,QAAQ,IAAI,yBAAyB,YAAY,MAAM,CAAC;gBACxD,QAAQ,IAAI,iCAAiC,CAAC;gBAE9C,QAAQ,IAAI,cAAc,CAAC;gBAC3B,QAAQ,IAAI,SAAS,YAAY,IAAI,CAAC;gBACtC,MAAM,OAAO,GAA6B;oBACzC,cAAc,EAAE,gCAAgC,YAAY,EAAE;iBAC9D,CAAC;gBAEF,mGAAmG;gBACnG,oGAAoG;gBACpG,0GAA0G;gBAC1G,6EAA6E;gBAC7E,kFAAkF;gBAClF,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;gBACpC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC;gBAErD,MAAM,QAAQ,GACb,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAC1C,OAAO,EACP;oBACC,OAAO;oBACP,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,KAAK,CAAC,MAAM;iBACpB,EACD,KAAK,EACL,IAAI,EACJ,YAAY,CACZ,CAAC;gBACH,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,MAAM,oBAAoB,GAAG,QAAQ,CAAC,OAAO,CAAC;gBAC9C,MAAM,QAAQ,GACb,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;oBACrC,IAAI,IAAI,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC;oBACpC,CAAC,CAAE,oBAAoB,CAAC,KAAoC,CAAC,GAAG,CAC9D,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAC1B;oBACH,CAAC,CAAE,oBAAoB,CAAC,KAAqC,CAAC;gBAEhE,KAAK,CAAC,GAAG,CAAC;oBACT,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;oBAC7D,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,GAAG,QAAQ,CAAC,UAAU;iBACtB,CAAC,CAAC;gBAEH,oGAAoG;gBACpG,4GAA4G;gBAC5G,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;YAC3C,CAAC,CACD,CAAC;QACH,CAAC,CAAC,CAAC;IACJ,CAAC;IAEM,QAAQ,CAAC,IAAY,EAAE,EAAU;QACvC,MAAM,MAAM,GAAG,kBAAkB,CAChC,qBAAqB,IAAI,0BAA0B,EAAE,GAAG,CAAC,EAAE,CAC3D,CAAC;QACF,MAAM,WAAW,GAAG,iBAAiB,MAAM,EAAE,CAAC;QAC9C,OAAO,GAAG,IAAI,CAAC,YAAY,GAAG,WAAW,EAAE,CAAC;IAC7C,CAAC;CACD;AAED,MAAM,OAAO,yBAAyB;IAGrC,YACS,WAAoD,EAC3C,MAA2B,EAC3B,SAAiB,EACjB,WAAmB,EACnB,cAKe,EACf,SAGwB,EACxB,iBAAqD,EACrD,WAAuD,EACvD,oBAAkE;QAhB3E,gBAAW,GAAX,WAAW,CAAyC;QAC3C,WAAM,GAAN,MAAM,CAAqB;QAC3B,cAAS,GAAT,SAAS,CAAQ;QACjB,gBAAW,GAAX,WAAW,CAAQ;QACnB,mBAAc,GAAd,cAAc,CAKC;QACf,cAAS,GAAT,SAAS,CAGe;QACxB,sBAAiB,GAAjB,iBAAiB,CAAoC;QACrD,gBAAW,GAAX,WAAW,CAA4C;QACvD,yBAAoB,GAApB,oBAAoB,CAA8C;QAnB5E,mBAAc,GAAG,IAAI,CAAC;IAoB3B,CAAC;IAEG,aAAa,CACnB,SAAiB,EACjB,OAA2B,EAC3B,WAAyB,EACzB,UAAoB,EACpB,WAAoB;QAEpB,mGAAmG;QACnG,yGAAyG;QACzG,4BAA4B;QAC5B,gGAAgG;QAChG,MAAM,CAAC,CAAC,UAAU,IAAI,OAAO,KAAK,SAAS,EAAE,KAAK,CAAC,CAAC;QAEpD,+FAA+F;QAC/F,IAAI,CAAC,cAAc;YAClB,IAAI,CAAC,cAAc;gBACnB,IAAI,CAAC,oBAAoB,EAAE,EAAE,0BAA0B,KAAK,KAAK,CAAC;QACnE,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,MAAM,eAAe,GAAG,KAAK,EAC5B,IAAY,EACZ,EAAU,EACV,cAAoC,EACnC,EAAE;YACH,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;gBACpE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CACvC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,cAAc,IAAI,IAAI,IAAI,EAAE,CAAC,cAAc,GAAG,EAAE,CAC3D,CAAC;gBACF,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACxD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,EAAE;oBAC/D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;oBAC5E,eAAe,IAAI,QAAQ,CAAC,MAAM,CAAC;oBACnC,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;iBACzC;gBACD,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;aAC7B;YAED,kDAAkD;YAClD,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAEjC,mFAAmF;YACnF,uCAAuC;YACvC,IAAI,IAAI,CAAC,cAAc,EAAE;gBACxB,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACzD,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACjE,oEAAoE;gBACpE,8EAA8E;gBAC9E,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,iBAAiB,CAAC,MAAM,IAAI,EAAE,CAAC;gBAC5D,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;oBACnC,YAAY,IAAI,iBAAiB,CAAC,MAAM,CAAC;oBACzC,OAAO;wBACN,QAAQ,EAAE,iBAAiB;wBAC3B,aAAa,EAAE,IAAI;qBACnB,CAAC;iBACF;aACD;YAED,IAAI,UAAU,EAAE;gBACf,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;aAC9C;YAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,cAAc,EAAE,WAAW,CAAC,CAAC;YAC7E,gBAAgB,CAAC,SAAS,EAAE,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7D,cAAc,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;YACtC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC/B,OAAO,GAAG,CAAC;QACZ,CAAC,CAAC;QAEF,MAAM,MAAM,GAAG,UAAU,CACxB,KAAK,EAAE,IAAY,EAAE,EAAU,EAAE,cAAoC,EAAE,EAAE;YACxE,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC;YAC/D,+BAA+B;YAC/B,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAClE,OAAO,MAAM,CAAC;QACf,CAAC;QACD,uEAAuE;QACvE,0DAA0D;QAC1D,IAAI,CAAC,WAAW,EAChB,SAAS,EAAE,YAAY;QACvB,OAAO,EAAE,YAAY;QACrB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,WAAW,EACX,WAAW,CACX,CAAC;QAEF,OAAO,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE;YACxC,IAAI,MAAM,CAAC,IAAI,IAAI,eAAe,GAAG,YAAY,GAAG,cAAc,KAAK,CAAC,EAAE;gBACzE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAChC,SAAS,EAAE,mBAAmB;oBAC9B,eAAe;oBACf,YAAY;oBACZ,cAAc;oBACd,MAAM,EAAE,WAAW;iBACnB,CAAC,CAAC;aACH;QACF,CAAC,CAAC,CAAC;IACJ,CAAC;CACD","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { v4 as uuid } from \"uuid\";\nimport { ITelemetryProperties } from \"@fluidframework/core-interfaces\";\nimport { validateMessages } from \"@fluidframework/driver-base\";\nimport { ITelemetryLoggerExt, PerformanceEvent } from \"@fluidframework/telemetry-utils\";\nimport { assert } from \"@fluidframework/core-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { InstrumentedStorageTokenFetcher } from \"@fluidframework/odsp-driver-definitions\";\nimport {\n\tIDeltasFetchResult,\n\tIDocumentDeltaStorageService,\n} from \"@fluidframework/driver-definitions\";\nimport { requestOps, streamObserver } from \"@fluidframework/driver-utils\";\nimport { IDeltaStorageGetResponse, ISequencedDeltaOpMessage } from \"./contracts\";\nimport { EpochTracker } from \"./epochTracker\";\nimport { getWithRetryForTokenRefresh } from \"./odspUtils\";\nimport { OdspDocumentStorageService } from \"./odspDocumentStorageManager\";\n\n/**\n * Provides access to the underlying delta storage on the server for sharepoint driver.\n */\nexport class OdspDeltaStorageService {\n\tconstructor(\n\t\tprivate readonly deltaFeedUrl: string,\n\t\tprivate readonly getStorageToken: InstrumentedStorageTokenFetcher,\n\t\tprivate readonly epochTracker: EpochTracker,\n\t\tprivate readonly logger: ITelemetryLoggerExt,\n\t) {}\n\n\t/**\n\t * Retrieves ops from storage\n\t * @param from - inclusive\n\t * @param to - exclusive\n\t * @param telemetryProps - properties to add when issuing telemetry events\n\t * @param scenarioName - reason for fetching ops\n\t * @returns ops retrieved & info if result was partial (i.e. more is available)\n\t */\n\tpublic async get(\n\t\tfrom: number,\n\t\tto: number,\n\t\ttelemetryProps: ITelemetryProperties,\n\t\tscenarioName?: string,\n\t): Promise<IDeltasFetchResult> {\n\t\treturn getWithRetryForTokenRefresh(async (options) => {\n\t\t\t// Note - this call ends up in getSocketStorageDiscovery() and can refresh token\n\t\t\t// Thus it needs to be done before we call getStorageToken() to reduce extra calls\n\t\t\tconst baseUrl = this.buildUrl(from, to);\n\t\t\tconst storageToken = await this.getStorageToken(options, \"DeltaStorage\");\n\n\t\t\treturn PerformanceEvent.timedExecAsync(\n\t\t\t\tthis.logger,\n\t\t\t\t{\n\t\t\t\t\teventName: \"OpsFetch\",\n\t\t\t\t\tattempts: options.refresh ? 2 : 1,\n\t\t\t\t\tfrom,\n\t\t\t\t\tto,\n\t\t\t\t\t...telemetryProps,\n\t\t\t\t\treason: scenarioName,\n\t\t\t\t},\n\t\t\t\tasync (event) => {\n\t\t\t\t\tconst formBoundary = uuid();\n\t\t\t\t\tlet postBody = `--${formBoundary}\\r\\n`;\n\t\t\t\t\tpostBody += `Authorization: Bearer ${storageToken}\\r\\n`;\n\t\t\t\t\tpostBody += `X-HTTP-Method-Override: GET\\r\\n`;\n\n\t\t\t\t\tpostBody += `_post: 1\\r\\n`;\n\t\t\t\t\tpostBody += `\\r\\n--${formBoundary}--`;\n\t\t\t\t\tconst headers: { [index: string]: any } = {\n\t\t\t\t\t\t\"Content-Type\": `multipart/form-data;boundary=${formBoundary}`,\n\t\t\t\t\t};\n\n\t\t\t\t\t// Some request take a long time (1-2 minutes) to complete, where telemetry shows very small amount\n\t\t\t\t\t// of time spent on server, and usually small payload sizes. I.e. all the time is spent somewhere in\n\t\t\t\t\t// networking. Even bigger problem - a lot of requests timeout (based on cursory look - after 1-2 minutes)\n\t\t\t\t\t// So adding some timeout to ensure we retry again in hope of faster success.\n\t\t\t\t\t// Please see https://github.com/microsoft/FluidFramework/issues/6997 for details.\n\t\t\t\t\tconst abort = new AbortController();\n\t\t\t\t\tconst timer = setTimeout(() => abort.abort(), 30000);\n\n\t\t\t\t\tconst response =\n\t\t\t\t\t\tawait this.epochTracker.fetchAndParseAsJSON<IDeltaStorageGetResponse>(\n\t\t\t\t\t\t\tbaseUrl,\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\theaders,\n\t\t\t\t\t\t\t\tbody: postBody,\n\t\t\t\t\t\t\t\tmethod: \"POST\",\n\t\t\t\t\t\t\t\tsignal: abort.signal,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\"ops\",\n\t\t\t\t\t\t\ttrue,\n\t\t\t\t\t\t\tscenarioName,\n\t\t\t\t\t\t);\n\t\t\t\t\tclearTimeout(timer);\n\t\t\t\t\tconst deltaStorageResponse = response.content;\n\t\t\t\t\tconst messages =\n\t\t\t\t\t\tdeltaStorageResponse.value.length > 0 &&\n\t\t\t\t\t\t\"op\" in deltaStorageResponse.value[0]\n\t\t\t\t\t\t\t? (deltaStorageResponse.value as ISequencedDeltaOpMessage[]).map(\n\t\t\t\t\t\t\t\t\t(operation) => operation.op,\n\t\t\t\t\t\t\t  )\n\t\t\t\t\t\t\t: (deltaStorageResponse.value as ISequencedDocumentMessage[]);\n\n\t\t\t\t\tevent.end({\n\t\t\t\t\t\theaders: Object.keys(headers).length !== 0 ? true : undefined,\n\t\t\t\t\t\tlength: messages.length,\n\t\t\t\t\t\t...response.propsToLog,\n\t\t\t\t\t});\n\n\t\t\t\t\t// It is assumed that server always returns all the ops that it has in the range that was requested.\n\t\t\t\t\t// This may change in the future, if so, we need to adjust and receive \"end\" value from server in such case.\n\t\t\t\t\treturn { messages, partialResult: false };\n\t\t\t\t},\n\t\t\t);\n\t\t});\n\t}\n\n\tpublic buildUrl(from: number, to: number) {\n\t\tconst filter = encodeURIComponent(\n\t\t\t`sequenceNumber ge ${from} and sequenceNumber le ${to - 1}`,\n\t\t);\n\t\tconst queryString = `?ump=1&filter=${filter}`;\n\t\treturn `${this.deltaFeedUrl}${queryString}`;\n\t}\n}\n\nexport class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {\n\tprivate useCacheForOps = true;\n\n\tpublic constructor(\n\t\tprivate snapshotOps: ISequencedDocumentMessage[] | undefined,\n\t\tprivate readonly logger: ITelemetryLoggerExt,\n\t\tprivate readonly batchSize: number,\n\t\tprivate readonly concurrency: number,\n\t\tprivate readonly getFromStorage: (\n\t\t\tfrom: number,\n\t\t\tto: number,\n\t\t\ttelemetryProps: ITelemetryProperties,\n\t\t\tfetchReason?: string,\n\t\t) => Promise<IDeltasFetchResult>,\n\t\tprivate readonly getCached: (\n\t\t\tfrom: number,\n\t\t\tto: number,\n\t\t) => Promise<ISequencedDocumentMessage[]>,\n\t\tprivate readonly requestFromSocket: (from: number, to: number) => void,\n\t\tprivate readonly opsReceived: (ops: ISequencedDocumentMessage[]) => void,\n\t\tprivate readonly storageManagerGetter: () => OdspDocumentStorageService | undefined,\n\t) {}\n\n\tpublic fetchMessages(\n\t\tfromTotal: number,\n\t\ttoTotal: number | undefined,\n\t\tabortSignal?: AbortSignal,\n\t\tcachedOnly?: boolean,\n\t\tfetchReason?: string,\n\t) {\n\t\t// We do not control what's in the cache. Current API assumes that fetchMessages() keeps banging on\n\t\t// storage / cache until it gets ops it needs. This would result in deadlock if fixed range is asked from\n\t\t// cache and it's not there.\n\t\t// Better implementation would be to return only what we have in cache, but that also breaks API\n\t\tassert(!cachedOnly || toTotal === undefined, 0x1e3);\n\n\t\t// Don't use cache for ops is snapshot is fetched from network or if it was not fetched at all.\n\t\tthis.useCacheForOps =\n\t\t\tthis.useCacheForOps &&\n\t\t\tthis.storageManagerGetter()?.isFirstSnapshotFromNetwork === false;\n\t\tlet opsFromSnapshot = 0;\n\t\tlet opsFromCache = 0;\n\t\tlet opsFromStorage = 0;\n\n\t\tconst requestCallback = async (\n\t\t\tfrom: number,\n\t\t\tto: number,\n\t\t\ttelemetryProps: ITelemetryProperties,\n\t\t) => {\n\t\t\tif (this.snapshotOps !== undefined && this.snapshotOps.length !== 0) {\n\t\t\t\tconst messages = this.snapshotOps.filter(\n\t\t\t\t\t(op) => op.sequenceNumber >= from && op.sequenceNumber < to,\n\t\t\t\t);\n\t\t\t\tvalidateMessages(\"cached\", messages, from, this.logger);\n\t\t\t\tif (messages.length > 0 && messages[0].sequenceNumber === from) {\n\t\t\t\t\tthis.snapshotOps = this.snapshotOps.filter((op) => op.sequenceNumber >= to);\n\t\t\t\t\topsFromSnapshot += messages.length;\n\t\t\t\t\treturn { messages, partialResult: true };\n\t\t\t\t}\n\t\t\t\tthis.snapshotOps = undefined;\n\t\t\t}\n\n\t\t\t// Kick out request to PUSH for ops if it has them\n\t\t\tthis.requestFromSocket(from, to);\n\n\t\t\t// Cache in normal flow is continuous. Once there is a miss, stop consulting cache.\n\t\t\t// This saves a bit of processing time.\n\t\t\tif (this.useCacheForOps) {\n\t\t\t\tconst messagesFromCache = await this.getCached(from, to);\n\t\t\t\tvalidateMessages(\"cached\", messagesFromCache, from, this.logger);\n\t\t\t\t// Set the firstCacheMiss as true in case we didn't get all the ops.\n\t\t\t\t// This will save an extra cache read on \"DocumentOpen\" or \"PostDocumentOpen\".\n\t\t\t\tthis.useCacheForOps = from + messagesFromCache.length >= to;\n\t\t\t\tif (messagesFromCache.length !== 0) {\n\t\t\t\t\topsFromCache += messagesFromCache.length;\n\t\t\t\t\treturn {\n\t\t\t\t\t\tmessages: messagesFromCache,\n\t\t\t\t\t\tpartialResult: true,\n\t\t\t\t\t};\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (cachedOnly) {\n\t\t\t\treturn { messages: [], partialResult: false };\n\t\t\t}\n\n\t\t\tconst ops = await this.getFromStorage(from, to, telemetryProps, fetchReason);\n\t\t\tvalidateMessages(\"storage\", ops.messages, from, this.logger);\n\t\t\topsFromStorage += ops.messages.length;\n\t\t\tthis.opsReceived(ops.messages);\n\t\t\treturn ops;\n\t\t};\n\n\t\tconst stream = requestOps(\n\t\t\tasync (from: number, to: number, telemetryProps: ITelemetryProperties) => {\n\t\t\t\tconst result = await requestCallback(from, to, telemetryProps);\n\t\t\t\t// Catch all case, just in case\n\t\t\t\tvalidateMessages(\"catch all\", result.messages, from, this.logger);\n\t\t\t\treturn result;\n\t\t\t},\n\t\t\t// Staging: starting with no concurrency, listening for feedback first.\n\t\t\t// In future releases we will switch to actual concurrency\n\t\t\tthis.concurrency,\n\t\t\tfromTotal, // inclusive\n\t\t\ttoTotal, // exclusive\n\t\t\tthis.batchSize,\n\t\t\tthis.logger,\n\t\t\tabortSignal,\n\t\t\tfetchReason,\n\t\t);\n\n\t\treturn streamObserver(stream, (result) => {\n\t\t\tif (result.done && opsFromSnapshot + opsFromCache + opsFromStorage !== 0) {\n\t\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\t\teventName: \"CacheOpsRetrieved\",\n\t\t\t\t\topsFromSnapshot,\n\t\t\t\t\topsFromCache,\n\t\t\t\t\topsFromStorage,\n\t\t\t\t\treason: fetchReason,\n\t\t\t\t});\n\t\t\t}\n\t\t});\n\t}\n}\n"]}

package/lib/{odspDocumentDeltaConnection.d.ts → odspDocumentDeltaConnection.d.mts}

@@ -2,21 +2,16 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { ITelemetryLogger, IEvent } from "@fluidframework/common-definitions";
+import { ITelemetryLoggerExt, IFluidErrorBase } from "@fluidframework/telemetry-utils";
 import { DocumentDeltaConnection } from "@fluidframework/driver-base";
+import { IAnyDriverError } from "@fluidframework/driver-definitions";
 import { OdspError } from "@fluidframework/odsp-driver-definitions";
-import { IAnyDriverError } from "@fluidframework/driver-utils";
-import { IFluidErrorBase } from "@fluidframework/telemetry-utils";
-import { IClient, IConnect } from "@fluidframework/protocol-definitions";
-import type { io as SocketIOClientStatic } from "socket.io-client";
-import { EpochTracker } from "./epochTracker";
+import { IClient, IConnect, IDocumentMessage } from "@fluidframework/protocol-definitions";
+import { EpochTracker } from "./epochTracker.mjs";
 export interface FlushResult {
     lastPersistedSequenceNumber?: number;
     retryAfter?: number;
 }
-export interface ISocketEvents extends IEvent {
-    (event: "server_disconnect", listener: (error: IFluidErrorBase & OdspError) => void): any;
-}
 /**
  * Represents a connection to a stream of delta updates
  */
@@ -29,7 +24,6 @@ export declare class OdspDocumentDeltaConnection extends DocumentDeltaConnection
      * @param tenantId - the ID of the tenant
      * @param documentId - document ID
      * @param token - authorization token for storage service
-     * @param io - websocket library
      * @param client - information about the client
      * @param mode - mode of the client
      * @param url - websocket URL
@@ -38,13 +32,14 @@ export declare class OdspDocumentDeltaConnection extends DocumentDeltaConnection
      * @param epochTracker - track epoch changes
      * @param socketReferenceKeyPrefix - (optional) prefix to isolate socket reuse cache
      */
-    static create(tenantId: string, documentId: string, token: string | null, io: typeof SocketIOClientStatic, client: IClient, url: string, telemetryLogger: ITelemetryLogger, timeoutMs: number, epochTracker: EpochTracker, socketReferenceKeyPrefix: string | undefined): Promise<OdspDocumentDeltaConnection>;
+    static create(tenantId: string, documentId: string, token: string | null, client: IClient, url: string, telemetryLogger: ITelemetryLoggerExt, timeoutMs: number, epochTracker: EpochTracker, socketReferenceKeyPrefix: string | undefined): Promise<OdspDocumentDeltaConnection>;
     private socketReference;
     private readonly requestOpsNoncePrefix;
     private pushCallCounter;
     private readonly getOpsMap;
     private flushOpNonce;
     private flushDeferred;
+    private connectionNotYetDisposedTimeout;
     /**
      * Error raising for socket.io issues
      */
@@ -69,12 +64,37 @@ export declare class OdspDocumentDeltaConnection extends DocumentDeltaConnection
      */
     requestOps(from: number, to: number): void;
     flush(): Promise<FlushResult>;
-    protected serverDisconnectHandler: (error: IFluidErrorBase & OdspError) => void;
+    protected disconnectHandler: (error: IFluidErrorBase & OdspError, clientId?: string) => void;
     protected initialize(connectMessage: IConnect, timeout: number): Promise<void>;
     protected addTrackedListener(event: string, listener: (...args: any[]) => void): void;
+    get disposed(): boolean;
+    /**
+     * Returns true in case the connection is not yet disposed and the socket is also connected. The expectation is
+     * that it will be called only after connection is fully established. i.e. there should no way to submit an op
+     * while we are connecting, as connection object is not exposed to Loader layer until connection is established.
+     */
+    private get connected();
+    protected emitMessages(type: string, messages: IDocumentMessage[][]): void;
+    /**
+     * Submits a new delta operation to the server
+     * @param message - delta operation to submit
+     */
+    submit(messages: IDocumentMessage[]): void;
+    /**
+     * Submits a new signal to the server
+     *
+     * @param content - Content of the signal.
+     * @param targetClientId - When specified, the signal is only sent to the provided client id.
+     */
+    submitSignal(content: IDocumentMessage, targetClientId?: string): void;
+    /**
+     * Critical path where we need to also close the socket for an error.
+     * @param error - Error causing the socket to close.
+     */
+    protected closeSocketCore(error: IAnyDriverError): void;
     /**
      * Disconnect from the websocket
      */
-    protected disconnect(socketProtocolError: boolean, reason: IAnyDriverError): void;
+    protected disconnectCore(): void;
 }
-//# sourceMappingURL=odspDocumentDeltaConnection.d.ts.map
+//# sourceMappingURL=odspDocumentDeltaConnection.d.mts.map

package/lib/odspDocumentDeltaConnection.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"odspDocumentDeltaConnection.d.mts","sourceRoot":"","sources":["../src/odspDocumentDeltaConnection.ts"],"names":[],"mappings":"AAAA;;;GAGG;OAGI,EACN,mBAAmB,EACnB,eAAe,EAEf,MAAM,iCAAiC;OAGjC,EAAE,uBAAuB,EAAE,MAAM,6BAA6B;OAC9D,EAAE,eAAe,EAAE,MAAM,oCAAoC;OAC7D,EAAE,SAAS,EAAE,MAAM,yCAAyC;OAC5D,EACN,OAAO,EACP,QAAQ,EACR,gBAAgB,EAKhB,MAAM,sCAAsC;OAKtC,EAAE,YAAY,EAAE;AAUvB,MAAM,WAAW,WAAW;IAC3B,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AA8LD;;GAEG;AACH,qBAAa,2BAA4B,SAAQ,uBAAuB;IA+KtE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IA9KrC;;;;;;;;;;;;;;OAcG;WACiB,MAAM,CACzB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAElB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,MAAM,EAAE,OAAO,EACf,GAAG,EAAE,MAAM,EACX,eAAe,EAAE,mBAAmB,EACpC,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,YAAY,EAC1B,wBAAwB,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,2BAA2B,CAAC;IAoFvC,OAAO,CAAC,eAAe,CAA8B;IAErD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS,CACf;IACX,OAAO,CAAC,YAAY,CAAqB;IACzC,OAAO,CAAC,aAAa,CAAoC;IACzD,OAAO,CAAC,+BAA+B,CAA4C;IAEnF;;OAEG;IACH,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,GAAG,EAAE,QAAQ,UAAO,GAAG,eAAe;IAS3F;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,4BAA4B;IA2B3C;;;;;;OAMG;IACH,OAAO;IAaP;;;;;OAKG;IACI,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM;IAuD7B,KAAK,IAAI,OAAO,CAAC,WAAW,CAAC;IAkC1C,SAAS,CAAC,iBAAiB,UAAW,eAAe,GAAG,SAAS,aAAa,MAAM,UAclF;cAEc,UAAU,CAAC,cAAc,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM;IAsGpE,SAAS,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;IAiE9E,IAAW,QAAQ,YAkBlB;IAED;;;;OAIG;IACH,OAAO,KAAK,SAAS,GAEpB;IAED,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,EAAE;IAOnE;;;OAGG;IACI,MAAM,CAAC,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAI;IAIjD;;;;;OAKG;IACI,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI;IAU7E;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,KAAK,EAAE,eAAe;IAUhD;;OAEG;IACH,SAAS,CAAC,cAAc;CAaxB"}