@fluidframework/odsp-driver 2.0.0-internal.2.1.2 → 2.0.0-internal.2.2.0

package/src/fetchSnapshot.ts

@@ -26,6 +26,8 @@ import {
     getWithRetryForTokenRefresh,
     getWithRetryForTokenRefreshRepeat,
     IOdspResponse,
+    measure,
+    measureP,
 } from "./odspUtils";
 import { ISnapshotContents } from "./odspPublicUtils";
 import { convertOdspSnapshotToSnapshotTreeAndBlobs } from "./odspSnapshotParser";
@@ -33,13 +35,6 @@ import { currentReadVersion, ISnapshotContentsWithProps, parseCompactSnapshotRes
 import { EpochTracker } from "./epochTracker";
 import { pkgVersion } from "./packageVersion";
 
-async function measure<T>(callback: () => T | Promise<T>): Promise<[T, number]> {
-    const start = performance.now();
-    const result = await callback();
-    const time = performance.now() - start;
-    return [result, time];
-}
-
 /**
  * Enum to support different types of snapshot formats.
  */
@@ -238,7 +233,7 @@ async function fetchLatestSnapshotCore(
                     );
                 }
 
-                const [response, fetchTime] = await measure(async () => snapshotDownloader(
+                const [response, fetchTime] = await measureP(async () => snapshotDownloader(
                     odspResolvedUrl,
                     storageToken,
                     snapshotOptions,
@@ -275,35 +270,43 @@ async function fetchLatestSnapshotCore(
                     switch (contentTypeToRead) {
                         case "application/json": {
                             let text: string;
-                            [text, receiveContentTime] = await measure(async () => odspResponse.content.text());
+                            [text, receiveContentTime] = await measureP(async () => odspResponse.content.text());
                             propsToLog.bodySize = text.length;
                             let content: IOdspSnapshot;
-                            [content, parseTime] = await measure( () => JSON.parse(text) as IOdspSnapshot);
+                            [content, parseTime] = measure( () => JSON.parse(text) as IOdspSnapshot);
                             validateBlobsAndTrees(content);
                             const snapshotContents: ISnapshotContents =
                                 convertOdspSnapshotToSnapshotTreeAndBlobs(content);
-                            parsedSnapshotContents = { ...odspResponse, content: snapshotContents };
+                            parsedSnapshotContents = {
+                                ...odspResponse,
+                                content: {
+                                    ...snapshotContents,
+                                    telemetryProps: {},
+                                },
+                            };
                             break;
                         }
                         case "application/ms-fluid": {
                             let content: ArrayBuffer;
                             [content, receiveContentTime] =
-                                await measure(async () => odspResponse.content.arrayBuffer());
+                                await measureP(async () => odspResponse.content.arrayBuffer());
                             propsToLog.bodySize = content.byteLength;
                             let snapshotContents: ISnapshotContentsWithProps;
-                            [snapshotContents, parseTime] = await measure(() => parseCompactSnapshotResponse(
+                            [snapshotContents, parseTime] = measure(() => parseCompactSnapshotResponse(
                                 new Uint8Array(content),
                                 logger));
                             if (snapshotContents.snapshotTree.trees === undefined ||
-                                snapshotContents.snapshotTree.blobs === undefined) {
-                                    throw new NonRetryableError(
-                                        "Returned odsp snapshot is malformed. No trees or blobs!",
-                                        DriverErrorType.incorrectServerResponse,
-                                        propsToLog,
-                                    );
-                                }
-                            const slowTreeParseCodePaths = snapshotContents.slowTreeStructureCount ?? 0;
-                            const slowBlobParseCodePaths = snapshotContents.slowBlobStructureCount ?? 0;
+                                    snapshotContents.snapshotTree.blobs === undefined) {
+                                throw new NonRetryableError(
+                                    "Returned odsp snapshot is malformed. No trees or blobs!",
+                                    DriverErrorType.incorrectServerResponse,
+                                    propsToLog,
+                                );
+                            }
+
+                            const props = snapshotContents.telemetryProps;
+                            const slowTreeParseCodePaths = props.slowTreeStructureCount ?? 0;
+                            const slowBlobParseCodePaths = props.slowBlobStructureCount ?? 0;
                             if (slowTreeParseCodePaths > 10 || slowBlobParseCodePaths > 10) {
                                 logger.sendErrorEvent({
                                     eventName: "SlowSnapshotParseCodePaths",
@@ -426,8 +429,6 @@ async function fetchLatestSnapshotCore(
                     encodedBlobsSize,
                     sequenceNumber,
                     ops: snapshot.ops?.length ?? 0,
-                    slowTreeStructureCount: snapshot.slowTreeStructureCount ?? 0,
-                    slowBlobStructureCount: snapshot.slowBlobStructureCount ?? 0,
                     userOps: snapshot.ops?.filter((op) => isRuntimeMessage(op)).length ?? 0,
                     headers: Object.keys(response.requestHeaders).length !== 0 ? true : undefined,
                     // Interval between the first fetch until the last byte of the last redirect.
@@ -468,6 +469,10 @@ async function fetchLatestSnapshotCore(
                     sltelemetry: odspResponse.headers.get("x-fluid-sltelemetry"),
                     // All other props
                     ...propsToLog,
+                    // Various perf counters and measures collected by binary parsing code:
+                    // slowTreeStructureCount, slowBlobStructureCount, durationStructure, durationStrings,
+                    // durationSnapshotTree, durationBlobs, etc.
+                    ...parsedSnapshotContents.content.telemetryProps,
                 });
                 return snapshot;
             },

package/src/localOdspDriver/localOdspDeltaStorageService.ts

@@ -0,0 +1,46 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { IDocumentDeltaStorageService, IStream } from "@fluidframework/driver-definitions";
+import { Queue, emptyMessageStream } from "@fluidframework/driver-utils";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { validateMessages } from "../odspUtils";
+
+/**
+ * Implementation of IDocumentDeltaStorageService that will return snapshot ops when fetching messages
+ */
+export class LocalOdspDeltaStorageService implements IDocumentDeltaStorageService {
+    constructor(
+        private readonly logger: ITelemetryLogger,
+        private snapshotOps: ISequencedDocumentMessage[],
+    ) { }
+
+    public fetchMessages(
+        from: number,
+        to: number | undefined,
+        _abortSignal?: AbortSignal,
+        _cachedOnly?: boolean,
+        _fetchReason?: string,
+    ): IStream<ISequencedDocumentMessage[]> {
+        if (this.snapshotOps.length === 0) {
+            return emptyMessageStream;
+        }
+
+        const queue = new Queue<ISequencedDocumentMessage[]>();
+        const messages = this.snapshotOps.filter((op) =>
+            op.sequenceNumber >= from && (to === undefined || op.sequenceNumber < to));
+        validateMessages("cached", messages, from, this.logger);
+
+        if (messages.length === 0 || messages[0].sequenceNumber !== from) {
+            this.snapshotOps = [];
+        }
+        this.snapshotOps = this.snapshotOps.filter((op) => to !== undefined && op.sequenceNumber >= to);
+
+        queue.pushValue(messages);
+        queue.pushDone();
+        return queue;
+    }
+}

package/src/localOdspDriver/localOdspDocumentService.ts

@@ -10,9 +10,10 @@ import {
     IDocumentStorageService,
     IResolvedUrl,
 } from "@fluidframework/driver-definitions";
-import { UsageError, EmptyDocumentDeltaStorageService } from "@fluidframework/driver-utils";
+import { UsageError } from "@fluidframework/driver-utils";
 import { IOdspResolvedUrl } from "@fluidframework/odsp-driver-definitions";
 import { IClient } from "@fluidframework/protocol-definitions";
+import { LocalOdspDeltaStorageService } from "./localOdspDeltaStorageService";
 import { LocalOdspDocumentStorageService } from "./localOdspDocumentStorageManager";
 
 /**
@@ -20,6 +21,7 @@ import { LocalOdspDocumentStorageService } from "./localOdspDocumentStorageManag
  */
 export class LocalOdspDocumentService implements IDocumentService {
     public policies = { storageOnly: true };
+    private storageManager?: LocalOdspDocumentStorageService;
 
     constructor(
         private readonly odspResolvedUrl: IOdspResolvedUrl,
@@ -32,14 +34,18 @@ export class LocalOdspDocumentService implements IDocumentService {
     }
 
     public async connectToStorage(): Promise<IDocumentStorageService> {
-        return new LocalOdspDocumentStorageService(
+        this.storageManager = new LocalOdspDocumentStorageService(
             this.logger,
             this.localSnapshot,
         );
+        return this.storageManager;
     }
 
     public async connectToDeltaStorage(): Promise<IDocumentDeltaStorageService> {
-        return new EmptyDocumentDeltaStorageService();
+        return new LocalOdspDeltaStorageService(
+            this.logger,
+            this.storageManager?.ops ?? [],
+        );
     }
 
     public connectToDeltaStream(_client: IClient): never {

package/src/odspDeltaStorageService.ts

@@ -16,7 +16,7 @@ import {
 } from "@fluidframework/driver-utils";
 import { IDeltaStorageGetResponse, ISequencedDeltaOpMessage } from "./contracts";
 import { EpochTracker } from "./epochTracker";
-import { getWithRetryForTokenRefresh } from "./odspUtils";
+import { getWithRetryForTokenRefresh, validateMessages } from "./odspUtils";
 
 /**
  * Provides access to the underlying delta storage on the server for sharepoint driver.
@@ -132,23 +132,6 @@ export class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {
     ) {
     }
 
-    protected validateMessages(reason: string, messages: ISequencedDocumentMessage[], from: number) {
-        if (messages.length !== 0) {
-            const start = messages[0].sequenceNumber;
-            const length = messages.length;
-            const last = messages[length - 1].sequenceNumber;
-            if (start !== from) {
-                this.logger.sendErrorEvent({ eventName: "OpsFetchViolation", reason, from, start, last, length });
-                messages.length = 0;
-            }
-            if (last + 1 !== from + length) {
-                this.logger.sendErrorEvent({ eventName: "OpsFetchViolation", reason, from, start, last, length });
-                // we can do better here by finding consecutive sub-block and return it
-                messages.length = 0;
-            }
-        }
-    }
-
     public fetchMessages(
         fromTotal: number,
         toTotal: number | undefined,
@@ -169,7 +152,7 @@ export class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {
             if (this.snapshotOps !== undefined && this.snapshotOps.length !== 0) {
                 const messages = this.snapshotOps.filter((op) =>
                     op.sequenceNumber >= from && op.sequenceNumber < to);
-                this.validateMessages("cached", messages, from);
+                validateMessages("cached", messages, from, this.logger);
                 if (messages.length > 0 && messages[0].sequenceNumber === from) {
                     this.snapshotOps = this.snapshotOps.filter((op) => op.sequenceNumber >= to);
                     opsFromSnapshot = messages.length;
@@ -185,7 +168,7 @@ export class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {
             // This saves a bit of processing time
             if (from < this.firstCacheMiss) {
                 const messagesFromCache = await this.getCached(from, to);
-                this.validateMessages("cached", messagesFromCache, from);
+                validateMessages("cached", messagesFromCache, from, this.logger);
                 if (messagesFromCache.length !== 0) {
                     opsFromCache += messagesFromCache.length;
                     return {
@@ -201,7 +184,7 @@ export class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {
             }
 
             const ops = await this.getFromStorage(from, to, telemetryProps, fetchReason);
-            this.validateMessages("storage", ops.messages, from);
+            validateMessages("storage", ops.messages, from, this.logger);
             opsFromStorage += ops.messages.length;
             this.opsReceived(ops.messages);
             return ops;
@@ -211,7 +194,7 @@ export class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {
             async (from: number, to: number, telemetryProps: ITelemetryProperties) => {
                 const result = await requestCallback(from, to, telemetryProps);
                 // Catch all case, just in case
-                this.validateMessages("catch all", result.messages, from);
+                validateMessages("catch all", result.messages, from, this.logger);
                 return result;
             },
             // Staging: starting with no concurrency, listening for feedback first.

package/src/odspDocumentDeltaConnection.ts

@@ -454,8 +454,8 @@ export class OdspDocumentDeltaConnection extends DocumentDeltaConnection {
 
     protected disconnectHandler = (error: IFluidErrorBase & OdspError, clientId?: string) => {
         if (clientId === undefined || clientId === this.clientId) {
+            this.logger.sendTelemetryEvent({ eventName: "ServerDisconnect", clientId: this.hasDetails ? this.clientId : undefined }, error);
             this.disconnect(error);
-            this.logger.sendTelemetryEvent({ eventName: "ServerDisconnect", clientId: this.clientId }, error);
         }
     };
 

package/src/odspDocumentService.ts

@@ -4,7 +4,7 @@
  */
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { performance } from "@fluidframework/common-utils";
+import { assert, performance } from "@fluidframework/common-utils";
 import {
     ChildLogger,
     IFluidErrorBase,
@@ -251,6 +251,7 @@ export class OdspDocumentService implements IDocumentService {
      * @returns returns the document delta stream service for onedrive/sharepoint driver.
      */
     public async connectToDeltaStream(client: IClient): Promise<IDocumentDeltaConnection> {
+        assert(this.currentConnection === undefined, 0x4ad /* Should not be called when connection is already present! */);
         // Attempt to connect twice, in case we used expired token.
         return getWithRetryForTokenRefresh<IDocumentDeltaConnection>(async (options) => {
             // Presence of getWebsocketToken callback dictates whether callback is used for fetching
@@ -296,13 +297,17 @@ export class OdspDocumentService implements IDocumentService {
                 });
                 // On disconnect with 401/403 error code, we can just clear the joinSession cache as we will again
                 // get the auth error on reconnecting and face latency.
-                connection.on("disconnect", (error: any) => {
+                connection.once("disconnect", (error: any) => {
                     // Clear the join session refresh timer so that it can be restarted on reconnection.
                     this.clearJoinSessionTimer();
                     if (typeof error === "object" && error !== null
                         && error.errorType === DriverErrorType.authorizationError) {
                         this.cache.sessionJoinCache.remove(this.joinSessionKey);
                     }
+                    // If we hit this assert, it means that "disconnect" event is emitted before the connection went through
+                    // dispose flow which is not correct and could lead to a bunch of erros.
+                    assert(connection.disposed, 0x4ae /* Connection should be disposed by now */);
+                    this.currentConnection = undefined;
                 });
                 this.currentConnection = connection;
                 return connection;
@@ -504,6 +509,8 @@ export class OdspDocumentService implements IDocumentService {
         }
         this._opsCache?.dispose();
         this.clearJoinSessionTimer();
+        this.currentConnection?.dispose();
+        this.currentConnection = undefined;
     }
 
     protected get opsCache() {

package/src/odspUtils.ts

@@ -34,6 +34,7 @@ import {
     InstrumentedStorageTokenFetcher,
     IOdspUrlParts,
 } from "@fluidframework/odsp-driver-definitions";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { fetch } from "./fetch";
 import { pkgVersion as driverVersion } from "./packageVersion";
 import { IOdspSnapshot } from "./contracts";
@@ -370,3 +371,39 @@ export function buildOdspShareLinkReqParams(shareLinkType: ShareLinkTypes | ISha
     shareLinkRequestParams = role ? `${shareLinkRequestParams}&createLinkRole=${role}` : shareLinkRequestParams;
     return shareLinkRequestParams;
 }
+
+export function measure<T>(callback: () => T): [T, number] {
+    const start = performance.now();
+    const result = callback();
+    const time = performance.now() - start;
+    return [result, time];
+}
+
+export async function measureP<T>(callback: () => Promise<T>): Promise<[T, number]> {
+    const start = performance.now();
+    const result = await callback();
+    const time = performance.now() - start;
+    return [result, time];
+}
+
+export function validateMessages(
+    reason: string,
+    messages: ISequencedDocumentMessage[],
+    from: number,
+    logger: ITelemetryLogger,
+) {
+    if (messages.length !== 0) {
+        const start = messages[0].sequenceNumber;
+        const length = messages.length;
+        const last = messages[length - 1].sequenceNumber;
+        if (start !== from) {
+            logger.sendErrorEvent({ eventName: "OpsFetchViolation", reason, from, start, last, length });
+            messages.length = 0;
+        }
+        if (last + 1 !== from + length) {
+            logger.sendErrorEvent({ eventName: "OpsFetchViolation", reason, from, start, last, length });
+            // we can do better here by finding consecutive sub-block and return it
+            messages.length = 0;
+        }
+    }
+}

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/odsp-driver";
-export const pkgVersion = "2.0.0-internal.2.1.2";
+export const pkgVersion = "2.0.0-internal.2.2.0";

package/src/zipItDataRepresentationUtils.ts

@@ -14,6 +14,7 @@ import { NonRetryableError } from "@fluidframework/driver-utils";
 import { DriverErrorType } from "@fluidframework/driver-definitions";
 import { ReadBuffer } from "./ReadBufferUtils";
 import { pkgVersion as driverVersion } from "./packageVersion";
+import { measure } from "./odspUtils";
 
 // eslint-disable-next-line max-len
 // https://onedrive.visualstudio.com/SharePoint%20Online/_git/SPO?path=/cobalt/Base/Property/BinaryEncodedPropertyReader.cs&version=GBmaster&_a=contents
@@ -388,6 +389,16 @@ export class NodeCore {
      * @param buffer - buffer to read from.
      */
     protected load(buffer: ReadBuffer, logger: ITelemetryLogger) {
+        const [stringsToResolve, durationStructure] = measure(() => this.loadStructure(buffer, logger));
+        const [, durationStrings] = measure(() => this.loadStrings(buffer, stringsToResolve, logger));
+        return { durationStructure, durationStrings };
+     }
+
+    /**
+     * Load and parse the buffer into a tree.
+     * @param buffer - buffer to read from.
+     */
+    protected loadStructure(buffer: ReadBuffer, logger: ITelemetryLogger) {
         const stack: NodeTypes[][] = [];
         const stringsToResolve: IStringElementInternal[] = [];
         const dictionary: IStringElement[] = [];
@@ -486,6 +497,10 @@ export class NodeCore {
         // This also ensures that stack.length === 0.
         assert(children === this.children, 0x3e7 /* Unpaired start/end list/set markers! */);
 
+        return stringsToResolve;
+    }
+
+    private loadStrings(buffer: ReadBuffer, stringsToResolve: IStringElementInternal[], logger: ITelemetryLogger) {
         /**
          * Process all the strings at once!
          */
@@ -532,11 +547,11 @@ export class NodeCore {
   * Provides loading and serialization capabilities.
   */
 export class TreeBuilder extends NodeCore {
-    static load(buffer: ReadBuffer, logger: ITelemetryLogger): TreeBuilder {
+    static load(buffer: ReadBuffer, logger: ITelemetryLogger) {
         const builder = new TreeBuilder();
-        builder.load(buffer, logger);
+        const telemetryProps = builder.load(buffer, logger);
         assert(buffer.eof, 0x233 /* "Unexpected data at the end of buffer" */);
-        return builder;
+        return { builder, telemetryProps };
     }
 }