@fluidframework/container-runtime 0.58.2001 → 0.59.1000-61898

package/src/blobManager.ts

@@ -5,13 +5,13 @@
 
 import { IFluidHandle, IFluidHandleContext } from "@fluidframework/core-interfaces";
 import { IDocumentStorageService } from "@fluidframework/driver-definitions";
-import { AttachmentTreeEntry, BlobTreeEntry } from "@fluidframework/protocol-base";
-import { ISnapshotTree, ITree, ITreeEntry } from "@fluidframework/protocol-definitions";
-import { generateHandleContextPath } from "@fluidframework/runtime-utils";
+import { ISnapshotTree } from "@fluidframework/protocol-definitions";
+import { generateHandleContextPath, SummaryTreeBuilder } from "@fluidframework/runtime-utils";
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert, Deferred } from "@fluidframework/common-utils";
 import { IContainerRuntime } from "@fluidframework/container-runtime-definitions";
 import { AttachState } from "@fluidframework/container-definitions";
+import { IGarbageCollectionData, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
 
 /**
  * This class represents blob (long string)
@@ -147,9 +147,12 @@ export class BlobManager {
     }
 
     public processBlobAttachOp(blobId: string, local: boolean) {
-        assert(!local || this.pendingBlobIds.has(blobId), 0x1f8 /* "local BlobAttach op with no pending blob" */);
-        this.pendingBlobIds.get(blobId)?.resolve();
-        this.pendingBlobIds.delete(blobId);
+        if (local) {
+            const pendingBlobP = this.pendingBlobIds.get(blobId);
+            assert(pendingBlobP !== undefined, 0x1f8 /* "local BlobAttach op with no pending blob" */);
+            pendingBlobP.resolve();
+            this.pendingBlobIds.delete(blobId);
+        }
         this.blobIds.add(blobId);
     }
 
@@ -202,19 +205,83 @@ export class BlobManager {
         });
     }
 
-    public snapshot(): ITree {
+    /**
+     * Generates data used for garbage collection. Each blob uploaded represents a node in the GC graph as it can be
+     * individually referenced by storing its handle in a referenced DDS. Returns the list of blob ids as GC nodes.
+     * @param fullGC - true to bypass optimizations and force full generation of GC data. BlobManager doesn't care
+     * about this for now because the data is a simple list of blob ids.
+     */
+    public getGCData(fullGC: boolean = false): IGarbageCollectionData {
+        const getGCNodePath = (blobId: string) => { return `/${BlobManager.basePath}/${blobId}`; };
+        const gcData: IGarbageCollectionData = { gcNodes: {} };
+        /**
+          * The node path is of the format `/_blobs/blobId`. This path must match the path of the blob handle returned
+          * by the createBlob API because blobs are marked referenced by storing these handles in a referenced DDS.
+          */
+        this.blobIds.forEach((blobId: string) => {
+            gcData.gcNodes[getGCNodePath(blobId)] = [];
+        });
+
+        /**
+         * For all blobs in the redirect table, the handle returned on creation is based off of the localId. So, these
+         * nodes can be referenced by storing the localId handle. When that happens, the corresponding storageId node
+         * must also be marked referenced. So, we add a route from the localId node to the storageId node.
+         * Note that because of de-duping, there can be multiple localIds that all redirect to the same storageId or
+         * a blob may be referenced via its storageId handle.
+         */
+        if (this.redirectTable !== undefined) {
+            for (const [localId, storageId] of this.redirectTable) {
+                // Add node for the localId and add a route to the storageId node. The storageId node will have been
+                // added above when adding nodes for this.blobIds.
+                gcData.gcNodes[getGCNodePath(localId)] = [getGCNodePath(storageId)];
+            }
+        }
+
+        return gcData;
+    }
+
+    /**
+     * When running GC in test mode, this is called to delete blobs that are unused.
+     * @param unusedRoutes - These are the blob node ids that are unused and should be deleted.
+     */
+    public deleteUnusedRoutes(unusedRoutes: string[]): void {
+        // The routes or blob node paths are in the same format as returned in getGCData - `/_blobs/blobId`.
+        for (const route of unusedRoutes) {
+            const pathParts = route.split("/");
+            assert(
+                pathParts.length === 3 && pathParts[1] === BlobManager.basePath,
+                0x2d5 /* "Invalid blob node id in unused routes." */,
+            );
+            const blobId = pathParts[2];
+
+            // The unused blobId could be a localId. If so, remove it from the redirect table and continue. The
+            // corresponding storageId may still be used either directly or via other localIds.
+            if (this.redirectTable?.has(blobId)) {
+                this.redirectTable.delete(blobId);
+                continue;
+            }
+            this.blobIds.delete(blobId);
+        }
+    }
+
+    public summarize(): ISummaryTreeWithStats {
         // If we have a redirect table it means the container is about to transition to "Attaching" state, so we need
         // to return an actual snapshot containing all the real storage IDs we know about.
         const attachingOrAttached = !!this.redirectTable || this.runtime.attachState !== AttachState.Detached;
         const blobIds = attachingOrAttached ? this.blobIds : this.detachedBlobIds;
-        const entries: ITreeEntry[] = [...blobIds].map((id) => new AttachmentTreeEntry(id, id));
+        const builder = new SummaryTreeBuilder();
+        blobIds.forEach((blobId) => {
+            builder.addAttachment(blobId);
+        });
+
         if (this.redirectTable && this.redirectTable.size > 0) {
-            entries.push(new BlobTreeEntry(
+            builder.addBlob(
                 BlobManager.redirectTableBlobName,
-                JSON.stringify(Array.from(this.redirectTable.entries()))),
+                JSON.stringify(Array.from(this.redirectTable.entries())),
             );
         }
-        return { entries };
+
+        return builder.getSummaryTree();
     }
 
     public setRedirectTable(table: Map<string, string>) {

package/src/connectionTelemetry.ts

@@ -9,6 +9,7 @@ import { IDeltaManager } from "@fluidframework/container-definitions";
 import {
     IDocumentMessage,
     ISequencedDocumentMessage,
+    MessageType,
 } from "@fluidframework/protocol-definitions";
 import { assert, performance } from "@fluidframework/common-utils";
 
@@ -17,17 +18,51 @@ import { assert, performance } from "@fluidframework/common-utils";
  */
 export const latencyThreshold = 5000;
 
+// Phases in OpPerfTelemetry:
+// 1.	Op sits in a buffer in DeltaManager (DM) queue, then in outbound queue for some time.
+// 	 - Note: We do not differentiate these two today in telemetry, but first one is due to batches,
+//           second one might happen due to outbound queue being paused.
+// 2.	Op is sent to service and back.
+// 3.	Op sits in inbound queue.
+// 4.	Op is processed.
+interface IOpPerfTelemetryProperties {
+    /** Measure time between (1) and (2) - Measure time outbound op is sitting in queue due to active batch */
+    durationOutboundQueue: number;
+    /** Measure time between (2) and (3) - Track how long op is sitting in inbound queue until it is processed */
+    durationInboundQueue: number;
+    /** Measure time between (3) and (4) - Time between DM's inbound "push" event until DM's "op" event */
+    durationInboundToProcessing: number;
+    /** Length of the DeltaManager's inbound queue at the time of the DM's inbound "push" event (3) */
+    lenghtInboundQueue: number;
+}
+
+/**
+ * Timings collected at various moments during the op processing.
+ */
+ interface IOpPerfTimings {
+     /** Starting time for (1) */
+    opStartTimeForLatencyStatistics: number;
+     /** Starting time for (2) */
+     opStartTimeSittingInboundQueue: number;
+     /** Starting time for (3) */
+     opStartTimeInboundPushEvent: number;
+}
+
 class OpPerfTelemetry {
     private pongCount: number = 0;
     private pingLatency: number | undefined;
 
     // Collab window tracking. This is timestamp of %1000 message.
-    private opSendTimeForLatencyStatisticsForMsnStatistics: number | undefined;
-
-    // To track round trip time for every %1000 client message.
-    private opSendTimeForLatencyStatistics: number | undefined;
+    private sequenceNumberForMsnTracking: number | undefined;
+    private msnTrackingTimestamp: number = 0;
+    // To track round trip time for every %500 client message.
     private clientSequenceNumberForLatencyStatistics: number | undefined;
 
+    private opProcessingTimes: Partial<IOpPerfTimings> = {};
+
+    // Performance Data to be reported for ops round trips and processing.
+    private opPerfData: Partial<IOpPerfTelemetryProperties> = {};
+
     private firstConnection = true;
     private connectionOpSeqNumber: number | undefined;
     private readonly bootTime = performance.now();
@@ -61,12 +96,49 @@ class OpPerfTelemetry {
             }
         });
         this.deltaManager.on("disconnect", () => {
-            this.opSendTimeForLatencyStatisticsForMsnStatistics = undefined;
+            this.sequenceNumberForMsnTracking = undefined;
             this.clientSequenceNumberForLatencyStatistics = undefined;
+            this.opProcessingTimes = {};
+            this.opPerfData = {};
             this.connectionOpSeqNumber = undefined;
             this.firstConnection = false;
         });
 
+        this.deltaManager.outbound.on("push", (messages) => {
+            for (const msg of messages) {
+                if (msg.type === MessageType.Operation &&
+                    this.clientSequenceNumberForLatencyStatistics === msg.clientSequenceNumber) {
+                    assert(this.opProcessingTimes.opStartTimeSittingInboundQueue === undefined,
+                        0x2c8 /* "opStartTimeSittingInboundQueue should be undefined" */);
+                    assert(this.opPerfData.durationInboundQueue === undefined,
+                        0x2c9 /* "durationInboundQueue should be undefined" */);
+                    this.opProcessingTimes.opStartTimeSittingInboundQueue = Date.now();
+
+                    assert(this.opPerfData.durationOutboundQueue === undefined,
+                        0x2ca /* "durationOutboundQueue should be undefined" */);
+
+                    assert(this.opProcessingTimes.opStartTimeForLatencyStatistics !== undefined,
+                        0x2cb /* "opStartTimeForLatencyStatistics should be undefined" */);
+
+                    this.opPerfData.durationOutboundQueue = this.opProcessingTimes.opStartTimeSittingInboundQueue
+                        - this.opProcessingTimes.opStartTimeForLatencyStatistics;
+                }
+            }
+        });
+
+        this.deltaManager.inbound.on("push", (message: ISequencedDocumentMessage) => {
+            if (this.clientId === message.clientId &&
+                message.type === MessageType.Operation &&
+                this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber &&
+                this.opProcessingTimes.opStartTimeSittingInboundQueue !== undefined) {
+                this.opProcessingTimes.opStartTimeInboundPushEvent = Date.now();
+                this.opPerfData.durationInboundQueue = this.opProcessingTimes.opStartTimeInboundPushEvent
+                                     - this.opProcessingTimes.opStartTimeSittingInboundQueue;
+                this.opProcessingTimes.opStartTimeSittingInboundQueue = undefined;
+                this.opPerfData.lenghtInboundQueue = this.deltaManager.inbound.length;
+            }
+        });
+
         this.deltaManager.inbound.on("idle", (count: number, duration: number) => {
             // Do not want to log zero for sure.
             // We are more interested in aggregates, so logging only if we are processing some number of ops
@@ -113,8 +185,13 @@ class OpPerfTelemetry {
 
     private beforeOpSubmit(message: IDocumentMessage) {
         // start with first client op and measure latency every 500 client ops
-        if (this.clientSequenceNumberForLatencyStatistics === undefined && message.clientSequenceNumber % 500 === 1) {
-            this.opSendTimeForLatencyStatistics = Date.now();
+        if (this.clientSequenceNumberForLatencyStatistics === undefined &&
+            message.clientSequenceNumber % 500 === 1) {
+            assert(this.opProcessingTimes.opStartTimeSittingInboundQueue === undefined,
+                0x2cc /* "OpTimeSittingInboundQueue should be undefined" */);
+            assert(this.opPerfData.durationInboundQueue === undefined,
+                0x2cd /* "durationInboundQueue should be undefined" */);
+            this.opProcessingTimes.opStartTimeForLatencyStatistics = Date.now();
             this.clientSequenceNumberForLatencyStatistics = message.clientSequenceNumber;
         }
     }
@@ -127,24 +204,35 @@ class OpPerfTelemetry {
         }
 
         // Record collab window max size after every 1000th op.
-        if (sequenceNumber % 1000 === 0) {
-            if (this.opSendTimeForLatencyStatisticsForMsnStatistics !== undefined) {
-                this.logger.sendPerformanceEvent({
-                    eventName: "MsnStatistics",
-                    sequenceNumber,
-                    msnDistance: this.deltaManager.lastSequenceNumber - this.deltaManager.minimumSequenceNumber,
-                    duration: message.timestamp - this.opSendTimeForLatencyStatisticsForMsnStatistics,
-                });
-            }
-            this.opSendTimeForLatencyStatisticsForMsnStatistics = message.timestamp;
+        if (this.sequenceNumberForMsnTracking === undefined && sequenceNumber % 1000 === 0) {
+            this.sequenceNumberForMsnTracking = sequenceNumber;
+            this.msnTrackingTimestamp = message.timestamp;
+        }
+        if (this.sequenceNumberForMsnTracking !== undefined &&
+                message.minimumSequenceNumber >= this.sequenceNumberForMsnTracking) {
+            assert(this.msnTrackingTimestamp !== undefined,
+                0x2ce /* "msnTrackingTimestamp should not be undefined" */);
+            this.logger.sendPerformanceEvent({
+                eventName: "MsnStatistics",
+                sequenceNumber,
+                msnDistance: sequenceNumber - this.sequenceNumberForMsnTracking,
+                duration: message.timestamp - this.msnTrackingTimestamp,
+            });
+            this.sequenceNumberForMsnTracking = undefined;
         }
 
         if (this.clientId === message.clientId &&
             this.clientSequenceNumberForLatencyStatistics === message.clientSequenceNumber) {
-            assert(this.opSendTimeForLatencyStatistics !== undefined,
+            assert(this.opProcessingTimes.opStartTimeForLatencyStatistics !== undefined,
                 0x120 /* "Undefined latency statistics (op send time)" */);
+            const currentTime = Date.now();
+
+            if (this.opProcessingTimes.opStartTimeInboundPushEvent !== undefined) {
+                this.opPerfData.durationInboundToProcessing = currentTime
+                - this.opProcessingTimes.opStartTimeInboundPushEvent;
+            }
 
-            const duration = Date.now() - this.opSendTimeForLatencyStatistics;
+            const duration = currentTime - this.opProcessingTimes.opStartTimeForLatencyStatistics;
 
             // One of the core expectations for Fluid service is to be fast.
             // When it's not the case, we want to learn about it and be able to investigate, so
@@ -162,8 +250,11 @@ class OpPerfTelemetry {
                 duration,
                 category,
                 pingLatency: this.pingLatency,
+                msnDistance: this.deltaManager.lastSequenceNumber - this.deltaManager.minimumSequenceNumber,
+                ...this.opPerfData,
             });
             this.clientSequenceNumberForLatencyStatistics = undefined;
+            this.opPerfData = {};
         }
     }
 }