@fluidframework/container-runtime 2.0.0-dev.4.4.0.162574 → 2.0.0-dev.5.3.2.178189

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-runtime",
-  "version": "2.0.0-dev.4.4.0.162574",
+  "version": "2.0.0-dev.5.3.2.178189",
   "description": "Fluid container runtime",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -37,19 +37,18 @@
   "dependencies": {
     "@fluidframework/common-definitions": "^0.20.1",
     "@fluidframework/common-utils": "^1.1.1",
-    "@fluidframework/container-definitions": "2.0.0-dev.4.4.0.162574",
-    "@fluidframework/container-runtime-definitions": "2.0.0-dev.4.4.0.162574",
-    "@fluidframework/container-utils": "2.0.0-dev.4.4.0.162574",
-    "@fluidframework/core-interfaces": "2.0.0-dev.4.4.0.162574",
-    "@fluidframework/datastore": "2.0.0-dev.4.4.0.162574",
-    "@fluidframework/driver-definitions": "2.0.0-dev.4.4.0.162574",
-    "@fluidframework/driver-utils": "2.0.0-dev.4.4.0.162574",
-    "@fluidframework/garbage-collector": "2.0.0-dev.4.4.0.162574",
-    "@fluidframework/protocol-base": "^0.1039.1000",
+    "@fluidframework/container-definitions": "2.0.0-dev.5.3.2.178189",
+    "@fluidframework/container-runtime-definitions": "2.0.0-dev.5.3.2.178189",
+    "@fluidframework/container-utils": "2.0.0-dev.5.3.2.178189",
+    "@fluidframework/core-interfaces": "2.0.0-dev.5.3.2.178189",
+    "@fluidframework/core-utils": "2.0.0-dev.5.3.2.178189",
+    "@fluidframework/datastore": "2.0.0-dev.5.3.2.178189",
+    "@fluidframework/driver-definitions": "2.0.0-dev.5.3.2.178189",
+    "@fluidframework/driver-utils": "2.0.0-dev.5.3.2.178189",
     "@fluidframework/protocol-definitions": "^1.1.0",
-    "@fluidframework/runtime-definitions": "2.0.0-dev.4.4.0.162574",
-    "@fluidframework/runtime-utils": "2.0.0-dev.4.4.0.162574",
-    "@fluidframework/telemetry-utils": "2.0.0-dev.4.4.0.162574",
+    "@fluidframework/runtime-definitions": "2.0.0-dev.5.3.2.178189",
+    "@fluidframework/runtime-utils": "2.0.0-dev.5.3.2.178189",
+    "@fluidframework/telemetry-utils": "2.0.0-dev.5.3.2.178189",
     "double-ended-queue": "^2.1.0-0",
     "events": "^3.1.0",
     "lz4js": "^0.2.0",
@@ -57,15 +56,15 @@
     "uuid": "^8.3.1"
   },
   "devDependencies": {
-    "@fluid-internal/stochastic-test-utils": "2.0.0-dev.4.4.0.162574",
-    "@fluid-tools/benchmark": "^0.46.0",
-    "@fluid-tools/build-cli": "^0.17.0",
-    "@fluidframework/build-common": "^1.1.0",
-    "@fluidframework/build-tools": "^0.17.0",
-    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.0.0-internal.4.1.0",
+    "@fluid-internal/stochastic-test-utils": "2.0.0-dev.5.3.2.178189",
+    "@fluid-tools/benchmark": "^0.48.0",
+    "@fluid-tools/build-cli": "^0.21.0",
+    "@fluidframework/build-common": "^1.2.0",
+    "@fluidframework/build-tools": "^0.21.0",
+    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.0.0-internal.5.2.0",
     "@fluidframework/eslint-config-fluid": "^2.0.0",
-    "@fluidframework/mocha-test-setup": "2.0.0-dev.4.4.0.162574",
-    "@fluidframework/test-runtime-utils": "2.0.0-dev.4.4.0.162574",
+    "@fluidframework/mocha-test-setup": "2.0.0-dev.5.3.2.178189",
+    "@fluidframework/test-runtime-utils": "2.0.0-dev.5.3.2.178189",
     "@microsoft/api-extractor": "^7.34.4",
     "@types/double-ended-queue": "^2.1.0",
     "@types/events": "^3.0.0",
@@ -87,29 +86,14 @@
     "typescript": "~4.5.5"
   },
   "typeValidation": {
-    "broken": {
-      "EnumDeclaration_ContainerMessageType": {
-        "backCompat": false
-      },
-      "ClassDeclaration_ContainerRuntime": {
-        "forwardCompat": false
-      },
-      "InterfaceDeclaration_ContainerRuntimeMessage": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_IChunkedOp": {
-        "backCompat": false
-      }
-    }
+    "broken": {}
   },
   "scripts": {
-    "build": "npm run build:genver && concurrently npm:build:compile npm:lint && npm run build:docs",
-    "build:commonjs": "npm run tsc && npm run typetests:gen && npm run build:test",
-    "build:compile": "concurrently npm:build:commonjs npm:build:esnext",
+    "build": "fluid-build . --task build",
+    "build:commonjs": "fluid-build . --task commonjs",
+    "build:compile": "fluid-build . --task compile",
     "build:docs": "api-extractor run --local --typescript-compiler-folder ../../../node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/doc-models/* ../../../_api-extractor-temp/",
     "build:esnext": "tsc --project ./tsconfig.esnext.json",
-    "build:full": "npm run build",
-    "build:full:compile": "npm run build:compile",
     "build:genver": "gen-version",
     "build:test": "tsc --project ./src/test/tsconfig.json",
     "ci:build:docs": "api-extractor run --typescript-compiler-folder ../../../node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/doc-models/* ../../../_api-extractor-temp/",
@@ -129,6 +113,6 @@
     "tsc": "tsc",
     "tsc:watch": "tsc --watch",
     "typetests:gen": "fluid-type-test-generator",
-    "typetests:prepare": "flub generate typetests --prepare --dir . --pin"
+    "typetests:prepare": "flub typetests --dir . --reset --previous --normalize"
   }
 }

package/src/batchTracker.ts

@@ -4,20 +4,19 @@
  */
 
 import { EventEmitter } from "events";
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ITelemetryLoggerExt, ChildLogger } from "@fluidframework/telemetry-utils";
 import { assert, performance } from "@fluidframework/common-utils";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ChildLogger } from "@fluidframework/telemetry-utils";
 
 export class BatchTracker {
-	private readonly logger: ITelemetryLogger;
+	private readonly logger: ITelemetryLoggerExt;
 	private startBatchSequenceNumber: number | undefined;
 	private trackedBatchCount: number = 0;
 	private batchProcessingStartTimeStamp: number | undefined;
 
 	constructor(
 		private readonly batchEventEmitter: EventEmitter,
-		logger: ITelemetryLogger,
+		logger: ITelemetryLoggerExt,
 		batchLengthThreshold: number,
 		batchCountSamplingRate: number,
 		dateTimeProvider: () => number = () => performance.now(),
@@ -72,13 +71,13 @@ export class BatchTracker {
  * Track batch sizes in terms of op counts and processing times
  *
  * @param batchEventEmitter - event emitter which tracks the lifecycle of batch operations
- * @param logger - See {@link @fluidframework/common-definitions#ITelemetryLogger}
+ * @param logger - See {@link @fluidframework/common-definitions#ITelemetryLoggerExt}
  * @param batchLengthThreshold - threshold for the length of a batch when to send an error event
  * @param batchCountSamplingRate - rate for batches for which to send an event with its characteristics
  */
 export const BindBatchTracker = (
 	batchEventEmitter: EventEmitter,
-	logger: ITelemetryLogger,
+	logger: ITelemetryLoggerExt,
 	batchLengthThreshold: number = 1000,
 	batchCountSamplingRate: number = 1000,
 ) => new BatchTracker(batchEventEmitter, logger, batchLengthThreshold, batchCountSamplingRate);

package/src/blobManager.ts

@@ -45,6 +45,7 @@ import { ContainerRuntime, TombstoneResponseHeaderKey } from "./containerRuntime
 import { sendGCUnexpectedUsageEvent, sweepAttachmentBlobsKey, throwOnTombstoneLoadKey } from "./gc";
 import { Throttler, formExponentialFn, IThrottler } from "./throttler";
 import { summarizerClientType } from "./summary";
+import { IBlobMetadata } from "./metadata";
 
 /**
  * This class represents blob (long string)
@@ -61,7 +62,7 @@ export class BlobHandle implements IFluidHandle<ArrayBufferLike> {
 	}
 
 	public get isAttached(): boolean {
-		return this.attached;
+		return this.routeContext.isAttached && this.attached;
 	}
 
 	public readonly absolutePath: string;
@@ -70,12 +71,16 @@ export class BlobHandle implements IFluidHandle<ArrayBufferLike> {
 		public readonly path: string,
 		public readonly routeContext: IFluidHandleContext,
 		public get: () => Promise<any>,
+		private readonly onAttachGraph?: () => void,
 	) {
 		this.absolutePath = generateHandleContextPath(path, this.routeContext);
 	}
 
 	public attachGraph() {
-		this.attached = true;
+		if (!this.attached) {
+			this.attached = true;
+			this.onAttachGraph?.();
+		}
 	}
 
 	public bind(handle: IFluidHandle) {
@@ -132,14 +137,22 @@ interface PendingBlob {
 	blob: ArrayBufferLike;
 	status: PendingBlobStatus;
 	storageId?: string;
-	handleP: Deferred<IFluidHandle<ArrayBufferLike>>;
+	handleP: Deferred<BlobHandle>;
 	uploadP?: Promise<ICreateBlobResponse>;
 	uploadTime?: number;
 	minTTLInSeconds?: number;
+	attached?: boolean;
+	acked?: boolean;
 }
 
 export interface IPendingBlobs {
-	[id: string]: { blob: string; uploadTime?: number; minTTLInSeconds?: number };
+	[id: string]: {
+		blob: string;
+		uploadTime?: number;
+		minTTLInSeconds?: number;
+		attached?: boolean;
+		acked?: boolean;
+	};
 }
 
 export interface IBlobManagerEvents {
@@ -227,11 +240,14 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 			this.runtime.clientDetails.type !== summarizerClientType;
 
 		this.runtime.on("disconnected", () => this.onDisconnected());
+
 		this.redirectTable = this.load(snapshot);
 
 		// Begin uploading stashed blobs from previous container instance
 		Object.entries(stashedBlobs).forEach(([localId, entry]) => {
 			const blob = stringToBuffer(entry.blob, "base64");
+			const attached = entry.attached;
+			const acked = entry.acked;
 			if (entry.minTTLInSeconds && entry.uploadTime) {
 				const timeLapseSinceLocalUpload = (Date.now() - entry.uploadTime) / 1000;
 				// stashed entries with more than half-life in storage will not be reuploaded
@@ -243,6 +259,8 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 						uploadP: undefined,
 						uploadTime: entry.uploadTime,
 						minTTLInSeconds: entry.minTTLInSeconds,
+						attached,
+						acked,
 					});
 					return;
 				}
@@ -252,6 +270,8 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 				status: PendingBlobStatus.OfflinePendingUpload,
 				handleP: new Deferred(),
 				uploadP: this.uploadBlob(localId, blob),
+				attached,
+				acked,
 			});
 		});
 
@@ -391,13 +411,23 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		);
 	}
 
-	private getBlobHandle(id: string): IFluidHandle<ArrayBufferLike> {
+	private getBlobHandle(id: string): BlobHandle {
 		assert(
 			this.redirectTable.has(id) || this.pendingBlobs.has(id),
 			0x384 /* requesting handle for unknown blob */,
 		);
-		return new BlobHandle(`${BlobManager.basePath}/${id}`, this.routeContext, async () =>
-			this.getBlob(id),
+		const pending = this.pendingBlobs.get(id);
+		const callback = pending
+			? () => {
+					pending.attached = true;
+					this.deletePendingBlobMaybe(id);
+			  }
+			: undefined;
+		return new BlobHandle(
+			`${BlobManager.basePath}/${id}`,
+			this.routeContext,
+			async () => this.getBlob(id),
+			callback,
 		);
 	}
 
@@ -433,6 +463,8 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 			status: PendingBlobStatus.OnlinePendingUpload,
 			handleP: new Deferred(),
 			uploadP: this.uploadBlob(localId, blob),
+			attached: false,
+			acked: false,
 		};
 		this.pendingBlobs.set(localId, pendingEntry);
 
@@ -459,18 +491,21 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		this.redirectTable.set(fromId, toId);
 	}
 
-	private deleteAndEmitsIfEmpty(id: string) {
+	private deletePendingBlobMaybe(id: string) {
 		if (this.pendingBlobs.has(id)) {
-			this.pendingBlobs.delete(id);
-			if (!this.hasPendingBlobs) {
-				this.emit("noPendingBlobs");
+			const entry = this.pendingBlobs.get(id);
+			if (entry?.attached && entry?.acked) {
+				this.pendingBlobs.delete(id);
+				if (!this.hasPendingBlobs) {
+					this.emit("noPendingBlobs");
+				}
 			}
 		}
 	}
 
 	private onUploadResolve(localId: string, response: ICreateBlobResponseWithTTL) {
 		const entry = this.pendingBlobs.get(localId);
-		assert(entry !== undefined, "pending blob entry not found for uploaded blob");
+		assert(entry !== undefined, 0x6c8 /* pending blob entry not found for uploaded blob */);
 		assert(
 			entry.status === PendingBlobStatus.OnlinePendingUpload ||
 				entry.status === PendingBlobStatus.OfflinePendingUpload,
@@ -493,7 +528,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 					// happened before and so, the server won't delete it.
 					this.setRedirection(localId, response.id);
 					entry.handleP.resolve(this.getBlobHandle(localId));
-					this.deleteAndEmitsIfEmpty(localId);
+					this.deletePendingBlobMaybe(localId);
 				} else {
 					// If there is already an op for this storage ID, append the local ID to the list. Once any op for
 					// this storage ID is ack'd, all pending blobs for it can be resolved since the op will keep the
@@ -594,8 +629,8 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	}
 
 	public processBlobAttachOp(message: ISequencedDocumentMessage, local: boolean) {
-		const localId = message.metadata?.localId;
-		const blobId = message.metadata?.blobId;
+		const localId = (message.metadata as IBlobMetadata | undefined)?.localId;
+		const blobId = (message.metadata as IBlobMetadata | undefined)?.blobId;
 		assert(blobId !== undefined, 0x12a /* "Missing blob id on metadata" */);
 
 		// Set up a mapping from local ID to storage ID. This is crucial since without this the blob cannot be
@@ -616,23 +651,29 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 				// This is safe because the server will keep the blob alive and the op containing the local ID to
 				// storage ID is already in flight and any op containing this local ID will be sequenced after that.
 				waitingBlobs.forEach((pendingLocalId) => {
-					const pendingBlobEntry = this.pendingBlobs.get(pendingLocalId);
+					const entry = this.pendingBlobs.get(pendingLocalId);
 					assert(
-						pendingBlobEntry !== undefined,
+						entry !== undefined,
 						0x38f /* local online BlobAttach op with no pending blob entry */,
 					);
 
 					// It's possible we transitioned to offline flow while waiting for this op.
-					if (pendingBlobEntry.status === PendingBlobStatus.OnlinePendingOp) {
+					if (entry.status === PendingBlobStatus.OnlinePendingOp) {
 						this.setRedirection(pendingLocalId, blobId);
-						pendingBlobEntry.handleP.resolve(this.getBlobHandle(pendingLocalId));
-						this.deleteAndEmitsIfEmpty(pendingLocalId);
+						entry.acked = true;
+						entry.handleP.resolve(this.getBlobHandle(pendingLocalId));
+						this.deletePendingBlobMaybe(pendingLocalId);
 					}
 				});
 				this.opsInFlight.delete(blobId);
 			}
-			// For blobs that were transitioned to offline flow while waiting for this op, the entry should be deleted.
-			this.deleteAndEmitsIfEmpty(localId);
+			// offline flow does not resolve the handle (since it was already resolved)
+			// but we still need to delete the entry in case is acked and attached.
+			const localEntry = this.pendingBlobs.get(localId);
+			if (localEntry) {
+				localEntry.acked = true;
+				this.deletePendingBlobMaybe(localId);
+			}
 		}
 	}
 
@@ -905,13 +946,13 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	public getPendingBlobs(): IPendingBlobs {
 		const blobs = {};
 		for (const [key, entry] of this.pendingBlobs) {
-			blobs[key] = entry.minTTLInSeconds
-				? {
-						blob: bufferToString(entry.blob, "base64"),
-						uploadTime: entry.uploadTime,
-						minTTLInSeconds: entry.minTTLInSeconds,
-				  }
-				: { blob: bufferToString(entry.blob, "base64") };
+			blobs[key] = {
+				blob: bufferToString(entry.blob, "base64"),
+				attached: entry.attached,
+				acked: entry.acked,
+				minTTLInSeconds: entry.minTTLInSeconds,
+				uploadTime: entry.uploadTime,
+			};
 		}
 		return blobs;
 	}

package/src/connectionTelemetry.ts

@@ -3,8 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { ChildLogger, TelemetryLogger } from "@fluidframework/telemetry-utils";
+import { ITelemetryLoggerExt, ChildLogger, TelemetryLogger } from "@fluidframework/telemetry-utils";
 import { IDeltaManager } from "@fluidframework/container-definitions";
 import {
 	IDocumentMessage,
@@ -69,12 +68,12 @@ class OpPerfTelemetry {
 	private connectionStartTime = 0;
 	private gap = 0;
 
-	private readonly logger: ITelemetryLogger;
+	private readonly logger: ITelemetryLoggerExt;
 
 	public constructor(
 		private clientId: string | undefined,
 		private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
-		logger: ITelemetryLogger,
+		logger: ITelemetryLoggerExt,
 	) {
 		this.logger = ChildLogger.create(logger, "OpPerf");
 
@@ -188,7 +187,16 @@ class OpPerfTelemetry {
 
 	private recordPingTime(latency: number) {
 		this.pingLatency = latency;
-		// logging one in every 1000 pongs, including the first time, if it is a "write" client.
+
+		// Log if latency is longer than 1 min
+		if (latency > 1000 * 60) {
+			this.logger.sendErrorEvent({
+				eventName: "LatencyTooLong",
+				duration: latency,
+			});
+		}
+
+		// logging one in every 100 pongs, including the first time, if it is a "write" client.
 		if (this.pongCount % 100 === 0 && this.deltaManager.active) {
 			this.logger.sendPerformanceEvent({
 				eventName: "DeltaLatency",
@@ -314,7 +322,7 @@ export interface IPerfSignalReport {
 export function ReportOpPerfTelemetry(
 	clientId: string | undefined,
 	deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
-	logger: ITelemetryLogger,
+	logger: ITelemetryLoggerExt,
 ) {
 	new OpPerfTelemetry(clientId, deltaManager, logger);
 }