@fluidframework/container-runtime 2.0.0-dev-rc.1.0.0.225277 → 2.0.0-dev-rc.1.0.0.228517

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-runtime",
-  "version": "2.0.0-dev-rc.1.0.0.225277",
+  "version": "2.0.0-dev-rc.1.0.0.228517",
   "description": "Fluid container runtime",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -14,7 +14,7 @@
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
-      "default": "./dist/index.cjs"
+      "default": "./dist/index.js"
     },
     "./test/containerRuntime": {
       "import": {
@@ -23,7 +23,7 @@
       },
       "require": {
         "types": "./dist/containerRuntime.d.ts",
-        "default": "./dist/containerRuntime.cjs"
+        "default": "./dist/containerRuntime.js"
       }
     },
     "./test/deltaScheduler": {
@@ -33,7 +33,7 @@
       },
       "require": {
         "types": "./dist/deltaScheduler.d.ts",
-        "default": "./dist/deltaScheduler.cjs"
+        "default": "./dist/deltaScheduler.js"
       }
     },
     "./test/scheduleManager": {
@@ -43,7 +43,7 @@
       },
       "require": {
         "types": "./dist/scheduleManager.d.ts",
-        "default": "./dist/scheduleManager.cjs"
+        "default": "./dist/scheduleManager.js"
       }
     },
     "./test/blobManager": {
@@ -53,7 +53,7 @@
       },
       "require": {
         "types": "./dist/blobManager.d.ts",
-        "default": "./dist/blobManager.cjs"
+        "default": "./dist/blobManager.js"
       }
     },
     "./test/summary": {
@@ -63,7 +63,7 @@
       },
       "require": {
         "types": "./dist/summary/index.d.ts",
-        "default": "./dist/summary/index.cjs"
+        "default": "./dist/summary/index.js"
       }
     },
     "./test/gc": {
@@ -73,7 +73,7 @@
       },
       "require": {
         "types": "./gc/index.d.ts",
-        "default": "./gc/index.cjs"
+        "default": "./gc/index.js"
       }
     },
     "./alpha": {
@@ -83,7 +83,7 @@
       },
       "require": {
         "types": "./dist/container-runtime-alpha.d.ts",
-        "default": "./dist/index.cjs"
+        "default": "./dist/index.js"
       }
     },
     "./beta": {
@@ -93,7 +93,7 @@
       },
       "require": {
         "types": "./dist/container-runtime-beta.d.ts",
-        "default": "./dist/index.cjs"
+        "default": "./dist/index.js"
       }
     },
     "./internal": {
@@ -103,7 +103,7 @@
       },
       "require": {
         "types": "./dist/index.d.ts",
-        "default": "./dist/index.cjs"
+        "default": "./dist/index.js"
       }
     },
     "./public": {
@@ -113,11 +113,11 @@
       },
       "require": {
         "types": "./dist/container-runtime-public.d.ts",
-        "default": "./dist/index.cjs"
+        "default": "./dist/index.js"
       }
     }
   },
-  "main": "dist/index.cjs",
+  "main": "dist/index.js",
   "module": "lib/index.mjs",
   "types": "dist/index.d.ts",
   "c8": {
@@ -141,36 +141,36 @@
     "temp-directory": "nyc/.nyc_output"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/container-definitions": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/container-runtime-definitions": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/core-interfaces": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/core-utils": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/datastore": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/driver-definitions": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/driver-utils": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/id-compressor": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/protocol-definitions": "^3.1.0-223007",
-    "@fluidframework/runtime-definitions": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/runtime-utils": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/telemetry-utils": "2.0.0-dev-rc.1.0.0.225277",
+    "@fluid-internal/client-utils": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/container-definitions": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/container-runtime-definitions": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/core-interfaces": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/core-utils": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/datastore": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/driver-definitions": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/driver-utils": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/id-compressor": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/protocol-definitions": "^3.1.0",
+    "@fluidframework/runtime-definitions": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/runtime-utils": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/telemetry-utils": "2.0.0-dev-rc.1.0.0.228517",
+    "@tylerbu/sorted-btree-es6": "^1.8.0",
     "double-ended-queue": "^2.1.0-0",
     "events": "^3.1.0",
     "lz4js": "^0.2.0",
-    "sorted-btree": "^1.8.0",
     "uuid": "^9.0.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.13.3",
-    "@fluid-private/stochastic-test-utils": "2.0.0-dev-rc.1.0.0.225277",
+    "@fluid-private/stochastic-test-utils": "2.0.0-dev-rc.1.0.0.228517",
     "@fluid-tools/benchmark": "^0.48.0",
-    "@fluid-tools/build-cli": "0.29.0-222379",
+    "@fluid-tools/build-cli": "^0.29.0",
     "@fluidframework/build-common": "^2.0.3",
-    "@fluidframework/build-tools": "0.29.0-222379",
+    "@fluidframework/build-tools": "^0.29.0",
     "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.0.0-internal.8.0.0",
-    "@fluidframework/eslint-config-fluid": "^3.1.0",
-    "@fluidframework/mocha-test-setup": "2.0.0-dev-rc.1.0.0.225277",
-    "@fluidframework/test-runtime-utils": "2.0.0-dev-rc.1.0.0.225277",
+    "@fluidframework/eslint-config-fluid": "^3.2.0",
+    "@fluidframework/mocha-test-setup": "2.0.0-dev-rc.1.0.0.228517",
+    "@fluidframework/test-runtime-utils": "2.0.0-dev-rc.1.0.0.228517",
     "@microsoft/api-extractor": "^7.38.3",
     "@types/double-ended-queue": "^2.1.0",
     "@types/events": "^3.0.0",
@@ -204,7 +204,11 @@
     }
   },
   "typeValidation": {
-    "broken": {}
+    "broken": {
+      "ClassDeclaration_ContainerRuntime": {
+        "forwardCompat": false
+      }
+    }
   },
   "scripts": {
     "api": "fluid-build . --task api",
@@ -216,7 +220,7 @@
     "build:docs": "fluid-build . --task api",
     "build:esnext": "tsc-multi --config ../../../common/build/build-common/tsc-multi.esm.json",
     "build:genver": "gen-version",
-    "build:test": "tsc-multi --config ./tsc-multi.test.json",
+    "build:test": "tsc --project ./src/test/tsconfig.json",
     "check:are-the-types-wrong": "attw --pack . --entrypoints .",
     "check:release-tags": "api-extractor run --local --config ./api-extractor-lint.json",
     "ci:build:docs": "api-extractor run",
@@ -233,7 +237,7 @@
     "test:coverage": "c8 npm test",
     "test:mocha": "mocha --ignore \"dist/test/types/*\" --recursive dist/test -r node_modules/@fluidframework/mocha-test-setup",
     "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
-    "tsc": "tsc-multi --config ../../../common/build/build-common/tsc-multi.cjs.json",
+    "tsc": "tsc",
     "tsc:watch": "tsc --watch",
     "typetests:gen": "fluid-type-test-generator",
     "typetests:prepare": "flub typetests --dir . --reset --previous --normalize"

package/src/blobManager.ts

@@ -772,10 +772,16 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		const maybeUnusedStorageIds: Set<string> = new Set();
 		for (const route of blobRoutes) {
 			const blobId = getBlobIdFromGCNodePath(route);
+			// If the blob hasn't already been deleted, log an error because this should never happen.
+			// If the blob has already been deleted, log a telemetry event. This can happen because multiple GC
+			// sweep ops can contain the same data store. It would be interesting to track how often this happens.
+			const alreadyDeleted = this.isBlobDeleted(route);
 			if (!this.redirectTable.has(blobId)) {
-				this.mc.logger.sendErrorEvent({
+				this.mc.logger.sendTelemetryEvent({
 					eventName: "DeletedAttachmentBlobNotFound",
+					category: alreadyDeleted ? "generic" : "error",
 					blobId,
+					details: { alreadyDeleted },
 				});
 				continue;
 			}

package/src/containerRuntime.ts

@@ -2268,6 +2268,7 @@ export class ContainerRuntime
 					messageWithContext.message,
 					local,
 					localOpMetadata,
+					(from, to) => this.garbageCollector.addedOutboundReference(from, to),
 				);
 				break;
 			case ContainerMessageType.BlobAttach:
@@ -2679,6 +2680,12 @@ export class ContainerRuntime
 			this.blobManager.setRedirectTable(blobRedirectTable);
 		}
 
+		// We can finalize any allocated IDs since we're the only client
+		const idRange = this.idCompressor?.takeNextCreationRange();
+		if (idRange !== undefined) {
+			this.idCompressor?.finalizeCreationRange(idRange);
+		}
+
 		const summarizeResult = this.dataStores.createSummary(telemetryContext);
 		// Wrap data store summaries in .channels subtree.
 		wrapSummaryInChannelsTree(summarizeResult);

package/src/dataStoreContext.ts

@@ -78,7 +78,7 @@ import {
 	summarizerClientType,
 } from "./summary";
 import { ContainerRuntime } from "./containerRuntime";
-import { sendGCUnexpectedUsageEvent } from "./gc";
+import { detectOutboundRoutesViaDDSKey, sendGCUnexpectedUsageEvent } from "./gc";
 
 function createAttributes(
 	pkg: readonly string[],
@@ -650,13 +650,20 @@ export abstract class FluidDataStoreContext
 	}
 
 	/**
+	 * @deprecated There is no replacement for this, its functionality is no longer needed.
+	 * It will be removed in a future release, sometime after 2.0.0-internal.8.0.0
+	 *
 	 * Called when a new outbound reference is added to another node. This is used by garbage collection to identify
 	 * all references added in the system.
 	 * @param srcHandle - The handle of the node that added the reference.
 	 * @param outboundHandle - The handle of the outbound node that is referenced.
 	 */
 	public addedGCOutboundReference(srcHandle: IFluidHandle, outboundHandle: IFluidHandle) {
-		this._containerRuntime.addedGCOutboundReference(srcHandle, outboundHandle);
+		// By default, skip this call since the ContainerRuntime will detect the outbound route directly.
+		if (this.mc.config.getBoolean(detectOutboundRoutesViaDDSKey) === true) {
+			// Note: The ContainerRuntime code will check this same setting to avoid double counting.
+			this._containerRuntime.addedGCOutboundReference(srcHandle, outboundHandle);
+		}
 	}
 
 	/**

package/src/dataStores.ts

@@ -33,6 +33,7 @@ import {
 	create404Response,
 	createResponseError,
 	GCDataBuilder,
+	isSerializedHandle,
 	responseToException,
 	SummaryTreeBuilder,
 	unpackChildNodesUsedRoutes,
@@ -61,7 +62,7 @@ import {
 } from "./dataStoreContext";
 import { StorageServiceWithAttachBlobs } from "./storageServiceWithAttachBlobs";
 import { IDataStoreAliasMessage, isDataStoreAliasMessage } from "./dataStore";
-import { GCNodeType, disableDatastoreSweepKey } from "./gc";
+import { GCNodeType, detectOutboundRoutesViaDDSKey, disableDatastoreSweepKey } from "./gc";
 import { IContainerRuntimeMetadata, nonDataStorePaths, rootHasIsolatedChannels } from "./summary";
 
 type PendingAliasResolve = (success: boolean) => void;
@@ -441,6 +442,7 @@ export class DataStores implements IDisposable {
 		message: ISequencedDocumentMessage,
 		local: boolean,
 		localMessageMetadata: unknown,
+		addedOutboundReference: (fromNodePath: string, toNodePath: string) => void,
 	) {
 		const envelope = message.contents as IEnvelope;
 		const transformed = { ...message, contents: envelope.contents };
@@ -462,6 +464,13 @@ export class DataStores implements IDisposable {
 		assert(!!context, 0x162 /* "There should be a store context for the op" */);
 		context.process(transformed, local, localMessageMetadata);
 
+		// By default, we use the new behavior of detecting outbound routes here.
+		// If this setting is true, then DataStoreContext would be notifying GC instead.
+		if (this.mc.config.getBoolean(detectOutboundRoutesViaDDSKey) !== true) {
+			// Notify GC of any outbound references that were added by this op.
+			detectOutboundReferences(envelope, addedOutboundReference);
+		}
+
 		// Notify that a GC node for the data store changed. This is used to detect if a deleted data store is
 		// being used.
 		this.gcNodeUpdated(
@@ -830,12 +839,15 @@ export class DataStores implements IDisposable {
 
 			const dataStoreContext = this.contexts.get(dataStoreId);
 			if (dataStoreContext === undefined) {
-				this.mc.logger.sendErrorEvent({
+				// If the data store hasn't already been deleted, log an error because this should never happen.
+				// If the data store has already been deleted, log a telemetry event. This can happen because multiple GC
+				// sweep ops can contain the same data store. It would be interesting to track how often this happens.
+				const alreadyDeleted = this.isDataStoreDeleted(`/${dataStoreId}`);
+				this.mc.logger.sendTelemetryEvent({
 					eventName: "DeletedDataStoreNotFound",
+					category: alreadyDeleted ? "generic" : "error",
 					...tagCodeArtifacts({ id: dataStoreId }),
-					details: {
-						alreadyDeleted: this.isDataStoreDeleted(dataStoreId),
-					},
+					details: { alreadyDeleted },
 				});
 				continue;
 			}
@@ -947,3 +959,42 @@ export function getSummaryForDatastores(
 		};
 	}
 }
+
+/**
+ * Traverse this op's contents and detect any outbound routes that were added by this op.
+ */
+export function detectOutboundReferences(
+	envelope: IEnvelope,
+	addedOutboundReference: (fromNodePath: string, toNodePath: string) => void,
+): void {
+	// These will be built up as we traverse the envelope contents
+	const outboundPaths: string[] = [];
+	let ddsAddress: string | undefined;
+
+	function recursivelyFindHandles(obj: unknown) {
+		if (typeof obj === "object" && obj !== null) {
+			for (const [key, value] of Object.entries(obj)) {
+				// If 'value' is a serialized IFluidHandle, it represents a new outbound route.
+				if (isSerializedHandle(value)) {
+					outboundPaths.push(value.url);
+				}
+
+				// NOTE: This is taking a hard dependency on the fact that in our DataStore implementation,
+				// the address of the DDS is stored in a property called "address".  This is not ideal.
+				// An alternative would be for the op envelope to include the absolute path (built up as it is submitted)
+				if (key === "address" && ddsAddress === undefined) {
+					ddsAddress = value;
+				}
+
+				recursivelyFindHandles(value);
+			}
+		}
+	}
+
+	recursivelyFindHandles(envelope.contents);
+
+	// GC node paths are all absolute paths, hence the "" prefix.
+	// e.g. this will yield "/dataStoreId/ddsId"
+	const fromPath = ["", envelope.address, ddsAddress].join("/");
+	outboundPaths.forEach((toPath) => addedOutboundReference(fromPath, toPath));
+}

package/src/gc/garbageCollection.ts

@@ -20,7 +20,9 @@ import {
 	ITelemetryLoggerExt,
 	MonitoringContext,
 	PerformanceEvent,
+	tagCodeArtifacts,
 } from "@fluidframework/telemetry-utils";
+import { BlobManager } from "../blobManager";
 import {
 	InactiveResponseHeaderKey,
 	RuntimeHeaderData,
@@ -261,7 +263,7 @@ export class GarbageCollector implements IGarbageCollector {
 			const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
 			assert(
 				currentReferenceTimestampMs !== undefined,
-				"Trying to initialize GC state without current timestamp",
+				0x8a4 /* Trying to initialize GC state without current timestamp */,
 			);
 
 			/**
@@ -1005,6 +1007,18 @@ export class GarbageCollector implements IGarbageCollector {
 			return;
 		}
 
+		if (!toNodePath.startsWith("/")) {
+			// A long time ago we stored handles with relatives paths. We don't expect to see these cases though
+			// because GC was enabled only after we made the switch to always using absolute paths.
+			this.mc.logger.sendErrorEvent({
+				eventName: "InvalidRelativeOutboundRoute",
+				...tagCodeArtifacts({ fromId: fromNodePath, id: toNodePath }),
+			});
+			return;
+		}
+
+		assert(fromNodePath.startsWith("/"), 0x8a5 /* fromNodePath must be an absolute path */);
+
 		const outboundRoutes = this.newReferencesSinceLastRun.get(fromNodePath) ?? [];
 		outboundRoutes.push(toNodePath);
 		this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
@@ -1120,9 +1134,25 @@ export class GarbageCollector implements IGarbageCollector {
 			deletedAttachmentBlobCount: 0,
 		};
 
+		// The runtime can't reliably identify the type of deleted nodes. So, get the type here. This should
+		// be good enough because the only types that participate in GC today are data stores, DDSes and blobs.
+		const getDeletedNodeType = (nodeId: string): GCNodeType => {
+			const pathParts = nodeId.split("/");
+			if (pathParts[1] === BlobManager.basePath) {
+				return GCNodeType.Blob;
+			}
+			if (pathParts.length === 2) {
+				return GCNodeType.DataStore;
+			}
+			if (pathParts.length === 3) {
+				return GCNodeType.SubDataStore;
+			}
+			return GCNodeType.Other;
+		};
+
 		for (const nodeId of deletedNodes) {
 			sweepPhaseStats.deletedNodeCount++;
-			const nodeType = this.runtime.getNodeType(nodeId);
+			const nodeType = getDeletedNodeType(nodeId);
 			if (nodeType === GCNodeType.DataStore) {
 				sweepPhaseStats.deletedDataStoreCount++;
 			} else if (nodeType === GCNodeType.Blob) {

package/src/gc/gcDefinitions.ts

@@ -74,6 +74,8 @@ export const gcVersionUpgradeToV4Key = "Fluid.GarbageCollection.GCVersionUpgrade
 export const disableDatastoreSweepKey = "Fluid.GarbageCollection.DisableDataStoreSweep";
 /** Config key to disable GC sweep for attachment blobs. */
 export const disableAttachmentBlobSweepKey = "Fluid.GarbageCollection.DisableAttachmentBlobSweep";
+/** Config key to revert new paradigm of detecting outbound routes in ContainerRuntime layer (use true) */
+export const detectOutboundRoutesViaDDSKey = "Fluid.GarbageCollection.DetectOutboundRoutesViaDDS";
 
 // One day in milliseconds.
 export const oneDayMs = 1 * 24 * 60 * 60 * 1000;

package/src/gc/index.ts

@@ -34,9 +34,11 @@ export {
 	stableGCVersion,
 	disableAttachmentBlobSweepKey,
 	disableDatastoreSweepKey,
+	detectOutboundRoutesViaDDSKey,
 	UnreferencedState,
 	throwOnTombstoneLoadOverrideKey,
 	GarbageCollectionMessage,
+	ISweepMessage,
 } from "./gcDefinitions";
 export {
 	cloneGCData,

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-dev-rc.1.0.0.225277";
+export const pkgVersion = "2.0.0-dev-rc.1.0.0.228517";

package/src/summary/runningSummarizer.ts

@@ -693,7 +693,9 @@ export class RunningSummarizer extends TypedEventEmitter<ISummarizerEvents> impl
 	 * Tries to summarize with retries where retry is based on the failure params.
 	 * For example, summarization may be retried for failures with "retryAfterSeconds" param.
 	 */
-	private async trySummarizeWithRetries(reason: SummarizeReason) {
+	private async trySummarizeWithRetries(
+		reason: SummarizeReason,
+	): Promise<ISummarizeResults | undefined> {
 		// Helper to set summarize options, telemetry properties and call summarize.
 		const attemptSummarize = (attemptNumber: number, finalAttempt: boolean) => {
 			const summarizeOptions: ISummarizeOptions = {
@@ -732,24 +734,23 @@ export class RunningSummarizer extends TypedEventEmitter<ISummarizerEvents> impl
 		let currentAttempt = 0;
 		let retryAfterSeconds: number | undefined;
 		let done = false;
-		let result: "success" | "failure" | "canceled" = "success";
+		let status: "success" | "failure" | "canceled" = "success";
+		let results: ISummarizeResults | undefined;
 		do {
 			currentAttempt++;
 			if (this.cancellationToken.cancelled) {
-				result = "canceled";
+				status = "canceled";
 				done = true;
 				break;
 			}
 
-			const { summarizeProps, summarizeResult } = attemptSummarize(
-				currentAttempt,
-				false /* finalAttempt */,
-			);
+			const attemptResult = attemptSummarize(currentAttempt, false /* finalAttempt */);
+			results = attemptResult.summarizeResult;
 
 			// Ack / nack is the final step, so if it succeeds we're done.
-			const ackNackResult = await summarizeResult.receivedSummaryAckOrNack;
+			const ackNackResult = await results.receivedSummaryAckOrNack;
 			if (ackNackResult.success) {
-				result = "success";
+				status = "success";
 				done = true;
 				break;
 			}
@@ -758,7 +759,7 @@ export class RunningSummarizer extends TypedEventEmitter<ISummarizerEvents> impl
 			// If submit summary failed, use the params from "summarySubmitted" result. Else, use the params
 			// from "receivedSummaryAckOrNack" result.
 			// Note: Check "summarySubmitted" result first because if it fails, ack nack would fail as well.
-			const submitSummaryResult = await summarizeResult.summarySubmitted;
+			const submitSummaryResult = await results.summarySubmitted;
 			if (!submitSummaryResult.success) {
 				maxAttempts = this.maxAttemptsForSubmitFailures;
 				retryAfterSeconds = submitSummaryResult.data?.retryAfterSeconds;
@@ -768,9 +769,9 @@ export class RunningSummarizer extends TypedEventEmitter<ISummarizerEvents> impl
 			}
 
 			// Emit "summarize" event for this failed attempt.
-			result = "failure";
+			status = "failure";
 			const eventProps: ISummarizeEventProps = {
-				result,
+				result: status,
 				currentAttempt,
 				maxAttempts,
 				error: ackNackResult.error,
@@ -791,16 +792,16 @@ export class RunningSummarizer extends TypedEventEmitter<ISummarizerEvents> impl
 					summaryNackDelay: ackNackResult.data?.retryAfterSeconds !== undefined,
 					stage: submitSummaryResult.data?.stage,
 					dynamicRetries: true, // To differentiate this telemetry from regular retry logic
-					...summarizeProps,
+					...attemptResult.summarizeProps,
 				});
 				await delay(retryAfterSeconds * 1000);
 			}
 		} while (!done);
 
 		// If summarize attempt did not fail, emit "summarize" event and return. A failed attempt may be retried below.
-		if (result !== "failure") {
-			this.emit("summarize", { result, currentAttempt, maxAttempts });
-			return;
+		if (status !== "failure") {
+			this.emit("summarize", { result: status, currentAttempt, maxAttempts });
+			return results;
 		}
 
 		// If summarization wasn't successful above and the failure contains "retryAfterSeconds", perform one last
@@ -809,20 +810,44 @@ export class RunningSummarizer extends TypedEventEmitter<ISummarizerEvents> impl
 			const { summarizeResult } = attemptSummarize(++currentAttempt, true /* finalAttempt */);
 			// Ack / nack is the final step, so if it succeeds we're done.
 			const ackNackResult = await summarizeResult.receivedSummaryAckOrNack;
-			result = ackNackResult.success ? "success" : "failure";
+			status = ackNackResult.success ? "success" : "failure";
 			const eventProps: ISummarizeEventProps = {
-				result,
+				result: status,
 				currentAttempt,
 				maxAttempts,
 				error: ackNackResult.success ? undefined : ackNackResult.error,
 			};
 			this.emit("summarize", eventProps);
+			results = summarizeResult;
 		}
 
 		// If summarization is still unsuccessful, stop the summarizer.
-		if (result === "failure") {
+		if (status === "failure") {
 			this.stopSummarizerCallback("failToSummarize");
 		}
+		return results;
+	}
+
+	/**
+	 * Attempts to generate a summary on demand with retries in case of failures. The retry logic is the same
+	 * as heuristics based summaries.
+	 */
+	private async summarizeOnDemandWithRetries(
+		reason: SummarizeReason,
+		resultsBuilder: SummarizeResultBuilder,
+	) {
+		const results = await this.trySummarizeWithRetries(reason);
+		if (results === undefined) {
+			resultsBuilder.fail("Summarization was canceled", undefined);
+			return resultsBuilder.build();
+		}
+		const submitResult = await results.summarySubmitted;
+		const summaryOpBroadcastedResult = await results.summaryOpBroadcasted;
+		const ackNackResult = await results.receivedSummaryAckOrNack;
+		resultsBuilder.summarySubmitted.resolve(submitResult);
+		resultsBuilder.summaryOpBroadcasted.resolve(summaryOpBroadcastedResult);
+		resultsBuilder.receivedSummaryAckOrNack.resolve(ackNackResult);
+		return resultsBuilder.build();
 	}
 
 	/** {@inheritdoc (ISummarizer:interface).summarizeOnDemand} */
@@ -842,12 +867,20 @@ export class RunningSummarizer extends TypedEventEmitter<ISummarizerEvents> impl
 		}
 
 		const { reason, ...summarizeOptions } = options;
-		const result = this.trySummarizeOnce(
-			{ summarizeReason: `onDemand/${reason}` },
-			summarizeOptions,
-			resultsBuilder,
-		);
-		return result;
+		if (options.retryOnFailure === true) {
+			this.summarizeOnDemandWithRetries(`onDemand;${reason}`, resultsBuilder).catch(
+				(error) => {
+					resultsBuilder.fail("summarize failed", error);
+				},
+			);
+		} else {
+			this.trySummarizeOnce(
+				{ summarizeReason: `onDemand/${reason}` },
+				summarizeOptions,
+				resultsBuilder,
+			);
+		}
+		return resultsBuilder.build();
 	}
 
 	/** {@inheritdoc (ISummarizer:interface).enqueueSummarize} */

package/src/summary/summarizerTypes.ts

@@ -149,6 +149,8 @@ export interface ISubmitSummaryOptions extends ISummarizeOptions {
 export interface IOnDemandSummarizeOptions extends ISummarizeOptions {
 	/** Reason for generating summary. */
 	readonly reason: string;
+	/** In case of a failure, will attempt to retry based on if the failure is retriable. */
+	readonly retryOnFailure?: boolean;
 }
 
 /**

package/dist/batchTracker.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"batchTracker.cjs","sourceRoot":"","sources":["../src/batchTracker.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,qEAAyF;AACzF,2DAAoD;AACpD,+DAA2D;AAI3D,MAAa,YAAY;IAMxB,YACkB,iBAA+B,EAChD,MAA4B,EAC5B,oBAA4B,EAC5B,sBAA8B,EAC9B,mBAAiC,GAAG,EAAE,CAAC,0BAAW,CAAC,GAAG,EAAE;QAJvC,sBAAiB,GAAjB,iBAAiB,CAAc;QAJzC,sBAAiB,GAAW,CAAC,CAAC;QAUrC,IAAI,CAAC,MAAM,GAAG,IAAA,mCAAiB,EAAC,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC;QAEnE,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,OAAkC,EAAE,EAAE;YAC9E,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,cAAc,CAAC;YACvD,IAAI,CAAC,6BAA6B,GAAG,gBAAgB,EAAE,CAAC;YACxD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,CAAC,EAAE,CACxB,UAAU,EACV,CAAC,KAAsB,EAAE,OAAkC,EAAE,EAAE;YAC9D,IAAA,mBAAM,EACL,IAAI,CAAC,wBAAwB,KAAK,SAAS;gBAC1C,IAAI,CAAC,6BAA6B,KAAK,SAAS,EACjD,KAAK,CAAC,4CAA4C,CAClD,CAAC;YAEF,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,wBAAwB,GAAG,CAAC,CAAC;YAC1E,IAAI,MAAM,IAAI,oBAAoB,EAAE;gBACnC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAChC,SAAS,EAAE,cAAc;oBACzB,MAAM;oBACN,SAAS,EAAE,oBAAoB;oBAC/B,sBAAsB,EAAE,OAAO,CAAC,cAAc;oBAC9C,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,6BAA6B;oBACjE,UAAU,EAAE,KAAK,KAAK,SAAS;iBAC/B,CAAC,CAAC;aACH;YAED,IAAI,IAAI,CAAC,iBAAiB,GAAG,sBAAsB,KAAK,CAAC,EAAE;gBAC1D,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAChC,SAAS,EAAE,QAAQ;oBACnB,MAAM;oBACN,YAAY,EAAE,sBAAsB;oBACpC,sBAAsB,EAAE,OAAO,CAAC,cAAc;oBAC9C,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,6BAA6B;iBACjE,CAAC,CAAC;aACH;YAED,IAAI,CAAC,wBAAwB,GAAG,SAAS,CAAC;YAC1C,IAAI,CAAC,6BAA6B,GAAG,SAAS,CAAC;QAChD,CAAC,CACD,CAAC;IACH,CAAC;CACD;AAzDD,oCAyDC;AAED;;;;;;;GAOG;AACI,MAAM,gBAAgB,GAAG,CAC/B,iBAA+B,EAC/B,MAA2B,EAC3B,uBAA+B,IAAI,EACnC,yBAAiC,IAAI,EACpC,EAAE,CAAC,IAAI,YAAY,CAAC,iBAAiB,EAAE,MAAM,EAAE,oBAAoB,EAAE,sBAAsB,CAAC,CAAC;AALlF,QAAA,gBAAgB,oBAKkE","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { EventEmitter } from \"events\";\nimport { ITelemetryLoggerExt, createChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { assert } from \"@fluidframework/core-utils\";\nimport { performance } from \"@fluid-internal/client-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\n\nexport class BatchTracker {\n\tprivate readonly logger: ITelemetryLoggerExt;\n\tprivate startBatchSequenceNumber: number | undefined;\n\tprivate trackedBatchCount: number = 0;\n\tprivate batchProcessingStartTimeStamp: number | undefined;\n\n\tconstructor(\n\t\tprivate readonly batchEventEmitter: EventEmitter,\n\t\tlogger: ITelemetryBaseLogger,\n\t\tbatchLengthThreshold: number,\n\t\tbatchCountSamplingRate: number,\n\t\tdateTimeProvider: () => number = () => performance.now(),\n\t) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"Batching\" });\n\n\t\tthis.batchEventEmitter.on(\"batchBegin\", (message: ISequencedDocumentMessage) => {\n\t\t\tthis.startBatchSequenceNumber = message.sequenceNumber;\n\t\t\tthis.batchProcessingStartTimeStamp = dateTimeProvider();\n\t\t\tthis.trackedBatchCount++;\n\t\t});\n\n\t\tthis.batchEventEmitter.on(\n\t\t\t\"batchEnd\",\n\t\t\t(error: any | undefined, message: ISequencedDocumentMessage) => {\n\t\t\t\tassert(\n\t\t\t\t\tthis.startBatchSequenceNumber !== undefined &&\n\t\t\t\t\t\tthis.batchProcessingStartTimeStamp !== undefined,\n\t\t\t\t\t0x2ba /* \"batchBegin must fire before batchEnd\" */,\n\t\t\t\t);\n\n\t\t\t\tconst length = message.sequenceNumber - this.startBatchSequenceNumber + 1;\n\t\t\t\tif (length >= batchLengthThreshold) {\n\t\t\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\t\t\teventName: \"LengthTooBig\",\n\t\t\t\t\t\tlength,\n\t\t\t\t\t\tthreshold: batchLengthThreshold,\n\t\t\t\t\t\tbatchEndSequenceNumber: message.sequenceNumber,\n\t\t\t\t\t\tduration: dateTimeProvider() - this.batchProcessingStartTimeStamp,\n\t\t\t\t\t\tbatchError: error !== undefined,\n\t\t\t\t\t});\n\t\t\t\t}\n\n\t\t\t\tif (this.trackedBatchCount % batchCountSamplingRate === 0) {\n\t\t\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\t\t\teventName: \"Length\",\n\t\t\t\t\t\tlength,\n\t\t\t\t\t\tsamplingRate: batchCountSamplingRate,\n\t\t\t\t\t\tbatchEndSequenceNumber: message.sequenceNumber,\n\t\t\t\t\t\tduration: dateTimeProvider() - this.batchProcessingStartTimeStamp,\n\t\t\t\t\t});\n\t\t\t\t}\n\n\t\t\t\tthis.startBatchSequenceNumber = undefined;\n\t\t\t\tthis.batchProcessingStartTimeStamp = undefined;\n\t\t\t},\n\t\t);\n\t}\n}\n\n/**\n * Track batch sizes in terms of op counts and processing times\n *\n * @param batchEventEmitter - event emitter which tracks the lifecycle of batch operations\n * @param logger - See {@link @fluidframework/core-interfaces#ITelemetryLoggerExt}\n * @param batchLengthThreshold - threshold for the length of a batch when to send an error event\n * @param batchCountSamplingRate - rate for batches for which to send an event with its characteristics\n */\nexport const BindBatchTracker = (\n\tbatchEventEmitter: EventEmitter,\n\tlogger: ITelemetryLoggerExt,\n\tbatchLengthThreshold: number = 1000,\n\tbatchCountSamplingRate: number = 1000,\n) => new BatchTracker(batchEventEmitter, logger, batchLengthThreshold, batchCountSamplingRate);\n"]}