@fluidframework/container-runtime 0.59.4000 → 1.0.1

package/lib/containerRuntime.js

@@ -1,12 +1,12 @@
 import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
 import { assert, Trace, TypedEventEmitter, unreachableCase, performance, } from "@fluidframework/common-utils";
-import { ChildLogger, raiseConnectedEvent, PerformanceEvent, normalizeError, TaggedLoggerAdapter, loggerToMonitoringContext, TelemetryDataTag, } from "@fluidframework/telemetry-utils";
+import { ChildLogger, raiseConnectedEvent, PerformanceEvent, TaggedLoggerAdapter, loggerToMonitoringContext, TelemetryDataTag, } from "@fluidframework/telemetry-utils";
 import { DriverHeader } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
 import { DataCorruptionError, GenericError, UsageError, extractSafePropertiesFromMessage, } from "@fluidframework/container-utils";
 import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
 import { FlushMode, channelsTreeName, } from "@fluidframework/runtime-definitions";
-import { addBlobToSummary, addTreeToSummary, createRootSummarizerNodeWithGC, RequestParser, create404Response, exceptionToResponse, requestFluidObject, responseToException, seqFromTree, calculateStats, addSummarizeResultToSummary, } from "@fluidframework/runtime-utils";
+import { addBlobToSummary, addSummarizeResultToSummary, addTreeToSummary, createRootSummarizerNodeWithGC, RequestParser, create404Response, exceptionToResponse, requestFluidObject, responseToException, seqFromTree, calculateStats, TelemetryContext, } from "@fluidframework/runtime-utils";
 import { GCDataBuilder, trimLeadingAndTrailingSlashes } from "@fluidframework/garbage-collector";
 import { v4 as uuid } from "uuid";
 import { ContainerFluidHandleContext } from "./containerHandleContext";
@@ -28,6 +28,7 @@ import { RunWhileConnectedCoordinator } from "./runWhileConnectedCoordinator";
 import { GarbageCollector, GCNodeType, gcTreeKey, } from "./garbageCollection";
 import { channelToDataStore, isDataStoreAliasMessage, } from "./dataStore";
 import { BindBatchTracker } from "./batchTracker";
+import { SerializedSnapshotStorage } from "./serializedSnapshotStorage";
 import { OpTracker } from "./opTelemetry";
 export var ContainerMessageType;
 (function (ContainerMessageType) {
@@ -44,17 +45,16 @@ export var ContainerMessageType;
     // Sets the alias of a root data store
     ContainerMessageType["Alias"] = "alias";
 })(ContainerMessageType || (ContainerMessageType = {}));
-// Consider idle 5s of no activity. And snapshot if a minute has gone by with no snapshot.
-const IdleDetectionTime = 5000;
-const DefaultSummaryConfiguration = {
-    idleTime: IdleDetectionTime * 3,
-    maxTime: IdleDetectionTime * 12,
-    // Summarize if 1000 ops received since last snapshot.
-    maxOps: 1000,
-    // Wait 10 minutes for summary ack
-    // this is less than maxSummarizeAckWaitTime
-    // the min of the two will be chosen
-    maxAckWaitTime: 600000,
+export const DefaultSummaryConfiguration = {
+    state: "enabled",
+    idleTime: 5000 * 3,
+    maxTime: 5000 * 12,
+    maxOps: 100,
+    minOpsForLastSummaryAttempt: 10,
+    maxAckWaitTime: 6 * 10 * 1000,
+    maxOpsSinceLastSummary: 7000,
+    initialSummarizerDelayMs: 5000,
+    summarizerClientElection: false,
 };
 /**
  * Accepted header keys for requests coming to the runtime.
@@ -372,44 +372,26 @@ export function getDeviceSpec() {
  * It will define the store level mappings.
  */
 export class ContainerRuntime extends TypedEventEmitter {
-    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, requestHandler) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l;
+    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, requestHandler, summaryConfiguration) {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j;
+        if (summaryConfiguration === void 0) { summaryConfiguration = Object.assign(Object.assign({}, DefaultSummaryConfiguration), (_a = runtimeOptions.summaryOptions) === null || _a === void 0 ? void 0 : _a.summaryConfigOverrides); }
         super();
         this.context = context;
         this.registry = registry;
         this.runtimeOptions = runtimeOptions;
         this.containerScope = containerScope;
         this.logger = logger;
+        this._storage = _storage;
         this.requestHandler = requestHandler;
+        this.summaryConfiguration = summaryConfiguration;
         this.defaultMaxConsecutiveReconnects = 15;
         this._orderSequentiallyCalls = 0;
         this.needsFlush = false;
         this.flushTrigger = false;
-        this.paused = false;
+        this.savedOps = [];
         this.consecutiveReconnects = 0;
         this._disposed = false;
         this.emitDirtyDocumentEvent = true;
-        /**
-         * Used to apply stashed ops at their reference sequence number.
-         * Normal op processing is synchronous, but applying stashed ops is async since the
-         * data store may not be loaded yet, so we pause DeltaManager between ops.
-         * It's also important that we see each op so we know all stashed ops have
-         * been applied by "connected" event, but process() doesn't see system ops,
-         * so we listen directly from DeltaManager instead.
-         */
-        this.onOp = (op) => {
-            assert(!this.paused, 0x128 /* "Container should not already be paused before applying stashed ops" */);
-            this.paused = true;
-            // eslint-disable-next-line @typescript-eslint/no-floating-promises
-            this.context.deltaManager.inbound.pause();
-            const stashP = this.pendingStateManager.applyStashedOpsAt(op.sequenceNumber);
-            stashP.then(() => {
-                this.paused = false;
-                this.context.deltaManager.inbound.resume();
-            }, (error) => {
-                this.closeFn(normalizeError(error));
-            });
-        };
         this.summarizeOnDemand = (...args) => {
             if (this.clientDetails.type === summarizerClientType) {
                 return this.summarizer.summarizeOnDemand(...args);
@@ -440,27 +422,34 @@ export class ContainerRuntime extends TypedEventEmitter {
         };
         this.messageAtLastSummary = metadata === null || metadata === void 0 ? void 0 : metadata.message;
         // Default to false (enabled).
-        this.disableIsolatedChannels = (_a = this.runtimeOptions.summaryOptions.disableIsolatedChannels) !== null && _a !== void 0 ? _a : false;
+        this.disableIsolatedChannels = (_b = this.runtimeOptions.summaryOptions.disableIsolatedChannels) !== null && _b !== void 0 ? _b : false;
         this._connected = this.context.connected;
         this.chunkMap = new Map(chunks);
         this.handleContext = new ContainerFluidHandleContext("", this);
         this.mc = loggerToMonitoringContext(ChildLogger.create(this.logger, "ContainerRuntime"));
+        this.summariesDisabled = this.isSummariesDisabled();
+        this.heuristicsDisabled = this.isHeuristicsDisabled();
+        this.summarizerClientElectionEnabled = this.isSummarizerClientElectionEnabled();
+        this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
+        this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
         this._aliasingEnabled =
-            ((_b = this.mc.config.getBoolean(useDataStoreAliasingKey)) !== null && _b !== void 0 ? _b : false) ||
-                ((_c = runtimeOptions.useDataStoreAliasing) !== null && _c !== void 0 ? _c : false);
-        this._maxOpSizeInBytes = ((_d = this.mc.config.getNumber(maxOpSizeInBytesKey)) !== null && _d !== void 0 ? _d : defaultMaxOpSizeInBytes);
+            ((_c = this.mc.config.getBoolean(useDataStoreAliasingKey)) !== null && _c !== void 0 ? _c : false) ||
+                ((_d = runtimeOptions.useDataStoreAliasing) !== null && _d !== void 0 ? _d : false);
+        this._maxOpSizeInBytes = ((_e = this.mc.config.getNumber(maxOpSizeInBytesKey)) !== null && _e !== void 0 ? _e : defaultMaxOpSizeInBytes);
         this.maxConsecutiveReconnects =
-            (_e = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _e !== void 0 ? _e : this.defaultMaxConsecutiveReconnects;
+            (_f = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _f !== void 0 ? _f : this.defaultMaxConsecutiveReconnects;
         this._flushMode = runtimeOptions.flushMode;
-        this.garbageCollector = GarbageCollector.create(this, this.runtimeOptions.gcOptions, (nodePath) => this.getGCNodePackagePath(nodePath), () => { var _a; return (_a = this.messageAtLastSummary) === null || _a === void 0 ? void 0 : _a.timestamp; }, context.baseSnapshot, async (id) => readAndParse(this.storage, id), this.mc.logger, existing, metadata, this.context.clientDetails.type === summarizerClientType);
+        const pendingRuntimeState = context.pendingLocalState;
+        const baseSnapshot = (_g = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _g !== void 0 ? _g : context.baseSnapshot;
+        this.garbageCollector = GarbageCollector.create(this, this.runtimeOptions.gcOptions, (nodePath) => this.getGCNodePackagePath(nodePath), () => { var _a; return (_a = this.messageAtLastSummary) === null || _a === void 0 ? void 0 : _a.timestamp; }, baseSnapshot, async (id) => readAndParse(this.storage, id), this.mc.logger, existing, metadata, this.context.clientDetails.type === summarizerClientType);
         const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
         this.summarizerNode = createRootSummarizerNodeWithGC(ChildLogger.create(this.logger, "SummarizerNode"), 
         // Summarize function to call when summarize is called. Summarizer node always tracks summary state.
-        async (fullTree, trackState) => this.summarizeInternal(fullTree, trackState), 
+        async (fullTree, trackState, telemetryContext) => this.summarizeInternal(fullTree, trackState, telemetryContext), 
         // Latest change sequence number, no changes since summary applied yet
         loadedFromSequenceNumber, 
         // Summary reference sequence number, undefined if no summary yet
-        context.baseSnapshot ? loadedFromSequenceNumber : undefined, {
+        baseSnapshot ? loadedFromSequenceNumber : undefined, {
             // Must set to false to prevent sending summary handle which would be pointing to
             // a summary with an older protocol state.
             canReuseHandle: false,
@@ -470,26 +459,31 @@ export class ContainerRuntime extends TypedEventEmitter {
             // If GC should not run, let the summarizer node know so that it does not track GC state.
             gcDisabled: !this.garbageCollector.shouldRunGC,
         });
-        if (this.context.baseSnapshot) {
-            this.summarizerNode.loadBaseSummaryWithoutDifferential(this.context.baseSnapshot);
+        if (baseSnapshot) {
+            this.summarizerNode.loadBaseSummaryWithoutDifferential(baseSnapshot);
         }
-        this.dataStores = new DataStores(getSummaryForDatastores(context.baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap), this.garbageCollector.writeDataAtRoot);
+        this.dataStores = new DataStores(getSummaryForDatastores(baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap), this.garbageCollector.writeDataAtRoot);
         this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (blobId) => this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId }), (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), this, this.logger);
         this.scheduleManager = new ScheduleManager(context.deltaManager, this, ChildLogger.create(this.logger, "ScheduleManager"));
         this.deltaSender = this.deltaManager;
-        this.pendingStateManager = new PendingStateManager(this, async (type, content) => this.applyStashedOp(type, content), this._flushMode, context.pendingLocalState);
+        this.pendingStateManager = new PendingStateManager({
+            applyStashedOp: this.applyStashedOp.bind(this),
+            clientId: () => this.clientId,
+            close: this.closeFn,
+            connected: () => this.connected,
+            flush: this.flush.bind(this),
+            flushMode: () => this.flushMode,
+            reSubmit: this.reSubmit.bind(this),
+            rollback: this.rollback.bind(this),
+            setFlushMode: (mode) => this.setFlushMode(mode),
+        }, this._flushMode, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pending);
         this.context.quorum.on("removeMember", (clientId) => {
             this.clearPartialChunks(clientId);
         });
         this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
-        const { attachState, pendingLocalState } = this.context;
-        this.dirtyContainer = attachState !== AttachState.Attached
-            || ((_f = pendingLocalState) === null || _f === void 0 ? void 0 : _f.pendingStates.length) > 0;
+        this.dirtyContainer = this.context.attachState !== AttachState.Attached
+            || this.pendingStateManager.hasPendingMessages();
         this.context.updateDirtyContainerState(this.dirtyContainer);
-        // Map the deprecated generateSummaries flag to disableSummaries.
-        if (this.runtimeOptions.summaryOptions.generateSummaries === false) {
-            this.runtimeOptions.summaryOptions.disableSummaries = true;
-        }
         if (this.summariesDisabled) {
             this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
         }
@@ -497,9 +491,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             const orderedClientLogger = ChildLogger.create(this.logger, "OrderedClientElection");
             const orderedClientCollection = new OrderedClientCollection(orderedClientLogger, this.context.deltaManager, this.context.quorum);
             const orderedClientElectionForSummarizer = new OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData !== null && electedSummarizerData !== void 0 ? electedSummarizerData : this.context.deltaManager.lastSequenceNumber, SummarizerClientElection.isClientEligible);
-            const summarizerClientElectionEnabled = (_g = this.mc.config.getBoolean("Fluid.ContainerRuntime.summarizerClientElection")) !== null && _g !== void 0 ? _g : ((_h = this.runtimeOptions.summaryOptions) === null || _h === void 0 ? void 0 : _h.summarizerClientElection) === true;
-            const maxOpsSinceLastSummary = (_j = this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary) !== null && _j !== void 0 ? _j : 7000;
-            this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, maxOpsSinceLastSummary, summarizerClientElectionEnabled);
+            this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary, this.summarizerClientElectionEnabled);
             if (this.context.clientDetails.type === summarizerClientType) {
                 this._summarizer = new Summarizer("/_summarizer", this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => RunWhileConnectedCoordinator.create(runtime));
             }
@@ -507,7 +499,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // Only create a SummaryManager and SummarizerClientElection
                 // if summaries are enabled and we are not the summarizer client.
                 const defaultAction = () => {
-                    if (this.summaryCollection.opsSinceLastAck > maxOpsSinceLastSummary) {
+                    if (this.summaryCollection.opsSinceLastAck > this.maxOpsSinceLastSummary) {
                         this.logger.sendErrorEvent({ eventName: "SummaryStatus:Behind" });
                         // unregister default to no log on every op after falling behind
                         // and register summary ack handler to re-register this handler
@@ -528,8 +520,8 @@ export class ContainerRuntime extends TypedEventEmitter {
                 30 * 1000, // 30 sec max delay
                 // throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
                 formExponentialFn({ coefficient: 20, initialDelay: 0 })), {
-                    initialDelayMs: this.runtimeOptions.summaryOptions.initialSummarizerDelayMs,
-                }, this.runtimeOptions.summaryOptions.summarizerOptions);
+                    initialDelayMs: this.initialSummarizerDelayMs,
+                }, this.heuristicsDisabled);
                 this.summaryManager.start();
             }
         }
@@ -552,9 +544,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             assert(!readonly || !this.connected, 0x125 /* "Unsafe to transition to read-only state!" */);
             this.replayPendingStates();
         });
-        if (context.pendingLocalState !== undefined) {
-            this.deltaManager.on("op", this.onOp);
-        }
         // logging hardware telemetry
         logger.sendTelemetryEvent(Object.assign({ eventName: "DeviceSpec" }, getDeviceSpec()));
         let loadSummaryNumber;
@@ -567,7 +556,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             };
             // back-compat 0.59.3000 - Older document may either write summaryCount or not write it at all. If it does
             // not write it, initialize summaryNumber to 0.
-            loadSummaryNumber = (_l = (_k = metadata === null || metadata === void 0 ? void 0 : metadata.summaryNumber) !== null && _k !== void 0 ? _k : metadata === null || metadata === void 0 ? void 0 : metadata.summaryCount) !== null && _l !== void 0 ? _l : 0;
+            loadSummaryNumber = (_j = (_h = metadata === null || metadata === void 0 ? void 0 : metadata.summaryNumber) !== null && _h !== void 0 ? _h : metadata === null || metadata === void 0 ? void 0 : metadata.summaryCount) !== null && _j !== void 0 ? _j : 0;
         }
         else {
             this.createContainerMetadata = {
@@ -603,13 +592,16 @@ export class ContainerRuntime extends TypedEventEmitter {
                 runtimeVersion: pkgVersion,
             },
         });
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", useDataStoreAliasing = false, flushMode = defaultFlushMode, } = runtimeOptions;
-        const storage = context.storage;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", useDataStoreAliasing = false, flushMode = defaultFlushMode, enableOfflineLoad = false, } = runtimeOptions;
+        const pendingRuntimeState = context.pendingLocalState;
+        const baseSnapshot = (_b = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _b !== void 0 ? _b : context.baseSnapshot;
+        const storage = !pendingRuntimeState ?
+            context.storage :
+            new SerializedSnapshotStorage(() => { return context.storage; }, pendingRuntimeState.snapshotBlobs);
         const registry = new FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
-            var _a;
-            const blobId = (_a = context.baseSnapshot) === null || _a === void 0 ? void 0 : _a.blobs[blobName];
-            if (context.baseSnapshot && blobId) {
+            const blobId = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.blobs[blobName];
+            if (baseSnapshot && blobId) {
                 // IContainerContext storage api return type still has undefined in 0.39 package version.
                 // So once we release 0.40 container-defn package we can remove this check.
                 assert(storage !== undefined, 0x1f5 /* "Attached state should have storage" */);
@@ -624,7 +616,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         ]);
         const loadExisting = existing === true || context.existing === true;
         // read snapshot blobs needed for BlobManager to load
-        const blobManagerSnapshot = await BlobManager.load((_b = context.baseSnapshot) === null || _b === void 0 ? void 0 : _b.trees[blobsTreeName], async (id) => {
+        const blobManagerSnapshot = await BlobManager.load(baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[blobsTreeName], async (id) => {
             // IContainerContext storage api return type still has undefined in 0.39 package version.
             // So once we release 0.40 container-defn package we can remove this check.
             assert(storage !== undefined, 0x256 /* "storage undefined in attached container" */);
@@ -654,7 +646,14 @@ export class ContainerRuntime extends TypedEventEmitter {
             loadSequenceNumberVerification,
             useDataStoreAliasing,
             flushMode,
-        }, containerScope, logger, loadExisting, blobManagerSnapshot, requestHandler);
+            enableOfflineLoad,
+        }, containerScope, logger, loadExisting, blobManagerSnapshot, storage, requestHandler);
+        if (pendingRuntimeState) {
+            await runtime.processSavedOps(pendingRuntimeState);
+            // delete these once runtime has seen them to save space
+            pendingRuntimeState.savedOps = [];
+        }
+        await runtime.getSnapshotBlobs();
         return runtime;
     }
     get options() {
@@ -670,7 +669,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         return this.context.deltaManager;
     }
     get storage() {
-        return this.context.storage;
+        return this._storage;
     }
     get reSubmitFn() {
         // eslint-disable-next-line @typescript-eslint/unbound-method
@@ -702,19 +701,70 @@ export class ContainerRuntime extends TypedEventEmitter {
         var _a;
         return (_a = this.summarizerClientElection) === null || _a === void 0 ? void 0 : _a.electedClientId;
     }
-    get summaryConfiguration() {
-        var _a;
-        return Object.assign(Object.assign({}, DefaultSummaryConfiguration), (_a = this.runtimeOptions.summaryOptions) === null || _a === void 0 ? void 0 : _a.summaryConfigOverrides);
-    }
     get disposed() { return this._disposed; }
     get summarizer() {
         assert(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
         return this._summarizer;
     }
-    get summariesDisabled() {
+    isSummariesDisabled() {
+        // back-compat: disableSummaries was moved from ISummaryRuntimeOptions
+        //   to ISummaryConfiguration in 0.60.
+        if (this.runtimeOptions.summaryOptions.disableSummaries === true) {
+            return true;
+        }
+        return this.summaryConfiguration.state === "disabled";
+    }
+    isHeuristicsDisabled() {
         var _a;
-        return this.runtimeOptions.summaryOptions.disableSummaries === true ||
-            ((_a = this.runtimeOptions.summaryOptions.summaryConfigOverrides) === null || _a === void 0 ? void 0 : _a.disableSummaries) === true;
+        // back-compat: disableHeuristics was moved from ISummarizerOptions
+        //   to ISummaryConfiguration in 0.60.
+        if (((_a = this.runtimeOptions.summaryOptions.summarizerOptions) === null || _a === void 0 ? void 0 : _a.disableHeuristics) === true) {
+            return true;
+        }
+        return this.summaryConfiguration.state === "disableHeuristics";
+    }
+    isSummarizerClientElectionEnabled() {
+        var _a;
+        if (this.mc.config.getBoolean("Fluid.ContainerRuntime.summarizerClientElection")) {
+            return (_a = this.mc.config.getBoolean("Fluid.ContainerRuntime.summarizerClientElection")) !== null && _a !== void 0 ? _a : true;
+        }
+        // back-compat: summarizerClientElection was moved from ISummaryRuntimeOptions
+        //   to ISummaryConfiguration in 0.60.
+        if (this.runtimeOptions.summaryOptions.summarizerClientElection === true) {
+            return true;
+        }
+        if (this.summaryConfiguration.state !== "disabled") {
+            return this.summaryConfiguration.summarizerClientElection === true;
+        }
+        else {
+            return false;
+        }
+    }
+    getMaxOpsSinceLastSummary() {
+        // back-compat: maxOpsSinceLastSummary was moved from ISummaryRuntimeOptions
+        //   to ISummaryConfiguration in 0.60.
+        if (this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary !== undefined) {
+            return this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary;
+        }
+        if (this.summaryConfiguration.state !== "disabled") {
+            return this.summaryConfiguration.maxOpsSinceLastSummary;
+        }
+        else {
+            return 0;
+        }
+    }
+    getInitialSummarizerDelayMs() {
+        // back-compat: initialSummarizerDelayMs was moved from ISummaryRuntimeOptions
+        //   to ISummaryConfiguration in 0.60.
+        if (this.runtimeOptions.summaryOptions.initialSummarizerDelayMs !== undefined) {
+            return this.runtimeOptions.summaryOptions.initialSummarizerDelayMs;
+        }
+        if (this.summaryConfiguration.state !== "disabled") {
+            return this.summaryConfiguration.initialSummarizerDelayMs;
+        }
+        else {
+            return 0;
+        }
     }
     dispose(error) {
         var _a;
@@ -855,7 +905,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             message: (_a = extractSummaryMetadataMessage(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.messageAtLastSummary });
         addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata));
     }
-    addContainerStateToSummary(summaryTree, fullTree, trackState) {
+    addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
         var _a;
         this.addMetadataToSummary(summaryTree);
         if (this.chunkMap.size > 0) {
@@ -877,7 +927,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             addTreeToSummary(summaryTree, blobsTreeName, blobManagerSummary);
         }
         if (this.garbageCollector.writeDataAtRoot) {
-            const gcSummary = this.garbageCollector.summarize(fullTree, trackState);
+            const gcSummary = this.garbageCollector.summarize(fullTree, trackState, telemetryContext);
             if (gcSummary !== undefined) {
                 addSummarizeResultToSummary(summaryTree, gcTreeKey, gcSummary);
             }
@@ -898,7 +948,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.resetReconnectCount();
             return true;
         }
-        this.consecutiveReconnects++;
         if (this.consecutiveReconnects === Math.floor(this.maxConsecutiveReconnects / 2)) {
             // If we're halfway through the max reconnects, send an event in order
             // to better identify false positives, if any. If the rate of this event
@@ -907,6 +956,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.mc.logger.sendTelemetryEvent({
                 eventName: "ReconnectsWithNoProgress",
                 attempts: this.consecutiveReconnects,
+                pendingMessages: this.pendingStateManager.pendingMessagesCount,
             });
         }
         return this.consecutiveReconnects < this.maxConsecutiveReconnects;
@@ -962,29 +1012,37 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.verifyNotClosed();
         // There might be no change of state due to Container calling this API after loading runtime.
         const changeOfState = this._connected !== connected;
+        const reconnection = changeOfState && connected;
         this._connected = connected;
-        if (changeOfState) {
-            this.deltaManager.off("op", this.onOp);
-            this.context.pendingLocalState = undefined;
+        if (reconnection) {
+            this.consecutiveReconnects++;
             if (!this.shouldContinueReconnecting()) {
                 this.closeFn(new GenericError(
                 // pre-0.58 error message: MaxReconnectsWithNoProgress
                 "Runtime detected too many reconnects with no progress syncing local ops", undefined, // error
-                { attempts: this.consecutiveReconnects }));
+                {
+                    attempts: this.consecutiveReconnects,
+                    pendingMessages: this.pendingStateManager.pendingMessagesCount,
+                }));
                 return;
             }
+        }
+        if (changeOfState) {
             this.replayPendingStates();
         }
         this.dataStores.setConnectionState(connected, clientId);
         raiseConnectedEvent(this.mc.logger, this, connected, clientId);
     }
     process(messageArg, local) {
-        var _a;
+        var _a, _b;
         this.verifyNotClosed();
         // If it's not message for runtime, bail out right away.
         if (!isRuntimeMessage(messageArg)) {
             return;
         }
+        if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
+            this.savedOps.push(messageArg);
+        }
         // Do shallow copy of message, as methods below will modify it.
         // There might be multiple container instances receiving same message
         // We do not need to make deep copy, as each layer will just replace message.content itself,
@@ -999,8 +1057,14 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Chunk processing must come first given that we will transform the message to the unchunked version
             // once all pieces are available
             message = this.processRemoteChunkedMessage(message);
-            // Call the PendingStateManager to process messages.
-            const { localAck, localOpMetadata } = this.pendingStateManager.processMessage(message, local);
+            let localOpMetadata;
+            if (local) {
+                // Call the PendingStateManager to process local messages.
+                // Do not process local chunked ops until all pieces are available.
+                if (message.type !== ContainerMessageType.ChunkedOp) {
+                    localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
+                }
+            }
             // If there are no more pending messages after processing a local message,
             // the document is no longer dirty.
             if (!this.pendingStateManager.hasPendingMessages()) {
@@ -1008,17 +1072,16 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
             switch (message.type) {
                 case ContainerMessageType.Attach:
-                    this.dataStores.processAttachMessage(message, local || localAck);
+                    this.dataStores.processAttachMessage(message, local);
                     break;
                 case ContainerMessageType.Alias:
                     this.processAliasMessage(message, localOpMetadata, local);
                     break;
                 case ContainerMessageType.FluidDataStoreOp:
-                    // if localAck === true, treat this as a local op because it's one we sent on a previous container
-                    this.dataStores.processFluidDataStoreOp(message, local || localAck, localOpMetadata);
+                    this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
                     break;
                 case ContainerMessageType.BlobAttach:
-                    assert((_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.blobId, 0x12a /* "Missing blob id on metadata" */);
+                    assert((_b = message === null || message === void 0 ? void 0 : message.metadata) === null || _b === void 0 ? void 0 : _b.blobId, 0x12a /* "Missing blob id on metadata" */);
                     this.blobManager.processBlobAttachOp(message.metadata.blobId, local);
                     break;
                 default:
@@ -1110,18 +1173,32 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         const savedFlushMode = this.flushMode;
         this.setFlushMode(FlushMode.TurnBased);
-        this.trackOrderSequentiallyCalls(callback);
-        this.flush();
-        this.setFlushMode(savedFlushMode);
+        try {
+            this.trackOrderSequentiallyCalls(callback);
+            this.flush();
+        }
+        finally {
+            this.setFlushMode(savedFlushMode);
+        }
     }
     trackOrderSequentiallyCalls(callback) {
+        let checkpoint;
+        if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
+            checkpoint = this.pendingStateManager.checkpoint();
+        }
         try {
             this._orderSequentiallyCalls++;
             callback();
         }
         catch (error) {
-            // pre-0.58 error message: orderSequentiallyCallbackException
-            this.closeFn(new GenericError("orderSequentially callback exception", error));
+            if (checkpoint) {
+                // This will throw and close the container if rollback fails
+                checkpoint.rollback();
+            }
+            else {
+                // pre-0.58 error message: orderSequentiallyCallbackException
+                this.closeFn(new GenericError("orderSequentially callback exception", error));
+            }
             throw error; // throw the original error for the consumer of the runtime
         }
         finally {
@@ -1293,17 +1370,18 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @param blobRedirectTable - A table passed during the attach process. While detached, blob upload is supported
      * using IDs generated locally. After attach, these IDs cannot be used, so this table maps the old local IDs to the
      * new storage IDs so requests can be redirected.
+     * @param telemetryContext - summary data passed through the layers for telemetry purposes
      */
-    createSummary(blobRedirectTable) {
+    createSummary(blobRedirectTable, telemetryContext) {
         if (blobRedirectTable) {
             this.blobManager.setRedirectTable(blobRedirectTable);
         }
-        const summarizeResult = this.dataStores.createSummary();
+        const summarizeResult = this.dataStores.createSummary(telemetryContext);
         if (!this.disableIsolatedChannels) {
             // Wrap data store summaries in .channels subtree.
             wrapSummaryInChannelsTree(summarizeResult);
         }
-        this.addContainerStateToSummary(summarizeResult, true /* fullTree */, false /* trackState */);
+        this.addContainerStateToSummary(summarizeResult, true /* fullTree */, false /* trackState */, telemetryContext);
         return summarizeResult.summary;
     }
     async getAbsoluteUrl(relativeUrl) {
@@ -1315,15 +1393,15 @@ export class ContainerRuntime extends TypedEventEmitter {
         }
         return this.context.getAbsoluteUrl(relativeUrl);
     }
-    async summarizeInternal(fullTree, trackState) {
-        const summarizeResult = await this.dataStores.summarize(fullTree, trackState);
+    async summarizeInternal(fullTree, trackState, telemetryContext) {
+        const summarizeResult = await this.dataStores.summarize(fullTree, trackState, telemetryContext);
         let pathPartsForChildren;
         if (!this.disableIsolatedChannels) {
             // Wrap data store summaries in .channels subtree.
             wrapSummaryInChannelsTree(summarizeResult);
             pathPartsForChildren = [channelsTreeName];
         }
-        this.addContainerStateToSummary(summarizeResult, fullTree, trackState);
+        this.addContainerStateToSummary(summarizeResult, fullTree, trackState, telemetryContext);
         return Object.assign(Object.assign({}, summarizeResult), { id: "", pathPartsForChildren });
     }
     /**
@@ -1336,7 +1414,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         if (runGC) {
             gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
         }
-        const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState);
+        const telemetryContext = new TelemetryContext();
+        const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
+        this.logger.sendTelemetryEvent({ eventName: "SummarizeTelemetry", details: telemetryContext.serialize() });
         assert(summary.type === SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
         return { stats, summary, gcStats };
     }
@@ -1700,9 +1780,6 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     submit(type, content, localOpMetadata = undefined, opMetadata = undefined) {
         this.verifyNotClosed();
-        if (this.context.pendingLocalState !== undefined) {
-            this.closeFn(new GenericError("containerRuntimeSubmitWithPendingLocalState"));
-        }
         // There should be no ops in detached container state!
         assert(this.attachState !== AttachState.Detached, 0x132 /* "sending ops in detached container" */);
         let clientSequenceNumber = -1;
@@ -1833,6 +1910,17 @@ export class ContainerRuntime extends TypedEventEmitter {
                 unreachableCase(type, `Unknown ContainerMessageType: ${type}`);
         }
     }
+    rollback(type, content, localOpMetadata) {
+        switch (type) {
+            case ContainerMessageType.FluidDataStoreOp:
+                // For operations, call rollbackDataStoreOp which will find the right store
+                // and trigger rollback on it.
+                this.dataStores.rollbackDataStoreOp(content, localOpMetadata);
+                break;
+            default:
+                throw new Error(`Can't rollback ${type}`);
+        }
+    }
     /** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
     async refreshLatestSummaryAck(proposalHandle, ackHandle, summaryRefSeq, summaryLogger) {
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
@@ -1877,8 +1965,43 @@ export class ContainerRuntime extends TypedEventEmitter {
             return maybeSnapshot;
         });
     }
+    notifyAttaching(snapshot) {
+        var _a;
+        if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
+            this.baseSnapshotBlobs = SerializedSnapshotStorage.serializeTreeWithBlobContents(snapshot);
+        }
+    }
+    async getSnapshotBlobs() {
+        var _a;
+        if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) ||
+            this.attachState !== AttachState.Attached || this.context.pendingLocalState) {
+            return;
+        }
+        assert(!!this.context.baseSnapshot, 0x2e5 /* "Must have a base snapshot" */);
+        this.baseSnapshotBlobs = await SerializedSnapshotStorage.serializeTree(this.context.baseSnapshot, this.storage);
+    }
     getPendingLocalState() {
-        return this.pendingStateManager.getLocalState();
+        var _a;
+        if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad)) {
+            throw new UsageError("can't get state when offline load disabled");
+        }
+        const previousPendingState = this.context.pendingLocalState;
+        if (previousPendingState) {
+            return {
+                pending: this.pendingStateManager.getLocalState(),
+                snapshotBlobs: previousPendingState.snapshotBlobs,
+                baseSnapshot: previousPendingState.baseSnapshot,
+                savedOps: this.savedOps,
+            };
+        }
+        assert(!!this.context.baseSnapshot, 0x2e6 /* "Must have a base snapshot" */);
+        assert(!!this.baseSnapshotBlobs, 0x2e7 /* "Must serialize base snapshot blobs before getting runtime state" */);
+        return {
+            pending: this.pendingStateManager.getLocalState(),
+            snapshotBlobs: this.baseSnapshotBlobs,
+            baseSnapshot: this.context.baseSnapshot,
+            savedOps: this.savedOps,
+        };
     }
     /**
      * * Forms a function that will request a Summarizer.
@@ -1906,6 +2029,15 @@ export class ContainerRuntime extends TypedEventEmitter {
             return summarizer;
         };
     }
+    async processSavedOps(state) {
+        for (const op of state.savedOps) {
+            this.process(op, false);
+            await this.pendingStateManager.applyStashedOpsAt(op.sequenceNumber);
+        }
+        // we may not have seen every sequence number (because of system ops) so apply everything once we
+        // don't have any more saved ops
+        await this.pendingStateManager.applyStashedOpsAt();
+    }
 }
 /**
  * Wait for a specific sequence number. Promise should resolve when we reach that number,