@fluidframework/container-runtime 0.59.4000 → 1.0.1

package/dist/containerRuntime.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ContainerRuntime = exports.getDeviceSpec = exports.agentSchedulerId = exports.ScheduleManager = exports.unpackRuntimeMessage = exports.isRuntimeMessage = exports.RuntimeMessage = exports.RuntimeHeaders = exports.ContainerMessageType = void 0;
+exports.ContainerRuntime = exports.getDeviceSpec = exports.agentSchedulerId = exports.ScheduleManager = exports.unpackRuntimeMessage = exports.isRuntimeMessage = exports.RuntimeMessage = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
 const container_definitions_1 = require("@fluidframework/container-definitions");
 const common_utils_1 = require("@fluidframework/common-utils");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
@@ -31,6 +31,7 @@ const runWhileConnectedCoordinator_1 = require("./runWhileConnectedCoordinator")
 const garbageCollection_1 = require("./garbageCollection");
 const dataStore_1 = require("./dataStore");
 const batchTracker_1 = require("./batchTracker");
+const serializedSnapshotStorage_1 = require("./serializedSnapshotStorage");
 const opTelemetry_1 = require("./opTelemetry");
 var ContainerMessageType;
 (function (ContainerMessageType) {
@@ -47,17 +48,16 @@ var ContainerMessageType;
     // Sets the alias of a root data store
     ContainerMessageType["Alias"] = "alias";
 })(ContainerMessageType = exports.ContainerMessageType || (exports.ContainerMessageType = {}));
-// Consider idle 5s of no activity. And snapshot if a minute has gone by with no snapshot.
-const IdleDetectionTime = 5000;
-const DefaultSummaryConfiguration = {
-    idleTime: IdleDetectionTime * 3,
-    maxTime: IdleDetectionTime * 12,
-    // Summarize if 1000 ops received since last snapshot.
-    maxOps: 1000,
-    // Wait 10 minutes for summary ack
-    // this is less than maxSummarizeAckWaitTime
-    // the min of the two will be chosen
-    maxAckWaitTime: 600000,
+exports.DefaultSummaryConfiguration = {
+    state: "enabled",
+    idleTime: 5000 * 3,
+    maxTime: 5000 * 12,
+    maxOps: 100,
+    minOpsForLastSummaryAttempt: 10,
+    maxAckWaitTime: 6 * 10 * 1000,
+    maxOpsSinceLastSummary: 7000,
+    initialSummarizerDelayMs: 5000,
+    summarizerClientElection: false,
 };
 /**
  * Accepted header keys for requests coming to the runtime.
@@ -379,44 +379,26 @@ exports.getDeviceSpec = getDeviceSpec;
  * It will define the store level mappings.
  */
 class ContainerRuntime extends common_utils_1.TypedEventEmitter {
-    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, requestHandler) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l;
+    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, requestHandler, summaryConfiguration) {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j;
+        if (summaryConfiguration === void 0) { summaryConfiguration = Object.assign(Object.assign({}, exports.DefaultSummaryConfiguration), (_a = runtimeOptions.summaryOptions) === null || _a === void 0 ? void 0 : _a.summaryConfigOverrides); }
         super();
         this.context = context;
         this.registry = registry;
         this.runtimeOptions = runtimeOptions;
         this.containerScope = containerScope;
         this.logger = logger;
+        this._storage = _storage;
         this.requestHandler = requestHandler;
+        this.summaryConfiguration = summaryConfiguration;
         this.defaultMaxConsecutiveReconnects = 15;
         this._orderSequentiallyCalls = 0;
         this.needsFlush = false;
         this.flushTrigger = false;
-        this.paused = false;
+        this.savedOps = [];
         this.consecutiveReconnects = 0;
         this._disposed = false;
         this.emitDirtyDocumentEvent = true;
-        /**
-         * Used to apply stashed ops at their reference sequence number.
-         * Normal op processing is synchronous, but applying stashed ops is async since the
-         * data store may not be loaded yet, so we pause DeltaManager between ops.
-         * It's also important that we see each op so we know all stashed ops have
-         * been applied by "connected" event, but process() doesn't see system ops,
-         * so we listen directly from DeltaManager instead.
-         */
-        this.onOp = (op) => {
-            (0, common_utils_1.assert)(!this.paused, 0x128 /* "Container should not already be paused before applying stashed ops" */);
-            this.paused = true;
-            // eslint-disable-next-line @typescript-eslint/no-floating-promises
-            this.context.deltaManager.inbound.pause();
-            const stashP = this.pendingStateManager.applyStashedOpsAt(op.sequenceNumber);
-            stashP.then(() => {
-                this.paused = false;
-                this.context.deltaManager.inbound.resume();
-            }, (error) => {
-                this.closeFn((0, telemetry_utils_1.normalizeError)(error));
-            });
-        };
         this.summarizeOnDemand = (...args) => {
             if (this.clientDetails.type === summarizerClientElection_1.summarizerClientType) {
                 return this.summarizer.summarizeOnDemand(...args);
@@ -447,27 +429,34 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         };
         this.messageAtLastSummary = metadata === null || metadata === void 0 ? void 0 : metadata.message;
         // Default to false (enabled).
-        this.disableIsolatedChannels = (_a = this.runtimeOptions.summaryOptions.disableIsolatedChannels) !== null && _a !== void 0 ? _a : false;
+        this.disableIsolatedChannels = (_b = this.runtimeOptions.summaryOptions.disableIsolatedChannels) !== null && _b !== void 0 ? _b : false;
         this._connected = this.context.connected;
         this.chunkMap = new Map(chunks);
         this.handleContext = new containerHandleContext_1.ContainerFluidHandleContext("", this);
         this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(this.logger, "ContainerRuntime"));
+        this.summariesDisabled = this.isSummariesDisabled();
+        this.heuristicsDisabled = this.isHeuristicsDisabled();
+        this.summarizerClientElectionEnabled = this.isSummarizerClientElectionEnabled();
+        this.maxOpsSinceLastSummary = this.getMaxOpsSinceLastSummary();
+        this.initialSummarizerDelayMs = this.getInitialSummarizerDelayMs();
         this._aliasingEnabled =
-            ((_b = this.mc.config.getBoolean(useDataStoreAliasingKey)) !== null && _b !== void 0 ? _b : false) ||
-                ((_c = runtimeOptions.useDataStoreAliasing) !== null && _c !== void 0 ? _c : false);
-        this._maxOpSizeInBytes = ((_d = this.mc.config.getNumber(maxOpSizeInBytesKey)) !== null && _d !== void 0 ? _d : defaultMaxOpSizeInBytes);
+            ((_c = this.mc.config.getBoolean(useDataStoreAliasingKey)) !== null && _c !== void 0 ? _c : false) ||
+                ((_d = runtimeOptions.useDataStoreAliasing) !== null && _d !== void 0 ? _d : false);
+        this._maxOpSizeInBytes = ((_e = this.mc.config.getNumber(maxOpSizeInBytesKey)) !== null && _e !== void 0 ? _e : defaultMaxOpSizeInBytes);
         this.maxConsecutiveReconnects =
-            (_e = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _e !== void 0 ? _e : this.defaultMaxConsecutiveReconnects;
+            (_f = this.mc.config.getNumber(maxConsecutiveReconnectsKey)) !== null && _f !== void 0 ? _f : this.defaultMaxConsecutiveReconnects;
         this._flushMode = runtimeOptions.flushMode;
-        this.garbageCollector = garbageCollection_1.GarbageCollector.create(this, this.runtimeOptions.gcOptions, (nodePath) => this.getGCNodePackagePath(nodePath), () => { var _a; return (_a = this.messageAtLastSummary) === null || _a === void 0 ? void 0 : _a.timestamp; }, context.baseSnapshot, async (id) => (0, driver_utils_1.readAndParse)(this.storage, id), this.mc.logger, existing, metadata, this.context.clientDetails.type === summarizerClientElection_1.summarizerClientType);
+        const pendingRuntimeState = context.pendingLocalState;
+        const baseSnapshot = (_g = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _g !== void 0 ? _g : context.baseSnapshot;
+        this.garbageCollector = garbageCollection_1.GarbageCollector.create(this, this.runtimeOptions.gcOptions, (nodePath) => this.getGCNodePackagePath(nodePath), () => { var _a; return (_a = this.messageAtLastSummary) === null || _a === void 0 ? void 0 : _a.timestamp; }, baseSnapshot, async (id) => (0, driver_utils_1.readAndParse)(this.storage, id), this.mc.logger, existing, metadata, this.context.clientDetails.type === summarizerClientElection_1.summarizerClientType);
         const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
         this.summarizerNode = (0, runtime_utils_1.createRootSummarizerNodeWithGC)(telemetry_utils_1.ChildLogger.create(this.logger, "SummarizerNode"), 
         // Summarize function to call when summarize is called. Summarizer node always tracks summary state.
-        async (fullTree, trackState) => this.summarizeInternal(fullTree, trackState), 
+        async (fullTree, trackState, telemetryContext) => this.summarizeInternal(fullTree, trackState, telemetryContext), 
         // Latest change sequence number, no changes since summary applied yet
         loadedFromSequenceNumber, 
         // Summary reference sequence number, undefined if no summary yet
-        context.baseSnapshot ? loadedFromSequenceNumber : undefined, {
+        baseSnapshot ? loadedFromSequenceNumber : undefined, {
             // Must set to false to prevent sending summary handle which would be pointing to
             // a summary with an older protocol state.
             canReuseHandle: false,
@@ -477,26 +466,31 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // If GC should not run, let the summarizer node know so that it does not track GC state.
             gcDisabled: !this.garbageCollector.shouldRunGC,
         });
-        if (this.context.baseSnapshot) {
-            this.summarizerNode.loadBaseSummaryWithoutDifferential(this.context.baseSnapshot);
+        if (baseSnapshot) {
+            this.summarizerNode.loadBaseSummaryWithoutDifferential(baseSnapshot);
         }
-        this.dataStores = new dataStores_1.DataStores((0, dataStores_1.getSummaryForDatastores)(context.baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap), this.garbageCollector.writeDataAtRoot);
+        this.dataStores = new dataStores_1.DataStores((0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id, createParam) => (summarizeInternal, getGCDataFn, getBaseGCDetailsFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn, getBaseGCDetailsFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, async () => this.garbageCollector.getBaseGCDetails(), (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), new Map(dataStoreAliasMap), this.garbageCollector.writeDataAtRoot);
         this.blobManager = new blobManager_1.BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (blobId) => this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId }), (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), this, this.logger);
         this.scheduleManager = new ScheduleManager(context.deltaManager, this, telemetry_utils_1.ChildLogger.create(this.logger, "ScheduleManager"));
         this.deltaSender = this.deltaManager;
-        this.pendingStateManager = new pendingStateManager_1.PendingStateManager(this, async (type, content) => this.applyStashedOp(type, content), this._flushMode, context.pendingLocalState);
+        this.pendingStateManager = new pendingStateManager_1.PendingStateManager({
+            applyStashedOp: this.applyStashedOp.bind(this),
+            clientId: () => this.clientId,
+            close: this.closeFn,
+            connected: () => this.connected,
+            flush: this.flush.bind(this),
+            flushMode: () => this.flushMode,
+            reSubmit: this.reSubmit.bind(this),
+            rollback: this.rollback.bind(this),
+            setFlushMode: (mode) => this.setFlushMode(mode),
+        }, this._flushMode, pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.pending);
         this.context.quorum.on("removeMember", (clientId) => {
             this.clearPartialChunks(clientId);
         });
         this.summaryCollection = new summaryCollection_1.SummaryCollection(this.deltaManager, this.logger);
-        const { attachState, pendingLocalState } = this.context;
-        this.dirtyContainer = attachState !== container_definitions_1.AttachState.Attached
-            || ((_f = pendingLocalState) === null || _f === void 0 ? void 0 : _f.pendingStates.length) > 0;
+        this.dirtyContainer = this.context.attachState !== container_definitions_1.AttachState.Attached
+            || this.pendingStateManager.hasPendingMessages();
         this.context.updateDirtyContainerState(this.dirtyContainer);
-        // Map the deprecated generateSummaries flag to disableSummaries.
-        if (this.runtimeOptions.summaryOptions.generateSummaries === false) {
-            this.runtimeOptions.summaryOptions.disableSummaries = true;
-        }
         if (this.summariesDisabled) {
             this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
         }
@@ -504,9 +498,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             const orderedClientLogger = telemetry_utils_1.ChildLogger.create(this.logger, "OrderedClientElection");
             const orderedClientCollection = new orderedClientElection_1.OrderedClientCollection(orderedClientLogger, this.context.deltaManager, this.context.quorum);
             const orderedClientElectionForSummarizer = new orderedClientElection_1.OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData !== null && electedSummarizerData !== void 0 ? electedSummarizerData : this.context.deltaManager.lastSequenceNumber, summarizerClientElection_1.SummarizerClientElection.isClientEligible);
-            const summarizerClientElectionEnabled = (_g = this.mc.config.getBoolean("Fluid.ContainerRuntime.summarizerClientElection")) !== null && _g !== void 0 ? _g : ((_h = this.runtimeOptions.summaryOptions) === null || _h === void 0 ? void 0 : _h.summarizerClientElection) === true;
-            const maxOpsSinceLastSummary = (_j = this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary) !== null && _j !== void 0 ? _j : 7000;
-            this.summarizerClientElection = new summarizerClientElection_1.SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, maxOpsSinceLastSummary, summarizerClientElectionEnabled);
+            this.summarizerClientElection = new summarizerClientElection_1.SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary, this.summarizerClientElectionEnabled);
             if (this.context.clientDetails.type === summarizerClientElection_1.summarizerClientType) {
                 this._summarizer = new summarizer_1.Summarizer("/_summarizer", this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => runWhileConnectedCoordinator_1.RunWhileConnectedCoordinator.create(runtime));
             }
@@ -514,7 +506,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 // Only create a SummaryManager and SummarizerClientElection
                 // if summaries are enabled and we are not the summarizer client.
                 const defaultAction = () => {
-                    if (this.summaryCollection.opsSinceLastAck > maxOpsSinceLastSummary) {
+                    if (this.summaryCollection.opsSinceLastAck > this.maxOpsSinceLastSummary) {
                         this.logger.sendErrorEvent({ eventName: "SummaryStatus:Behind" });
                         // unregister default to no log on every op after falling behind
                         // and register summary ack handler to re-register this handler
@@ -535,8 +527,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 30 * 1000, // 30 sec max delay
                 // throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
                 (0, throttler_1.formExponentialFn)({ coefficient: 20, initialDelay: 0 })), {
-                    initialDelayMs: this.runtimeOptions.summaryOptions.initialSummarizerDelayMs,
-                }, this.runtimeOptions.summaryOptions.summarizerOptions);
+                    initialDelayMs: this.initialSummarizerDelayMs,
+                }, this.heuristicsDisabled);
                 this.summaryManager.start();
             }
         }
@@ -559,9 +551,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             (0, common_utils_1.assert)(!readonly || !this.connected, 0x125 /* "Unsafe to transition to read-only state!" */);
             this.replayPendingStates();
         });
-        if (context.pendingLocalState !== undefined) {
-            this.deltaManager.on("op", this.onOp);
-        }
         // logging hardware telemetry
         logger.sendTelemetryEvent(Object.assign({ eventName: "DeviceSpec" }, getDeviceSpec()));
         let loadSummaryNumber;
@@ -574,7 +563,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             };
             // back-compat 0.59.3000 - Older document may either write summaryCount or not write it at all. If it does
             // not write it, initialize summaryNumber to 0.
-            loadSummaryNumber = (_l = (_k = metadata === null || metadata === void 0 ? void 0 : metadata.summaryNumber) !== null && _k !== void 0 ? _k : metadata === null || metadata === void 0 ? void 0 : metadata.summaryCount) !== null && _l !== void 0 ? _l : 0;
+            loadSummaryNumber = (_j = (_h = metadata === null || metadata === void 0 ? void 0 : metadata.summaryNumber) !== null && _h !== void 0 ? _h : metadata === null || metadata === void 0 ? void 0 : metadata.summaryCount) !== null && _j !== void 0 ? _j : 0;
         }
         else {
             this.createContainerMetadata = {
@@ -610,13 +599,16 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 runtimeVersion: packageVersion_1.pkgVersion,
             },
         });
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", useDataStoreAliasing = false, flushMode = defaultFlushMode, } = runtimeOptions;
-        const storage = context.storage;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", useDataStoreAliasing = false, flushMode = defaultFlushMode, enableOfflineLoad = false, } = runtimeOptions;
+        const pendingRuntimeState = context.pendingLocalState;
+        const baseSnapshot = (_b = pendingRuntimeState === null || pendingRuntimeState === void 0 ? void 0 : pendingRuntimeState.baseSnapshot) !== null && _b !== void 0 ? _b : context.baseSnapshot;
+        const storage = !pendingRuntimeState ?
+            context.storage :
+            new serializedSnapshotStorage_1.SerializedSnapshotStorage(() => { return context.storage; }, pendingRuntimeState.snapshotBlobs);
         const registry = new dataStoreRegistry_1.FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
-            var _a;
-            const blobId = (_a = context.baseSnapshot) === null || _a === void 0 ? void 0 : _a.blobs[blobName];
-            if (context.baseSnapshot && blobId) {
+            const blobId = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.blobs[blobName];
+            if (baseSnapshot && blobId) {
                 // IContainerContext storage api return type still has undefined in 0.39 package version.
                 // So once we release 0.40 container-defn package we can remove this check.
                 (0, common_utils_1.assert)(storage !== undefined, 0x1f5 /* "Attached state should have storage" */);
@@ -631,7 +623,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         ]);
         const loadExisting = existing === true || context.existing === true;
         // read snapshot blobs needed for BlobManager to load
-        const blobManagerSnapshot = await blobManager_1.BlobManager.load((_b = context.baseSnapshot) === null || _b === void 0 ? void 0 : _b.trees[summaryFormat_1.blobsTreeName], async (id) => {
+        const blobManagerSnapshot = await blobManager_1.BlobManager.load(baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[summaryFormat_1.blobsTreeName], async (id) => {
             // IContainerContext storage api return type still has undefined in 0.39 package version.
             // So once we release 0.40 container-defn package we can remove this check.
             (0, common_utils_1.assert)(storage !== undefined, 0x256 /* "storage undefined in attached container" */);
@@ -661,7 +653,14 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             loadSequenceNumberVerification,
             useDataStoreAliasing,
             flushMode,
-        }, containerScope, logger, loadExisting, blobManagerSnapshot, requestHandler);
+            enableOfflineLoad,
+        }, containerScope, logger, loadExisting, blobManagerSnapshot, storage, requestHandler);
+        if (pendingRuntimeState) {
+            await runtime.processSavedOps(pendingRuntimeState);
+            // delete these once runtime has seen them to save space
+            pendingRuntimeState.savedOps = [];
+        }
+        await runtime.getSnapshotBlobs();
         return runtime;
     }
     get options() {
@@ -677,7 +676,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         return this.context.deltaManager;
     }
     get storage() {
-        return this.context.storage;
+        return this._storage;
     }
     get reSubmitFn() {
         // eslint-disable-next-line @typescript-eslint/unbound-method
@@ -709,19 +708,70 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         var _a;
         return (_a = this.summarizerClientElection) === null || _a === void 0 ? void 0 : _a.electedClientId;
     }
-    get summaryConfiguration() {
-        var _a;
-        return Object.assign(Object.assign({}, DefaultSummaryConfiguration), (_a = this.runtimeOptions.summaryOptions) === null || _a === void 0 ? void 0 : _a.summaryConfigOverrides);
-    }
     get disposed() { return this._disposed; }
     get summarizer() {
         (0, common_utils_1.assert)(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
         return this._summarizer;
     }
-    get summariesDisabled() {
+    isSummariesDisabled() {
+        // back-compat: disableSummaries was moved from ISummaryRuntimeOptions
+        //   to ISummaryConfiguration in 0.60.
+        if (this.runtimeOptions.summaryOptions.disableSummaries === true) {
+            return true;
+        }
+        return this.summaryConfiguration.state === "disabled";
+    }
+    isHeuristicsDisabled() {
         var _a;
-        return this.runtimeOptions.summaryOptions.disableSummaries === true ||
-            ((_a = this.runtimeOptions.summaryOptions.summaryConfigOverrides) === null || _a === void 0 ? void 0 : _a.disableSummaries) === true;
+        // back-compat: disableHeuristics was moved from ISummarizerOptions
+        //   to ISummaryConfiguration in 0.60.
+        if (((_a = this.runtimeOptions.summaryOptions.summarizerOptions) === null || _a === void 0 ? void 0 : _a.disableHeuristics) === true) {
+            return true;
+        }
+        return this.summaryConfiguration.state === "disableHeuristics";
+    }
+    isSummarizerClientElectionEnabled() {
+        var _a;
+        if (this.mc.config.getBoolean("Fluid.ContainerRuntime.summarizerClientElection")) {
+            return (_a = this.mc.config.getBoolean("Fluid.ContainerRuntime.summarizerClientElection")) !== null && _a !== void 0 ? _a : true;
+        }
+        // back-compat: summarizerClientElection was moved from ISummaryRuntimeOptions
+        //   to ISummaryConfiguration in 0.60.
+        if (this.runtimeOptions.summaryOptions.summarizerClientElection === true) {
+            return true;
+        }
+        if (this.summaryConfiguration.state !== "disabled") {
+            return this.summaryConfiguration.summarizerClientElection === true;
+        }
+        else {
+            return false;
+        }
+    }
+    getMaxOpsSinceLastSummary() {
+        // back-compat: maxOpsSinceLastSummary was moved from ISummaryRuntimeOptions
+        //   to ISummaryConfiguration in 0.60.
+        if (this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary !== undefined) {
+            return this.runtimeOptions.summaryOptions.maxOpsSinceLastSummary;
+        }
+        if (this.summaryConfiguration.state !== "disabled") {
+            return this.summaryConfiguration.maxOpsSinceLastSummary;
+        }
+        else {
+            return 0;
+        }
+    }
+    getInitialSummarizerDelayMs() {
+        // back-compat: initialSummarizerDelayMs was moved from ISummaryRuntimeOptions
+        //   to ISummaryConfiguration in 0.60.
+        if (this.runtimeOptions.summaryOptions.initialSummarizerDelayMs !== undefined) {
+            return this.runtimeOptions.summaryOptions.initialSummarizerDelayMs;
+        }
+        if (this.summaryConfiguration.state !== "disabled") {
+            return this.summaryConfiguration.initialSummarizerDelayMs;
+        }
+        else {
+            return 0;
+        }
     }
     dispose(error) {
         var _a;
@@ -862,7 +912,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             message: (_a = (0, summaryFormat_1.extractSummaryMetadataMessage)(this.deltaManager.lastMessage)) !== null && _a !== void 0 ? _a : this.messageAtLastSummary });
         (0, runtime_utils_1.addBlobToSummary)(summaryTree, summaryFormat_1.metadataBlobName, JSON.stringify(metadata));
     }
-    addContainerStateToSummary(summaryTree, fullTree, trackState) {
+    addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
         var _a;
         this.addMetadataToSummary(summaryTree);
         if (this.chunkMap.size > 0) {
@@ -884,7 +934,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             (0, runtime_utils_1.addTreeToSummary)(summaryTree, summaryFormat_1.blobsTreeName, blobManagerSummary);
         }
         if (this.garbageCollector.writeDataAtRoot) {
-            const gcSummary = this.garbageCollector.summarize(fullTree, trackState);
+            const gcSummary = this.garbageCollector.summarize(fullTree, trackState, telemetryContext);
             if (gcSummary !== undefined) {
                 (0, runtime_utils_1.addSummarizeResultToSummary)(summaryTree, garbageCollection_1.gcTreeKey, gcSummary);
             }
@@ -905,7 +955,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this.resetReconnectCount();
             return true;
         }
-        this.consecutiveReconnects++;
         if (this.consecutiveReconnects === Math.floor(this.maxConsecutiveReconnects / 2)) {
             // If we're halfway through the max reconnects, send an event in order
             // to better identify false positives, if any. If the rate of this event
@@ -914,6 +963,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this.mc.logger.sendTelemetryEvent({
                 eventName: "ReconnectsWithNoProgress",
                 attempts: this.consecutiveReconnects,
+                pendingMessages: this.pendingStateManager.pendingMessagesCount,
             });
         }
         return this.consecutiveReconnects < this.maxConsecutiveReconnects;
@@ -969,29 +1019,37 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.verifyNotClosed();
         // There might be no change of state due to Container calling this API after loading runtime.
         const changeOfState = this._connected !== connected;
+        const reconnection = changeOfState && connected;
         this._connected = connected;
-        if (changeOfState) {
-            this.deltaManager.off("op", this.onOp);
-            this.context.pendingLocalState = undefined;
+        if (reconnection) {
+            this.consecutiveReconnects++;
             if (!this.shouldContinueReconnecting()) {
                 this.closeFn(new container_utils_1.GenericError(
                 // pre-0.58 error message: MaxReconnectsWithNoProgress
                 "Runtime detected too many reconnects with no progress syncing local ops", undefined, // error
-                { attempts: this.consecutiveReconnects }));
+                {
+                    attempts: this.consecutiveReconnects,
+                    pendingMessages: this.pendingStateManager.pendingMessagesCount,
+                }));
                 return;
             }
+        }
+        if (changeOfState) {
             this.replayPendingStates();
         }
         this.dataStores.setConnectionState(connected, clientId);
         (0, telemetry_utils_1.raiseConnectedEvent)(this.mc.logger, this, connected, clientId);
     }
     process(messageArg, local) {
-        var _a;
+        var _a, _b;
         this.verifyNotClosed();
         // If it's not message for runtime, bail out right away.
         if (!isRuntimeMessage(messageArg)) {
             return;
         }
+        if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
+            this.savedOps.push(messageArg);
+        }
         // Do shallow copy of message, as methods below will modify it.
         // There might be multiple container instances receiving same message
         // We do not need to make deep copy, as each layer will just replace message.content itself,
@@ -1006,8 +1064,14 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // Chunk processing must come first given that we will transform the message to the unchunked version
             // once all pieces are available
             message = this.processRemoteChunkedMessage(message);
-            // Call the PendingStateManager to process messages.
-            const { localAck, localOpMetadata } = this.pendingStateManager.processMessage(message, local);
+            let localOpMetadata;
+            if (local) {
+                // Call the PendingStateManager to process local messages.
+                // Do not process local chunked ops until all pieces are available.
+                if (message.type !== ContainerMessageType.ChunkedOp) {
+                    localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
+                }
+            }
             // If there are no more pending messages after processing a local message,
             // the document is no longer dirty.
             if (!this.pendingStateManager.hasPendingMessages()) {
@@ -1015,17 +1079,16 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
             switch (message.type) {
                 case ContainerMessageType.Attach:
-                    this.dataStores.processAttachMessage(message, local || localAck);
+                    this.dataStores.processAttachMessage(message, local);
                     break;
                 case ContainerMessageType.Alias:
                     this.processAliasMessage(message, localOpMetadata, local);
                     break;
                 case ContainerMessageType.FluidDataStoreOp:
-                    // if localAck === true, treat this as a local op because it's one we sent on a previous container
-                    this.dataStores.processFluidDataStoreOp(message, local || localAck, localOpMetadata);
+                    this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
                     break;
                 case ContainerMessageType.BlobAttach:
-                    (0, common_utils_1.assert)((_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.blobId, 0x12a /* "Missing blob id on metadata" */);
+                    (0, common_utils_1.assert)((_b = message === null || message === void 0 ? void 0 : message.metadata) === null || _b === void 0 ? void 0 : _b.blobId, 0x12a /* "Missing blob id on metadata" */);
                     this.blobManager.processBlobAttachOp(message.metadata.blobId, local);
                     break;
                 default:
@@ -1117,18 +1180,32 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         }
         const savedFlushMode = this.flushMode;
         this.setFlushMode(runtime_definitions_1.FlushMode.TurnBased);
-        this.trackOrderSequentiallyCalls(callback);
-        this.flush();
-        this.setFlushMode(savedFlushMode);
+        try {
+            this.trackOrderSequentiallyCalls(callback);
+            this.flush();
+        }
+        finally {
+            this.setFlushMode(savedFlushMode);
+        }
     }
     trackOrderSequentiallyCalls(callback) {
+        let checkpoint;
+        if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
+            checkpoint = this.pendingStateManager.checkpoint();
+        }
         try {
             this._orderSequentiallyCalls++;
             callback();
         }
         catch (error) {
-            // pre-0.58 error message: orderSequentiallyCallbackException
-            this.closeFn(new container_utils_1.GenericError("orderSequentially callback exception", error));
+            if (checkpoint) {
+                // This will throw and close the container if rollback fails
+                checkpoint.rollback();
+            }
+            else {
+                // pre-0.58 error message: orderSequentiallyCallbackException
+                this.closeFn(new container_utils_1.GenericError("orderSequentially callback exception", error));
+            }
             throw error; // throw the original error for the consumer of the runtime
         }
         finally {
@@ -1300,17 +1377,18 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * @param blobRedirectTable - A table passed during the attach process. While detached, blob upload is supported
      * using IDs generated locally. After attach, these IDs cannot be used, so this table maps the old local IDs to the
      * new storage IDs so requests can be redirected.
+     * @param telemetryContext - summary data passed through the layers for telemetry purposes
      */
-    createSummary(blobRedirectTable) {
+    createSummary(blobRedirectTable, telemetryContext) {
         if (blobRedirectTable) {
             this.blobManager.setRedirectTable(blobRedirectTable);
         }
-        const summarizeResult = this.dataStores.createSummary();
+        const summarizeResult = this.dataStores.createSummary(telemetryContext);
         if (!this.disableIsolatedChannels) {
             // Wrap data store summaries in .channels subtree.
             (0, summaryFormat_1.wrapSummaryInChannelsTree)(summarizeResult);
         }
-        this.addContainerStateToSummary(summarizeResult, true /* fullTree */, false /* trackState */);
+        this.addContainerStateToSummary(summarizeResult, true /* fullTree */, false /* trackState */, telemetryContext);
         return summarizeResult.summary;
     }
     async getAbsoluteUrl(relativeUrl) {
@@ -1322,15 +1400,15 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         }
         return this.context.getAbsoluteUrl(relativeUrl);
     }
-    async summarizeInternal(fullTree, trackState) {
-        const summarizeResult = await this.dataStores.summarize(fullTree, trackState);
+    async summarizeInternal(fullTree, trackState, telemetryContext) {
+        const summarizeResult = await this.dataStores.summarize(fullTree, trackState, telemetryContext);
         let pathPartsForChildren;
         if (!this.disableIsolatedChannels) {
             // Wrap data store summaries in .channels subtree.
             (0, summaryFormat_1.wrapSummaryInChannelsTree)(summarizeResult);
             pathPartsForChildren = [runtime_definitions_1.channelsTreeName];
         }
-        this.addContainerStateToSummary(summarizeResult, fullTree, trackState);
+        this.addContainerStateToSummary(summarizeResult, fullTree, trackState, telemetryContext);
         return Object.assign(Object.assign({}, summarizeResult), { id: "", pathPartsForChildren });
     }
     /**
@@ -1343,7 +1421,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         if (runGC) {
             gcStats = await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC });
         }
-        const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState);
+        const telemetryContext = new runtime_utils_1.TelemetryContext();
+        const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
+        this.logger.sendTelemetryEvent({ eventName: "SummarizeTelemetry", details: telemetryContext.serialize() });
         (0, common_utils_1.assert)(summary.type === protocol_definitions_1.SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
         return { stats, summary, gcStats };
     }
@@ -1707,9 +1787,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     }
     submit(type, content, localOpMetadata = undefined, opMetadata = undefined) {
         this.verifyNotClosed();
-        if (this.context.pendingLocalState !== undefined) {
-            this.closeFn(new container_utils_1.GenericError("containerRuntimeSubmitWithPendingLocalState"));
-        }
         // There should be no ops in detached container state!
         (0, common_utils_1.assert)(this.attachState !== container_definitions_1.AttachState.Detached, 0x132 /* "sending ops in detached container" */);
         let clientSequenceNumber = -1;
@@ -1840,6 +1917,17 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 (0, common_utils_1.unreachableCase)(type, `Unknown ContainerMessageType: ${type}`);
         }
     }
+    rollback(type, content, localOpMetadata) {
+        switch (type) {
+            case ContainerMessageType.FluidDataStoreOp:
+                // For operations, call rollbackDataStoreOp which will find the right store
+                // and trigger rollback on it.
+                this.dataStores.rollbackDataStoreOp(content, localOpMetadata);
+                break;
+            default:
+                throw new Error(`Can't rollback ${type}`);
+        }
+    }
     /** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
     async refreshLatestSummaryAck(proposalHandle, ackHandle, summaryRefSeq, summaryLogger) {
         const readAndParseBlob = async (id) => (0, driver_utils_1.readAndParse)(this.storage, id);
@@ -1884,8 +1972,43 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             return maybeSnapshot;
         });
     }
+    notifyAttaching(snapshot) {
+        var _a;
+        if ((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) {
+            this.baseSnapshotBlobs = serializedSnapshotStorage_1.SerializedSnapshotStorage.serializeTreeWithBlobContents(snapshot);
+        }
+    }
+    async getSnapshotBlobs() {
+        var _a;
+        if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad) ||
+            this.attachState !== container_definitions_1.AttachState.Attached || this.context.pendingLocalState) {
+            return;
+        }
+        (0, common_utils_1.assert)(!!this.context.baseSnapshot, 0x2e5 /* "Must have a base snapshot" */);
+        this.baseSnapshotBlobs = await serializedSnapshotStorage_1.SerializedSnapshotStorage.serializeTree(this.context.baseSnapshot, this.storage);
+    }
     getPendingLocalState() {
-        return this.pendingStateManager.getLocalState();
+        var _a;
+        if (!((_a = this.mc.config.getBoolean("enableOfflineLoad")) !== null && _a !== void 0 ? _a : this.runtimeOptions.enableOfflineLoad)) {
+            throw new container_utils_1.UsageError("can't get state when offline load disabled");
+        }
+        const previousPendingState = this.context.pendingLocalState;
+        if (previousPendingState) {
+            return {
+                pending: this.pendingStateManager.getLocalState(),
+                snapshotBlobs: previousPendingState.snapshotBlobs,
+                baseSnapshot: previousPendingState.baseSnapshot,
+                savedOps: this.savedOps,
+            };
+        }
+        (0, common_utils_1.assert)(!!this.context.baseSnapshot, 0x2e6 /* "Must have a base snapshot" */);
+        (0, common_utils_1.assert)(!!this.baseSnapshotBlobs, 0x2e7 /* "Must serialize base snapshot blobs before getting runtime state" */);
+        return {
+            pending: this.pendingStateManager.getLocalState(),
+            snapshotBlobs: this.baseSnapshotBlobs,
+            baseSnapshot: this.context.baseSnapshot,
+            savedOps: this.savedOps,
+        };
     }
     /**
      * * Forms a function that will request a Summarizer.
@@ -1913,6 +2036,15 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             return summarizer;
         };
     }
+    async processSavedOps(state) {
+        for (const op of state.savedOps) {
+            this.process(op, false);
+            await this.pendingStateManager.applyStashedOpsAt(op.sequenceNumber);
+        }
+        // we may not have seen every sequence number (because of system ops) so apply everything once we
+        // don't have any more saved ops
+        await this.pendingStateManager.applyStashedOpsAt();
+    }
 }
 exports.ContainerRuntime = ContainerRuntime;
 /**