@fluidframework/container-runtime 0.59.4002 → 1.0.2

package/src/runningSummarizer.ts

@@ -8,17 +8,18 @@ import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/common-ut
 import { UsageError } from "@fluidframework/container-utils";
 import {
     ISequencedDocumentMessage,
-    ISummaryConfiguration,
     MessageType,
 } from "@fluidframework/protocol-definitions";
 import { ChildLogger } from "@fluidframework/telemetry-utils";
+import {
+    ISummaryConfiguration,
+} from "./containerRuntime";
 import { SummarizeHeuristicRunner } from "./summarizerHeuristics";
 import {
     IEnqueueSummarizeOptions,
     ISummarizeOptions,
     ISummarizeHeuristicData,
     ISummarizeHeuristicRunner,
-    ISummarizerOptions,
     IOnDemandSummarizeOptions,
     EnqueueSummarizeResult,
     SummarizerStopReason,
@@ -57,7 +58,6 @@ export class RunningSummarizer implements IDisposable {
         summaryCollection: SummaryCollection,
         cancellationToken: ISummaryCancellationToken,
         stopSummarizerCallback: (reason: SummarizerStopReason) => void,
-        options?: Readonly<Partial<ISummarizerOptions>>,
     ): Promise<RunningSummarizer> {
         const summarizer = new RunningSummarizer(
             logger,
@@ -68,8 +68,7 @@ export class RunningSummarizer implements IDisposable {
             raiseSummarizingError,
             summaryCollection,
             cancellationToken,
-            stopSummarizerCallback,
-            options);
+            stopSummarizerCallback);
 
         await summarizer.waitStart();
 
@@ -107,7 +106,6 @@ export class RunningSummarizer implements IDisposable {
         private readonly summaryCollection: SummaryCollection,
         private readonly cancellationToken: ISummaryCancellationToken,
         private readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,
-        { disableHeuristics = false }: Readonly<Partial<ISummarizerOptions>> = {},
     ) {
         const telemetryProps: ISummarizeRunnerTelemetry = {
             summarizeCount: () => this.summarizeCount,
@@ -121,15 +119,23 @@ export class RunningSummarizer implements IDisposable {
             },
         );
 
-        if (!disableHeuristics) {
+        if (configuration.state !== "disableHeuristics") {
+            assert(this.configuration.state === "enabled", 0x2ea /* "Configuration state should be enabled" */);
             this.heuristicRunner = new SummarizeHeuristicRunner(
                 heuristicData,
-                configuration,
-                (reason) => this.trySummarize(reason));
+                this.configuration,
+                (reason) => this.trySummarize(reason),
+                this.logger);
         }
 
+        assert(
+            this.configuration.state !== "disabled",
+            0x2eb /* "Summary not supported with configuration disabled" */,
+        );
+
         // Cap the maximum amount of time client will wait for a summarize op ack to maxSummarizeAckWaitTime
         // configuration.maxAckWaitTime is composed from defaults, server values, and runtime overrides
+
         const maxAckWaitTime = Math.min(this.configuration.maxAckWaitTime, maxSummarizeAckWaitTime);
 
         this.pendingAckTimer = new PromiseTimer(
@@ -360,7 +366,11 @@ export class RunningSummarizer implements IDisposable {
                     return;
                 }
 
-                summaryAttempts++;
+                // We only want to attempt 1 summary when reason is "lastSummary"
+                if (++summaryAttempts > 1 && reason === "lastSummary") {
+                    return;
+                }
+
                 summaryAttemptsPerPhase++;
 
                 const { delaySeconds: regularDelaySeconds = 0, ...options } = attempts[summaryAttemptPhase];

package/src/serializedSnapshotStorage.ts

@@ -0,0 +1,146 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert, bufferToString, stringToBuffer } from "@fluidframework/common-utils";
+import { IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
+import {
+    ICreateBlobResponse,
+    ISnapshotTree,
+    ISummaryHandle,
+    ISummaryTree,
+    IVersion,
+} from "@fluidframework/protocol-definitions";
+import { ISnapshotTreeWithBlobContents } from "@fluidframework/container-definitions";
+
+/**
+ * Serialized blobs from a snapshot. Used to load offline.
+ */
+export interface ISerializedBaseSnapshotBlobs {
+    [id: string]: string;
+}
+
+/**
+ * A storage wrapper that can serialize blobs from a snapshot tree and then use them to rehydrate.
+ * Used in offline load/attached dehydration to save snapshot blobs that are still needed but may have been deleted.
+ */
+export class SerializedSnapshotStorage implements IDocumentStorageService {
+    constructor(
+        private readonly storageGetter: () => IDocumentStorageService,
+        private readonly blobs: ISerializedBaseSnapshotBlobs,
+    ) { }
+
+    public static async serializeTree(
+        snapshot: ISnapshotTree,
+        storage: IDocumentStorageService,
+    ): Promise<ISerializedBaseSnapshotBlobs> {
+        const blobs = {};
+        await this.serializeTreeCore(snapshot, blobs, storage);
+        return blobs;
+    }
+
+    private static async serializeTreeCore(
+        tree: ISnapshotTree,
+        blobs: ISerializedBaseSnapshotBlobs,
+        storage: IDocumentStorageService,
+    ) {
+        const treePs: Promise<any>[] = [];
+        for (const subTree of Object.values(tree.trees)) {
+            treePs.push(this.serializeTreeCore(subTree, blobs, storage));
+        }
+        for (const id of Object.values(tree.blobs)) {
+            const blob = await storage.readBlob(id);
+            // ArrayBufferLike will not survive JSON.stringify()
+            blobs[id] = bufferToString(blob, "utf8");
+        }
+        return Promise.all(treePs);
+    }
+
+    public static serializeTreeWithBlobContents(
+        snapshot: ISnapshotTreeWithBlobContents,
+    ): ISerializedBaseSnapshotBlobs {
+        const blobs = {};
+        this.serializeTreeWithBlobContentsCore(snapshot, blobs);
+        return blobs;
+    }
+
+    private static serializeTreeWithBlobContentsCore(
+        tree: ISnapshotTreeWithBlobContents,
+        blobs: ISerializedBaseSnapshotBlobs,
+    ) {
+        for (const subTree of Object.values(tree.trees)) {
+            this.serializeTreeWithBlobContentsCore(subTree, blobs);
+        }
+        for (const id of Object.values(tree.blobs)) {
+            const blob = tree.blobsContents[id];
+            assert(!!blob, 0x2ec /* "Blob must be present in blobsContents" */);
+            // ArrayBufferLike will not survive JSON.stringify()
+            blobs[id] = bufferToString(blob, "utf8");
+        }
+    }
+
+    private _storage?: IDocumentStorageService;
+    private get storage(): IDocumentStorageService {
+        // avoid calling it until we need it since it will be undefined if we're not connected
+        // and we shouldn't need it in this case anyway
+        if (this._storage) {
+            return this._storage;
+        }
+        this._storage = this.storageGetter();
+        return this._storage;
+    }
+
+    public get repositoryUrl(): string { return this.storage.repositoryUrl; }
+
+    /**
+     * Reads the object with the given ID, returns content in arrayBufferLike
+     */
+    public async readBlob(id: string): Promise<ArrayBufferLike> {
+        if (this.blobs[id] !== undefined) {
+            return stringToBuffer(this.blobs[id], "utf8");
+        }
+        return this.storage.readBlob(id);
+    }
+
+    /**
+     * Returns the snapshot tree.
+     */
+    // eslint-disable-next-line @rushstack/no-new-null
+    public async getSnapshotTree(version?: IVersion): Promise<ISnapshotTree | null> {
+        return this.storage.getSnapshotTree(version);
+    }
+
+    /**
+     * Retrieves all versions of the document starting at the specified versionId - or null if from the head
+     */
+    // eslint-disable-next-line @rushstack/no-new-null
+    public async getVersions(versionId: string | null, count: number): Promise<IVersion[]> {
+        return this.storage.getVersions(versionId, count);
+    }
+
+    /**
+     * Creates a blob out of the given buffer
+     */
+    public async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {
+        return this.storage.createBlob(file);
+    }
+
+    /**
+     * Uploads a summary tree to storage using the given context for reference of previous summary handle.
+     * The ISummaryHandles in the uploaded tree should have paths to indicate which summary object they are
+     * referencing from the previously acked summary.
+     * Returns the uploaded summary handle.
+     */
+    public async uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise<string> {
+        return this.storage.uploadSummaryWithContext(summary, context);
+    }
+
+    /**
+     * Retrieves the commit that matches the packfile handle. If the packfile has already been committed and the
+     * server has deleted it this call may result in a broken promise.
+     */
+    public async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {
+        return this.storage.downloadSummary(handle);
+    }
+}

package/src/summarizer.ts

@@ -19,8 +19,8 @@ import {
 } from "@fluidframework/core-interfaces";
 import {
     ISequencedDocumentMessage,
-    ISummaryConfiguration,
 } from "@fluidframework/protocol-definitions";
+import { ISummaryConfiguration } from "./containerRuntime";
 import { ICancellableSummarizerController } from "./runWhileConnectedCoordinator";
 import { summarizerClientType } from "./summarizerClientElection";
 import { SummaryCollection } from "./summaryCollection";
@@ -29,7 +29,6 @@ import { RunningSummarizer } from "./runningSummarizer";
 import {
     ISummarizer,
     ISummarizerInternalsProvider,
-    ISummarizerOptions,
     ISummarizerRuntime,
     ISummarizingWarning,
     SummarizerStopReason,
@@ -136,11 +135,9 @@ export class Summarizer extends EventEmitter implements ISummarizer {
         return fluidObject.ISummarizer;
     }
 
-    public async run(
-        onBehalfOf: string,
-        options?: Readonly<Partial<ISummarizerOptions>>): Promise<SummarizerStopReason> {
+    public async run(onBehalfOf: string): Promise<SummarizerStopReason> {
         try {
-            return await this.runCore(onBehalfOf, options);
+            return await this.runCore(onBehalfOf);
         } catch (error) {
             this.stop("summarizerException");
             throw SummarizingWarning.wrap(error, false /* logged */, this.logger);
@@ -165,9 +162,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
         this.runtime.closeFn();
     }
 
-    private async runCore(
-        onBehalfOf: string,
-        options?: Readonly<Partial<ISummarizerOptions>>): Promise<SummarizerStopReason> {
+    private async runCore(onBehalfOf: string): Promise<SummarizerStopReason> {
         const runCoordinator: ICancellableSummarizerController = await this.runCoordinatorCreateFn(this.runtime);
 
         // Wait for either external signal to cancel, or loss of connectivity.
@@ -184,7 +179,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
             return runCoordinator.waitCancelled;
         }
 
-        const runningSummarizer = await this.start(onBehalfOf, runCoordinator, options);
+        const runningSummarizer = await this.start(onBehalfOf, runCoordinator);
 
         // Wait for either external signal to cancel, or loss of connectivity.
         const stopReason = await stopP;
@@ -206,7 +201,8 @@ export class Summarizer extends EventEmitter implements ISummarizer {
         // cons of #2 substantially.
 
         // Cleanup after running
-        await runningSummarizer.waitStop(!runCoordinator.cancelled /* allowLastSummary */);
+        await runningSummarizer.waitStop(
+            !runCoordinator.cancelled && Summarizer.stopReasonCanRunLastSummary(stopReason));
 
         // Propagate reason and ensure that if someone is waiting for cancellation token, they are moving to exit
         runCoordinator.stop(stopReason);
@@ -214,19 +210,28 @@ export class Summarizer extends EventEmitter implements ISummarizer {
         return stopReason;
     }
 
+    /**
+     * Should we try to run a last summary for the given stop reason?
+     * Currently only allows "parentNotConnected"
+     * @param stopReason - SummarizerStopReason
+     * @returns - true if the stop reason can run a last summary
+     */
+    public static stopReasonCanRunLastSummary(stopReason: SummarizerStopReason): boolean {
+        return stopReason === "parentNotConnected";
+    }
+
     /**
      * Put the summarizer in a started state, including creating and initializing the RunningSummarizer.
      * The start request can come either from the SummaryManager (in the auto-summarize case) or from the user
      * (in the on-demand case).
      * @param onBehalfOf - ID of the client that requested that the summarizer start
      * @param runCoordinator - cancellation token
-     * @param options - options to forward to the RunningSummarizer
+     * @param newConfig - Summary configuration to override the existing config when invoking the RunningSummarizer.
      * @returns - Promise that is fulfilled when the RunningSummarizer is ready
      */
     private async start(
         onBehalfOf: string,
-        runCoordinator: ICancellableSummarizerController,
-        options?: Readonly<Partial<ISummarizerOptions>>): Promise<RunningSummarizer> {
+        runCoordinator: ICancellableSummarizerController): Promise<RunningSummarizer> {
         if (this.runningSummarizer) {
             if (this.runningSummarizer.disposed) {
                 throw new UsageError("Starting a disposed summarizer");
@@ -272,7 +277,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
             this.summaryCollection,
             runCoordinator /* cancellationToken */,
             (reason) => runCoordinator.stop(reason), /* stopSummarizerCallback */
-            options,
         );
         this.runningSummarizer = runningSummarizer;
         this.starting = false;
@@ -343,7 +347,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
             coordinatorCreateP.then((runCoordinator) => {
                 // Successully created the cancellation token. Start the summarizer.
                 // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-                const startP = this.start(this.runtime.clientId!, runCoordinator, { disableHeuristics: true });
+                const startP = this.start(this.runtime.clientId!, runCoordinator);
                 startP.then(async (runningSummarizer) => {
                     // Successfully started the summarizer. Run it.
                     runningSummarizer.summarizeOnDemand(builder, ...args);

package/src/summarizerHeuristics.ts

@@ -3,9 +3,15 @@
  * Licensed under the MIT License.
  */
 
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { Timer } from "@fluidframework/common-utils";
-import { ISummaryConfiguration } from "@fluidframework/protocol-definitions";
-import { ISummarizeHeuristicData, ISummarizeHeuristicRunner, ISummarizeAttempt } from "./summarizerTypes";
+import { ISummaryConfigurationHeuristics } from "./containerRuntime";
+
+import {
+    ISummarizeHeuristicData,
+    ISummarizeHeuristicRunner,
+    ISummarizeAttempt,
+} from "./summarizerTypes";
 import { SummarizeReason } from "./summaryGenerator";
 
 /** Simple implementation of class for tracking summarize heuristic data. */
@@ -51,16 +57,18 @@ export class SummarizeHeuristicData implements ISummarizeHeuristicData {
  */
 export class SummarizeHeuristicRunner implements ISummarizeHeuristicRunner {
     private readonly idleTimer: Timer;
+    private readonly minOpsForLastSummaryAttempt: number;
 
     public constructor(
         private readonly heuristicData: ISummarizeHeuristicData,
-        private readonly configuration: ISummaryConfiguration,
+        private readonly configuration: ISummaryConfigurationHeuristics,
         private readonly trySummarize: (reason: SummarizeReason) => void,
-        private readonly minOpsForAttemptOnClose = 50,
+        private readonly logger: ITelemetryLogger,
     ) {
         this.idleTimer = new Timer(
             this.configuration.idleTime,
             () => this.trySummarize("idle"));
+        this.minOpsForLastSummaryAttempt = this.configuration.minOpsForLastSummaryAttempt;
     }
 
     public get opsSinceLastAck(): number {
@@ -83,7 +91,15 @@ export class SummarizeHeuristicRunner implements ISummarizeHeuristicRunner {
 
     public shouldRunLastSummary(): boolean {
         const opsSinceLastAck = this.opsSinceLastAck;
-        return (opsSinceLastAck > this.minOpsForAttemptOnClose);
+        const minOpsForLastSummaryAttempt = this.minOpsForLastSummaryAttempt;
+
+        this.logger.sendTelemetryEvent({
+            eventName: "ShouldRunLastSummary",
+            opsSinceLastAck,
+            minOpsForLastSummaryAttempt,
+        });
+
+        return opsSinceLastAck >= minOpsForLastSummaryAttempt;
     }
 
     public dispose() {

package/src/summarizerTypes.ts

@@ -67,7 +67,9 @@ export interface ISummarizerInternalsProvider {
     ): Promise<void>;
 }
 
-/** Options that control the behavior of a running summarizer. */
+/**
+ * @deprecated Options that control the behavior of a running summarizer.
+ * */
 export interface ISummarizerOptions {
     /**
      * Set to true to disable the default heuristics from running; false by default.
@@ -309,7 +311,7 @@ export interface ISummarizer extends
     /* Closes summarizer. Any pending processes (summary in flight) are abandoned. */
     close(): void;
 
-    run(onBehalfOf: string, options?: Readonly<Partial<ISummarizerOptions>>): Promise<SummarizerStopReason>;
+    run(onBehalfOf: string, disableHeuristics?: boolean): Promise<SummarizerStopReason>;
 
     /**
      * Attempts to generate a summary on demand. If already running, takes no action.

package/src/summaryManager.ts

@@ -11,7 +11,6 @@ import { ISummarizerClientElection } from "./summarizerClientElection";
 import { IThrottler } from "./throttler";
 import {
     ISummarizer,
-    ISummarizerOptions,
     SummarizerStopReason,
 } from "./summarizerTypes";
 import { SummaryCollection } from "./summaryCollection";
@@ -97,7 +96,7 @@ export class SummaryManager implements IDisposable {
             initialDelayMs = defaultInitialDelayMs,
             opsToBypassInitialDelay = defaultOpsToBypassInitialDelay,
         }: Readonly<Partial<ISummaryManagerConfig>> = {},
-        private readonly summarizerOptions?: Readonly<Partial<ISummarizerOptions>>,
+        private readonly disableHeuristics?: boolean,
     ) {
         this.logger = ChildLogger.create(
             parentLogger,
@@ -141,12 +140,12 @@ export class SummaryManager implements IDisposable {
         // enforce connectedState.clientId === clientElection.electedClientId. But once we're Running, we should
         // only transition to Stopping when the electedParentId changes. Stopping the summarizer without
         // changing the electedParent will just cause us to transition to Starting again.
-        if (!this.connectedState.connected) {
-            return { shouldSummarize: false, stopReason: "parentNotConnected" };
-        } else if (this.connectedState.clientId !== this.clientElection.electedParentId ||
+        if (this.connectedState.clientId !== this.clientElection.electedParentId ||
             (this.state !== SummaryManagerState.Running &&
                 this.connectedState.clientId !== this.clientElection.electedClientId)) {
             return { shouldSummarize: false, stopReason: "parentShouldNotSummarize" };
+        } else if (!this.connectedState.connected) {
+            return { shouldSummarize: false, stopReason: "parentNotConnected" };
         } else if (this.disposed) {
             assert(false, 0x260 /* "Disposed should mean disconnected!" */);
         } else {
@@ -227,7 +226,7 @@ export class SummaryManager implements IDisposable {
             return PerformanceEvent.timedExecAsync(
                 this.logger,
                 { eventName: "RunningSummarizer", attempt: this.startThrottler.numAttempts },
-                async () => summarizer.run(clientId, this.summarizerOptions),
+                async () => summarizer.run(clientId, this.disableHeuristics),
             );
         }).then((reason: string) => {
             this.logger.sendTelemetryEvent({