@fluidframework/runtime-utils 1.2.3-83900 → 2.0.0-internal.1.0.0

package/src/summarizerNode/summarizerNode.ts

@@ -24,9 +24,6 @@ import { assert, unreachableCase } from "@fluidframework/common-utils";
 import { mergeStats, convertToSummaryTree, calculateStats } from "../summaryUtils";
 import { ReadAndParseBlob } from "../utils";
 import {
-    decodeSummary,
-    encodeSummary,
-    EncodeSummaryParam,
     EscapedPath,
     ICreateChildDetails,
     IInitialSummary,
@@ -63,7 +60,6 @@ export class SummarizerNode implements IRootSummarizerNode {
 
     protected readonly children = new Map<string, SummarizerNode>();
     protected readonly pendingSummaries = new Map<string, SummaryNode>();
-    private readonly outstandingOps: ISequencedDocumentMessage[] = [];
     private wipReferenceSequenceNumber: number | undefined;
     private wipLocalPaths: { localPath: EscapedPath; additionalPath?: EscapedPath; } | undefined;
     private wipSkipRecursion = false;
@@ -112,52 +108,12 @@ export class SummarizerNode implements IRootSummarizerNode {
             }
         }
 
-        try {
-            const result = await this.summarizeInternalFn(fullTree, true, telemetryContext);
-            this.wipLocalPaths = { localPath: EscapedPath.create(result.id) };
-            if (result.pathPartsForChildren !== undefined) {
-                this.wipLocalPaths.additionalPath = EscapedPath.createAndConcat(result.pathPartsForChildren);
-            }
-            return { summary: result.summary, stats: result.stats };
-        } catch (error) {
-            if (this.throwOnError || this.trackingSequenceNumber < this._changeSequenceNumber) {
-                throw error;
-            }
-            const latestSummary = this._latestSummary;
-            const initialSummary = this.initialSummary;
-
-            let encodeParam: EncodeSummaryParam;
-            let localPath: EscapedPath;
-            if (latestSummary !== undefined) {
-                // Create using handle of latest acked summary
-                encodeParam = {
-                    fromSummary: true,
-                    summaryNode: latestSummary,
-                };
-                localPath = latestSummary.localPath;
-            } else if (initialSummary?.summary !== undefined) {
-                // Create using initial summary from attach op
-                encodeParam = {
-                    fromSummary: false,
-                    initialSummary: initialSummary.summary,
-                };
-                localPath = EscapedPath.create(initialSummary.id);
-            } else {
-                // No base summary to reference
-                throw error;
-            }
-            this.wipSummaryLogger.sendErrorEvent({
-                eventName: "SummarizingWithBasePlusOps",
-            },
-            error);
-            const summary = encodeSummary(encodeParam, this.outstandingOps);
-            this.wipLocalPaths = {
-                localPath,
-                additionalPath: summary.additionalPath,
-            };
-            this.wipSkipRecursion = true;
-            return { summary: summary.summary, stats: summary.stats };
+        const result = await this.summarizeInternalFn(fullTree, true, telemetryContext);
+        this.wipLocalPaths = { localPath: EscapedPath.create(result.id) };
+        if (result.pathPartsForChildren !== undefined) {
+            this.wipLocalPaths.additionalPath = EscapedPath.createAndConcat(result.pathPartsForChildren);
         }
+        return { summary: result.summary, stats: result.stats };
     }
 
     /**
@@ -286,7 +242,12 @@ export class SummarizerNode implements IRootSummarizerNode {
         );
         return { latestSummaryUpdated: true, wasSummaryTracked: false, snapshot: snapshotTree };
     }
-
+/**
+ * Called when we get an ack from the server for a summary we've just sent. Updates the reference state of this node
+ * from the state in the pending summary queue.
+ * @param proposalHandle - Handle for the current proposal.
+ * @param referenceSequenceNumber -  reference sequence number of sent summary.
+ */
     protected refreshLatestSummaryFromPending(
         proposalHandle: string,
         referenceSequenceNumber: number,
@@ -312,7 +273,6 @@ export class SummarizerNode implements IRootSummarizerNode {
         this.refreshLatestSummaryCore(referenceSequenceNumber);
 
         this._latestSummary = summaryNode;
-
         // Propagate update to all child nodes
         for (const child of this.children.values()) {
             child.refreshLatestSummaryFromPending(proposalHandle, referenceSequenceNumber);
@@ -334,15 +294,14 @@ export class SummarizerNode implements IRootSummarizerNode {
 
         this.refreshLatestSummaryCore(referenceSequenceNumber);
 
-        const { baseSummary, pathParts } = decodeSummary(snapshotTree, correlatedSummaryLogger);
-
         this._latestSummary = new SummaryNode({
             referenceSequenceNumber,
             basePath,
             localPath,
         });
 
-        const { childrenTree, childrenPathPart } = parseSummaryForSubtrees(baseSummary);
+        const pathParts: string[] = [];
+        const { childrenTree, childrenPathPart } = parseSummaryForSubtrees(snapshotTree);
         if (childrenPathPart !== undefined) {
             pathParts.push(childrenPathPart);
         }
@@ -376,14 +335,6 @@ export class SummarizerNode implements IRootSummarizerNode {
                 this.pendingSummaries.delete(key);
             }
         }
-
-        // Clear earlier outstanding ops
-        while (
-            this.outstandingOps.length > 0
-            && this.outstandingOps[0].sequenceNumber <= referenceSequenceNumber
-        ) {
-            this.outstandingOps.shift();
-        }
     }
 
     public loadBaseSummaryWithoutDifferential(snapshot: ISnapshotTree) {
@@ -398,46 +349,22 @@ export class SummarizerNode implements IRootSummarizerNode {
     public async loadBaseSummary(
         snapshot: ISnapshotTree,
         readAndParseBlob: ReadAndParseBlob,
-    ): Promise<{ baseSummary: ISnapshotTree; outstandingOps: ISequencedDocumentMessage[]; }> {
-        const decodedSummary = decodeSummary(snapshot, this.defaultLogger);
-        const outstandingOps = await decodedSummary.getOutstandingOps(readAndParseBlob);
-
-        const { childrenPathPart } = parseSummaryForSubtrees(decodedSummary.baseSummary);
+    ): Promise<ISnapshotTree> {
+        const pathParts: string[] = [];
+        const { childrenPathPart } = parseSummaryForSubtrees(snapshot);
         if (childrenPathPart !== undefined) {
-            decodedSummary.pathParts.push(childrenPathPart);
-        }
-
-        if (decodedSummary.pathParts.length > 0 && this._latestSummary !== undefined) {
-            this._latestSummary.additionalPath = EscapedPath.createAndConcat(decodedSummary.pathParts);
+            pathParts.push(childrenPathPart);
         }
 
-        // Defensive assertion: tracking number should already exceed this number.
-        // This is probably a little excessive; can remove when stable.
-        if (outstandingOps.length > 0) {
-            const newOpsLatestSeq = outstandingOps[outstandingOps.length - 1].sequenceNumber;
-            assert(
-                newOpsLatestSeq <= this.trackingSequenceNumber,
-                0x1a9 /* "When loading base summary, expected outstanding ops <= tracking sequence number" */,
-            );
+        if (pathParts.length > 0 && this._latestSummary !== undefined) {
+            this._latestSummary.additionalPath = EscapedPath.createAndConcat(pathParts);
         }
 
-        return {
-            baseSummary: decodedSummary.baseSummary,
-            outstandingOps,
-        };
+        return snapshot;
     }
 
     public recordChange(op: ISequencedDocumentMessage): void {
-        const lastOp = this.outstandingOps[this.outstandingOps.length - 1];
-        if (lastOp !== undefined) {
-            assert(
-                lastOp.sequenceNumber < op.sequenceNumber,
-                0x1aa /* Out of order change recorded */,
-            );
-        }
         this.invalidate(op.sequenceNumber);
-        this.trackingSequenceNumber = op.sequenceNumber;
-        this.outstandingOps.push(op);
     }
 
     public invalidate(sequenceNumber: number): void {
@@ -460,13 +387,6 @@ export class SummarizerNode implements IRootSummarizerNode {
     }
 
     private readonly canReuseHandle: boolean;
-    private readonly throwOnError: boolean;
-    /**
-     * Sequence number of latest tracked op. This updates during recordChange,
-     * but not for invalidate since we don't have the op. If this drifts from
-     * changeSequenceNumber and we try to create a differential summary we assert.
-     */
-    private trackingSequenceNumber: number;
 
     /**
      * Do not call constructor directly.
@@ -483,11 +403,6 @@ export class SummarizerNode implements IRootSummarizerNode {
         protected wipSummaryLogger?: ITelemetryLogger,
     ) {
         this.canReuseHandle = config.canReuseHandle ?? true;
-        // BUGBUG: Seeing issues with differential summaries.
-        // this will disable them, and throw instead
-        // while we continue to investigate
-        this.throwOnError = true; // config.throwOnFailure ?? false;
-        this.trackingSequenceNumber = this._changeSequenceNumber;
     }
 
     public createChild(
@@ -517,7 +432,8 @@ export class SummarizerNode implements IRootSummarizerNode {
         );
 
         // There may be additional state that has to be updated in this child. For example, if a summary is being
-        // tracked, the child's summary tracking state needs to be updated too.
+        // tracked, the child's summary tracking state needs to be updated too. Same goes for pendingSummaries we might
+        // have outstanding on the parent in case we realize nodes in between Summary Op and Summary Ack.
         this.maybeUpdateChildState(child);
 
         this.children.set(id, child);
@@ -620,6 +536,8 @@ export class SummarizerNode implements IRootSummarizerNode {
     /**
      * Updates the state of the child if required. For example, if a summary is currently being  tracked, the child's
      * summary tracking state needs to be updated too.
+     * Also, in case a child node gets realized in between Summary Op and Summary Ack, let's initialize the child's
+     * pending summary as well.
      * @param child - The child node whose state is to be updated.
      */
     protected maybeUpdateChildState(child: SummarizerNode) {
@@ -628,8 +546,23 @@ export class SummarizerNode implements IRootSummarizerNode {
         if (this.isTrackingInProgress()) {
             child.wipReferenceSequenceNumber = this.wipReferenceSequenceNumber;
         }
+        // In case we have pending summaries on the parent, let's initialize it on the child.
+        if (child._latestSummary !== undefined) {
+            for (const [key, value] of this.pendingSummaries.entries()) {
+                const newLatestSummaryNode = new SummaryNode({
+                    referenceSequenceNumber: value.referenceSequenceNumber,
+                    basePath: child._latestSummary.basePath,
+                    localPath: child._latestSummary.localPath,
+                });
+
+                child.addPendingSummary(key, newLatestSummaryNode);
+            }
+        }
     }
 
+    protected addPendingSummary(key: string, summary: SummaryNode) {
+        this.pendingSummaries.set(key, summary);
+    }
     /**
      * Tells whether summary tracking is in progress. True if "startSummary" API is called before summarize.
      */

package/src/summarizerNode/summarizerNodeUtils.ts

@@ -4,22 +4,14 @@
  */
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { assert } from "@fluidframework/common-utils";
 import {
     ISnapshotTree,
-    ISequencedDocumentMessage,
-    SummaryType,
     ISummaryTree,
     SummaryObject,
 } from "@fluidframework/protocol-definitions";
 import { channelsTreeName, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
-import { SummaryTreeBuilder } from "../summaryUtils";
 import { ReadAndParseBlob } from "../utils";
 
-const baseSummaryTreeKey = "_baseSummary";
-const outstandingOpsBlobKey = "_outstandingOps";
-const maxDecodeDepth = 100;
-
 /**
  * Return value of refreshSummaryAck function. There can be three different scenarios based on the passed params:
  * 1. The latest summary was not udpated.
@@ -137,88 +129,6 @@ export class SummaryNode {
     }
 }
 
-/** Result from decoding summary which may have been a differential summary. */
-interface IDecodedSummary {
-    /** The innermost base summary which is not itself a differential summary */
-    readonly baseSummary: ISnapshotTree;
-    /** The entire path name to the innermost base summary */
-    readonly pathParts: string[];
-    /** Function to fetch all outstanding ops since the innermost base summary */
-    getOutstandingOps(readAndParseBlob: ReadAndParseBlob): Promise<ISequencedDocumentMessage[]>;
-}
-
-/**
- * Checks if the snapshot is created by referencing a previous successful
- * summary plus outstanding ops. If so, it will recursively "decode" it until
- * it gets to the last successful summary (the base summary) and returns that
- * as well as a function for fetching the outstanding ops. Also returns the
- * full path to the previous base summary for child summarizer nodes to use as
- * their base path when necessary.
- * @param snapshot - snapshot tree to decode
- */
-export function decodeSummary(
-    snapshot: ISnapshotTree,
-    logger: Pick<ITelemetryLogger, "sendTelemetryEvent">,
-): IDecodedSummary {
-    let baseSummary = snapshot;
-    const pathParts: string[] = [];
-    const opsBlobs: string[] = [];
-
-    for (let i = 0; ; i++) {
-        if (i > maxDecodeDepth) {
-            logger.sendTelemetryEvent({
-                eventName: "DecodeSummaryMaxDepth",
-                maxDecodeDepth,
-            });
-        }
-        const outstandingOpsBlob = baseSummary.blobs[outstandingOpsBlobKey];
-        const newBaseSummary = baseSummary.trees[baseSummaryTreeKey];
-        if (outstandingOpsBlob === undefined && newBaseSummary === undefined) {
-            return {
-                baseSummary,
-                pathParts,
-                async getOutstandingOps(readAndParseBlob: ReadAndParseBlob) {
-                    let outstandingOps: ISequencedDocumentMessage[] = [];
-                    for (const opsBlob of opsBlobs) {
-                        const newOutstandingOps = await readAndParseBlob<ISequencedDocumentMessage[]>(opsBlob);
-                        if (outstandingOps.length > 0 && newOutstandingOps.length > 0) {
-                            const latestSeq = outstandingOps[outstandingOps.length - 1].sequenceNumber;
-                            const newEarliestSeq = newOutstandingOps[0].sequenceNumber;
-                            if (newEarliestSeq <= latestSeq) {
-                                logger.sendTelemetryEvent({
-                                    eventName: "DuplicateOutstandingOps",
-                                    // eslint-disable-next-line max-len
-                                    message: `newEarliestSeq <= latestSeq in decodeSummary: ${newEarliestSeq} <= ${latestSeq}`,
-                                });
-                                while (newOutstandingOps.length > 0
-                                    && newOutstandingOps[0].sequenceNumber <= latestSeq) {
-                                    newOutstandingOps.shift();
-                                }
-                            }
-                        }
-                        outstandingOps = outstandingOps.concat(newOutstandingOps);
-                    }
-                    return outstandingOps;
-                },
-            };
-        }
-
-        assert(!!outstandingOpsBlob, 0x1af /* "Outstanding ops blob missing, but base summary tree exists" */);
-        assert(newBaseSummary !== undefined, 0x1b0 /* "Base summary tree missing, but outstanding ops blob exists" */);
-        baseSummary = newBaseSummary;
-        pathParts.push(baseSummaryTreeKey);
-        opsBlobs.unshift(outstandingOpsBlob);
-    }
-}
-
-/**
- * Summary tree which is a handle of the previous successfully acked summary
- * and a blob of the outstanding ops since that summary.
- */
-interface IEncodedSummary extends ISummaryTreeWithStats {
-    readonly additionalPath: EscapedPath;
-}
-
 /**
  * Parameter to help encode summary with conditional behavior.
  * When fromSummary is true, it will contain the SummaryNode of
@@ -234,42 +144,6 @@ export type EncodeSummaryParam = {
     initialSummary: ISummaryTreeWithStats;
 };
 
-/**
- * Creates a summary tree which is a handle of the previous successfully acked summary
- * and a blob of the outstanding ops since that summary. If there is no acked summary yet,
- * it will create with the tree found in the initial attach op and the blob of outstanding ops.
- * @param summaryParam - information about last acked summary and paths to encode if from summary,
- * otherwise the initial summary from the attach op.
- * @param outstandingOps - outstanding ops since last acked summary
- */
-export function encodeSummary(
-    summaryParam: EncodeSummaryParam,
-    outstandingOps: ISequencedDocumentMessage[],
-): IEncodedSummary {
-    let additionalPath = EscapedPath.create(baseSummaryTreeKey);
-
-    const builder = new SummaryTreeBuilder();
-    builder.addBlob(outstandingOpsBlobKey, JSON.stringify(outstandingOps));
-
-    if (summaryParam.fromSummary) {
-        // Create using handle of latest acked summary
-        const summaryNode = summaryParam.summaryNode;
-        if (summaryNode.additionalPath !== undefined) {
-            additionalPath = additionalPath.concat(summaryNode.additionalPath);
-        }
-        builder.addHandle(baseSummaryTreeKey, SummaryType.Tree, summaryNode.fullPath.path);
-    } else {
-        // Create using initial summary from attach op
-        builder.addWithStats(baseSummaryTreeKey, summaryParam.initialSummary);
-    }
-
-    const summary = builder.getSummaryTree();
-    return {
-        ...summary,
-        additionalPath,
-    };
-}
-
 /**
  * Information about the initial summary tree found from an attach op.
  */